pax_global_header00006660000000000000000000000064152701413760014520gustar00rootroot0000000000000052 comment=a2f2415c86f7d35212ae8c0d5dc5e4a22a6c6191 zarr-developers-zarr-python-a2f2415/000077500000000000000000000000001527014137600174655ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/.git-blame-ignore-revs000066400000000000000000000002321527014137600235620ustar00rootroot00000000000000# lint codebase with black and ruff 4e348d6b80c96da461fd866576c971b8a659ba15 # migrate from black to ruff format 22cea005629913208a85799372e045f353744add zarr-developers-zarr-python-a2f2415/.git_archival.txt000066400000000000000000000002011527014137600227310ustar00rootroot00000000000000node: a2f2415c86f7d35212ae8c0d5dc5e4a22a6c6191 node-date: 2026-10-27T16:27:58+01:00 describe-name: v3.4.2 ref-names: tag: v3.4.2 zarr-developers-zarr-python-a2f2415/.gitattributes000066400000000000000000000005141527014137600223600ustar00rootroot00000000000000*.py linguist-language=python *.ipynb linguist-documentation .git_archival.txt export-subst # just refuses to format a CRLF file, so `just just-check` would fail on a Windows # checkout with core.autocrlf=true before the contributor has changed anything. Justfile text eol=lf */justfile text eol=lf packages/*/justfile text eol=lf zarr-developers-zarr-python-a2f2415/.github/000077500000000000000000000000001527014137600210255ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/.github/CODEOWNERS000066400000000000000000000000701527014137600224150ustar00rootroot00000000000000zarr/_storage/absstore.py @zarr-developers/azure-team zarr-developers-zarr-python-a2f2415/.github/CONTRIBUTING.md000066400000000000000000000002621527014137600232560ustar00rootroot00000000000000Contributing ============ Please see the [project documentation](https://zarr.readthedocs.io/en/stable/developers/contributing.html) for information about contributing to Zarr. zarr-developers-zarr-python-a2f2415/.github/ISSUE_TEMPLATE/000077500000000000000000000000001527014137600232105ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/.github/ISSUE_TEMPLATE/bug_report.yml000066400000000000000000000051331527014137600261050ustar00rootroot00000000000000name: Bug Report description: Report incorrect behaviour in the library. labels: ["bug"] body: - type: markdown attributes: value: | Please provide the following information. - type: input id: Zarr-version attributes: label: Zarr version description: Value of ``zarr.__version__`` placeholder: v2.10.2, v2.11.3, v2.12.0, etc. validations: required: true - type: input id: Numcodecs-version attributes: label: Numcodecs version description: Value of ``numcodecs.__version__`` placeholder: v0.8.1, v0.9.0, v0.10.0, etc. validations: required: true - type: input id: Python-version attributes: label: Python Version description: Version of Python interpreter placeholder: 3.10, 3.11, 3.12 etc. validations: required: true - type: input id: OS attributes: label: Operating System description: Operating System placeholder: (Linux/Windows/Mac) validations: required: true - type: input id: installation attributes: label: Installation description: How was Zarr installed? placeholder: e.g., "using pip into virtual environment", or "using conda" validations: required: true - type: textarea id: description attributes: label: Description description: Explain why the current behavior is a problem, what the expected output/behaviour is, and why the expected output/behaviour is a better solution. validations: required: true - type: textarea id: reproduce attributes: label: Steps to reproduce description: Minimal, reproducible code sample. Must list dependencies in [inline script metadata](https://packaging.python.org/en/latest/specifications/inline-script-metadata/#example). When put in a file named `issue.py` calling `uv run issue.py` should show the issue. value: | ```python # /// script # requires-python = ">=3.12" # dependencies = [ # "zarr@git+https://github.com/zarr-developers/zarr-python.git@main", # ] # /// # # This script automatically imports the development branch of zarr to check for issues import zarr # your reproducer code # zarr.print_debug_info() ``` validations: required: true - type: textarea id: additional-output attributes: label: Additional output description: If you think it might be relevant, please provide the output from ``pip freeze`` or ``conda env export`` depending on which was used to install Zarr. zarr-developers-zarr-python-a2f2415/.github/ISSUE_TEMPLATE/config.yml000066400000000000000000000012221527014137600251750ustar00rootroot00000000000000blank_issues_enabled: true contact_links: - name: Propose a new Zarr specification feature url: https://github.com/zarr-developers/zarr-specs about: A new feature for the Zarr storage specification should be opened on the zarr-specs repository. - name: Discuss something on ZulipChat url: https://ossci.zulipchat.com/ about: For questions like "How do I do X with Zarr?", consider posting your question to our developer chat. - name: Discuss something on GitHub Discussions url: https://github.com/zarr-developers/zarr-python/discussions about: For questions like "How do I do X with Zarr?", you can move to GitHub Discussions. zarr-developers-zarr-python-a2f2415/.github/ISSUE_TEMPLATE/documentation.yml000066400000000000000000000012021527014137600265770ustar00rootroot00000000000000name: Documentation Improvement description: Report missing or wrong documentation. Alternatively, you can just open a pull request with the suggested change. title: "DOC: " labels: [documentation, help wanted] body: - type: textarea attributes: label: Describe the issue linked to the documentation description: > Please provide a description of what documentation you believe needs to be fixed/improved. validations: required: true - type: textarea attributes: label: Suggested fix for documentation description: > Please explain the suggested fix and why it's better than the existing documentation. zarr-developers-zarr-python-a2f2415/.github/ISSUE_TEMPLATE/feature_request.yml000066400000000000000000000005111527014137600271330ustar00rootroot00000000000000name: Feature Request description: Request a new feature for zarr-python # labels: [] body: - type: textarea attributes: label: Describe the new feature you'd like description: > Please provide a description of what new feature or functionality you'd like to see in zarr-python. validations: required: true zarr-developers-zarr-python-a2f2415/.github/ISSUE_TEMPLATE/release-checklist.md000066400000000000000000000057061527014137600271310ustar00rootroot00000000000000--- name: Zarr-Python release checklist about: Checklist for a new Zarr-Python release. [For project maintainers only!] title: Release Zarr-Python vX.Y.Z labels: release-checklist assignees: '' --- **Release**: [v3.x.x](https://github.com/zarr-developers/zarr-python/milestones/?) **Scheduled Date**: 20YY/MM/DD **Priority PRs/issues to complete prior to release** - [ ] Priority pull request #X **Before release**: - [ ] Check [SPEC 0](https://scientific-python.org/specs/spec-0000/#support-window) to see if the minimum supported version of Python or NumPy needs bumping. - [ ] Verify that the latest CI workflows on `main` are passing: [Tests](https://github.com/zarr-developers/zarr-python/actions/workflows/test.yml), [GPU Tests](https://github.com/zarr-developers/zarr-python/actions/workflows/gpu_test.yml), [Hypothesis](https://github.com/zarr-developers/zarr-python/actions/workflows/hypothesis.yaml), [Docs](https://github.com/zarr-developers/zarr-python/actions/workflows/docs.yml), [Lint](https://github.com/zarr-developers/zarr-python/actions/workflows/lint.yml), [Wheels](https://github.com/zarr-developers/zarr-python/actions/workflows/releases.yml). - [ ] Run the [downstream tests](https://github.com/zarr-developers/zarr-python/actions/workflows/downstream.yml) against `main`: go to the workflow page, click "Run workflow", and select the `main` branch. Verify that the Xarray and numcodecs integration tests pass. - [ ] Open a release PR with the changelog entries for the upcoming release, generated with `uv run --only-group release towncrier build --version x.y.z`. - [ ] Review the release PR and verify the changelog in `docs/release-notes.md` looks correct. - [ ] Merge the release PR. **Release**: - [ ] [Draft a new GitHub Release](https://github.com/zarr-developers/zarr-python/releases/new) with tag `vX.Y.Z` targeting `main`. Use "Generate release notes" for the description. - [ ] Verify the release is published on [PyPI](https://pypi.org/project/zarr/) and [ReadTheDocs](https://zarr.readthedocs.io/en/stable/). **After release**: - [ ] Review and merge the pull request on the conda-forge [zarr-feedstock](https://github.com/conda-forge/zarr-feedstock) that will be automatically generated. --- - [ ] Party :tada: ---
Releasing from a branch other than main In rare cases (e.g. patch releases for an older minor version), you may need to release from a dedicated release branch (e.g. `3.1.x`): - Create the release branch from the appropriate tag if it doesn't already exist. - Cherry-pick or backport the necessary commits onto the branch. - Run `towncrier build --version x.y.z` and open the release PR against the release branch instead of `main`. - Run the downstream tests against the release branch instead of `main`. - When drafting the GitHub Release, set the target to the release branch instead of `main`. - After the release, ensure any relevant changelog updates are also reflected on `main`.
zarr-developers-zarr-python-a2f2415/.github/PULL_REQUEST_TEMPLATE.md000066400000000000000000000025401527014137600246270ustar00rootroot00000000000000 ## Summary [Describe what this PR changes and why, in your own words.] ## For reviewers [What would you most value a second look at? What are you already confident in? For a refactor, say whether behavior is meant to be unchanged.] ## Author attestation - [ ] I am a human, these are my changes, and I have reviewed and understood every change and can explain why each is correct. ## TODO * [ ] Add unit tests and/or doctests in docstrings * [ ] Add docstrings and API docs for any new/modified user-facing classes and functions * [ ] New/modified features documented in `docs/user-guide/*.md` * [ ] Changes documented as a new file in `changes/` * [ ] GitHub Actions have all passed * [ ] Test coverage is 100% (Codecov passes) zarr-developers-zarr-python-a2f2415/.github/dependabot.yml000066400000000000000000000016151527014137600236600ustar00rootroot00000000000000--- version: 2 updates: # Updates for main - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" groups: actions: patterns: - "*" cooldown: default-days: 7 # Keep the pinned dev tooling in pyproject.toml's [dependency-groups] and the # uv.lock current. Without this the exact pins (e.g. pytest) would never be # bumped automatically and would silently rot. # # `allow: dependency-type: direct` restricts updates to dependencies declared # in pyproject.toml. Transitive deps in uv.lock are then only updated as a # side effect of a direct bump, never via a standalone PR. - package-ecosystem: "uv" directory: "/" allow: - dependency-type: "direct" schedule: interval: "weekly" groups: python-dependencies: patterns: - "*" cooldown: default-days: 7 zarr-developers-zarr-python-a2f2415/.github/labeler.yml000066400000000000000000000011771527014137600231640ustar00rootroot00000000000000needs release notes: - all: - changed-files: - all-globs-to-all-files: '!changes/*.md' # Subpackage labels: a pull request whose changes all live under one package # directory gets that package's label, which .github/release.yml uses to keep # subpackage work out of the generated `zarr` release notes. zarr-metadata: - all: - changed-files: - all-globs-to-all-files: 'packages/zarr-metadata/**' zarr-indexing: - all: - changed-files: - all-globs-to-all-files: 'packages/zarr-indexing/**' zarr-http-server: - all: - changed-files: - all-globs-to-all-files: 'packages/zarr-http-server/**' zarr-developers-zarr-python-a2f2415/.github/release.yml000066400000000000000000000013271527014137600231730ustar00rootroot00000000000000# Configuration for GitHub's "Generate release notes" button. # # The zarr-python repository also hosts the companion packages under packages/ # (zarr-metadata, zarr-indexing, zarr-http-server), which have their own release # cadence, tags and changelogs. Pull requests that only touch a subpackage are # labelled by the labeler workflow (.github/labeler.yml) and excluded here, so a # `zarr` release lists only the changes that ship in `zarr`. changelog: exclude: labels: - zarr-metadata - zarr-indexing - zarr-http-server authors: - pre-commit-ci[bot] categories: - title: Dependency updates labels: - dependencies - title: What's Changed labels: - "*" zarr-developers-zarr-python-a2f2415/.github/workflows/000077500000000000000000000000001527014137600230625ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/.github/workflows/big-endian.yml000066400000000000000000000051451527014137600256070ustar00rootroot00000000000000# Run the byte-order-sensitive tests on a big-endian host: s390x, emulated with QEMU. # Emulation makes the full suite take about 40 minutes, so this job runs only the tests # that exercise data types, codecs and metadata, and only when that code changes (plus # a weekly run to catch anything the path filter misses). name: Big-endian on: pull_request: branches: [ main ] paths: - "src/zarr/codecs/**" - "src/zarr/core/buffer/**" - "src/zarr/core/dtype/**" - "src/zarr/core/metadata/**" - "src/zarr/metadata/**" - "tests/test_codecs/**" - "tests/test_dtype/**" - "tests/test_metadata/**" - "ci/big-endian/**" - ".github/workflows/big-endian.yml" schedule: - cron: "0 3 * * 1" workflow_dispatch: permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: s390x: name: s390x (QEMU) runs-on: ubuntu-latest timeout-minutes: 90 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 # hatch-vcs needs the tags to resolve a version persist-credentials: false - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 with: platforms: s390x - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Build test image uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: ci/big-endian platforms: linux/s390x tags: zarr-big-endian load: true cache-from: type=gha,scope=big-endian cache-to: type=gha,scope=big-endian,mode=max - name: Run tests run: | docker run --rm --platform linux/s390x -v "$PWD":/src:ro zarr-big-endian bash -euc ' cp -r /src /work && cd /work # Resolve dependencies against pyproject.toml rather than trusting whatever the # image provides, then fail if any installed package falls outside the constraints. pip install -q --no-build-isolation . pip check python -c "import sys; assert sys.byteorder == \"big\", sys.byteorder" python -m pytest -p no:cacheprovider --hypothesis-profile ci -n auto \ tests/test_dtype tests/test_dtype_registry.py \ tests/test_codecs tests/test_metadata \ tests/test_array.py tests/test_v2.py tests/test_info.py \ tests/test_pipeline_parity.py tests/test_cli/test_migrate_v3.py ' zarr-developers-zarr-python-a2f2415/.github/workflows/check_changelogs.yml000066400000000000000000000022211527014137600270510ustar00rootroot00000000000000name: Check changelog entries on: pull_request: workflow_dispatch: permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: check-changelogs: name: Check changelog entries runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Check zarr-python changelog entries run: just check-changelogs - name: Check zarr-metadata changelog entries run: just check-changelogs packages/zarr-metadata/changes - name: Check zarr-indexing changelog entries run: just check-changelogs packages/zarr-indexing/changes - name: Check zarr-http-server changelog entries run: just check-changelogs packages/zarr-http-server/changes zarr-developers-zarr-python-a2f2415/.github/workflows/codspeed.yml000066400000000000000000000031301527014137600253700ustar00rootroot00000000000000name: CodSpeed Benchmarks on: schedule: - cron: '0 9 * * 1' # Every Monday at 9am UTC pull_request: types: [labeled] workflow_dispatch: permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: benchmarks: name: Run benchmarks runs-on: codspeed-macro if: | github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'benchmark')) steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - name: Install Hatch uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: version: '1.16.5' # No interpreter is set up in this job on purpose. Benchmarks are only comparable # across runs if the interpreter underneath them does not move: adding a pinned # Python here measured a uniform ~13% slowdown on every benchmark, because hatch # then built the environment on that interpreter instead of the runner's own. - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Run the benchmarks uses: CodSpeedHQ/action@821a40cd6c80e3cfe387caf55668287addd808da # v5.5.0 env: ZARR_BENCHMARK_CLEAR_CACHE: '1' with: mode: walltime run: HATCH_ENV=test.py3.12-minimal just benchmark-codspeed zarr-developers-zarr-python-a2f2415/.github/workflows/docs.yml000066400000000000000000000026471527014137600245460ustar00rootroot00000000000000name: Docs on: push: branches: [main] pull_request: branches: [main] workflow_dispatch: permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: docs: name: Check docs runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 # Fast source-level guards that need no built site, so they run before the (slower) # build for a quick failure: every public export is in the API reference, and no # docstring/Markdown carries reStructuredText markup that MkDocs won't render. - run: just check-doc-exports - run: just lint-docs # --strict turns warnings into errors, so a docs code block that fails to execute # at build time (e.g. a non-exec python fence disrupting a later exec="true" block) # fails CI instead of merging as a silent warning. - run: just docs-build env: DISABLE_MKDOCS_2_WARNING: "true" NO_MKDOCS_2_WARNING: "true" - run: just check-doc-links continue-on-error: true zarr-developers-zarr-python-a2f2415/.github/workflows/downstream.yml000066400000000000000000000112301527014137600257650ustar00rootroot00000000000000name: Downstream on: workflow_dispatch: pull_request: types: [labeled, synchronize, opened, reopened] permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: xarray: name: Xarray zarr backend tests if: | github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-downstream') runs-on: ubuntu-latest steps: - name: Check out zarr-python uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - name: Check out xarray uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: pydata/xarray path: xarray persist-credentials: false # We install xarray with plain pip/uv rather than pixi. pixi solves # xarray's entire manifest (it has no committed lockfile), which drags in # the `mypy-upstream` environment; that environment sources numcodecs from # git and fails to build under newer pixi with # `meson-python: error: Unknown option "pixi-conda-environment"`, breaking # the job before any test runs. Tests that need a backend we don't install # are skipped via xarray's `requires_*` markers, not failed. - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.13' - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - name: Install xarray and test dependencies working-directory: xarray run: | uv venv # xarray's pytest tooling lives in the PEP 735 `dev` dependency group; # the zarr-relevant backends come from the `io` and `parallel` extras. uv pip install --group dev ".[io,parallel,accel]" - name: Override zarr-python with branch version working-directory: xarray run: uv pip install --no-deps .. # Keep the branch override: syncing xarray would reinstall zarr from PyPI. - name: Show versions working-directory: xarray run: | uv run --no-sync python -c " import zarr; print(f'zarr {zarr.__version__}') import xarray; print(f'xarray {xarray.__version__}') " # xarray keeps its backend tests in the `xarray/tests/backends` package # (pydata/xarray#11633 split the former `test_backends*.py` modules into # it). Running the package covers the zarr store, region/append, API and # DataTree tests; the other backends are installed by the `io` extra, so # their tests run too, as they did before the split. - name: Run xarray backend tests working-directory: xarray run: | uv run --no-sync python -m pytest --no-header -q \ xarray/tests/backends numcodecs: name: numcodecs zarr3 codec tests if: | github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-downstream') runs-on: ubuntu-latest steps: - name: Check out zarr-python uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - name: Check out numcodecs uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: zarr-developers/numcodecs fetch-depth: 0 path: numcodecs submodules: recursive persist-credentials: false - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.13' - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - name: Install numcodecs with test-zarr-main group working-directory: numcodecs run: | uv venv uv pip install --group dev uv sync --group dev --group test-zarr-main uv pip install --no-build-isolation -e . - name: Override zarr-python with branch version working-directory: numcodecs run: uv pip install --no-deps .. - name: Show versions working-directory: numcodecs run: | uv run python -c " import zarr; print(f'zarr {zarr.__version__}') import numcodecs; print(f'numcodecs {numcodecs.__version__}') " - name: Run numcodecs zarr3 tests working-directory: numcodecs run: uv run python -m pytest -x --no-header -q tests/test_zarr3.py zarr-developers-zarr-python-a2f2415/.github/workflows/gpu_test.yml000066400000000000000000000056551527014137600254520ustar00rootroot00000000000000# This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: GPU Test on: push: branches: [ main, 3.1.x ] pull_request: branches: [ main, 3.1.x ] workflow_dispatch: env: LD_LIBRARY_PATH: /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64 # Use the uv from astral-sh/setup-uv instead of hatch's bundled (pyapp) uv. HATCH_ENV_TYPE_VIRTUAL_UV_PATH: uv permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: test: name: py=${{ matrix.python-version }} environment: name: codecov-upload deployment: false runs-on: gpu-runner strategy: matrix: python-version: ['3.12'] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 # grab all branches and tags persist-credentials: false # - name: cuda-toolkit # uses: Jimver/cuda-toolkit@v0.2.16 # id: cuda-toolkit # with: # cuda: '12.4.1' - name: Set up CUDA run: | wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.1-1_all.deb sudo dpkg -i cuda-keyring_1.1-1_all.deb sudo apt-get update sudo apt-get -y install cuda-toolkit-12-6 echo "/usr/local/cuda/bin" >> $GITHUB_PATH - name: GPU check run: | nvidia-smi echo $PATH echo $LD_LIBRARY_PATH nvcc -V - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - name: Install Hatch uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: version: '1.16.5' - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 # Two names for the same environment on purpose: `just setup` builds whatever # HATCH_ENV points at, while `just gpu` reads GPU_HATCH_ENV so that an # ambient HATCH_ENV can never redirect `pytest -m gpu` into a CPU environment. - name: Set Up Hatch Env env: HATCH_ENV: gputest.py${{ matrix.python-version }} run: | just setup - name: Run Tests env: GPU_HATCH_ENV: gputest.py${{ matrix.python-version }} run: | just gpu - name: Upload coverage uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: gpu verbose: true # optional (default = false) zarr-developers-zarr-python-a2f2415/.github/workflows/hypothesis.yaml000066400000000000000000000077261527014137600261610ustar00rootroot00000000000000name: Slow Hypothesis CI on: push: branches: [main, 3.1.x] pull_request: branches: [main, 3.1.x] types: [opened, reopened, synchronize, labeled] schedule: - cron: "0 0 * * *" # Daily “At 00:00” UTC workflow_dispatch: # allows you to trigger manually permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true env: FORCE_COLOR: 3 # Use the uv from astral-sh/setup-uv instead of hatch's bundled (pyapp) uv. HATCH_ENV_TYPE_VIRTUAL_UV_PATH: uv jobs: hypothesis: name: Slow Hypothesis Tests permissions: contents: read issues: write environment: name: codecov-upload deployment: false runs-on: "ubuntu-latest" defaults: run: shell: bash -l {0} strategy: matrix: python-version: ['3.12'] dependency-set: ["optional"] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Set HYPOTHESIS_PROFILE based on trigger env: EVENT_NAME: ${{ github.event_name }} run: | if [[ "$EVENT_NAME" == "schedule" || "$EVENT_NAME" == "workflow_dispatch" ]]; then echo "HYPOTHESIS_PROFILE=nightly" >> $GITHUB_ENV else echo "HYPOTHESIS_PROFILE=ci" >> $GITHUB_ENV fi - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - name: Install Hatch uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: version: '1.16.5' - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Set Up Hatch Env env: HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} run: | just setup # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache - name: Restore cached hypothesis directory id: restore-hypothesis-cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: .hypothesis/ key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} restore-keys: | cache-hypothesis- - name: Run slow Hypothesis tests if: success() id: status env: HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} PYTEST_ADDOPTS: "--report-log=output-${{ matrix.python-version }}-log.jsonl" run: | echo "Using Hypothesis profile: $HYPOTHESIS_PROFILE" just hypothesis # explicitly save the cache so it gets updated, also do this even if it fails. - name: Save cached hypothesis directory id: save-hypothesis-cache if: always() && steps.status.outcome != 'skipped' uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: .hypothesis/ key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} - name: Upload coverage uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: tests verbose: true # optional (default = false) - name: Generate and publish the report if: | failure() && steps.status.outcome == 'failure' && github.event_name == 'schedule' && github.repository_owner == 'zarr-developers' uses: scientific-python/issue-from-pytest-log-action@35b4e0a9e06f8e7e261778289cf4b968b722662d # v1.6.2 with: log-path: output-${{ matrix.python-version }}-log.jsonl issue-title: "Nightly Hypothesis tests failed" issue-label: "automated issue" zarr-developers-zarr-python-a2f2415/.github/workflows/issue-metrics.yml000066400000000000000000000026731527014137600264110ustar00rootroot00000000000000name: Monthly issue metrics on: workflow_dispatch: schedule: - cron: '3 2 1 * *' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build: name: issue metrics runs-on: ubuntu-latest permissions: issues: write # Required to create the metrics report issue pull-requests: read # Required to read PR metrics steps: - name: Get dates for last month shell: bash run: | # Calculate the first day of the previous month first_day=$(date -d "last month" +%Y-%m-01) # Calculate the last day of the previous month last_day=$(date -d "$first_day +1 month -1 day" +%Y-%m-%d) #Set an environment variable with the date range echo "$first_day..$last_day" echo "last_month=$first_day..$last_day" >> "$GITHUB_ENV" - name: Run issue-metrics tool uses: github-community-projects/issue-metrics@671528652402b9b8a33da44b5c693b3161bc090f # v5.0.3 env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} SEARCH_QUERY: 'repo:zarr-developers/zarr-python is:issue created:${{ env.last_month }} -reason:"not planned"' - name: Create issue uses: peter-evans/create-issue-from-file@fca9117c27cdc29c6c4db3b86c48e4115a786710 # v6.0.0 with: title: Monthly issue metrics report token: ${{ secrets.GITHUB_TOKEN }} content-filepath: ./issue_metrics.md zarr-developers-zarr-python-a2f2415/.github/workflows/links.yml000066400000000000000000000016401527014137600247260ustar00rootroot00000000000000name: Check links on: repository_dispatch: workflow_dispatch: # pull_request: schedule: - cron: "00 18 * * 1" # weekly, Mondays at 18:00 UTC jobs: linkChecker: runs-on: ubuntu-latest permissions: issues: write # required for peter-evans/create-issue-from-file steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Link Checker id: lychee uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0 with: fail: false - name: Create Issue From File if: steps.lychee.outputs.exit_code != 0 uses: peter-evans/create-issue-from-file@fca9117c27cdc29c6c4db3b86c48e4115a786710 # v6.0.0 with: title: Link Checker Report content-filepath: ./lychee/out.md labels: report, automated issue zarr-developers-zarr-python-a2f2415/.github/workflows/lint.yml000066400000000000000000000036041527014137600245560ustar00rootroot00000000000000name: Lint on: push: branches: [main, 3.1.x] pull_request: branches: [main, 3.1.x] workflow_dispatch: permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: lint: name: Lint runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: true - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 # `uvx prek` builds each hook's environment from scratch, so cache them the # way the prek action used to. Keyed on the hook config: a new pinned rev or # a new hook is exactly when the cached environments stop being valid. - name: Cache prek hook environments uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/prek # Justfile is hashed too: prek_version lives there, and a bumped prek # must not restore a store built by the previous one. No restore-keys # fallback for the same reason. key: prek-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml', 'Justfile') }} - name: Lint run: just lint - name: Check justfile formatting # After the linters, and never masking them: this is cosmetic, and a # mis-formatted recipe should not cost someone their ruff/mypy results. if: always() run: just just-check zarr-developers-zarr-python-a2f2415/.github/workflows/needs_release_notes.yml000066400000000000000000000016371527014137600276220ustar00rootroot00000000000000name: "Pull Request Labeler" on: # pull_request_target is needed to label PRs from forks. # This workflow only runs actions/labeler (no code checkout), so it's safe. pull_request_target: # zizmor: ignore[dangerous-triggers] types: [opened, reopened, synchronize] permissions: {} concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: true jobs: labeler: name: Label pull request if: ${{ github.event.pull_request.user.login != 'dependabot[bot]' && github.event.pull_request.user.login != 'pre-commit-ci[bot]' }} permissions: contents: read # Required to read label configuration pull-requests: write # Required to add labels to PRs runs-on: ubuntu-latest steps: - uses: actions/labeler@bf12e9b00b37c5c0ca2b87b79b2daf7891dbda13 # v7.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} sync-labels: true zarr-developers-zarr-python-a2f2415/.github/workflows/nightly_wheels.yml000066400000000000000000000023151527014137600266330ustar00rootroot00000000000000name: Nightly Wheels on: schedule: # Run nightly at 2 AM UTC - cron: '0 2 * * *' workflow_dispatch: permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build_and_upload_nightly: name: Build and upload nightly wheels environment: name: nightly-wheel-upload deployment: false runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: submodules: true fetch-depth: 0 persist-credentials: false - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 name: Install Python with: python-version: '3.14' - name: Install Hatch uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: version: '1.16.5' - name: Build wheel and sdist run: hatch build - name: Upload nightly wheels uses: scientific-python/upload-nightly-action@16fa02eacee1655195143de09f03676e60ef2bf5 with: artifacts_path: dist anaconda_nightly_upload_token: ${{ secrets.ANACONDA_ORG_UPLOAD_TOKEN }} zarr-developers-zarr-python-a2f2415/.github/workflows/releases.yml000066400000000000000000000044561527014137600254210ustar00rootroot00000000000000name: Wheels on: release: types: - published push: branches: [main] pull_request: branches: [main] workflow_dispatch: permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build_artifacts: name: Build wheel on ubuntu-latest runs-on: ubuntu-latest strategy: fail-fast: false steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: submodules: true fetch-depth: 0 persist-credentials: false - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 name: Install Python with: python-version: '3.12' - name: Install Hatch uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: version: '1.16.5' - name: Build wheel and sdist run: hatch build - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: releases path: dist test_dist_pypi: name: Test distribution artifacts needs: [build_artifacts] runs-on: ubuntu-latest steps: - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: releases path: dist - name: test run: | ls ls dist upload_pypi: name: Upload to PyPI needs: [build_artifacts, test_dist_pypi] runs-on: ubuntu-latest if: github.event_name == 'release' environment: name: releases url: https://pypi.org/p/zarr permissions: id-token: write # Required for OIDC trusted publishing to PyPI attestations: write # Required for artifact attestation artifact-metadata: write # Required for artifact attestation metadata steps: - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: releases path: dist - name: Generate artifact attestation uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-path: dist/* - name: Publish package to PyPI uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 zarr-developers-zarr-python-a2f2415/.github/workflows/test.yml000066400000000000000000000152701527014137600245710ustar00rootroot00000000000000# This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Test on: push: branches: [ main, 3.1.x ] pull_request: branches: [ main, 3.1.x ] workflow_dispatch: permissions: contents: read env: # Use the uv from astral-sh/setup-uv; without an explicit path hatch # bootstraps its own (pyapp) uv, which fails on non-3.12 runners. HATCH_ENV_TYPE_VIRTUAL_UV_PATH: uv concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: test: name: os=${{ matrix.os }}, py=${{ matrix.python-version }}, deps=${{ matrix.dependency-set }} environment: name: codecov-upload deployment: false defaults: run: shell: bash strategy: matrix: python-version: ['3.12', '3.13', '3.14'] dependency-set: ["minimal", "optional"] os: ["ubuntu-latest"] include: - python-version: '3.12' dependency-set: 'optional' os: 'macos-latest' - python-version: '3.14' dependency-set: 'optional' os: 'macos-latest' - python-version: '3.12' dependency-set: 'optional' os: 'windows-latest' - python-version: '3.14' dependency-set: 'optional' os: 'windows-latest' runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 # grab all branches and tags persist-credentials: false - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - name: Install Hatch run: python -m pip install hatch==1.16.5 - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Set Up Hatch Env env: HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} run: | just setup - name: Run Tests env: HYPOTHESIS_PROFILE: ci HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} run: | just coverage - name: Upload coverage if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }} uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: tests verbose: true # optional (default = false) test-upstream-and-min-deps: name: py=${{ matrix.python-version }}-${{ matrix.dependency-set }} environment: name: codecov-upload deployment: false runs-on: ubuntu-latest strategy: matrix: python-version: ['3.12', "3.14"] dependency-set: ["upstream", "min_deps"] exclude: - python-version: "3.14" dependency-set: min_deps - python-version: "3.12" dependency-set: upstream steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - name: Install Hatch run: python -m pip install hatch==1.16.5 - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Set Up Hatch Env env: HATCH_ENV: ${{ matrix.dependency-set }} run: | just setup - name: Run Tests env: HATCH_ENV: ${{ matrix.dependency-set }} run: | just coverage - name: Upload coverage uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: tests verbose: true # optional (default = false) doctests: name: doctests runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 # required for hatch version discovery, which is needed for numcodecs.zarr3 persist-credentials: false - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.13' cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - name: Install Hatch run: python -m pip install hatch==1.16.5 - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Set Up Hatch Env run: | HATCH_ENV=doctest just setup - name: Run Tests run: | just doctest benchmarks: name: Benchmark smoke test runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.13' cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - name: Install Hatch run: python -m pip install hatch==1.16.5 - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Run Benchmarks env: ZARR_BENCHMARK_CLEAR_CACHE: '1' run: | HATCH_ENV=test.py3.13-minimal just benchmark test-complete: name: Test complete needs: [ test, test-upstream-and-min-deps, doctests, benchmarks ] if: always() runs-on: ubuntu-latest steps: - name: Check failure if: | contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') run: exit 1 - name: Success run: echo Success! zarr-developers-zarr-python-a2f2415/.github/workflows/zarr-http-server-release.yml000066400000000000000000000111061527014137600304610ustar00rootroot00000000000000name: zarr-http-server release on: workflow_dispatch: push: tags: - 'zarr_http_server-v*' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: build: name: Build wheel and sdist runs-on: ubuntu-latest defaults: run: shell: bash working-directory: packages/zarr-http-server steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 0 # hatch-vcs needs full history + tags - name: Install Hatch uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: version: '1.16.5' - name: Build run: hatch build - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: zarr-http-server-dist path: packages/zarr-http-server/dist test_artifacts: name: Test built artifacts needs: [build] runs-on: ubuntu-latest steps: - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: zarr-http-server-dist path: dist - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: false - name: Set up Python run: uv python install 3.12 - name: Install built wheel and run import smoke test run: | wheel=$(ls dist/*.whl) uv run --with "${wheel}" --python 3.12 --no-project \ python -c "import zarr_http_server; print('zarr_http_server', zarr_http_server.__version__)" # The sdist has to stand alone, and `hatch build` above builds the wheel # from the source tree, not from the sdist. So build a wheel from the # sdist with pip, which unpacks it under the sdist format's rules (no # links out of the archive), then run the package's own tests and strict # docs build from the unpacked sdist. This job has no checkout, so # nothing comes from the repository; zarr, where a package uses it, # resolves from PyPI. - name: Build a wheel from the sdist with pip run: | sdist=$(ls dist/*.tar.gz) uvx --python 3.12 pip@26.2.1 wheel --no-deps --wheel-dir sdist-wheel "${sdist}" - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Test and build docs from the unpacked sdist env: UV_PYTHON: '3.12' run: | mkdir sdist tar -xzf dist/*.tar.gz -C sdist cd sdist/*/ just test just docs-check upload_pypi: name: Upload to PyPI needs: [build, test_artifacts] if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/zarr_http_server-v') runs-on: ubuntu-latest environment: name: zarr-http-server-releases url: https://pypi.org/p/zarr-http-server permissions: id-token: write # required for OIDC trusted publishing attestations: write # required for artifact attestations steps: - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: zarr-http-server-dist path: dist - name: Generate artifact attestation uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-path: dist/* - name: Publish package to PyPI uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 upload_testpypi: name: Upload to TestPyPI needs: [build, test_artifacts] if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest environment: name: zarr-http-server-releases-test url: https://test.pypi.org/p/zarr-http-server permissions: id-token: write attestations: write steps: - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: zarr-http-server-dist path: dist - name: Generate artifact attestation uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-path: dist/* - name: Publish package to TestPyPI uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 with: repository-url: https://test.pypi.org/legacy/ zarr-developers-zarr-python-a2f2415/.github/workflows/zarr-http-server.yml000066400000000000000000000117241527014137600270510ustar00rootroot00000000000000name: zarr-http-server # Job steps delegate to packages/zarr-http-server/justfile, the single source # of truth for this package's verbs; CI owns only the python matrix and # caching. Keeping the commands in one place is what makes `just check` # locally mean the same thing as a green run here. on: push: branches: [main] paths: - 'packages/zarr-http-server/**' - '.github/workflows/zarr-http-server.yml' # The docs config and header script shared by all package sites. - 'packages/mkdocs-base.yml' - 'packages/source-version.js' # The package resolves zarr from the repo root for its own tests, so a # core change can break it. Run this suite when core changes too. - 'src/zarr/**' pull_request: paths: - 'packages/zarr-http-server/**' - '.github/workflows/zarr-http-server.yml' # The docs config and header script shared by all package sites. - 'packages/mkdocs-base.yml' - 'packages/source-version.js' # The package resolves zarr from the repo root for its own tests, so a # core change can break it. Run this suite when core changes too. - 'src/zarr/**' workflow_dispatch: permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: test: name: pytest py=${{ matrix.python-version }} runs-on: ubuntu-latest defaults: run: shell: bash working-directory: packages/zarr-http-server strategy: fail-fast: false matrix: python-version: ['3.12', '3.13', '3.14'] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: true - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Sync test dependency groups # The examples group carries the deps the README examples need, so the # test that reads a served array back with a zarr client runs here # instead of silently skipping. run: uv sync --group test --group examples --python ${{ matrix.python-version }} - name: Run pytest run: just test ruff: name: ruff runs-on: ubuntu-latest defaults: run: shell: bash working-directory: packages/zarr-http-server steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Run ruff run: just lint mypy: name: mypy runs-on: ubuntu-latest defaults: run: shell: bash working-directory: packages/zarr-http-server steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: true - name: Set up Python run: uv python install 3.12 - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Sync test dependency group run: uv sync --group test --python 3.12 - name: Run mypy run: just typecheck docs: name: docs runs-on: ubuntu-latest defaults: run: shell: bash working-directory: packages/zarr-http-server steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: true - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Build docs run: just docs-check zarr-http-server-complete: name: zarr-http-server complete needs: [test, ruff, mypy, docs] if: always() runs-on: ubuntu-latest steps: - name: Check failure if: | contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') run: exit 1 - name: Success run: echo Success! zarr-developers-zarr-python-a2f2415/.github/workflows/zarr-indexing-release.yml000066400000000000000000000110261527014137600300040ustar00rootroot00000000000000name: zarr-indexing release on: workflow_dispatch: push: tags: - 'zarr_indexing-v*' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: build: name: Build wheel and sdist runs-on: ubuntu-latest defaults: run: shell: bash working-directory: packages/zarr-indexing steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 0 # hatch-vcs needs full history + tags - name: Install Hatch uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: version: '1.16.5' - name: Build run: hatch build - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: zarr-indexing-dist path: packages/zarr-indexing/dist test_artifacts: name: Test built artifacts needs: [build] runs-on: ubuntu-latest steps: - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: zarr-indexing-dist path: dist - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: false - name: Set up Python run: uv python install 3.12 - name: Install built wheel and run import smoke test run: | wheel=$(ls dist/*.whl) uv run --with "${wheel}" --python 3.12 --no-project \ python -c "import zarr_indexing; print('zarr_indexing', zarr_indexing.__version__)" # The sdist has to stand alone, and `hatch build` above builds the wheel # from the source tree, not from the sdist. So build a wheel from the # sdist with pip, which unpacks it under the sdist format's rules (no # links out of the archive), then run the package's own tests and strict # docs build from the unpacked sdist. This job has no checkout, so # nothing comes from the repository; zarr, where a package uses it, # resolves from PyPI. - name: Build a wheel from the sdist with pip run: | sdist=$(ls dist/*.tar.gz) uvx --python 3.12 pip@26.2.1 wheel --no-deps --wheel-dir sdist-wheel "${sdist}" - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Test and build docs from the unpacked sdist env: UV_PYTHON: '3.12' run: | mkdir sdist tar -xzf dist/*.tar.gz -C sdist cd sdist/*/ just test just docs-check upload_pypi: name: Upload to PyPI needs: [build, test_artifacts] if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/zarr_indexing-v') runs-on: ubuntu-latest environment: name: zarr-indexing-releases url: https://pypi.org/p/zarr-indexing permissions: id-token: write # required for OIDC trusted publishing attestations: write # required for artifact attestations steps: - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: zarr-indexing-dist path: dist - name: Generate artifact attestation uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-path: dist/* - name: Publish package to PyPI uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 upload_testpypi: name: Upload to TestPyPI needs: [build, test_artifacts] if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest environment: name: zarr-indexing-releases-test url: https://test.pypi.org/p/zarr-indexing permissions: id-token: write attestations: write steps: - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: zarr-indexing-dist path: dist - name: Generate artifact attestation uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-path: dist/* - name: Publish package to TestPyPI uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 with: repository-url: https://test.pypi.org/legacy/ zarr-developers-zarr-python-a2f2415/.github/workflows/zarr-indexing.yml000066400000000000000000000123341527014137600263710ustar00rootroot00000000000000name: zarr-indexing on: push: branches: [main] paths: - 'packages/zarr-indexing/**' - '.github/workflows/zarr-indexing.yml' # The docs config and header script shared by all package sites. - 'packages/mkdocs-base.yml' - 'packages/source-version.js' pull_request: paths: - 'packages/zarr-indexing/**' - '.github/workflows/zarr-indexing.yml' # The docs config and header script shared by all package sites. - 'packages/mkdocs-base.yml' - 'packages/source-version.js' workflow_dispatch: permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: test: name: pytest py=${{ matrix.python-version }} runs-on: ubuntu-latest defaults: run: shell: bash working-directory: packages/zarr-indexing strategy: fail-fast: false matrix: python-version: ['3.12', '3.13', '3.14'] # UV_PYTHON reaches every uv call inside the recipes, selecting the matrix # interpreter while keeping the justfile interpreter-agnostic for local use. env: UV_PYTHON: ${{ matrix.python-version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: true - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} # The suite imports nothing from `zarr`; it runs against the repo-root # environment, with this package as an editable overlay, to share the # parent project's pinned test toolchain. The recipes carry that # invocation; this step installs dependencies up front so their # install time is reported separately from the tests. - name: Sync test dependency group run: uv sync --group test - name: Check interpreter run: >- uv run --group test python -c "import sys; v = '%d.%d' % sys.version_info[:2]; assert v == '${{ matrix.python-version }}', v" - name: Run pytest # Suites and invocation live in packages/zarr-indexing/justfile. run: just test - name: Run pytest (tensorstore parity) run: just test-tensorstore ruff: name: ruff runs-on: ubuntu-latest defaults: run: shell: bash working-directory: packages/zarr-indexing steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Run ruff # The ruff version pin lives in packages/zarr-indexing/justfile. run: just lint pyright: name: pyright runs-on: ubuntu-latest defaults: run: shell: bash working-directory: packages/zarr-indexing steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: true - name: Set up Python run: uv python install 3.12 - name: Sync test dependency group run: uv sync --group test --python 3.12 - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Run pyright # The pyright invocation lives in packages/zarr-indexing/justfile. run: just typecheck docs: name: docs runs-on: ubuntu-latest defaults: run: shell: bash working-directory: packages/zarr-indexing steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: true - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Build docs # The strict mkdocs build lives in packages/zarr-indexing/justfile. run: just docs-check zarr-indexing-complete: name: zarr-indexing complete needs: [test, ruff, pyright, docs] if: always() runs-on: ubuntu-latest steps: - name: Check failure if: | contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') run: exit 1 - name: Success run: echo Success! zarr-developers-zarr-python-a2f2415/.github/workflows/zarr-metadata-release.yml000066400000000000000000000110261527014137600277570ustar00rootroot00000000000000name: zarr-metadata release on: workflow_dispatch: push: tags: - 'zarr_metadata-v*' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: build: name: Build wheel and sdist runs-on: ubuntu-latest defaults: run: shell: bash working-directory: packages/zarr-metadata steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 0 # hatch-vcs needs full history + tags - name: Install Hatch uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: version: '1.16.5' - name: Build run: hatch build - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: zarr-metadata-dist path: packages/zarr-metadata/dist test_artifacts: name: Test built artifacts needs: [build] runs-on: ubuntu-latest steps: - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: zarr-metadata-dist path: dist - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: false - name: Set up Python run: uv python install 3.12 - name: Install built wheel and run import smoke test run: | wheel=$(ls dist/*.whl) uv run --with "${wheel}" --python 3.12 --no-project \ python -c "import zarr_metadata; print('zarr_metadata', zarr_metadata.__version__)" # The sdist has to stand alone, and `hatch build` above builds the wheel # from the source tree, not from the sdist. So build a wheel from the # sdist with pip, which unpacks it under the sdist format's rules (no # links out of the archive), then run the package's own tests and strict # docs build from the unpacked sdist. This job has no checkout, so # nothing comes from the repository; zarr, where a package uses it, # resolves from PyPI. - name: Build a wheel from the sdist with pip run: | sdist=$(ls dist/*.tar.gz) uvx --python 3.12 pip@26.2.1 wheel --no-deps --wheel-dir sdist-wheel "${sdist}" - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Test and build docs from the unpacked sdist env: UV_PYTHON: '3.12' run: | mkdir sdist tar -xzf dist/*.tar.gz -C sdist cd sdist/*/ just test just docs-check upload_pypi: name: Upload to PyPI needs: [build, test_artifacts] if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/zarr_metadata-v') runs-on: ubuntu-latest environment: name: zarr-metadata-releases url: https://pypi.org/p/zarr-metadata permissions: id-token: write # required for OIDC trusted publishing attestations: write # required for artifact attestations steps: - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: zarr-metadata-dist path: dist - name: Generate artifact attestation uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-path: dist/* - name: Publish package to PyPI uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 upload_testpypi: name: Upload to TestPyPI needs: [build, test_artifacts] if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest environment: name: zarr-metadata-releases-test url: https://test.pypi.org/p/zarr-metadata permissions: id-token: write attestations: write steps: - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: zarr-metadata-dist path: dist - name: Generate artifact attestation uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-path: dist/* - name: Publish package to TestPyPI uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 with: repository-url: https://test.pypi.org/legacy/ zarr-developers-zarr-python-a2f2415/.github/workflows/zarr-metadata.yml000066400000000000000000000106051527014137600263430ustar00rootroot00000000000000name: zarr-metadata # Job steps delegate to packages/zarr-metadata/justfile, the single source of # truth for this package's verbs; CI owns only the python matrix and caching. on: push: branches: [main] paths: - 'packages/zarr-metadata/**' - '.github/workflows/zarr-metadata.yml' # The docs config and header script shared by all package sites, and # zarr-python's docs hook, which this site uses. - 'packages/mkdocs-base.yml' - 'packages/source-version.js' - 'mkdocs_hooks.py' pull_request: paths: - 'packages/zarr-metadata/**' - '.github/workflows/zarr-metadata.yml' # The docs config and header script shared by all package sites, and # zarr-python's docs hook, which this site uses. - 'packages/mkdocs-base.yml' - 'packages/source-version.js' - 'mkdocs_hooks.py' workflow_dispatch: permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: test: name: pytest py=${{ matrix.python-version }} runs-on: ubuntu-latest defaults: run: shell: bash working-directory: packages/zarr-metadata strategy: fail-fast: false matrix: python-version: ['3.11', '3.12', '3.13', '3.14'] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: true - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} - name: Sync test dependency group run: uv sync --group test --python ${{ matrix.python-version }} - name: Run pytest run: just test ruff: name: ruff runs-on: ubuntu-latest defaults: run: shell: bash working-directory: packages/zarr-metadata steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Run ruff run: just lint pyright: name: pyright runs-on: ubuntu-latest defaults: run: shell: bash working-directory: packages/zarr-metadata steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: true - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Run pyright # Pyright runs unpinned; the interpreter pin lives in the justfile. run: just typecheck docs: name: docs runs-on: ubuntu-latest defaults: run: shell: bash working-directory: packages/zarr-metadata steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: true - name: Install just uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: repo: casey/just version: 1.58.0 - name: Build docs run: just docs-check zarr-metadata-complete: name: zarr-metadata complete needs: [test, ruff, pyright, docs] if: always() runs-on: ubuntu-latest steps: - name: Check failure if: | contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') run: exit 1 - name: Success run: echo Success! zarr-developers-zarr-python-a2f2415/.github/workflows/zizmor.yml000066400000000000000000000015021527014137600251350ustar00rootroot00000000000000name: GitHub Actions Security Analysis on: push: branches: [main] paths: - '.github/workflows/**' - '.github/actions/**' pull_request: branches: ["**"] paths: - '.github/workflows/**' - '.github/actions/**' workflow_dispatch: permissions: {} concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: zizmor: name: Run zizmor runs-on: ubuntu-latest permissions: security-events: write # Required by zizmor-action to upload SARIF files steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Run zizmor uses: zizmorcore/zizmor-action@85b2b9fb8e3325e7a23d7d0a9d7ba5ae0f999e3e # v0.7.1 zarr-developers-zarr-python-a2f2415/.gitignore000066400000000000000000000021041527014137600214520ustar00rootroot00000000000000# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] # C extensions *.so # Distribution / packaging .Python env/ .venv/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .coverage .coverage.* .cache coverage.xml *,cover # Translations *.mo *.pot # Django stuff: *.log # Documentation site/ docs/_build/ docs/data data data.zip # PyBuilder target/ # PyCharm .idea # Jupyter .ipynb_checkpoints/ # VCS versioning src/zarr/_version.py # emacs *~ # VSCode .vscode/ # test data #*.zarr #*.zip #example* #doesnotexist #test_sync* data/* src/fixture/ fixture/ junit.xml .DS_Store tests/.hypothesis .hypothesis/ zarr/version.py zarr.egg-info/ # Local agent / planning notes (not versioned) .claude/ CLAUDE.md docs/superpowers/ zarr-developers-zarr-python-a2f2415/.markdownlint-cli2.jsonc000066400000000000000000000046651527014137600241540ustar00rootroot00000000000000// markdownlint-cli2 configuration for zarr-python docs. // // We keep the rules that catch real rendering/structure problems and disable those that // are pure style, conflict with house conventions, or fire false positives against our // MkDocs/mkdocstrings + pymdownx toolchain. Complementary, not overlapping, with // ci/lint_docs.py (RST residue + list-breaking fences) and `mkdocs build --strict`. { "config": { "default": true, // House style: Markdown paragraphs are single unwrapped lines, so line length is not // a meaningful constraint. "MD013": false, // Purely stylistic marker/emphasis choices -- not worth the churn across existing docs. "MD004": false, // ul bullet style (-, *, +) "MD007": false, // ul indentation width "MD050": false, // strong (bold) style "MD035": false, // hr style // False positives from our toolchain: // mkdocstrings cross-refs `[`X`][zarr.X]` read as undefined reference links (MD052); // pymdownx.magiclink auto-links bare URLs (MD034); // md_in_html lets us embed intentional raw HTML (MD033); // generated/included files (api stubs, snippets) need not open with an H1 (MD041). "MD052": false, "MD034": false, "MD033": false, "MD041": false, // Duplicate headings are legitimate under different sections (e.g. repeated // "Documentation"); only flag true sibling duplicates. "MD024": { "siblings_only": true }, // Opinionated table/link/command rules with low value for these docs. "MD055": false, // table pipe style "MD060": false, // table column style "MD059": false, // "descriptive" link text (no "click here") "MD014": false, // $ before commands without shown output // markdownlint does not understand MkDocs `!!!` admonitions, so it reads their // 4-space-indented bodies as indented code blocks and flags them (and, via inferred // file style, flags real fenced blocks too). Cannot coexist with our admonitions. "MD046": false // code block style (fenced vs indented) // Kept on (structural / real rendering bugs): MD012 (multiple blanks), MD022/MD031/MD032 // (blanks around headings/fences/lists), MD025 (single H1), MD029 (ordered-list prefix), // MD040 (fenced code language), MD042 (empty links), // MD047 (trailing newline), MD056 (table column count), among others. }, "globs": ["docs/**/*.md"], "ignores": [ "docs/api/**" // mkdocstrings stubs (`::: zarr.X`) ] } zarr-developers-zarr-python-a2f2415/.pre-commit-config.yaml000066400000000000000000000070351527014137600237530ustar00rootroot00000000000000ci: autoupdate_commit_msg: "chore: update pre-commit hooks" autoupdate_schedule: "monthly" autofix_prs: false # Both of these are `language: system` hooks that shell out to the local # toolchain — `uv` for mypy, `uv` and `just` for the docs build — and need # the repo checkout to resolve their environments from `uv.lock`. Neither is # available on pre-commit.ci's runners. Each is covered instead by a GitHub # Actions job (Lint for mypy, zarr-http-server for the docs build) and by # local prek runs. skip: [mypy, zarr-http-server-docs] default_stages: [pre-commit, pre-push] default_language_version: python: python3.12 repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.16.10 hooks: - id: ruff-check args: ["--fix", "--show-fixes"] - id: ruff-format - repo: https://github.com/codespell-project/codespell rev: v2.4.3 hooks: - id: codespell args: ["-L", "fo,ihs,kake,te", "-S", "fixture"] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: - id: check-yaml exclude: mkdocs.yml - id: trailing-whitespace - repo: https://github.com/DavidAnson/markdownlint-cli2 rev: v0.23.3 hooks: # Markdown structure/hygiene. Rule selection and ignores are in # .markdownlint-cli2.jsonc; complements ci/lint_docs.py (RST residue, # list-breaking fences) and `mkdocs build --strict`. Scoped to docs/ to # match the config's globs (pre-commit passes filenames, which would # otherwise override that scoping and lint all repo Markdown). - id: markdownlint-cli2 files: ^docs/ - repo: local hooks: - id: mypy name: mypy language: system entry: uv run --frozen mypy pass_filenames: false always_run: true types_or: [python, pyi] # Builds the zarr-http-server docs site with warnings as errors, which # catches a dead cross-reference or a nav entry pointing at a file that # no longer exists before it reaches CI. # # `stages: [pre-push]` overrides the default of running on every commit: # this is a whole-site build, too slow to pay per commit and only # actionable before the code leaves the machine. `files:` limits it to # changes that touch the package, and `pass_filenames: false` because # mkdocs builds the site, not a list of files. Delegating to the # justfile keeps one definition of the build shared with CI. - id: zarr-http-server-docs name: zarr-http-server docs build language: system entry: >- just --justfile packages/zarr-http-server/justfile --working-directory packages/zarr-http-server docs-check pass_filenames: false files: ^packages/zarr-http-server/ stages: [pre-push] - repo: https://github.com/scientific-python/cookie rev: 2026.08.14 hooks: - id: sp-repo-review - repo: https://github.com/numpy/numpydoc rev: v1.11.0 hooks: - id: numpydoc-validation - repo: local hooks: - id: ban-lstrip-rstrip name: ban lstrip/rstrip language: pygrep # Matches .lstrip() or .rstrip() where the string argument is 2+ characters. entry: "\\.(lstrip|rstrip)\\([\"'][^\"']{2,}[\"']\\)" types: [python] files: ^(src|tests)/ - repo: https://github.com/zizmorcore/zizmor-pre-commit rev: v1.30.1 hooks: - id: zizmor - repo: https://github.com/twisted/towncrier rev: 26.9.0 hooks: - id: towncrier-check zarr-developers-zarr-python-a2f2415/.python-version000066400000000000000000000000051527014137600224650ustar00rootroot000000000000003.12 zarr-developers-zarr-python-a2f2415/.pyup.yml000066400000000000000000000005131527014137600212620ustar00rootroot00000000000000# pyup.io config file # see https://pyup.io/docs/configuration/ for all available options schedule: every month requirements: - requirements_dev_minimal.txt: pin: True update: all - requirements_dev_numpy.txt: pin: True update: all - requirements_dev_optional.txt: pin: True update: all zarr-developers-zarr-python-a2f2415/.readthedocs.yaml000066400000000000000000000053741527014137600227250ustar00rootroot00000000000000version: 2 build: os: ubuntu-22.04 tools: python: "3.12" jobs: post_checkout: # Cancel pull request builds whose changes are confined to the packages # that have their own Read the Docs projects. Exit code 183 cancels the # build and reports success to the Git provider. Scoped to PR builds # ("external" versions) because origin/main is only a meaningful diff # base there. Compare against the merge base (origin/main...HEAD), not # against the tip of main: a two-dot diff also reports everything that # landed on main after the PR branched, which made every PR that was # behind main build every site. The checkout is shallow, so fetch the # full history first; without a merge base the diff fails and the build # runs, which is the safe default. Read the Docs strips shell quoting from commands, so the # exclude pathspecs must use the quote-free :! form, not ':(exclude)'. # The diff is taken against the merge base (origin/main...HEAD), not # against the tip of main: a two-dot diff also reports everything that # landed on main after the PR branched, which made every PR that was # behind main build every site. The checkout is shallow, so fetch the # full history first; without a merge base the diff fails and the build # runs, which is the safe default. - | if [ "$READTHEDOCS_VERSION_TYPE" = "external" ]; then git fetch --quiet --unshallow origin main pull/$READTHEDOCS_VERSION_NAME/head || true; if git diff --quiet origin/main...HEAD -- :!packages/zarr-metadata :!packages/zarr-indexing :!packages/zarr-http-server; then exit 183; fi fi install: - pip install --upgrade pip # The docs recipes resolve their toolchain from uv.lock, so uv is all that is # needed here besides just itself. - pip install uv==0.12.9 # No GitHub Action is available on Read the Docs, so fetch the official just # release and verify it against the checksum casey/just publishes. Installed # into the build virtualenv's bin, which is already on PATH. - | curl -fsSL -o /tmp/just.tar.gz https://github.com/casey/just/releases/download/1.58.0/just-1.58.0-x86_64-unknown-linux-musl.tar.gz echo "4a5cc2f53e6f0f8c59092a6cc38291eb729d46a7dd95d3ae582008881b84931d /tmp/just.tar.gz" | sha256sum -c - tar -xzf /tmp/just.tar.gz -C "$READTHEDOCS_VIRTUALENV_PATH/bin" just pre_build: - | if [ "$READTHEDOCS_VERSION_TYPE" != "tag" ]; then just changelog-build --version Unreleased --yes; fi build: html: - just docs-build --site-dir $READTHEDOCS_OUTPUT/html mkdocs: configuration: mkdocs.yml zarr-developers-zarr-python-a2f2415/FUNDING.yml000066400000000000000000000001031527014137600212740ustar00rootroot00000000000000github: [numfocus] custom: ['https://numfocus.org/donate-to-zarr'] zarr-developers-zarr-python-a2f2415/Justfile000066400000000000000000000144621527014137600212040ustar00rootroot00000000000000# Development and CI verbs live here; Hatch owns Python environments in pyproject.toml. # Install: uv tool install hatch==1.16.5 && uv tool install rust-just==1.58.0 # Select test dependencies/interpreter: HATCH_ENV=test.py3.13-minimal just test # On Windows, use Git Bash (the same shell used by the test workflow). set shell := ["bash", "-eu", "-o", "pipefail", "-c"] set windows-shell := ["bash", "-eu", "-o", "pipefail", "-c"] set positional-arguments hatch_env := env("HATCH_ENV", "test.py3.12-optional") # Deliberately a different variable from HATCH_ENV: `just gpu` must not inherit a # CPU test environment that happens to be exported in the caller's shell, which # would run `pytest -m gpu` against an environment built without the gpu feature. gpu_env := env("GPU_HATCH_ENV", "gputest.py3.12") # Pinned so a prek release cannot change what CI lints without a commit here. prek_version := "0.5.3" # Documentation and changelog tooling resolves from uv.lock, not a hatch environment. # There is only ever one docs toolchain, so it can be locked and hash-verified, and # dependabot's uv ecosystem keeps it current. Hatch still owns the test environments, # which exist per interpreter and per dependency set and cannot live in one lockfile. docs_run := "uv run --frozen --group docs" # The hatch docs environment used to set these; they belong with the mkdocs calls now. mkdocs_env := "DISABLE_MKDOCS_2_WARNING=true NO_MKDOCS_2_WARNING=true" # List available recipes default: @just --list # List available Python environments envs: hatch env show # Create the selected Python environment and list its installed packages setup: && list-env hatch env create {{ quote(hatch_env) }} # List packages in the selected Python environment list-env: hatch run {{ quote(hatch_env) }}:pip list # Run unit tests; pass pytest arguments, e.g. just test -k 'array and resize' test *args: hatch run {{ quote(hatch_env) }}:pytest --ignore tests/benchmarks "$@" # Run unit tests and write coverage.xml and junit.xml coverage *args: hatch run {{ quote(hatch_env) }}:coverage run --source=src -m pytest --ignore tests/benchmarks --junitxml=junit.xml -o junit_family=legacy "$@" hatch run {{ quote(hatch_env) }}:coverage xml # Run unit tests and generate an HTML coverage report coverage-html *args: hatch run {{ quote(hatch_env) }}:coverage run --source=src -m pytest --ignore tests/benchmarks "$@" hatch run {{ quote(hatch_env) }}:coverage html # Serve the HTML coverage report (default port 8000) coverage-serve *args: hatch run {{ quote(hatch_env) }}:python -m http.server -d htmlcov "$@" # Run slow Hypothesis tests and write coverage.xml hypothesis *args: hatch run {{ quote(hatch_env) }}:coverage run --source=src -m pytest -nauto --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful* tests/test_array_stateful.py "$@" hatch run {{ quote(hatch_env) }}:coverage xml # Validate executable documentation code blocks doctest *args: hatch run doctest:pytest tests/test_docs.py -v "$@" # Run the benchmark suite benchmark *args: hatch run {{ quote(hatch_env) }}:pytest --benchmark-enable tests/benchmarks "$@" # Run benchmarks under CodSpeed benchmark-codspeed *args: hatch run {{ quote(hatch_env) }}:pytest tests/benchmarks --codspeed "$@" # Run GPU tests with coverage; select the environment with GPU_HATCH_ENV gpu *args: HATCH_ENV={{ quote(gpu_env) }} just coverage -m gpu "$@" # Build documentation (warnings are errors) docs-build *args: {{ mkdocs_env }} {{ docs_run }} mkdocs build --strict "$@" # Serve documentation with live reload docs-serve *args: {{ mkdocs_env }} {{ docs_run }} mkdocs serve --watch src "$@" # Check that every public export has API documentation check-doc-exports *args: {{ docs_run }} python ci/check_documented_exports.py docs/api "$@" # Check documentation source conventions lint-docs *args: {{ docs_run }} python ci/lint_docs.py "$@" # Report unlinked types in built documentation check-doc-links *args: {{ docs_run }} python ci/check_unlinked_types.py "$@" # Run source documentation checks followed by a strict build docs-check: check-doc-exports lint-docs docs-build # Run all pre-commit hooks (ruff, codespell, mypy, repo-review, ...) lint *args: uvx prek@{{ prek_version }} run --show-diff-on-failure --color=always --all-files "$@" # Run hooks with a custom selection, e.g. just hooks run --last-commit hooks +args: uvx prek@{{ prek_version }} "$@" # prek is installed as a persistent uv tool rather than run through uvx: the hook shim # prek writes into .git/hooks hard-codes the binary path it was installed from and falls # back to `prek` on PATH, and a uvx archive path stops existing at the next cache prune. # Install local pre-commit hooks hooks-install: uv tool install prek=={{ prek_version }} prek install # Type-check the library using the locked tooling environment typecheck *args: uv run --frozen mypy "$@" # Check that uv.lock is in sync with pyproject.toml lock-check: uv lock --check # Update the dependency lockfile lock *args: uv lock "$@" # Build the source distribution and wheel build *args: hatch build "$@" # Create a changelog fragment (interactive without arguments) changelog *args: {{ docs_run }} towncrier create "$@" # Preview the next release's changelog changelog-draft *args: {{ docs_run }} towncrier build --draft --version Unreleased "$@" # Build release notes; pass --version and --yes when preparing a release changelog-build *args: {{ docs_run }} towncrier build "$@" # Check changelog filenames (default: changes/; accepts a package changes directory) check-changelogs *args: uv run --no-project python ci/check_changelog_entries.py "$@" # Check recipe formatting of the root Justfile and every package justfile just-check: just --fmt --check shopt -s nullglob; for f in packages/*/justfile; do just --justfile "$f" --fmt --check; done # Run a zarr-metadata recipe, or list its recipes with no arguments zarr-metadata *args: just --justfile packages/zarr-metadata/justfile "$@" # Run a zarr-indexing recipe, or list its recipes with no arguments zarr-indexing *args: just --justfile packages/zarr-indexing/justfile "$@" # Run a zarr-http-server recipe, or list its recipes with no arguments zarr-http-server *args: just --justfile packages/zarr-http-server/justfile "$@" zarr-developers-zarr-python-a2f2415/LICENSE.txt000066400000000000000000000021441527014137600213110ustar00rootroot00000000000000The MIT License (MIT) Copyright (c) 2015-2025 Zarr Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. zarr-developers-zarr-python-a2f2415/README.md000066400000000000000000000067151527014137600207550ustar00rootroot00000000000000

# Zarr [![Latest Release](https://badge.fury.io/py/zarr.svg)](https://pypi.org/project/zarr/) [![CondaForge](https://anaconda.org/conda-forge/zarr/badges/version.svg)](https://anaconda.org/anaconda/zarr/) [![Package Status](https://img.shields.io/pypi/status/zarr.svg)](https://pypi.org/project/zarr/) [![License](https://img.shields.io/pypi/l/zarr.svg)](https://github.com/zarr-developers/zarr-python/blob/main/LICENSE.txt) [![Coverage](https://codecov.io/gh/zarr-developers/zarr-python/branch/main/graph/badge.svg)](https://app.codecov.io/gh/zarr-developers/zarr-python) [![Downloads](https://static.pepy.tech/badge/zarr)](https://zarr.readthedocs.io/en/stable/) [![Developer Chat](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://ossci.zulipchat.com/#narrow/channel/423692-Zarr-Python) [![Citation](https://zenodo.org/badge/DOI/10.5281/zenodo.3773450.svg)](https://doi.org/10.5281/zenodo.3773450) ## What is it? The `zarr` library is a Python implementation of the [Zarr storage format](https://zarr.dev/). `zarr` delivers compressed, chunked, N-dimensional arrays that work well for parallel computing and object storage. See the [documentation](https://zarr.readthedocs.io/en/stable/) for more information. ## Main Features - [**Create**](https://zarr.readthedocs.io/en/stable/user-guide/arrays/#creating-an-array) N-dimensional arrays with NumPy-compatible `dtype`s. - [**Chunk arrays**](https://zarr.readthedocs.io/en/stable/user-guide/performance/#chunk-optimizations) along any dimension. - [**Encode**](https://zarr.readthedocs.io/en/stable/user-guide/arrays/#compressors) chunks using a variety of useful encodings (e.g., compression). - [**Store arrays**](https://zarr.readthedocs.io/en/stable/user-guide/storage/) in memory, on disk, inside a zip file, on S3, etc... - [**Read**](https://zarr.readthedocs.io/en/stable/user-guide/arrays/#reading-and-writing-data) an array [**concurrently**](https://zarr.readthedocs.io/en/stable/user-guide/performance/#parallel-computing-and-synchronization) from multiple threads or processes. - [**Write**](https://zarr.readthedocs.io/en/stable/user-guide/arrays/#reading-and-writing-data) to an array concurrently from multiple threads or processes. - Organize arrays into hierarchies via [**groups**](https://zarr.readthedocs.io/en/stable/quick-start/#hierarchical-groups). ## Where to get it Zarr can be installed from PyPI using `pip`: ```bash pip install zarr ``` or via `conda`: ```bash conda install -c conda-forge zarr ``` For more details, including how to install from source, see the [installation documentation](https://zarr.readthedocs.io/en/stable/#installation). ## Repository sub-packages In addition to the primary `zarr` implementation, this repository contains other packages that provide specialized functionality with minimal dependencies: - [`zarr-metadata`](https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata): Tools for Zarr metadata. Install with `pip install zarr-metadata`. - [`zarr-indexing`](https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-indexing): Tools for lazily indexing chunked arrays. Install with `pip install zarr-indexing`. - [`zarr-http-server`](https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-http-server): An HTTP server implementation targeting Zarr data. Install with `pip install zarr-http-server`. zarr-developers-zarr-python-a2f2415/TEAM.md000066400000000000000000000014031527014137600205330ustar00rootroot00000000000000## Active core-developers - @joshmoore (Josh Moore) - @jni (Juan Nunez-Iglesias) - @rabernat (Ryan Abernathey) - @jhamman (Joe Hamman) - @d-v-b (Davis Bennett) - @jakirkham (jakirkham) - @martindurant (Martin Durant) - @normanrz (Norman Rzepka) - @dstansby (David Stansby) - @dcherian (Deepak Cherian) - @TomAugspurger (Tom Augspurger) - @maxrjones (Max Jones) - @ilan-gold (Ilan Gold) ## Emeritus core-developers - @alimanfoo (Alistair Miles) - @shoyer (Stephan Hoyer) - @ryan-williams (Ryan Williams) - @jrbourbeau (James Bourbeau) - @mzjp2 (Zain Patel) - @grlee77 (Gregory Lee) ## Former core-developers - @jeromekelleher (Jerome Kelleher) - @tjcrone (Tim Crone) - @funkey (Jan Funke) - @shikharsg - @Carreau (Matthias Bussonnier) - @dazzag24 - @WardF (Ward Fisher) zarr-developers-zarr-python-a2f2415/bench/000077500000000000000000000000001527014137600205445ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/bench/compress_normal.py000066400000000000000000000016551527014137600243300ustar00rootroot00000000000000import sys import timeit import blosc import line_profiler import numpy as np import zarr if __name__ == "__main__": sys.path.insert(0, "..") # setup a = np.random.normal(2000, 1000, size=200000000).astype("u2") z = zarr.empty_like( a, chunks=1000000, compression="blosc", compression_opts={"cname": "lz4", "clevel": 5, "shuffle": 2}, ) print(z) print("*" * 79) # time t = timeit.repeat("z[:] = a", repeat=10, number=1, globals=globals()) print(t) print(min(t)) print(z) # profile profile = line_profiler.LineProfiler(blosc.compress) profile.run("z[:] = a") profile.print_stats() print("*" * 79) # time t = timeit.repeat("z[:]", repeat=10, number=1, globals=globals()) print(t) print(min(t)) # profile profile = line_profiler.LineProfiler(blosc.decompress) profile.run("z[:]") profile.print_stats() zarr-developers-zarr-python-a2f2415/bench/compress_normal.txt000066400000000000000000000252341527014137600245160ustar00rootroot00000000000000zarr.core.Array((200000000,), uint16, chunks=(1000000,), order=C) compression: blosc; compression_opts: {'clevel': 5, 'cname': 'lz4', 'shuffle': 2} nbytes: 381.5M; nbytes_stored: 294; ratio: 1360544.2; initialized: 0/200 store: builtins.dict ******************************************************************************* [0.27119584499996563, 0.2855067059999783, 0.2887747180002407, 0.3058794240005227, 0.3139041080003153, 0.3021271820007314, 0.31543190899992624, 0.31403100900024583, 0.3272544129995367, 0.31834129100025166] 0.27119584499996563 zarr.core.Array((200000000,), uint16, chunks=(1000000,), order=C) compression: blosc; compression_opts: {'clevel': 5, 'cname': 'lz4', 'shuffle': 2} nbytes: 381.5M; nbytes_stored: 314.1M; ratio: 1.2; initialized: 200/200 store: builtins.dict Timer unit: 1e-06 s Total time: 0.297223 s File: /home/aliman/code/github/alimanfoo/zarr/zarr/blosc.pyx Function: compress at line 137 Line # Hits Time Per Hit % Time Line Contents ============================================================== 137 def compress(source, char* cname, int clevel, int shuffle): 138 """Compress data in a numpy array. 139 140 Parameters 141 ---------- 142 source : array-like 143 Data to be compressed. 144 cname : bytes 145 Name of compression library to use. 146 clevel : int 147 Compression level. 148 shuffle : int 149 Shuffle filter. 150 151 Returns 152 ------- 153 dest : bytes-like 154 Compressed data. 155 156 """ 157 158 cdef: 159 char *source_ptr 160 char *dest_ptr 161 Py_buffer source_buffer 162 size_t nbytes, cbytes, itemsize 163 200 506 2.5 0.2 array.array char_array_template = array.array('b', []) 164 array.array dest 165 166 # setup source buffer 167 200 458 2.3 0.2 PyObject_GetBuffer(source, &source_buffer, PyBUF_ANY_CONTIGUOUS) 168 200 119 0.6 0.0 source_ptr = source_buffer.buf 169 170 # setup destination 171 200 239 1.2 0.1 nbytes = source_buffer.len 172 200 103 0.5 0.0 itemsize = source_buffer.itemsize 173 200 2286 11.4 0.8 dest = array.clone(char_array_template, nbytes + BLOSC_MAX_OVERHEAD, 174 zero=False) 175 200 129 0.6 0.0 dest_ptr = dest.data.as_voidptr 176 177 # perform compression 178 200 1734 8.7 0.6 if _get_use_threads(): 179 # allow blosc to use threads internally 180 200 167 0.8 0.1 compressor_set = blosc_set_compressor(cname) 181 200 94 0.5 0.0 if compressor_set < 0: 182 raise ValueError('compressor not supported: %r' % cname) 183 200 288570 1442.8 97.1 with nogil: 184 cbytes = blosc_compress(clevel, shuffle, itemsize, nbytes, 185 source_ptr, dest_ptr, 186 nbytes + BLOSC_MAX_OVERHEAD) 187 188 else: 189 with nogil: 190 cbytes = blosc_compress_ctx(clevel, shuffle, itemsize, nbytes, 191 source_ptr, dest_ptr, 192 nbytes + BLOSC_MAX_OVERHEAD, cname, 193 0, 1) 194 195 # release source buffer 196 200 616 3.1 0.2 PyBuffer_Release(&source_buffer) 197 198 # check compression was successful 199 200 120 0.6 0.0 if cbytes <= 0: 200 raise RuntimeError('error during blosc compression: %d' % cbytes) 201 202 # resize after compression 203 200 1896 9.5 0.6 array.resize(dest, cbytes) 204 205 200 186 0.9 0.1 return dest ******************************************************************************* [0.24293352799941204, 0.2324290420001489, 0.24935673900017719, 0.25716222699975333, 0.24246313799994823, 0.23272456500035332, 0.2636815870000646, 0.2576046349995522, 0.2781278639995435, 0.23824110699933954] 0.2324290420001489 Timer unit: 1e-06 s Total time: 0.240178 s File: /home/aliman/code/github/alimanfoo/zarr/zarr/blosc.pyx Function: decompress at line 75 Line # Hits Time Per Hit % Time Line Contents ============================================================== 75 def decompress(source, dest): 76 """Decompress data. 77 78 Parameters 79 ---------- 80 source : bytes-like 81 Compressed data, including blosc header. 82 dest : array-like 83 Object to decompress into. 84 85 Notes 86 ----- 87 Assumes that the size of the destination buffer is correct for the size of 88 the uncompressed data. 89 90 """ 91 cdef: 92 int ret 93 char *source_ptr 94 char *dest_ptr 95 Py_buffer source_buffer 96 array.array source_array 97 Py_buffer dest_buffer 98 size_t nbytes 99 100 # setup source buffer 101 200 573 2.9 0.2 if PY2 and isinstance(source, array.array): 102 # workaround fact that array.array does not support new-style buffer 103 # interface in PY2 104 release_source_buffer = False 105 source_array = source 106 source_ptr = source_array.data.as_voidptr 107 else: 108 200 112 0.6 0.0 release_source_buffer = True 109 200 144 0.7 0.1 PyObject_GetBuffer(source, &source_buffer, PyBUF_ANY_CONTIGUOUS) 110 200 98 0.5 0.0 source_ptr = source_buffer.buf 111 112 # setup destination buffer 113 200 552 2.8 0.2 PyObject_GetBuffer(dest, &dest_buffer, 114 PyBUF_ANY_CONTIGUOUS | PyBUF_WRITEABLE) 115 200 100 0.5 0.0 dest_ptr = dest_buffer.buf 116 200 84 0.4 0.0 nbytes = dest_buffer.len 117 118 # perform decompression 119 200 1856 9.3 0.8 if _get_use_threads(): 120 # allow blosc to use threads internally 121 200 235286 1176.4 98.0 with nogil: 122 ret = blosc_decompress(source_ptr, dest_ptr, nbytes) 123 else: 124 with nogil: 125 ret = blosc_decompress_ctx(source_ptr, dest_ptr, nbytes, 1) 126 127 # release buffers 128 200 754 3.8 0.3 if release_source_buffer: 129 200 326 1.6 0.1 PyBuffer_Release(&source_buffer) 130 200 165 0.8 0.1 PyBuffer_Release(&dest_buffer) 131 132 # handle errors 133 200 128 0.6 0.1 if ret <= 0: 134 raise RuntimeError('error during blosc decompression: %d' % ret) zarr-developers-zarr-python-a2f2415/changes/000077500000000000000000000000001527014137600210755ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/changes/.gitignore000066400000000000000000000000141527014137600230600ustar00rootroot00000000000000!.gitignore zarr-developers-zarr-python-a2f2415/changes/README.md000066400000000000000000000005751527014137600223630ustar00rootroot00000000000000Writing a changelog entry ------------------------- Please put a new file in this directory named `xxxx..md`, where - `xxxx` is the pull request number associated with this entry - `` is one of: - feature - bugfix - doc - removal - misc Inside the file, please write a short description of what you have changed, and how it impacts users of `zarr-python`. zarr-developers-zarr-python-a2f2415/ci/000077500000000000000000000000001527014137600200605ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/ci/big-endian/000077500000000000000000000000001527014137600220555ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/ci/big-endian/Dockerfile000066400000000000000000000054411527014137600240530ustar00rootroot00000000000000# Test image for a big-endian host (s390x, run under QEMU by .github/workflows/big-endian.yml). # # Debian packages are used as a source of prebuilt binaries for s390x, which PyPI does not # have, so that numpy and friends do not compile under emulation. They do not define the # test environment: zarr's pyproject.toml does. The workflow installs zarr with its # dependencies resolved and runs `pip check`, so a Debian package that falls outside the # declared constraints fails the job at install time instead of through unrelated-looking # test failures. Anything Debian cannot satisfy is built here from source so the compile is # cached as an image layer rather than repeated in every job. # # Debian does not package Google's crc32c library, so it is built here for the # google-crc32c C extension. FROM debian:sid RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \ build-essential cmake cython3 git meson ninja-build pkgconf python3 python3-dev \ python3-mesonpy python3-pip python3-setuptools-scm python3-venv \ python3-crc32c python3-donfig python3-fsspec python3-hatch-vcs python3-hatchling \ python3-hypothesis python3-numcodecs python3-numpy python3-numpydoc python3-packaging \ python3-pytest python3-pytest-asyncio python3-pytest-benchmark python3-pytest-xdist \ python3-tomlkit python3-typer python3-typing-extensions \ && rm -rf /var/lib/apt/lists/* RUN git clone -q --depth 1 --branch 1.1.2 https://github.com/google/crc32c /tmp/crc32c \ && cmake -S /tmp/crc32c -B /tmp/crc32c/build -DBUILD_SHARED_LIBS=1 \ -DCRC32C_BUILD_TESTS=0 -DCRC32C_BUILD_BENCHMARKS=0 -DCRC32C_USE_GLOG=0 \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ && cmake --build /tmp/crc32c/build -j"$(nproc)" \ && cmake --install /tmp/crc32c/build \ && ldconfig \ && rm -rf /tmp/crc32c RUN python3 -m venv --system-site-packages /venv ENV PATH=/venv/bin:$PATH RUN pip install -q --no-binary google-crc32c google-crc32c msgspec pytest-accept # Debian sid's python3-dateutil 3.0.0 declares a dependency on the `tzdata` distribution, # which no Debian package installs, so `pip check` fails on the image as built. tzdata is # pure Python; install it from PyPI. RUN pip install -q tzdata # Debian sid ships numcodecs 0.17.0, which zarr excludes (see pyproject.toml; GHSA-jqc8-qjfv-5m6r), # and PyPI has no s390x wheels, so build a satisfying numcodecs from source into the venv, where # it shadows the Debian package. Built without isolation so the Debian meson, Cython and numpy # are used instead of compiling a build-time numpy under emulation. Remove once Debian ships a # version that satisfies zarr's constraint; `pip check` in the workflow will say when it does not. RUN pip install -q --no-binary numcodecs --no-build-isolation "numcodecs>=0.16.6,!=0.17.0" zarr-developers-zarr-python-a2f2415/ci/check_changelog_entries.py000066400000000000000000000043071527014137600252530ustar00rootroot00000000000000""" Check changelog entries have the correct filename structure. Usage: python check_changelog_entries.py [DIRECTORY] DIRECTORY defaults to the repo-root `changes/`. """ import sys from pathlib import Path VALID_CHANGELOG_TYPES = ["feature", "bugfix", "doc", "removal", "misc"] REPO_ROOT = Path(__file__).parent.parent.resolve() DEFAULT_DIRECTORY = REPO_ROOT / "changes" def is_int(s: str) -> bool: try: int(s) except ValueError: return False else: return True def check(directory: Path) -> int: print(f"Looking for changelog entries in {directory}") entries = list(directory.glob("*")) entries = [e for e in entries if e.name not in [".gitignore", "README.md"]] print(f"Found {len(entries)} entries") print() bad_suffix = [e for e in entries if e.suffix != ".md"] bad_issue_no = [e for e in entries if not is_int(e.name.split(".")[0])] # Only flag bad_type for files that have already passed the prior two # checks; otherwise `e.name.split(".")[1]` may raise IndexError on a # malformed name like `notes.md`. bad_type = [ e for e in entries if e.suffix == ".md" and is_int(e.name.split(".")[0]) and e.name.split(".")[1] not in VALID_CHANGELOG_TYPES ] if bad_suffix or bad_issue_no or bad_type: if bad_suffix: print("Changelog entries without .md suffix") print("-------------------------------------") print("\n".join(p.name for p in bad_suffix)) print() if bad_issue_no: print("Changelog entries without integer issue number") print("----------------------------------------------") print("\n".join(p.name for p in bad_issue_no)) print() if bad_type: print("Changelog entries without valid type") print("------------------------------------") print("\n".join(p.name for p in bad_type)) print(f"Valid types are: {VALID_CHANGELOG_TYPES}") print() return 1 return 0 if __name__ == "__main__": directory = Path(sys.argv[1]).resolve() if len(sys.argv) > 1 else DEFAULT_DIRECTORY sys.exit(check(directory)) zarr-developers-zarr-python-a2f2415/ci/check_documented_exports.py000066400000000000000000000135301527014137600255040ustar00rootroot00000000000000"""Check that every public top-level export is in the API reference. The API reference is authored as explicit mkdocstrings directives (``::: target``) under ``docs/api/`` -- one per documented symbol -- rather than autodoc, so a newly added ``zarr.__all__`` entry will not appear in the docs until someone writes a page for it (or it becomes a rendered member of an already-documented module). This script catches that gap: it resolves every ``:::`` target, expands module directives into the members they render (honoring ``members: false``), and asserts each name in ``zarr.__all__`` resolves to a documented object. Usage: python ci/check_documented_exports.py [API_DOCS_DIR] API_DOCS_DIR defaults to the repo-root ``docs/api``. Exits non-zero (and prints the undocumented exports to stderr) if any public export is missing from the reference. """ from __future__ import annotations import importlib import re import sys from pathlib import Path from types import ModuleType from typing import TYPE_CHECKING, Any import zarr if TYPE_CHECKING: from collections.abc import Iterator REPO_ROOT = Path(__file__).parent.parent.resolve() DEFAULT_API_DOCS_ROOT = REPO_ROOT / "docs" / "api" # Names in zarr.__all__ that are intentionally absent from the API reference. # Keep this list short and justified -- it is the only escape hatch from the guard. EXEMPT_EXPORTS = { "__version__", # version string, not an API symbol "print_debug_info", # debugging helper, deliberately not in the reference } # A mkdocstrings autodoc directive: `::: some.dotted.target` at the start of a line. DIRECTIVE_RE = re.compile(r"^:::[ \t]+(?P\S+)") # `members: false` (or `members: []`) within a directive's option block disables # rendering of a module's members. MEMBERS_DISABLED_RE = re.compile(r"^\s+members:\s*(false|\[\s*\])\s*$") def resolve(target: str) -> Any: """Resolve a `:::` target (a dotted path) to the Python object it documents.""" try: return importlib.import_module(target) except ImportError: pass module_path, _, attr = target.rpartition(".") try: return getattr(importlib.import_module(module_path), attr) except (ImportError, AttributeError): return None def iter_directives(text: str) -> Iterator[tuple[str, bool]]: """Yield ``(target, members_enabled)`` for each ``:::`` directive in ``text``. The file is split into lines once; for each directive we scan its indented option block -- stopping at the first non-indented line, which ends the block -- so options belonging to a later directive are never consulted. ``members_enabled`` is False when that block sets ``members: false`` (or ``members: []``).""" lines = text.splitlines() i = 0 while i < len(lines): match = DIRECTIVE_RE.match(lines[i]) if match is None: i += 1 continue members_enabled = True i += 1 while i < len(lines): line = lines[i] if line.strip() == "": i += 1 continue if not line.startswith((" ", "\t")): break # non-indented line: end of this directive's option block if MEMBERS_DISABLED_RE.match(line): members_enabled = False i += 1 yield match.group("target"), members_enabled def module_member_ids(module: ModuleType) -> Iterator[int]: """Yield the id() of each public member a module directive renders. The rendered members are the module's ``__all__`` if defined, else its public (non-underscore) attributes.""" member_names = getattr(module, "__all__", None) or [ name for name in dir(module) if not name.startswith("_") ] for name in member_names: member = getattr(module, name, None) if member is not None: yield id(member) def documented_object_ids(api_docs_root: Path) -> set[int]: """Collect the id()s of every object rendered by a `:::` directive under api_docs_root. A directive pointing at an object documents that object. A directive pointing at a module documents the module's public members unless the directive sets ``members: false``.""" documented: set[int] = set() for md_file in sorted(api_docs_root.rglob("*.md")): for target, members_enabled in iter_directives(md_file.read_text(encoding="utf-8")): obj = resolve(target) if obj is None: continue documented.add(id(obj)) if isinstance(obj, ModuleType) and members_enabled: documented.update(module_member_ids(obj)) return documented def find_undocumented_exports(api_docs_root: Path) -> list[str]: documented = documented_object_ids(api_docs_root) return sorted( name for name in zarr.__all__ if name not in EXEMPT_EXPORTS and id(getattr(zarr, name)) not in documented ) def main() -> int: args = sys.argv[1:] api_docs_root = Path(args[0]).resolve() if args else DEFAULT_API_DOCS_ROOT if not api_docs_root.exists(): print(f"{api_docs_root} does not exist.", file=sys.stderr) return 1 missing = find_undocumented_exports(api_docs_root) if not missing: print(f"All {len(zarr.__all__)} public exports are documented.") return 0 print( f"Found {len(missing)} public export(s) in zarr.__all__ missing from the API " "reference (docs/api/):\n", file=sys.stderr, ) for name in missing: print(f" - zarr.{name}", file=sys.stderr) print( "\nAdd a `::: zarr.` page under docs/api/zarr/ (and register it in " "mkdocs.yml and docs/api/zarr/index.md), or -- if the export is intentionally " "undocumented -- add it to EXEMPT_EXPORTS in this script with a reason.", file=sys.stderr, ) return 1 if __name__ == "__main__": sys.exit(main()) zarr-developers-zarr-python-a2f2415/ci/check_unlinked_types.py000066400000000000000000000055171527014137600246340ustar00rootroot00000000000000"""Check for unlinked type annotations in built documentation. mkdocstrings renders resolved types as links and unresolved types as Name without an anchor. This script finds all such unlinked types in the built HTML and reports them. Usage: python ci/check_unlinked_types.py [site_dir] Raises ValueError if unlinked types are found. """ from __future__ import annotations import re import sys from pathlib import Path # Matches the griffe/mkdocstrings pattern for unlinked cross-references: # Name UNLINKED_PATTERN = re.compile( r'(?P[^<]+)' ) # Patterns to exclude from the report EXCLUDE_PATTERNS = [ # TypeVars and type parameters (single brackets like Foo[T]) re.compile(r"\[.+\]$"), # Dataclass field / namedtuple field references (contain parens) re.compile(r"\("), # Private names re.compile(r"\._"), # Dunder attributes re.compile(r"\.__\w+__$"), # Testing utilities re.compile(r"^zarr\.testing\."), # Third-party types (hypothesis, pytest, etc.) re.compile(r"^(hypothesis|pytest|typing_extensions|builtins|dataclasses)\."), ] def should_exclude(qualname: str) -> bool: return any(p.search(qualname) for p in EXCLUDE_PATTERNS) def find_unlinked_types(site_dir: Path) -> dict[str, set[str]]: """Find all unlinked types in built HTML files. Returns a dict mapping qualified type names to the set of pages where they appear. """ api_dir = site_dir / "api" if not api_dir.exists(): raise FileNotFoundError(f"{api_dir} does not exist. Run 'mkdocs build' first.") unlinked: dict[str, set[str]] = {} for html_file in api_dir.rglob("*.html"): content = html_file.read_text(errors="replace") rel_path = str(html_file.relative_to(site_dir)) for match in UNLINKED_PATTERN.finditer(content): qualname = match.group("qualname") if not should_exclude(qualname): unlinked.setdefault(qualname, set()).add(rel_path) return unlinked def main() -> None: site_dir = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("site") unlinked = find_unlinked_types(site_dir) if not unlinked: print("No unlinked types found.") return lines = [f"Found {len(unlinked)} unlinked types:\n"] for qualname in sorted(unlinked): pages = sorted(unlinked[qualname]) lines.append(f" {qualname}") lines.extend(f" - {page}" for page in pages) all_pages = {p for ps in unlinked.values() for p in ps} lines.append(f"\nTotal: {len(unlinked)} unlinked types across {len(all_pages)} pages") report = "\n".join(lines) raise ValueError(report) if __name__ == "__main__": main() zarr-developers-zarr-python-a2f2415/ci/lint_docs.py000066400000000000000000000346161527014137600224220ustar00rootroot00000000000000"""Lint docstrings and Markdown for reStructuredText markup that won't render. This project renders API docs with mkdocstrings (``docstring_style: numpy``) and prose with MkDocs + Markdown -- not Sphinx/reStructuredText. RST constructs that survive from older docstrings (or muscle memory) are not interpreted: a Sphinx role passes through as literal text instead of becoming a link, an ``.. note::`` directive renders as a stray line, and a ``:param:`` field list never becomes a documented parameter. Crucially, none of this is caught by the rest of the docs CI. ``mkdocs build --strict`` sees the residue as ordinary prose (no warning), and ``ci/check_unlinked_types.py`` only finds cross-references mkdocstrings *attempted* to resolve -- a raw ``:class:`` role is never attempted, so it leaves no unlinked-type span. This linter fills that gap with a fast, source-level check that needs no docs build. Checks fall into two groups -- RST markup that silently fails under MkDocs/mkdocstrings, and Markdown structural problems that render as valid-but-wrong HTML (so `mkdocs build` emits no warning): sphinx-role :class:`X`, :func:`X`, :py:meth:`X` -> [`X`][zarr.X] rst-directive .. note:: / .. code-block:: python -> MkDocs admonition / fenced code rst-field :param x:, :returns:, :rtype: -> numpydoc Parameters/Returns/Raises rst-link `text `_ -> [text](https://example) list-break unindented code fence between list items -> indent the fence under its item list-indent continuation block indented < 4 spaces -> indent it 4 spaces list-blank list item directly after indented block -> blank line before the item The ``list-break`` check catches a fenced code block at column 0 placed *between* two list items: because the fence is not indented into the preceding item, Markdown ends the list at the fence and the following item starts a fresh list -- renumbering an ordered list (1, 1, 2 instead of 1, 2, 3) or breaking the grouping/spacing of any list. markdownlint's MD029 only notices this for sequentially-numbered ordered lists; lazily-numbered (1., 1.) and unordered lists slip past it, so this structural check covers the gap. The ``list-indent`` and ``list-blank`` checks catch the two halves of Python-Markdown's strict list-continuation rules, which differ from CommonMark. A blank-line-separated block (paragraph, nested list, table) belongs to a list item only when indented at least 4 spaces; at the 2-space indent other renderers accept, Python-Markdown ends the list and the block escapes to the top level (``list-indent``). And a new list item can not start directly after an indented continuation block: without a blank line first, the ``- `` line is lazily absorbed into the preceding paragraph as literal text (``list-blank``). Both produced silently-broken changelog rendering in ``docs/release-notes.md``. Usage: python ci/lint_docs.py [PATH ...] PATH defaults to the repo-root ``src/zarr`` and ``docs``. Each PATH may be a file or a directory (directories are searched for ``*.py`` and ``*.md``). Exits non-zero if any issues are found. """ from __future__ import annotations import ast import re import sys from dataclasses import dataclass from pathlib import Path from typing import NamedTuple REPO_ROOT = Path(__file__).parent.parent.resolve() DEFAULT_PATHS = (REPO_ROOT / "src" / "zarr", REPO_ROOT / "docs") # A Sphinx interpreted-text role: an optional domain, a role name, then a backtick # target -- e.g. :class:`Foo` or :py:meth:`Foo.bar`. Requires the trailing backtick so # plain "::" (RST literal markers, time strings, mkdocs-material :icon: shortcodes) and # URLs ("https://") never match. SPHINX_ROLE = re.compile(r":[a-zA-Z_]\w*(?::[a-zA-Z_]\w*)?:`[^`\n]+`") # An RST directive line: ".. name::" (with or without an argument after it). RST hyperlink # targets (".. _label:") and comments (".. text") lack the "::" and are not flagged. RST_DIRECTIVE = re.compile(r"^\s*\.\.[ \t]+[\w-]+::") # An RST field-list entry used for docstring fields. The role names above (class, func, # ...) are deliberately excluded so a role is reported as a role, not a field. RST_FIELD = re.compile( r"^\s*:(param|parameter|arg|argument|key|keyword|kwarg|type|returns?|rtype" r"|raises?|except|exception|yields?|ytype|var|cvar|ivar)\b[^:]*:" ) # An RST external hyperlink: `text `_ RST_LINK = re.compile(r"`[^`\n]+\n]+>`_") # A list item at column 0: an ordered marker (1. / 1)) or a bullet (-, *, +) followed by # whitespace and content. Leading-whitespace (nested/continuation) lines are intentionally # not matched -- the list-break check only fires on top-level items. LIST_ITEM = re.compile(r"^(?:\d+[.)]|[-*+])\s+\S") class Check(NamedTuple): """One docs-residue check: its category, the line pattern that flags it (None for a structural check matched outside ``_scan_line``), and the user-facing remediation shown by ``main()``. Keeping ``example``/``fix`` here makes this the single source for the help text, so adding a check can't leave the help out of date.""" category: str pattern: re.Pattern[str] | None example: str fix: str # The ``list-*`` checks carry no pattern -- they are detected structurally, not by scanning # a single line -- but they appear here so they share the remediation help. CHECKS = ( Check("sphinx-role", SPHINX_ROLE, ":class:`X`", "[`X`][zarr.X]"), Check("rst-directive", RST_DIRECTIVE, ".. note::", "MkDocs admonition (!!! note)"), Check("rst-field", RST_FIELD, ":param x:", "numpydoc Parameters/Returns/Raises section"), Check("rst-link", RST_LINK, "`text `_", "[text](url)"), Check("list-break", None, "fence between items", "indent the fence 4 spaces to nest it"), Check("list-indent", None, "2-space continuation", "indent the block 4 spaces under its item"), Check("list-blank", None, "item after indented block", "add a blank line before the item"), ) @dataclass(frozen=True) class Finding: path: Path line: int category: str snippet: str def format(self) -> str: try: location: Path | str = self.path.relative_to(REPO_ROOT) except ValueError: location = self.path return f" {location}:{self.line}: [{self.category}] {self.snippet.strip()}" def _scan_line(text: str) -> list[str]: """Return every RST-residue category found in a single line (a line can carry more than one, e.g. a role and an external link).""" return [c.category for c in CHECKS if c.pattern is not None and c.pattern.search(text)] def lint_python(path: Path) -> list[Finding]: """Scan the docstrings (module, classes, functions) of a Python file. Only docstrings are checked -- they are what mkdocstrings renders -- so RST-looking text inside ordinary code or string literals is never misreported.""" source = path.read_text(encoding="utf-8") try: tree = ast.parse(source) except SyntaxError as exc: # pragma: no cover - surfaced, not silently skipped return [Finding(path, exc.lineno or 0, "syntax-error", str(exc.msg))] doc_nodes = (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef) # node.body[0].value is the docstring literal; its lineno is the line the string opens # on, so content line i maps to source line (start + i). docstrings = [ (docstring, node.body[0].value.lineno) # type: ignore[attr-defined] for node in ast.walk(tree) if isinstance(node, doc_nodes) if (docstring := ast.get_docstring(node, clean=False)) ] return [ Finding(path, start + offset, category, line) for docstring, start in docstrings for offset, line in enumerate(docstring.splitlines()) for category in _scan_line(line) ] class Fence(NamedTuple): """A fenced code block, by 0-based line index. ``terminated`` is False when the fence has no closing delimiter before EOF, in which case ``close`` is the last line.""" open: int close: int terminated: bool def fenced_blocks(lines: list[str]) -> list[Fence]: """Index every fenced code block in ``lines``. An unterminated fence is malformed Markdown that `mkdocs build` surfaces anyway; it is still returned (with ``terminated=False``, ``close`` at the last line) so callers that skip code can skip to EOF.""" blocks: list[Fence] = [] fence: str | None = None open_idx = -1 for i, line in enumerate(lines): stripped = line.lstrip() if fence is None: if stripped.startswith(("```", "~~~")): fence, open_idx = stripped[:3], i elif stripped.startswith(fence): blocks.append(Fence(open_idx, i, terminated=True)) fence = None if fence is not None: blocks.append(Fence(open_idx, len(lines) - 1, terminated=False)) return blocks def find_list_breaking_fences(lines: list[str], blocks: list[Fence]) -> list[tuple[int, str]]: """Return ``(lineno, snippet)`` for each fenced code block at column 0 that splits a list -- i.e. one whose nearest non-blank neighbours on both sides are top-level list items. Such a fence is not indented into the preceding item, so Markdown closes the list at the fence and the following item starts a new one. The fix is to indent the fence (4 spaces) so it nests inside its list item. See the module docstring. Conservative on purpose: it requires a list item *directly* before and after (a continuation line or paragraph in between is not matched), keeping false positives low for a check that fails CI. Unterminated fences are ignored.""" def neighbour(start: int, step: int) -> str | None: j = start + step while 0 <= j < len(lines): if lines[j].strip(): return lines[j] j += step return None def splits_a_list(open_i: int, close_i: int) -> bool: if lines[open_i][:1].isspace(): return False # indented fence: already nested in the list item, not a break before = neighbour(open_i, -1) after = neighbour(close_i, +1) return bool(before and after and LIST_ITEM.match(before) and LIST_ITEM.match(after)) return [ (fence.open + 1, lines[fence.open]) for fence in blocks if fence.terminated and splits_a_list(fence.open, fence.close) ] def find_list_continuation_issues( lines: list[str], in_code: set[int] ) -> list[tuple[int, str, str]]: """Return ``(lineno, category, snippet)`` for list continuations Python-Markdown will mis-render (see the module docstring): - ``list-indent``: a blank-line-separated block inside a list item indented 1-3 spaces. Python-Markdown requires 4; at less, the block escapes the list. - ``list-blank``: a top-level list item directly after a line indented 4+ spaces. Without a blank line in between, the item is absorbed into the preceding paragraph as literal ``- `` text. Lazy continuations (an indented line with no blank line before it) are valid at any indent and are not flagged. Fenced-code lines are opaque: never flagged themselves, but they keep the item scope open and their indent feeds the ``list-blank`` check so an item directly after an indented fence is still caught.""" findings: list[tuple[int, str, str]] = [] in_item = False # inside a top-level list item's scope prev_blank = True prev_indent = 0 for i, line in enumerate(lines): stripped = line.strip() if not stripped: prev_blank = True continue indent = len(line) - len(line.lstrip(" ")) if i not in in_code: if indent == 0: is_item = bool(LIST_ITEM.match(line)) if is_item and in_item and not prev_blank and prev_indent >= 4: findings.append((i + 1, "list-blank", line)) in_item = is_item elif in_item and prev_blank and indent < 4: findings.append((i + 1, "list-indent", line)) prev_blank = False prev_indent = indent return findings def lint_markdown(path: Path) -> list[Finding]: """Scan a Markdown file: RST residue in prose (skipping fenced code blocks), plus list-structure problems (see find_list_breaking_fences and find_list_continuation_issues).""" lines = path.read_text(encoding="utf-8").splitlines() blocks = fenced_blocks(lines) in_code = {i for fence in blocks for i in range(fence.open, fence.close + 1)} prose = [ Finding(path, lineno, category, line) for lineno, line in enumerate(lines, start=1) if lineno - 1 not in in_code for category in _scan_line(line) ] breaks = [ Finding(path, lineno, "list-break", snippet) for lineno, snippet in find_list_breaking_fences(lines, blocks) ] continuations = [ Finding(path, lineno, category, snippet) for lineno, category, snippet in find_list_continuation_issues(lines, in_code) ] return prose + breaks + continuations def iter_files(paths: tuple[Path, ...]) -> list[Path]: files: list[Path] = [] for path in paths: if path.is_file(): files.append(path) elif path.is_dir(): files.extend(sorted(path.rglob("*.py"))) files.extend(sorted(path.rglob("*.md"))) else: raise FileNotFoundError(f"{path} does not exist") return files LINTERS = {".py": lint_python, ".md": lint_markdown} def lint(paths: tuple[Path, ...]) -> list[Finding]: return [ finding for file in iter_files(paths) if file.suffix in LINTERS for finding in LINTERS[file.suffix](file) ] def main() -> int: args = sys.argv[1:] paths = tuple(Path(a).resolve() for a in args) if args else DEFAULT_PATHS findings = lint(paths) if not findings: print("No reStructuredText residue or list-breaking fences found in docs.") return 0 print( f"Found {len(findings)} docs issue(s) -- RST markup that will not render under " "MkDocs/mkdocstrings, or Markdown that renders as valid-but-wrong HTML:\n", file=sys.stderr, ) for finding in findings: print(finding.format(), file=sys.stderr) remediation = "\n".join(f" {c.category:<13} {c.example:<19} -> {c.fix}" for c in CHECKS) print( f"\nFix each issue (see ci/lint_docs.py header):\n{remediation}", file=sys.stderr, ) return 1 if __name__ == "__main__": sys.exit(main()) zarr-developers-zarr-python-a2f2415/codecov.yml000066400000000000000000000012771527014137600216410ustar00rootroot00000000000000coverage: status: patch: default: target: auto informational: true project: default: target: auto threshold: 0.1 flags: - tests flags: tests: paths: - src/ carryforward: true gpu: paths: - src/ carryforward: true codecov: notify: # 6 = test.yml: 3 (optional+ubuntu) + 2 (upstream + min_deps), hypothesis: 1 after_n_builds: 6 wait_for_ci: yes comment: layout: "diff, files" behavior: default require_changes: true # if true: only post the comment if coverage changes branches: # branch names that can post comment - "main" github_checks: annotations: false zarr-developers-zarr-python-a2f2415/design/000077500000000000000000000000001527014137600207365ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/design/chunk-grid.md000066400000000000000000001330231527014137600233150ustar00rootroot00000000000000# Unified Chunk Grid Version: 6 Design document for adding rectilinear (variable) chunk grid support to **zarr-python**, conforming to the [rectilinear chunk grid extension spec](https://github.com/zarr-developers/zarr-extensions/pull/25). **Related:** - [#3750](https://github.com/zarr-developers/zarr-python/issues/3750) (single ChunkGrid proposal) - [#3534](https://github.com/zarr-developers/zarr-python/pull/3534) (rectilinear implementation) - [#3735](https://github.com/zarr-developers/zarr-python/pull/3735) (chunk grid module/registry) - [ZEP0003](https://github.com/zarr-developers/zeps/blob/main/draft/ZEP0003.md) (variable chunking spec) - [zarr-specs#370](https://github.com/zarr-developers/zarr-specs/pull/370) (sharding v1.1: non-divisible subchunks) - [zarr-extensions#25](https://github.com/zarr-developers/zarr-extensions/pull/25) (rectilinear extension) - [zarr-extensions#34](https://github.com/zarr-developers/zarr-extensions/issues/34) (sharding + rectilinear) ## Problem Chunk grids form a hierarchy — the rectilinear grid is strictly more general than the regular grid. Any regular grid is expressible as a rectilinear grid. There is no known chunk grid that is both (a) more general than rectilinear and (b) retains the axis-aligned tessellation properties Zarr assumes. All known grids are special cases: | Grid type | Description | Example | |---|---|---| | Regular | Uniform chunk size, boundary chunks padded with fill_value | `[10, 10, 10, 10]` | | Regular-bounded (zarrs) | Uniform chunk size, boundary chunks trimmed to array extent | `[10, 10, 10, 5]` | | HPC boundary-padded | Regular interior, larger boundary chunks ([VirtualiZarr#217](https://github.com/zarr-developers/VirtualiZarr/issues/217)) | `[10, 8, 8, 8, 10]` | | Fully variable | Arbitrary per-chunk sizes | `[5, 12, 3, 20]` | Prior iterations on the chunk grid design were based on the Zarr V3 spec's definition of chunk grids as an extension point alongside codecs, dtypes, etc. Therefore, we started designing the chunk grid implementation following a similar registry-based approach. However, in practice chunk grids are fundamentally different than codecs. Codecs are independent; supporting `zstd` tells you nothing about `gzip`. Chunk grids are not: every regular grid is a valid rectilinear grid. A registry-based plugin system makes sense for codecs but adds complexity without clear benefit for chunk grids. Here we start from some basic goals and propose a more fitting design for supporting different chunk grids in zarr-python. ## Goals 1. **Follow the zarr extension proposal.** The implementation should conform to the [rectilinear chunk grid spec](https://github.com/zarr-developers/zarr-extensions/tree/main/chunk-grids/rectilinear), not innovate on the metadata format. 2. **Minimize changes to the public API.** Users creating regular arrays should see no difference. Rectilinear is additive. 3. **Maintain backwards compatibility.** Existing code using `.chunks`, `isinstance` checks, or importing `RegularChunkGrid`/`RectilinearChunkGrid` from `zarr.core.chunk_grids` should continue to work where practical (with deprecation warnings where appropriate). Internal code paths/imports may be broken with justification. 4. **Design for future iteration.** The internal architecture should allow refactoring (e.g., metadata/array separation, new dimension types) without breaking the public API. 5. **Minimize downstream changes.** xarray, VirtualiZarr, Icechunk, Cubed, etc. should need minimal updates. 6. **Minimize time to stable release.** Ship behind a feature flag, stabilize through real-world usage, promote to stable API. 7. **The new API should be useful.** `read_chunk_sizes`/`write_chunk_sizes`, `ChunkGrid.__getitem__`, `is_regular` — these should solve real problems, not just expose internals. 8. **Extensible for other serialization structures.** The per-dimension design should support future encodings (tile, temporal) without changes to indexing or codecs. ## Design ### Design choices 1. **A chunk grid is a concrete arrangement of chunks.** Not an abstract tiling pattern. This means that the chunk grid is bound to specific array dimensions, which enables the chunk grid to answer any question about any chunk (offset, size, count) without external parameters. 2. **One implementation, multiple serialization forms.** A single `ChunkGrid` class handles all chunking logic. The serialization format (`"regular"` vs `"rectilinear"`) is chosen by the metadata layer, not the grid. 3. **No chunk grid registry.** Simple name-based dispatch in the metadata layer's `parse_chunk_grid()`. 4. **Fixed vs Varying per dimension.** `FixedDimension(size, extent)` for uniform chunks; `VaryingDimension(edges, extent)` for per-chunk edge lengths with precomputed prefix sums. Avoids expanding regular dimensions into lists of identical values. 5. **Transparent transitions.** Operations like `resize()` can move an array from regular to rectilinear chunking. ### Internal representation ```python @dataclass(frozen=True) class FixedDimension: """Uniform chunk size. Boundary chunks contain less data but are encoded at full size by the codec pipeline.""" size: int # chunk edge length (>= 0) extent: int # array dimension length def __post_init__(self) -> None: # validates size >= 0 and extent >= 0 @property def nchunks(self) -> int: if self.size == 0: return 0 return ceildiv(self.extent, self.size) def index_to_chunk(self, idx: int) -> int: return idx // self.size # raises IndexError if OOB def chunk_offset(self, chunk_ix: int) -> int: return chunk_ix * self.size # raises IndexError if OOB def chunk_size(self, chunk_ix: int) -> int: return self.size # always uniform; raises IndexError if OOB def data_size(self, chunk_ix: int) -> int: return max(0, min(self.size, self.extent - chunk_ix * self.size)) # raises IndexError if OOB @property def unique_edge_lengths(self) -> Iterable[int]: return (self.size,) # O(1) def indices_to_chunks(self, indices: NDArray) -> NDArray: return indices // self.size def with_extent(self, new_extent: int) -> FixedDimension: return FixedDimension(size=self.size, extent=new_extent) def resize(self, new_extent: int) -> FixedDimension: return FixedDimension(size=self.size, extent=new_extent) @dataclass(frozen=True) class VaryingDimension: """Explicit per-chunk sizes. The last chunk may extend past the array extent (extent < sum(edges)), in which case data_size clips to the valid region while chunk_size returns the full edge length for codec processing. This underflow is allowed to match how regular grids handle boundary chunks, and to support shrinking an array without rewriting chunk edges (the spec allows trailing edges beyond the extent).""" edges: tuple[int, ...] # per-chunk edge lengths (all > 0) cumulative: tuple[int, ...] # prefix sums for O(log n) lookup extent: int # array dimension length (may be < sum(edges)) def __init__(self, edges: Sequence[int], extent: int) -> None: # validates edges non-empty, all > 0, extent >= 0, extent <= sum(edges) # computes cumulative via itertools.accumulate # uses object.__setattr__ for frozen dataclass @property def nchunks(self) -> int: # number of chunks that overlap [0, extent) if extent == 0: return 0 return bisect.bisect_left(self.cumulative, extent) + 1 @property def ngridcells(self) -> int: return len(self.edges) def index_to_chunk(self, idx: int) -> int: return bisect.bisect_right(self.cumulative, idx) # raises IndexError if OOB def chunk_offset(self, chunk_ix: int) -> int: return self.cumulative[chunk_ix - 1] if chunk_ix > 0 else 0 # raises IndexError if OOB def chunk_size(self, chunk_ix: int) -> int: return self.edges[chunk_ix] # raises IndexError if OOB def data_size(self, chunk_ix: int) -> int: offset = self.chunk_offset(chunk_ix) return max(0, min(self.edges[chunk_ix], self.extent - offset)) # raises IndexError if OOB @property def unique_edge_lengths(self) -> Iterable[int]: # lazy generator: yields unseen values, short-circuits deduplication def indices_to_chunks(self, indices: NDArray) -> NDArray: return np.searchsorted(self.cumulative, indices, side='right') def with_extent(self, new_extent: int) -> VaryingDimension: # validates cumulative[-1] >= new_extent (O(1)), re-binds extent return VaryingDimension(self.edges, extent=new_extent) def resize(self, new_extent: int) -> VaryingDimension: # grow past edge sum: append chunk of size (new_extent - sum(edges)) # shrink or grow within edge sum: preserve all edges, re-bind extent ``` Both types implement the `DimensionGrid` protocol: `nchunks`, `extent`, `index_to_chunk`, `chunk_offset`, `chunk_size`, `data_size`, `indices_to_chunks`, `unique_edge_lengths`, `with_extent`, `resize`. Memory usage scales with the number of *varying* dimensions, not total chunks. All per-chunk methods (`chunk_offset`, `chunk_size`, `data_size`) raise `IndexError` for out-of-bounds chunk indices, providing consistent fail-fast behavior across both dimension types. The two size methods serve different consumers: | Method | Returns | Consumer | |---|---|---| | `chunk_size` | Buffer size for codec processing | Codec pipeline (`ArraySpec.shape`) | | `data_size` | Valid data region within the buffer | Indexing pipeline (`chunk_selection` slicing) | For `FixedDimension`, these differ only at the boundary. For `VaryingDimension`, these differ only when the last chunk extends past the extent (i.e., `extent < sum(edges)`). This matches current zarr-python behavior: `get_chunk_spec` passes the full `chunk_shape` to the codec for all chunks, and the indexer generates a `chunk_selection` that clips the decoded buffer. ### DimensionGrid Protocol ```python @runtime_checkable class DimensionGrid(Protocol): """Structural interface shared by FixedDimension and VaryingDimension.""" @property def nchunks(self) -> int: ... @property def ngridcells(self) -> int: ... @property def extent(self) -> int: ... def index_to_chunk(self, idx: int) -> int: ... def chunk_offset(self, chunk_ix: int) -> int: ... # raises IndexError if OOB def chunk_size(self, chunk_ix: int) -> int: ... # raises IndexError if OOB def data_size(self, chunk_ix: int) -> int: ... # raises IndexError if OOB def indices_to_chunks(self, indices: NDArray[np.intp]) -> NDArray[np.intp]: ... @property def unique_edge_lengths(self) -> Iterable[int]: ... def with_extent(self, new_extent: int) -> DimensionGrid: ... def resize(self, new_extent: int) -> DimensionGrid: ... ``` The protocol is `@runtime_checkable`, enabling polymorphic handling of both dimension types without `isinstance` checks. `nchunks` and `ngridcells` differ when `extent < sum(edges)`: `nchunks` counts only chunks that overlap `[0, extent)`, while `ngridcells` counts total defined grid cells (i.e., `len(edges)`). For `FixedDimension`, both are equal. For `VaryingDimension`, they differ after a resize that shrinks the extent below the edge sum. ### ChunkSpec ```python @dataclass(frozen=True) class ChunkSpec: slices: tuple[slice, ...] # valid data region in array coordinates codec_shape: tuple[int, ...] # buffer shape for codec processing @property def shape(self) -> tuple[int, ...]: return tuple(s.stop - s.start for s in self.slices) @property def is_boundary(self) -> bool: return self.shape != self.codec_shape ``` For interior chunks, `shape == codec_shape`. For boundary chunks of a regular grid, `codec_shape` is the full declared chunk size while `shape` is clipped. For rectilinear grids, `shape == codec_shape` unless the last chunk extends past the extent. ### API ```python # Creating arrays arr = zarr.create_array(shape=(100, 200), chunks=(10, 20)) # regular arr = zarr.create_array(shape=(60, 100), chunks=[[10, 20, 30], [25, 25, 25, 25]]) # rectilinear # ChunkGrid as a collection grid = arr._chunk_grid # ChunkGrid (bound to array shape) grid.grid_shape # (10, 10) — number of chunks per dimension grid.ndim # 2 grid.is_regular # True if all dimensions are Fixed spec = grid[0, 1] # ChunkSpec for chunk at grid position (0, 1) spec.slices # (slice(0, 10), slice(20, 40)) spec.shape # (10, 20) — data shape spec.codec_shape # (10, 20) — same for interior chunks boundary = grid[9, 0] # boundary chunk (extent=100, size=10) boundary.shape # (10, 20) — data shape boundary.codec_shape # (10, 20) — codec sees full buffer grid[99, 99] # None — out of bounds for spec in grid: # iterate all chunks ... # .chunks property: retained for regular grids, raises NotImplementedError for rectilinear arr.chunks # (10, 20) # .read_chunk_sizes / .write_chunk_sizes: works for all grids (dask-style) arr.write_chunk_sizes # ((10, 10, ..., 10), (20, 20, ..., 20)) ``` `ChunkGrid.__getitem__` constructs `ChunkSpec` using `chunk_size` for `codec_shape` and `data_size` for `slices`: ```python def __getitem__(self, coords: int | tuple[int, ...]) -> ChunkSpec | None: if isinstance(coords, int): coords = (coords,) slices = [] codec_shape = [] for dim, ix in zip(self.dimensions, coords): if ix < 0 or ix >= dim.nchunks: return None offset = dim.chunk_offset(ix) slices.append(slice(offset, offset + dim.data_size(ix))) codec_shape.append(dim.chunk_size(ix)) return ChunkSpec(tuple(slices), tuple(codec_shape)) ``` #### Construction `from_sizes` requires `array_shape`, binding the extent per dimension at construction time. This is a core design choice: a chunk grid is a concrete arrangement for a specific array, not an abstract tiling pattern. ```python # Regular grid — all FixedDimension grid = ChunkGrid.from_sizes(array_shape=(100, 200), chunk_sizes=(10, 20)) # Rectilinear grid — extent = sum(edges) when shape matches grid = ChunkGrid.from_sizes(array_shape=(60, 100), chunk_sizes=[[10, 20, 30], [25, 25, 25, 25]]) # Rectilinear grid with boundary clipping — last chunk extends past array extent # e.g., shape=(55, 90) but edges sum to (60, 100): data_size clips at extent grid = ChunkGrid.from_sizes(array_shape=(55, 90), chunk_sizes=[[10, 20, 30], [25, 25, 25, 25]]) # Direct construction grid = ChunkGrid(dimensions=(FixedDimension(10, 100), VaryingDimension([10, 20, 30], 55))) ``` When `extent < sum(edges)`, the dimension is always stored as `VaryingDimension` (even if all edges are identical) to preserve the explicit edge count. The last chunk's `chunk_size` returns the full declared edge (codec buffer) while `data_size` clips to the extent. This mirrors how `FixedDimension` handles boundary chunks in regular grids. #### Serialization ```python # Regular grid: {"name": "regular", "configuration": {"chunk_shape": [10, 20]}} # Rectilinear grid (with RLE compression and "kind" field): { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[10, 20, 30], [[25, 4]]]}, } ``` Both names deserialize to the same `ChunkGrid` class. The serialized form does not include the array extent — that comes from `shape` in array metadata and is combined with the chunk grid when constructing a `ChunkGrid` via `ChunkGrid.from_metadata()`. **The `ChunkGrid` does not serialize itself.** The format choice (`"regular"` vs `"rectilinear"`) belongs to `ArrayV3Metadata`. Serialization and deserialization are handled by the metadata-layer chunk grid classes (`RegularChunkGridMetadata` and `RectilinearChunkGridMetadata` in `metadata/v3.py`), which provide `to_dict()` and `from_dict()` methods. For `create_array`, the format is inferred from the `chunks` argument: a flat tuple produces `"regular"`, a nested list produces `"rectilinear"`. The `_is_rectilinear_chunks()` helper detects nested sequences like `[[10, 20], [5, 5]]`. ##### Rectilinear spec compliance The rectilinear format requires `"kind": "inline"` (validated by `validate_rectilinear_kind()`). Per the spec, each element of `chunk_shapes` can be: - A bare integer `m`: repeated until `sum >= array_extent` - A list of bare integers: explicit per-chunk sizes - A mixed array of bare integers and `[value, count]` RLE pairs RLE compression is used when serializing: runs of identical sizes become `[value, count]` pairs, singletons stay as bare integers. ```python # compress_rle([10, 10, 10, 5]) -> [[10, 3], 5] # expand_rle([[10, 3], 5]) -> [10, 10, 10, 5] ``` For a single-element `chunk_shapes` tuple like `(10,)`, `RectilinearChunkGridMetadata.to_dict()` serializes it as a bare integer `10`. Per the rectilinear spec, a bare integer is repeated until the sum >= extent, preserving the full codec buffer size for boundary chunks. **Zero-extent handling:** Regular grids serialize zero-extent dimensions without issue (the format encodes only `chunk_shape`, no edges). Rectilinear grids cannot represent zero-extent dimensions because the spec requires at least one positive-integer edge length per axis. #### read_chunk_sizes / write_chunk_sizes The `read_chunk_sizes` and `write_chunk_sizes` properties provide universal access to per-dimension chunk data sizes, matching the dask `Array.chunks` convention. They work for both regular and rectilinear grids: - `write_chunk_sizes`: always returns outer (storage) chunk sizes - `read_chunk_sizes`: returns inner chunk sizes when sharding is used, otherwise same as `write_chunk_sizes` ```python >>> arr = zarr.create_array(store, shape=(100, 80), chunks=(30, 40)) >>> arr.write_chunk_sizes ((30, 30, 30, 10), (40, 40)) >>> arr = zarr.create_array(store, shape=(60, 100), chunks=[[10, 20, 30], [50, 50]]) >>> arr.write_chunk_sizes ((10, 20, 30), (50, 50)) ``` The underlying `ChunkGrid.chunk_sizes` property (on the grid, not the array) returns the same as `write_chunk_sizes`. #### Resize ```python arr.resize((80, 100)) # re-binds extent; FixedDimension stays fixed arr.resize((200, 100)) # VaryingDimension grows by appending a new chunk arr.resize((30, 100)) # VaryingDimension shrinks: preserves all edges, re-binds extent ``` Resize uses `ChunkGrid.update_shape(new_shape)`, which delegates to each dimension's `.resize()` method: - `FixedDimension.resize()`: simply re-binds the extent (identical to `with_extent`) - `VaryingDimension.resize()`: grow past `sum(edges)` appends a chunk covering the gap; shrink or grow within `sum(edges)` preserves all edges and re-binds the extent (the spec allows trailing edges beyond the array extent) **Known limitation (deferred):** When growing a `VaryingDimension`, the current implementation always appends a single chunk covering the new region. For example, `[10, 10, 10]` resized from 30 to 45 produces `[10, 10, 10, 15]` instead of the more natural `[10, 10, 10, 10, 10]`. A future improvement should add an optional `chunks` parameter to `resize()` that controls how the new region is partitioned, with a sane default (e.g., repeating the last chunk size). This is safely deferrable because: - `FixedDimension` already handles resize correctly (regular grids stay regular) - The single-chunk default produces valid state, just suboptimal chunk layout - Rectilinear arrays are behind an experimental feature flag - Adding an optional parameter is backwards-compatible Open design questions for the `chunks` parameter: - Does it describe the new region only, or the entire post-resize array? - Must the overlapping portion agree with existing chunks (no rechunking)? - What is the type? Same as `chunks` in `create_array`? #### from_array The `from_array()` function handles both regular and rectilinear source arrays: ```python src = zarr.create_array(store, shape=(60, 100), chunks=[[10, 20, 30], [50, 50]]) new = zarr.from_array(data=src, store=new_store, chunks="keep") # Preserves rectilinear structure: new.write_chunk_sizes == ((10, 20, 30), (50, 50)) ``` When `chunks="keep"`, the logic checks `data._chunk_grid.is_regular`: - Regular: extracts `data.chunks` (flat tuple) and preserves shards - Rectilinear: extracts `data.write_chunk_sizes` (nested tuples) and forces shards to None ### Indexing The indexing pipeline is coupled to regular grid assumptions — every per-dimension indexer takes a scalar `dim_chunk_len: int` and uses `//` and `*`: ```python dim_chunk_ix = self.dim_sel // self.dim_chunk_len # IntDimIndexer dim_offset = dim_chunk_ix * self.dim_chunk_len # SliceDimIndexer ``` Replace `dim_chunk_len: int` with the dimension object (`FixedDimension | VaryingDimension`). The shared interface means the indexer code structure stays the same — `dim_sel // dim_chunk_len` becomes `dim_grid.index_to_chunk(dim_sel)`. O(1) for regular, binary search for varying. ### Codec pipeline Today, `get_chunk_spec()` returns the same `ArraySpec(shape=chunk_grid.chunk_shape)` for every chunk. For rectilinear grids, each chunk has a different codec shape: ```python def get_chunk_spec(self, chunk_coords, array_config, prototype) -> ArraySpec: spec = self._chunk_grid[chunk_coords] return ArraySpec(shape=spec.codec_shape, ...) ``` Note `spec.codec_shape`, not `spec.shape`. For regular grids, `codec_shape` is uniform (preserving current behavior). The boundary clipping flow is unchanged: ``` Write: user data → pad to codec_shape with fill_value → encode → store Read: store → decode to codec_shape → slice via chunk_selection → user data ``` ### Sharding The `ShardingCodec` constructs a `ChunkGrid` per shard using the shard shape as extent and the subchunk shape as `FixedDimension`. Each shard is self-contained — it doesn't need to know whether the outer grid is regular or rectilinear. Validation checks that every unique edge length per dimension is divisible by the inner chunk size, using `dim.unique_edge_lengths` for efficient polymorphic iteration (O(1) for fixed dimensions, lazy-deduplicated for varying). ``` Level 1 — Outer chunk grid (shard boundaries): regular or rectilinear Level 2 — Inner subchunk grid (within each shard): always regular Level 3 — Shard index: ceil(shard_dim / subchunk_dim) entries per dimension ``` [zarr-specs#370](https://github.com/zarr-developers/zarr-specs/pull/370) lifts the requirement that subchunk shapes evenly divide the shard shape. With the proposed `ChunkGrid`, this just means removing the `shard_shape % subchunk_shape == 0` validation — `FixedDimension` already handles boundary clipping via `data_size`. | Outer grid | Subchunk divisibility | Required change | |---|---|---| | Regular | Evenly divides (v1.0) | None | | Regular | Non-divisible (v1.1) | Remove divisibility validation | | Rectilinear | Evenly divides | Remove "sharding incompatible" guard | | Rectilinear | Non-divisible | Both changes | ### What this replaces | Current | Proposed | |---|---| | `ChunkGrid` ABC + `RegularChunkGrid` subclass | Single concrete `ChunkGrid` with `is_regular` | | `RectilinearChunkGrid` (#3534) | Same `ChunkGrid` class | | Chunk grid registry + entrypoints (#3735) | Direct name dispatch | | `arr.chunks` | Retained for regular; `arr.read_chunk_sizes`/`arr.write_chunk_sizes` for general use | | `get_chunk_shape(shape, coord)` | `grid[coord].codec_shape` or `grid[coord].shape` | ## Design decisions ### Why store the extent in ChunkGrid? The chunk grid is a concrete arrangement, not an abstract tiling pattern. A finite collection naturally has an extent. Storing it enables `__getitem__`, eliminates `dim_len` parameters from every method, and makes the grid self-describing. This does *not* mean `ArrayV3Metadata.shape` should delegate to the grid. The array shape remains an independent field in metadata. The extent is passed into the grid at construction time so it can answer boundary questions without external parameters. It is **not** serialized as part of the chunk grid JSON — it comes from the `shape` field in array metadata and is combined with the chunk grid configuration in `ChunkGrid.from_metadata()`. ### Why distinguish chunk_size from data_size? A chunk in a regular grid has two sizes. `chunk_size` is the buffer size the codec processes — always `size` for `FixedDimension`, even at the boundary (padded with `fill_value`). `data_size` is the valid data region — clipped to `extent % size` at the boundary. The indexing layer uses `data_size` to generate `chunk_selection` slices. This matches current zarr-python behavior and matters for: 1. **Backward compatibility.** Existing stores have boundary chunks encoded at full `chunk_shape`. 2. **Codec simplicity.** Codecs assume uniform input shapes for regular grids. 3. **Shard index correctness.** The index assumes `subchunk_dim`-sized entries. For `VaryingDimension`, `chunk_size == data_size` when `extent == sum(edges)`. When `extent < sum(edges)` (e.g., after a resize that keeps the last chunk oversized), `data_size` clips the last chunk. This is the fundamental difference: `FixedDimension` has a declared size plus an extent that clips data; `VaryingDimension` has explicit sizes that normally *are* the extent but can also extend past it. ### Why not a chunk grid registry? There is no known chunk grid outside the rectilinear family that retains the tessellation properties zarr-python assumes. A `match` on the grid name is sufficient. ### Why a single ChunkGrid class instead of RegularChunkGrid + RectilinearChunkGrid? [Discussed in #3534.](https://github.com/zarr-developers/zarr-python/pull/3534) @d-v-b argued that `RegularChunkGrid` is unnecessary since rectilinear is more general; @dcherian argued that downstream libraries need a fast way to detect regular grids without inspecting potentially millions of chunk edges (see [xarray#9808](https://github.com/pydata/xarray/pull/9808)). The resolution: a single `ChunkGrid` class with an `is_regular` property (O(1), cached at construction). This gives downstream code the fast-path detection @dcherian needed without the class hierarchy complexity @d-v-b wanted to avoid. The metadata document's `name` field (`"regular"` vs `"rectilinear"`) is also available for clients who inspect JSON directly. A backwards-compatibility shim in `chunk_grids.py` preserves the old `RegularChunkGrid` / `RectilinearChunkGrid` import paths with deprecation warnings — see [Backwards compatibility](#backwards-compatibility). ### Why is ChunkGrid a concrete class instead of a Protocol/ABC? The old design had `ChunkGrid` as an ABC with `RegularChunkGrid` as its only subclass. #3534 added `RectilinearChunkGrid` as a second subclass. This branch makes `ChunkGrid` a single concrete class instead, with separate metadata DTOs (`RegularChunkGridMetadata` and `RectilinearChunkGridMetadata` in `metadata/v3.py`) for serialization. All known grids are special cases of rectilinear, so there's no need for a class hierarchy at the grid level. A `ChunkGrid` Protocol/ABC would mean every caller programs against an abstract interface and adding a grid type requires implementing ~15 methods. A single class is simpler. Note: the *dimension* types (`FixedDimension`, `VaryingDimension`) do use a `DimensionGrid` Protocol — that's where the polymorphism lives. The grid-level class is concrete; the dimension-level types are polymorphic. If a genuinely novel grid type emerges that can't be expressed as a combination of per-dimension types, a grid-level Protocol can be extracted. ### Why `.chunks` raises for rectilinear grids [Debated in #3534.](https://github.com/zarr-developers/zarr-python/pull/3534) @d-v-b suggested making `.chunks` return `tuple[tuple[int, ...], ...]` (dask-style) for all grids. @dcherian strongly objected: every downstream consumer expects `tuple[int, ...]`, and silently returning a different type would be worse than raising. Materializing O(10M) chunk edges into a Python tuple is also a real performance risk ([xarray#8902](https://github.com/pydata/xarray/issues/8902#issuecomment-2546127373)). The resolution: - `.chunks` is retained for regular grids (returns `tuple[int, ...]` as before) - `.chunks` raises `NotImplementedError` for rectilinear grids with a message pointing to `.read_chunk_sizes`/`.write_chunk_sizes` - `.read_chunk_sizes` and `.write_chunk_sizes` return `tuple[tuple[int, ...], ...]` (dask convention) for all grids @maxrjones noted in review that deprecating `.chunks` for regular grids was not desirable. The current branch does not deprecate it. ### User control over grid serialization format @d-v-b raised in #3534 that users need a way to say "these chunks are regular, but serialize as rectilinear" (e.g., to allow future append/extend workflows without format changes). @jhamman initially made nested-list input always produce `RectilinearChunkGridMetadata`. The current branch resolves this via the metadata-layer chunk grid classes. When metadata is deserialized, the original name (from `{"name": "regular"}` or `{"name": "rectilinear"}`) determines which metadata class is instantiated (`RegularChunkGridMetadata` or `RectilinearChunkGridMetadata`), and that class handles serialization via `to_dict()`. Current inference behavior for `create_array`: - `chunks=(10, 20)` (flat tuple) → infers `"regular"` - `chunks=[[10, 20], [5, 5]]` (nested lists with varying sizes) → infers `"rectilinear"` - `chunks=[[10, 10], [20, 20]]` (nested lists with uniform sizes) → `from_sizes` collapses to `FixedDimension`, so `is_regular=True` and infers `"regular"` **Open question:** Should uniform nested lists preserve `"rectilinear"` to support future append workflows without a format change? This could be addressed by checking the input form before collapsing, or by allowing users to pass `chunk_grid_name` explicitly through the `create_array` API. ### Deferred: Tiled/periodic chunk patterns [#3750 discussion](https://github.com/zarr-developers/zarr-python/issues/3750) identified periodic chunk patterns as a use case not efficiently served by RLE alone. RLE compresses runs of identical values (`np.repeat`), but periodic patterns like days-per-month (`[31, 28, 31, 30, ...]` repeated 30 years) need a tile encoding (`np.tile`). Real-world examples include: - **Oceanographic models** (ROMS): HPC boundary-padded chunks like `[10, 8, 8, 8, 10]` — handled by RLE - **Temporal axes**: days-per-month, hours-per-day — need tile encoding for compact metadata - **Temporal-aware grids**: date/time-aware chunk grids that layer over other axes (raised by @LDeakin) A `TiledDimension` prototype was built ([commit 9c0f582](https://github.com/maxrjones/zarr-python/commit/9c0f582f)) demonstrating that the per-dimension design supports this without changes to indexing or the codec pipeline. However, it was intentionally excluded from this release because: 1. **Metadata format must come first.** Tile encoding requires a new `kind` value in the rectilinear spec (currently only `"inline"` is defined). This should go through [zarr-extensions#25](https://github.com/zarr-developers/zarr-extensions/pull/25), not zarr-python unilaterally. 2. **The per-dimension architecture doesn't preclude it.** A future `TiledDimension` can implement the `DimensionGrid` protocol alongside `FixedDimension` and `VaryingDimension` with no changes to indexing, codecs, or the `ChunkGrid` class. 3. **RLE covers the MVP.** Most real-world variable chunk patterns (HPC boundaries, irregular partitions) are efficiently encoded with RLE. Tile encoding is an optimization for a specific (temporal) subset. ### Metadata / Array separation (partially implemented) An earlier design doc proposed decoupling `ChunkGrid` (runtime) from `ArrayV3Metadata` (serialization), so that metadata would store only a plain dict and the array layer would construct the `ChunkGrid`. The current implementation partially realizes this separation: - **Metadata DTOs** (`RegularChunkGridMetadata`, `RectilinearChunkGridMetadata` in `metadata/v3.py`): Pure data, frozen dataclasses, no array shape. These live on `ArrayV3Metadata.chunk_grid` and represent only what goes into `zarr.json`. - **`ChunkGrid`** (`chunk_grids.py`): Shape-bound, supports indexing, iteration, and chunk specs. Lives on `AsyncArray._chunk_grid`, constructed from metadata + `shape` via `ChunkGrid.from_metadata()`. This means `ArrayV3Metadata.chunk_grid` is now a `ChunkGridMetadata` (the DTO union type), **not** the runtime `ChunkGrid`. Code that previously accessed runtime methods on `metadata.chunk_grid` (e.g., `all_chunk_coords()`, `__getitem__`) must now use the grid from the array layer instead. The name controls serialization format; each metadata DTO class provides its own `to_dict()` method for serialization. The `ChunkGrid` handles all runtime queries. ## Prior art **zarrs (Rust):** Three independent grid types behind a `ChunkGridTraits` trait. Key patterns adopted: Fixed vs Varying per dimension, prefix sums + binary search, `Option` for out-of-bounds, `NonZeroU64` for chunk dimensions, separate subchunk grid per shard, array shape at construction. **TensorStore (C++):** Stores only `chunk_shape` — boundary clipping via `valid_data_bounds` at query time. Both `RegularGridRef` and `IrregularGrid` internally. No registry. ## Migration ### Public API compatibility The user-facing API is fully backward-compatible. Existing code that creates, opens, reads, and writes zarr arrays continues to work without changes: - `zarr.create_array`, `zarr.open`, `zarr.open_array`, `zarr.open_group` -- unchanged signatures. The `chunks` parameter type is *widened* (now also accepts nested sequences for rectilinear grids), but all existing call patterns still work. - `arr.chunks` -- returns `tuple[int, ...]` for regular arrays, same as before. - `arr.shape`, `arr.dtype`, `arr.ndim`, `arr.shards` -- unchanged. - Top-level `zarr` exports -- unchanged. - Rectilinear chunks are gated behind `zarr.config.set({'array.rectilinear_chunks': True})`, so they cannot be created accidentally. New additions (purely additive): `arr.read_chunk_sizes`, `arr.write_chunk_sizes`, `zarr.experimental.ChunkGrid`, `zarr.experimental.ChunkSpec`. The breaking changes discussed below are confined to **internal modules** (`zarr.core.chunk_grids`, `zarr.core.metadata.v3`, `zarr.core.indexing`) that downstream libraries like cubed and VirtualiZarr access directly. ### Internal API compatibility trade-off analysis This section analyzes the internal breaking changes from the metadata/array separation and evaluates two strategies: (A) add backward-compatibility shims in zarr-python, vs. (B) require downstream packages to update. The baseline is **no shims at all**. #### What breaks without any shims Three API changes affect downstream code: 1. **`RegularChunkGrid` class removed from `zarr.core.chunk_grids`.** On `main`, `RegularChunkGrid` is defined in `chunk_grids.py` as a `Metadata` subclass. This branch replaces it with `RegularChunkGridMetadata` in `metadata/v3.py`. Without a shim, `from zarr.core.chunk_grids import RegularChunkGrid` raises `ImportError`. 2. **`RegularChunkGrid` no longer available from `zarr.core.metadata.v3`.** On `main`, `v3.py` imports `RegularChunkGrid` from `chunk_grids.py` for internal use. VirtualiZarr imports it from this location (`from zarr.core.metadata.v3 import RegularChunkGrid`). Without the internal import, this raises `ImportError`. 3. **`OrthogonalIndexer` constructor expects `ChunkGrid`, not `RegularChunkGrid`/`RegularChunkGridMetadata`.** Even if the import shims above resolve to `RegularChunkGridMetadata`, the indexer constructors access `chunk_grid._dimensions`, which only exists on the runtime `ChunkGrid` class. Cubed constructs `OrthogonalIndexer(selection, shape, RegularChunkGrid(chunk_shape=chunks))` directly. #### Downstream impact without shims **VirtualiZarr** (5 line changes across 2 files): ```python # manifests/array.py (line 6): import - from zarr.core.metadata.v3 import ArrayV3Metadata, RegularChunkGrid + from zarr.core.metadata.v3 import ArrayV3Metadata, RegularChunkGridMetadata # manifests/array.py (line 53): isinstance check - if not isinstance(_metadata.chunk_grid, RegularChunkGrid): + if not isinstance(_metadata.chunk_grid, RegularChunkGridMetadata): # parsers/zarr.py (line 16): import - from zarr.core.chunk_grids import RegularChunkGrid + from zarr.core.metadata.v3 import RegularChunkGridMetadata # parsers/zarr.py (line 270): isinstance check - if not isinstance(array_v3_metadata.chunk_grid, RegularChunkGrid): + if not isinstance(array_v3_metadata.chunk_grid, RegularChunkGridMetadata): # parsers/zarr.py (line 390): cast - cast(RegularChunkGrid, metadata.chunk_grid).chunk_shape + cast(RegularChunkGridMetadata, metadata.chunk_grid).chunk_shape ``` The `manifests/array.py` import is from `zarr.core.metadata.v3` (never a documented export; VirtualiZarr relied on a transitive import). The `parsers/zarr.py` import is from `zarr.core.chunk_grids` (the canonical location on `main`). Both are straightforward renames. The `.chunk_shape` attribute is unchanged on the new class. If VirtualiZarr needs to support both old and new zarr-python, a version-conditional import adds ~5 more lines. **Cubed** (3 line changes in 1 file): ```python # core/ops.py (lines 626-631) def _create_zarr_indexer(selection, shape, chunks): if zarr.__version__[0] == "3": - from zarr.core.chunk_grids import RegularChunkGrid + from zarr.core.chunk_grids import ChunkGrid from zarr.core.indexing import OrthogonalIndexer - return OrthogonalIndexer(selection, shape, RegularChunkGrid(chunk_shape=chunks)) + return OrthogonalIndexer(selection, shape, ChunkGrid.from_sizes(shape, chunks)) ``` Note that `ChunkGrid` is *not* a renamed class. `RegularChunkGrid(chunk_shape=chunks)` took only chunk sizes; `ChunkGrid.from_sizes(shape, chunks)` also requires the array shape. The `shape` parameter is already available at this call site. If cubed needs to support both old and new zarr-python: ```python def _create_zarr_indexer(selection, shape, chunks): if zarr.__version__[0] == "3": from zarr.core.indexing import OrthogonalIndexer try: from zarr.core.chunk_grids import ChunkGrid return OrthogonalIndexer(selection, shape, ChunkGrid.from_sizes(shape, chunks)) except ImportError: from zarr.core.chunk_grids import RegularChunkGrid return OrthogonalIndexer(selection, shape, RegularChunkGrid(chunk_shape=chunks)) else: from zarr.indexing import OrthogonalIndexer return OrthogonalIndexer(selection, ZarrArrayIndexingAdaptor(shape, chunks)) ``` #### What shims can cover **Shim 1: `__getattr__` in `chunk_grids.py`** (~15 lines) Maps `RegularChunkGrid` to `RegularChunkGridMetadata` with a deprecation warning. Covers: - The `from zarr.core.chunk_grids import RegularChunkGrid` import pattern (used by cubed and VirtualiZarr's `parsers/zarr.py`) - `isinstance(x, RegularChunkGrid)` checks (because the name resolves to the actual class) - `RegularChunkGrid(chunk_shape=(...))` construction (because `RegularChunkGridMetadata` accepts the same arguments) Does **not** cover: passing the result to `OrthogonalIndexer`, because `RegularChunkGridMetadata` lacks `._dimensions`. **Shim 2: `__getattr__` in `metadata/v3.py`** (~12 lines) Same pattern, covers VirtualiZarr's import from `zarr.core.metadata.v3`. Mirrors Shim 1 for a different import path. **Shim 3: Auto-coerce `ChunkGridMetadata` in indexer constructors** (~30 lines) A helper function + 1-line insertion in each of `BasicIndexer`, `OrthogonalIndexer`, `CoordinateIndexer`, and `MaskIndexer`: ```python def _resolve_chunk_grid(chunk_grid, shape): """Coerce ChunkGridMetadata to runtime ChunkGrid if needed.""" from zarr.core.chunk_grids import ChunkGrid as _ChunkGrid from zarr.core.metadata.v3 import ChunkGridMetadata if isinstance(chunk_grid, _ChunkGrid): return chunk_grid if isinstance(chunk_grid, ChunkGridMetadata): warnings.warn( "Passing ChunkGridMetadata to indexers is deprecated. " "Use ChunkGrid.from_sizes() instead.", DeprecationWarning, stacklevel=2, ) if hasattr(chunk_grid, "chunk_shape"): return _ChunkGrid.from_sizes(shape, tuple(chunk_grid.chunk_shape)) return _ChunkGrid.from_sizes(shape, chunk_grid.chunk_shapes) raise TypeError(f"Expected ChunkGrid or ChunkGridMetadata, got {type(chunk_grid)}") ``` This covers cubed's `OrthogonalIndexer(selection, shape, RegularChunkGrid(...))` pattern end-to-end (combined with Shim 1). #### Comparison | | No shims | Shims 1+2 only | Shims 1+2+3 | |---|---|---|---| | **zarr-python additions** | 0 lines | ~27 lines | ~57 lines | | **VirtualiZarr changes** | 5 lines | 0 lines | 0 lines | | **Cubed changes** | 3 lines | 3 lines | 0 lines | | **Maintenance burden** | None | Low (deprecation shims are well-understood) | Medium (indexer coercion blurs metadata/runtime boundary) | | **API clarity** | Clean (metadata DTOs and runtime types are distinct) | Good (old names redirect to new names) | Weaker (indexers implicitly accept two type families) | With Shims 1+2 only, VirtualiZarr's `manifests/array.py` import from `zarr.core.metadata.v3` is covered by Shim 2, and the `parsers/zarr.py` import from `zarr.core.chunk_grids` is covered by Shim 1. The `isinstance` checks work because both shims resolve to `RegularChunkGridMetadata`. The `cast` works because `.chunk_shape` is unchanged. So VirtualiZarr needs 0 changes with Shims 1+2. The 3 lines for cubed remain because Shim 1 resolves the import but `OrthogonalIndexer` still needs a runtime `ChunkGrid`. ### Downstream migration Migration from `main` (where only `RegularChunkGrid` and the abstract `ChunkGrid` ABC exist): | Old pattern (on `main`) | New pattern | |---|---| | `from zarr.core.chunk_grids import RegularChunkGrid` | `from zarr.core.metadata.v3 import RegularChunkGridMetadata` | | `from zarr.core.chunk_grids import ChunkGrid` (ABC) | `from zarr.core.chunk_grids import ChunkGrid` (concrete class, different API) | | `isinstance(cg, RegularChunkGrid)` | `isinstance(cg, RegularChunkGridMetadata)` or `grid.is_regular` on the runtime `ChunkGrid` | | `cg.chunk_shape` on `RegularChunkGrid` | `cg.chunk_shape` on `RegularChunkGridMetadata` (unchanged) | | `ChunkGrid.from_dict(data)` | `parse_chunk_grid(data)` from `zarr.core.metadata.v3` | | `chunk_grid.all_chunk_coords(array_shape)` | `chunk_grid.all_chunk_coords()` (shape now stored in grid) | | `chunk_grid.get_nchunks(array_shape)` | `chunk_grid.get_nchunks()` (shape now stored in grid) | During the earlier [#3534](https://github.com/zarr-developers/zarr-python/pull/3534) effort (which used separate `RegularChunkGrid`/`RectilinearChunkGrid` classes), downstream PRs and issues were opened to explore compatibility: - xarray ([#10880](https://github.com/pydata/xarray/pull/10880)), VirtualiZarr ([#877](https://github.com/zarr-developers/VirtualiZarr/pull/877)), Icechunk ([#1338](https://github.com/earth-mover/icechunk/issues/1338)), cubed ([#876](https://github.com/cubed-dev/cubed/issues/876)) These target #3534's API, not this branch's unified `ChunkGrid` design. New downstream POC branches for this design are linked in [Proofs of concepts](#proofs-of-concepts). ### Credits This implementation builds on prior work: - **[#3534](https://github.com/zarr-developers/zarr-python/pull/3534)** (@jhamman) — RLE helpers, validation logic, test cases, and the review discussion that shaped the architecture. - **[#3737](https://github.com/zarr-developers/zarr-python/pull/3737)** — extent-in-grid idea (adopted per-dimension). - **[#1483](https://github.com/zarr-developers/zarr-python/pull/1483)** — original variable chunking POC. - **[#3736](https://github.com/zarr-developers/zarr-python/pull/3736)** — resolved by storing extent per-dimension. ## Open questions 1. **Resize defaults (deferred):** When growing a rectilinear array, should `resize()` accept an optional `chunks` parameter? See the [Resize section](#resize) for details and open design questions. Regular arrays already stay regular on resize. 2. **`ChunkSpec` complexity:** `ChunkSpec` carries both `slices` and `codec_shape`. Should the grid expose separate methods for codec vs data queries instead? 3. **`__getitem__` with slices:** Should `grid[0, :]` or `grid[0:3, :]` return a sub-grid or an iterator of `ChunkSpec`s? 4. **Uniform nested lists:** Should `chunks=[[10, 10], [20, 20]]` serialize as `"rectilinear"` (preserving user intent for future append) or `"regular"` (current behavior, collapses uniform edges)? See [User control over grid serialization format](#user-control-over-grid-serialization-format). 5. **`zarr.open` with rectilinear:** @tomwhite noted in #3534 that `zarr.open(mode="w")` doesn't support rectilinear chunks directly. This could be addressed in a follow-up. ## Proofs of concepts - Zarr-Python: - branch - https://github.com/maxrjones/zarr-python/tree/poc/unified-chunk-grid - diff - https://github.com/zarr-developers/zarr-python/compare/main...maxrjones:zarr-python:poc/unified-chunk-grid?expand=1 - Xarray: - branch - https://github.com/maxrjones/xarray/tree/poc/unified-zarr-chunk-grid - diff - https://github.com/pydata/xarray/compare/main...maxrjones:xarray:poc/unified-zarr-chunk-grid?expand=1 - VirtualiZarr: - branch - https://github.com/maxrjones/VirtualiZarr/tree/poc/unified-chunk-grid - diff - https://github.com/zarr-developers/VirtualiZarr/compare/main...maxrjones:VirtualiZarr:poc/unified-chunk-grid?expand=1 - Virtual TIFF: - branch - https://github.com/virtual-zarr/virtual-tiff/tree/poc/unified-chunk-grid - diff - https://github.com/virtual-zarr/virtual-tiff/compare/main...poc/unified-chunk-grid?expand=1 - Cubed: - branch - https://github.com/maxrjones/cubed/tree/poc/unified-chunk-grid - Microbenchmarks: - https://github.com/maxrjones/zarr-chunk-grid-tests/tree/unified-chunk-grid zarr-developers-zarr-python-a2f2415/docs/000077500000000000000000000000001527014137600204155ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/docs/_static/000077500000000000000000000000001527014137600220435ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/docs/_static/favicon-96x96.png000066400000000000000000000306521527014137600250070ustar00rootroot00000000000000PNG  IHDR_`{` pHYsodtEXtSoftwarewww.inkscape.org< IDATx}y|Uι7&lU* Ep\@[kTlUYVZpׯ-l, [s}~$dAk|>瓛s3g33̼!_L;[wG4kڽ3>B~I Nj4!oPD1۔Ec.:Џ[O觎/B}J1 8]Q?~jkPp_zkg-~޽)1 Ժ\Q6e`HJܗ6s\3_|77=el0"d~z"oeOU05uम8?:ՑLs\E!A @*G`EG3nyv,Ƽy``JZ{SNJOV:sϏbgav LI++?j]-HL2 @m4Pe?  W=SRG*>s~RL+Wzh##yx(f'Ҥf3Uu(w"T<ʍ1qsR*ں|QQI]g̘Ƭxibhӭ 5ulɭOqJ[;;Ң*& g@TYU2yj,P>x{ xɩcֵʚ`?}6x )鷉Ӳ+\ [Ϙ:D$bU)+[0{qwE7pg#;HNCP@.' Y譠XKʜ/P/B8Am (9ɩ=a 8m/SJP@䌽i臦 ,ŰaN~RMI @9Tp&Ӧ4pĽPP6JP@: 5a Q2X qYsABj:X<.+cFK5*U(bjpWc?!}t'SNNnZOBL،!\*PxkO (,VhUTJAIDêKTh=]OO0j܂{Rn+f q/' ܝ:Vw *DX`ЗWǧ6i,}nS{{0* / UU (B 0ķ'VZר@AțT^ Zma ~-RYok{"|ܓr;STp8M1= N]xԩ^DJ4OǍkC~)ό,E6p nV+WT,b*5By D:Ú aJw05q1QO ;~q僸d51}b,VG0J??Dl7.2 ot@⪺ZyKlJ!U-zXBc}(A@8|FͣJKߕ:2Q [<6()2N8` sZ,)eq)缚;kȖJs|Zeו7*n2o7.7.E-lϩW%aŷ64 Hxq5 z8Фp>tQ$ d47&DLݿx4?)DZV "y6Xg( ~e<@ߙRYR "H FZwqoѲ*t|2W|Ci>F?0{vwzB7~/0*9h+0C ͡##)OhNT >w`h#% '`( DD!Pט+cP( 'xT*a_Z5e x'Zע;y6D4- ϪЮL|rOmiЂ;ҴᢶOM-/6?ߞziB9P*04S Eu[EG'T 6"ʩɇ4RBѾ@c |d!,T]CEd)W>Z/17=S.ʽl^pGڈcǛwFA `KڠȦd]?9 e< ($!.wΚy`9nP MAαl>Xt6)4(!BԪqwP0ɢʐ29R/0yPp|YhV9֬TV4)hZS FE,VGZ^ aè7rN6@S@h~"꧍JE&oP4!olʱFbTI`w礼a"bEzͅbN~ b(dTJZuN`IQ%~Im1aӞmܜ'x0[Ӯ0/9Pg:B՚MH*:٬c O|>TjoMPgO~NMy$5PGřlmo+w4gE:gPz*QxǧaQ((/pTLe!7ՎP&ݒr[ȴUWCT,#FNu8~3?EřTD<6:oTUNLFk>7W5`ѨuY[9OzaFaNC_B y<  "T=ϟ"F>2U`(+Ð!bJEWK@ ^PbaO Fv!haM1~sj]vuK[?X?2'h kB-nB4Ӓ⛩R0ECi,Fbj a: Tx9 ŹUv@nr1}N`JDOl']Ee+P@ c^\(^v*|V֌V[ ot8Qv-(1#(ڏtQTWVy^H{5,n9 _hcn"332 iDR +AѼTO+EDs?Ӳ-9EUcmeybǍV1< ࣇ]-b!D(1fsS GԀ } \!hy!^֎@?cRD!!1<:-$({“YK/]/ynq A5\c%a͑Yn44հxZ_ +jpt q`&irOF:w/~*\ 2WXz9M-^*pݣQȱy_?@zcf^N0hr86\)x[‚ugXZ_(e~ kS{Ѱ#do OCo4/{Mפ-=={>+n-I >6v̈́Nt|P : z>('9"*!  KoJHd)GE\S9fŽ YT: 9b~9;}Æ9y^N`EG']s6Wj%7匵Y;44Vϻ6/+ŏ Jc"JƊ3&K:pgm˽&9/lkTEIR\.}UCRA _5aW( RϘm]?lﶻl;ImVtd ux÷uO ax$r!jY2Hv5`DK(o—o.E_'dP!lTQ2kjw%{$6cg%W"WGp:aPdCؓsXSH a5*Q<Ab1mJ|E_0. r,Q$?P`(R_m DQbuTk0jḡř g &Lrx0P_!1jʹ&&&Ð(xD,C·X|*FQo AH>[N1Q ^-[FLO`̵ KcJAVN:q*_F~qu) [~?BcwxhI]q.B,(ixsyE9Z=[Rb|@C9(HX[l_&)]Ȁ7aGoi,:U\|dgH3Q =GV7{tP5 7~E֢WCAi ֬R;8A#Ty>MZ1s. |ǮFq}qYv qlr`с|EnRs!Q:Xf+xEx~u\1FH8(bǯ-iJIdR'D++prx8GI):.{ K__xXرaK"6 X9K^w߻ӰмE(AhXDT mOcbRQ|@8MD$ @ @b\$bt8״Pk& fOtBە~ո6-}W$_0&@1VeίJKSCŪhZJ.IJM5r3 U6F"ʭ\h`!?PDLZ_("\͸[gToP}KHKT嶫fm-zn;#6v@Ҫ̑'>r/NYrq1RUw|_Օ(P\t= H;Ϫ)Zճ1lS+Ix'ňK#]}6u6e;!du_C pX**kK! ⥊ CT.\/ f Ljoʏ+δ5u+z*H@L\xәG[gVgV铲5Y=>^1B!K|?MH^仔)iܾq 5B .O|?=}Kڛ蚤)\PĔ x8!ѫG@(3h ^!r}y@LB0 @O(e @  &nF0a]:^0bcdJ@e8u'^A+rW6볖$&R>ԛhfv^ \UZJU/۩Ԭ|%u A_cؔ r)aIiD4% ;J\[<_Ghx| aY1G@IBX,p։׋h*^A;aY̱'>䁹|7`3qmf6벖" & #2J %oj$|9Ǹq8m<}k3tBPgkάWi&~[ˡ ~`\[mȮm? 4ZE%RQyD-y+ ˡӫ19B0"==/#ȦUt4]G*e/XDZuygUP-&l=9cLwOy\y f'3I0y&;7_ ~ygϲI\T­q"T+`Z8*ēˡ;ib ]4ŠsP#ė;;ܳ'~!Z8B\2VnkCNipeE‡Y*9gJ>=8+쉷>kbg1/I,*-D\@=v;{mZ;*Uv/?g9,ļ'aFV\C :ʤL6Q1miZ dhaJ<,GYb5;_X ⨴X*VvM^(EPq|3b( @a/vgc"_fTšt= H_%m^ `uNKA?kXk7V T3_d|ܮ/hQ:6Uv[2Ƽ.)'M@FtMF?jqgАٱ <F@\<6$Y/,J ,!Ey%,σEZW><[Q"-l"X4wŚQ 5C{Q+v:g25dlX)DXv>9ּNą3/rD?P*th/9cmpVmV,PLcIb@ E),h Og2@b1w">*+iCz2h7lYxfb ,$]Nu$ V&^tGtvfΧE;%/NT^M{[pUcM=UxNW(QM(ׯM+@5>I 8Aܤ1՛UpQJ'C` g7 J#!JhL[(1 p !&7cs˹CL*NH9@nSwN,}hy92XptxP͵`PkRxl(( p˜/`uE=%bVLObBkZbU>AOg2@M>6SyQ-,z(-[Z|V8S_hKKP U`)[O򛤼&5f썟\U62 6qbU>"%kN [ZΟZ&7rZ{ZOnTR*]7N-OIO=c]sZyШ+IDATC/ez0jk&Q1Xl'7œpuPϱchx$i<#=LLbn5`F\*MdkU1ݱfHӊp6f= yֈ@?fTW=T8Y|V!6C }%mfX/]*QlX@oqB6]~~!"cU 9J6튲'9 E^''|0?x4I1K|7"'69wMy6)IۃY𼈳F;|o29 1EN y@rH845r@9MdQǻ ]y쨈l-%1nͤRxai}R R N1*qn1eNN;u<2wǤ& 3<#|#Ltb '#IK`Mvib"'q@>oS43J׭QEW $x-0 !v&LwERQ"tbα4_y[B^ IROF8eNns |3m ^~~G L}5' ؽ>|Jq\[¬`u7I0bgl&`˕>%1)n>s%?d.|hS]Q\z9|(no|a-9f+ʚK1,q՞;ɗU%*_Ew t]PV' C Sꞕp6-$ \-*;j! &8CWp=4ri6aM>Pó%c*h jΘ.|Q۵=9-YYs-o$ulC+e"¸=FĵPTx>l@a I^<I#K;2/uIjn,=&(XW)F<|xڢHkt8\wJ7w(̚-:#"먦l06"ozL]CKk!kO㼢YA7$|8^ jMLr.gԵ b8AN}pEYaszulI/;\U@JҫZf#\Xs$:f?4|`-Q,B޼Nز%6h#y}UU#ו7"hֹFQW8%"$h4 OA]㳏Յ}K@ Եp|;~=:Ʀ@5\-ƴH+Q-:N?@H:75z":Ŷش3}ug$n "5rК֯gj&.TDU\C]5o -άʲ56 @vDo8r8if4[KH*B:8`-6"\E]JH)(hSTAYu^ Z\$EꉾBy񱣱җ:ޢWu$Mgͨ1]2>V ^ojr#&K7 4II &]=}w5Zlp|C;(Dt й£a@m롚s(X `@zqf:8FL6*3[F8:KB+_$1\'|z90\}t_QK.qW* N)] JjTf, V䁮p&$h6G^/" o|yƷ3RR0oGt"sLrM| "za/xOD2IċJ0ӎ՘JFu}]4.< K57L#o B):N͖ `w](́J\aoZl} # M;9~{!}| (G+ Њ4mE00/Qrt zi3\ߤqYUfSlZ4D`rb ÜM1F V VpХh[HuG6ŤjU݋BPū$k(-(kec =\}a da"N*.2˻ɊP " "(pw?xe%Fw<:iPk.6.S0ad=•K闱G2ĹVZwK}@;7Lic<l;Q{wX%A_@ML ŀB]xc'FkTOAVSo M49ENUO1KC9f}t; /kf>wUo~MiK,"po $*ec&jTzhJ,Ö˻MIfa}6 }#qB,{F߂_&xUŴ<0@@Ǵu|{CɁ5V'Tn~/.K꿬롩l$BdgC@9@HK)>(oSG]m09_//l`ZAUC{i[3lAl`vEE3=~GĔ,":jƝ@@VO2.HLD\rNԢ#~'$Q8@|v56/Üc:'E;)籓Y='Y-O&9}yF~ $ho$J߳MZe10*b{p2NG /7G{?occ7V#Qğ퐠(3*D8GoI* G{C Ю_>V̥a@sϟs:+"Aa"U;~1AnkL~0Muwc$W}Chsw0ۧ}raCB A.@yWS?AdژJoRT>pMO'#ek"Hȫ :PRO)J!K޷Afm $xQRϢ{ߣ>Tom _WXڻhNN:Ax*)zʚ?ڷW!ekYн42E"T>?U>F~l/+ LS1:F+}KNKh:P XtQ| ?X;^A~(V/Wf%SVg7و_U y9D>wpIIk~FޟX;/[ˊ욹?[+*vZ_ %S^8P= 0+/-ӷ^Jч >i}r$S_Oi*2ފ"z%~qWbE>PP|EStEGIENDB`zarr-developers-zarr-python-a2f2415/docs/_static/logo1.png000066400000000000000000001416141527014137600236010ustar00rootroot00000000000000PNG  IHDR[~gAMA a cHRMz&u0`:pQ<bKGD pHYs+tIME ~IDATxwUչ߾wfl%[4jb {5Qaf`W5k F{G`Ez{^}ff; 0{v9oﯬ $H A $H A $H A $H A $H A $H A $H A $H A $H A $H A $H A $H A $H A $H A $H A $H A $H A $H A $H A $H A $H A $H A $H A $H A $H A $H A $H A ga lL4y L{pfD(4dUw3AU8$ɃI7 ejkJ 9 ] *Ab@ՠwAڌ퀓Dž4/`o#Uc`ȌA׭!$HRӨTaf)Ik4j`1QL JAbO/m7 ŗe$2+k#~~  QXF _W\ qH?R}d䝜0b"ua Qv@cP+wSM$!=&^43HCPsr~kT?9 /]5":]o!1 k1寗 !p1p m)kztNc& d`$C!-ֲH$@zUw A Χ}xE@:,룭jB0tKa9Sc(FoLF!mT]6KW4%H\HN kՠAU";: GU7[qpr@y'Q!!Ev6G|5M9.Ћp"8x9l&nJꑜT6r)Gxc\v8cW*qڛxp|`bcpzl(hsB[{ow3x{lOtvO>9ÙqHRPu-#|tsh < !xv΢~u!fK0'3e)ISE%Lt&ѸNEb0L).Fx2QD*S^[ryc >g+B`u7ҳ S>PV5p)T/x /ƛLϰi`0r񉪋K 46"<uL˓Ǩ0s& 3HCԁ,ZkI}RԱ`"EΏ-Fp{&܊Xg^23 UyE8ZcGp{0<1 BTat\RzTߓ.UD.!P)&Py* {O!I"Eb,3& ȃr&CHK<^\Ro`#nfp#ASaP=ӧ(PQ\LAN;hPՋnC8  E!gR<8C^WO0a^쟠?cTOF_oN\q ݇ [:K$!2E,H"N"4E{710 T6S*TDjJ2C;4qsa>ǘqW4 c>.cؐ)4q".0<хL>=|E7&z I\68l !ޣ0h#` \[WN⽪su1q6*i=<r֔x8v0`H` KqH,&ÈbQ0tYླྀNy: 9նgԅ5xħ{} ģ51G#` %Pg\FO-btkS4Z  `ıPq}CB`HJ LyFqb40 BuQ8tGH:̮){|x_đ@:5!<Sg< 'qbZek(k݇.6] J)ظj?Lz87`2PK&cSR$Pѯ@rI8qp*KBj)%"$<~N8Jϱ _bN([v|8>W}"Ӂv5@b\(ln=8[H89Ip+0 Qd5OH#x ~";[cJyT^PN7&5 "1Qy~ L5!݀ÄNClEKrdx>n: )8R;g j,0e`l |\VA`ԠH&,N};=:j"0y^S8TRG`*N6ϘƬ.gnz.nl}q"_b"K,ac-<LQfU `Ya\~\4 xy{0,?Ӊ| ^|Cg0{YQzy8 xgp\tK>-4N3m1h6pn:O?!>n&%\sL;עa'`#;yx8`WD~qYQfc1oÆ >OLYj,1傁L:?I&ݏEݺg̥`Z׌#:#zȆ <<s1e92P$w*EfTFOtk2RSnd_}=":5 ,l<  a'1!ӵNukn< 0}'V[8o* \<23cyYnSʱWK߄? ϼz >`< tm(^Xڒ!d#.3 `BF/k=)u5Qqz[ } {40: q"mG Zmq&"Tįs}iFl ӌQvn@M׳ efv>гֵǓs;*Ͻpsw!99*|":ΜhA !W|R ˁ ;Tp:!UwF!cf{35 qMV3)aYs.Wшj@*n{Ay(up30{1@C|܉:.U7NTigaKjcVJ3r6/ 4=*g΢=nq<?w$ 畢("QYۤ7 ˊ}o`v- * RBqW?l`$0!40f~ omh.)*,B^F6n Y A嗁*-AmfH,Ifp'rs dψϣ✁,]x!@$aw  FkV"O#c)z= \/#1#:9kӀ_ L5Ǭ_c bE+(DG}G%?F^5ǿ6&&0sS謁T}K?i &#B\J{HQq@ cfnpڔxFhq;!}yO-/KQq@ov,΋t( Qv9ʯSyN3j rTE5wz=Jd;yKɒ1c$[c.A؁֞r;wqV>pXaa>KlaP$KWc":p{誔)䡪!EG(e0*ϿBs8"-G2|OST`T\&2+^U ܄9p6y)Sjy &49 Rv?q0{y%HO!1c 56  ȝv4ߒٞέQ20ALBC|G[I1$ƝA8||X^>!n{M78"?^ bcBb|q];sDL$-9v-4BXk |INb1#Gx[[gɗe̟^&i/cl87JS<}m)1͇My3)kh1xΕ B?:q wk@5b-YD鱋Kn4f܍NO'].0TF Ƀ B }#E0 oc%oSEZ߬C.ho!DKح]݆gvOb^fSd=k0~PySby'Lj8(pVzg|DR0Qxb4'[!taA7mlQpN*F։> ZRHSsn;P<pJ`ݭ$ky&X$cieOt޸8BXd&;_9E]<Ů˨8Ix܏3G:0kk#+!ģ.q["7mǽC)w85CXWf9ۈ[na kXz7A5`$́Iae"|:,]j ;;0WXDdG3 Tw_!vA`CۈZ9xÀsÂƱ4Nx7E/"[u Sh Z&ޫ:(2jǸgY`+H`2p<&7^b@=﫯ʕ4En_v;#^JL_"[y]h1T@}Us|ȝ1  Fb,$d2S1SKq*p$aeB mH; ` Cו/Â[ft@ l{Twoiګ #9ܜh%ژ#zX9.|d{?P^y{W̝{IH2HϦ' 춣 pڐkD/\kĽj~cb#+e">^sk{tC=Q 5SNci'֥.$ DbV&$,`z~!ӿmv7ieEST*3/!}q:-iGy;0FYCƅEOk9/x jjb҃}O5lX]{15i 8?ZΌb9/Z6oC%mӪ6rwiLQR:a>ai7xJ+e(Wx=44 p)Gݍ]4Eń{G}91urbp.:z`;5x:*>k)?jT ep 3 Q&8c~w5[fڔI_2zO\@Qy3-"p#YW[ނpes'&XHʄZxR''\Z&#Am1406D/ߍaFsO$3 K3KYy_SiLmxz*, lL16%Чu/0^-=}+,^‡G}KBF#dYI({-?k8Ռ˛n:W=?@QpRY 5`o/NڡLr: am-Kf(RL< ϛkNbzo$#1+M_1 dqy5^W##T6߻a:P.ߛW|Ja++#HБ@K8,w6U%u<yd:ց# H)K^lGb8Ȅz[(;QKAO<v0 c{Dt[_[B_.~?} p0bnF)j4W->2cJp>-$dI`#?+Y_Sxӌ)aޤWmޟi; }#lgB-c c-`GTVeXl+( GqHI4 3(c4{#K^D\^z}l&ke S٧evIj3vrl^ D76LZ0j"R)^G3prwpL1ǺZ3ՍSct%lZE ,H&PT_P +r{|M5) L'#bV!?s5&*h:g64f_1q AaH 7;f/6j*Z;~/^PϿA$QU8|*к.>2xVIC`쵌կa.!vcdz:wˊ *G1Aʘ= lۣVoDЇK[m4M?Wkt2uJ*zLmVhtCVq`:N/جڧN t6Ǔ=a~n^ Q`|Xp|eL}bo']-fDwcdȦx 4l'~>=R`Z`_"61B <Ⱂ0Wx,dpx셬T,t7QR[4G A'6м.]DgQFv̶T=:.C]\acYELY'N+h,z :̳OSuhĺҜNLǠӂL_sn6q.a"b˘UmqZ1Si/'.w2S /x'hP42pJ' gwBo6_p?8? 6#, 7M7S;/ά1iLC/גw|lmQ 2u?SFp vہ#iHl-TQU8L, ^gQ=T`mo!pE-">u1z.VF7w;Bn+-X{-paI[spa#iC b+Mn]/x[08<[ek)G3 \ 7 vOQ8:n%J(| `'Lv=%BvTBϑK=8 x9x~4xHFiBsR;,B$aem/|(O,`~QX>0׭׶!b o@.X o5BGal FawnE bJYEL8jZr)mC-Ć+b:eu?!F&*_h[~֫[wD!=G/vɲs/}oB@D`5dxo)y6b;?xL0x!y>M Cl>>jߙ9jHֿ {aj (rQ/Z\˻,(7w1$^VINYs+n1$\ԎyKlRd#j Ȕ66D(7@m.C[wDHE1)z>m  ¥ !XxC] En%p9<AYߋIg c` \ωSjn&EBxx8GUN|$YJAQ V 2]c˵@5ؖb٩U_%0`hPg*p5M劁1 bs#o's&bXJv?=(RL6oL FaT0 ZcaZtӔbRd3Yȅ.`;w&Kns96#oKm8),ʻ>{R*dLٯ -~g~d}^(32@xEe4b\fU}Klb6 x"c<0 A>MIrA!1+-0A<3N7t=A-pjR2|=b4ʹ*;&=0 U#L: #Hal@2}Q5\fUFrF !>7LdvÒ\h<,dcQ0`7 dz&絟K^X{<{CŤT#K]OǙB}^^w Rzʾ%(MAT\Gbl85mA$ae++Ȧ8˸c?aՖe{`NHQlhW|Tgg2ĝC>.m…x~pPw (gji**rc .~Y-L:#Q дX ,5 p)nN'd:7`<}&e ohL몚( O",m @P4\@DQ{ dmD#$6KCܷLa X,#SkPs]"G0F>-RTV sK3nE/H^%dl;%~eyBf/*fFlAtI^O>%rHP/VM|2b2vK(l6%QK]f/WAwnQ+#;MPo#@tpVojBB8Yvo g,⼁#{l"Ȣ0h^^j\&MEc^\[ H|FH; q$(sm(S{iK0aB`gMܽ?T Z74 @@\ jLsU-gZHnKpf" 65i=! 8)g$>LGGט!LsY^%XF$aeBo9&1>b )m &N=`KSxQa ez, ^SܡȾB E=6$O)?6hPM޵p3!3=9܎`TiQfHeRBoK%pB*quy= ٳ}nI'b\)E۬7ka]@עs:ԜJ| +"W1-CQ\Qg)`AVÂwQ8tq|D=㼃GL$a%Ba%dZfP-`٬Z(YoK| j=M:]|My"OTZ,tcR;]i6xq(b=";Дs4dR a[+$%|g籴.z:&6OZod-N `{=_Ln%p ^V|<aNRJA^˵g8L, ^xq~<0!ϒoV3d0#Fk]qлsRW]*"h:< ښvt0~tKQk >P$Vsn|q]EMa2y6zᖥvbpuoOʨ0_>2u~Fy{,$;IJX1'>p !`U|^jD!,:q8QMpF`L-#凐mGѻU XHq:b2D֠A;V-lRL=ipj>C~ڨ4 *Jr!*NVs Yyϵp+lֿqQ-2v~Ck[׎>xL q!6Z^HȘ%cyWHU0@틸kP9=̙]4BA\xx *$R DKƶ.hHj)R!." ' y/~ \?kd|j!J#9,QJ<mNϥgM.KQ 4>=ߍǓ4ER'fNȟ@ { 4!+זܹZ/#ВE!;6J;H8-(nI2O Y@Z3jп4j06aG^RǙ{ WJ[LT";!yc(@1)s V!2T#'w!}=s'&[V3lg>f?&&HmuB=؎%N>CY7vGZ^ź;73H2Ls!ě|v3ʥyT#U+lwcKS]" B1^Ԙ`"1+N%(ԉE5n+;Кgܺ wF0 Ӫ6ኢRm1 {|[܂ B}#dWA7L<x}{Cú_ukEX^ğ`#v|U F^)u-U0qӆ {oN3ɤ] y7ܢ(ۺMb}y#!o_BhN &gFn\;Wλy͌彘cb|n}! :a V2!@ӊ2 H?ʳ){"~>-v74V Y1Rsy筁qp8Tl]3G#n2J(P D -n4oD䑒Ǎ C%AR؝yy3X* {x./ ȧ^EN3MQ[V㰨oW` lc< Enyd#ogl64ʹ*/9A8EG'OWUvvgЎB-Sb}Eh{`HDp,54 LxJ#,gO ^I`_5x.3FB<8et~dKQB#Gwbt"(mZom/s}cvn%O9+ ?I[F6f`U"1+疆Paiʄv1V,Ƨ\{x\.WAmoz"-BA [e/KHuRn#anQ-Jy4=S_T zQi9uROiujV` .7T!v}*RQ1>0pO<aGR܇k1eYn+ oQh0 Ήi8X8аc/^RC)ߪB"cap'Sf)K-6,Fa ƝonB1~Gc5ew?L1 B/tc8B@5B˫?SYM RTM'R]~ֱ-JP({s~}Kq4pbsy*X!_#ἱw5jt&Bisc'y3ݮZ$GϏ/ΫCQ&fol؏%(A(΂'NPe lF. jɘ,7@ ;p0bM*;6O>[vy6 4y]0#_-W@D%`"1+j'Qo  vBJ ̨U^~&=N]_9⽏0N0xI!a BwM 3Q]ܼXjL廍c 6Ax(#4 wfVgUd1 xυ_DzMEvi&bTm\ qQ]^``R\v ʣbbQTyc]uU4XHD_2Siv~^||GX#a)-o#K<+yL!khBl 8x"-յ3\;qh>ǺB@ZbLi'z,bR)z&P9Ra@SR.pC,ݼޕٮFBb$;MCPux&R}yk?y23xPEB o?H`6{)ʷ(faֱw[|ZG~'o}޵:ogwE*s8;>6r3ȬτΥ0;˶JBbV&φTn{8t4ぇ0Sn+*@r9>E<*e}d:&V8LU` b[;}k[!WNwoah6""ef#$#6Öh"CpG܊8թul0p(P |"VZb'- {cP!6IsC^G`B8m]eb9O/Clb$a&9p2ٵ2/#>Հ~x C7u5B c|ND7/fNrSM_B; B ܊Wk39Xcp THNF>OC b1bdm{M2'Xypo3pax<#F AtBqvTؗaD@܍.m0Ύ9tb 4K[B<;#ن@W<7?/FGx#Ղe^ .s f0\+bg!v^4U,-  A)"aApOlܠm,0;@RWtkJ[{ڄת#\Pa120h/q19|:* s]&~~8vlqPqtN!^73U8\hSb3Z~0]pJ%qM["*W}oAx?ίNS>>gj vg`"1+ !pM _Q38/xkת˾ 'pKd:Zxr ioXR"uF OXȖw`z=6arӋ\8c/k-Ac^981x}`_<a@Lܠ?6bIXiG:7jq;x4tP~zofd]afI}Żx F1>x-GN5~1r(FID:?#"P*p aC`"1+CPB𛯳]^ܳc#{ov-ֵѺvdq|8O(|mXQ4On0[jM:py(lh6PE ƕ\(~AMp^ lgxW*"aؚxbӳA?|]_ <׊êDbV&BuWt˟?NX\vIhK7cZ 8:v!mɊ r"Ǹ8ucY2 adz~tG]ֻ8796ה)F> vL c(4<ce+1ֿt|*oiZ٭lS]ЎY1?-B+'[+,޽ʆ6=940 K7V8{:Hi1,`OBbđmoׇ2Fw*2 1+zZop܉w`[d:K_10jvZa첵c [ :Tϖƍ='>⛱e{R ßJc A®g@eQ;7.ư L-o,^sҾW\63׻ʋ/Xqy3Y_Fu6A1gb!!v\u-2q-Tl$P+̰ą8{+7/d}0-D?$[ie"S[,ăAx5܃R^6 5r,!Fw] -4A@9}8_q) D> hsx*u`Ţad!;w6M*gQq{ ̤{-uךe /Ϣhb(+* H0'b.b$OP]x5z-BC{nDv{'C| Wx"aU)8sX^ ބ !oӴ܈^:6aHsm$XEHD[z0P9};_a\'xx@_3ΤhPi{;.죑xV.85N/k&f4\ggSE?F=+gn2#3ʂa_TJF+B`B-=5ADSE>SKB{04A{\k5&Ex>cS\K'* J˛t,7L^1b BlDpeuڍ+j$aeC^xJ> wz?ٶT};!^08 1FZv!M+)yu4s6we"_3g;iB eI׾|`⹐PKq9TR(|e+[0; $YGil 556@͒YZB`{"N'ɒ 4#S,S82{G$H$/~I @~p cScif͵AԼ\涎C ?Rg jL>?kFOy 1+j=]e3T@!p3AH]m )|i0nE 4SQ (|XyOL8I/S_#~"8^5os$mq Pj8C,$JlEvi'㍅1zO B2]M g'`̍ Qn?>|F {g)R9py6xGZᘰ^)zM B9׽_j0w;.[!ŠCRyM\4姾x*Z@+hڐ̯9 2A/,ػ救<)3WQ`&SB*zȕMh_❻8. 5致gxrU51׈?sx|R_fLvaB3BbV&Z^ Fw5ΥFje K#, xn"{b3 BB c_R_%`{bHl`Ʌ~ڧ} >:0ĺK9zMi̝4q~d"q*r`&Ns_ Խ?rF<6xcm g w^Y?Z|L/ڛЭ?d}2EB"^la.GE5hהz]ȚK-tB1ٵ ,f'{qXh+%Pfثz"t%dNWx0vwAx-RvA=Q`gk"ޥ zՖ֋Dr hMnZz1So{u9 ЁgďD3`~'ɹaW![h&_)fUޞ"J,%; J47 {xqUk>u0fa<8?_=_8)P=5kV3'|G}U>%p8bH !1#( lLS(R@ -p,yLmƴ6- V8aYPBXlz-q̑ HpJp7R-kC$g"R+0k{B; AeTq#ufehMMnXA |D G+ p|1s_J:م)<lfɆ#D"{؞>8^K}7kͅ@K䙈 e*fw#G"nr n>ڧ>o(b1A9Úo/aRH)ˈVvֽ ' q@~sAW|ץ3ͼ gV g,n3󎓫.33PRh *~EERE+4`)HZCSDDAH $dC%ےl9sfɦl5MB3gf}nGb@wġNa3щ/@#6>u4>d+ۉԩ^~< B'>K8JCZf2|TT1b JP-f}›d<߼e'sLb#?o@D-lK&*&yXܢؿ}cɈ)HSb_ 24V81O ܭ:<8N) c{AVט7&74V"2q~c*KPє=s\.M1uW7f}px)(s>պ#0xbrb2K%ǮΡ%^GD`NS5@/ġٗ Yܯh\7Nadk y]Dl49JvHTMᇱ%zU"ج&'Nى8^ANWkʻ 1J[D5 ZEThѵװdȪh*A> z&p)orFz_mSyp)͗ 2XsՄ|ă^f2wM&!b#d]01<zUfvmmY@,{;Mn{Z"GYA8"뀙8]g..4Hy9; qʞW`q6Wh]Zw,ږ^"/QlT\n;Փկ L>y^3Ox8M] }@ zӋ˹Iљ; Td'&/晊iypǃ< ڞg'(jm=^,BwxdѲ']ӜE.[ZDbOk"{$:QlZl6ߤf;LͱH#f9>0k"e&sQQ]Y] s׹޹A>;4pYmj{?Jj*'SJ<޷ xTbMG8xa{ڛn{K}qA&Xu˯ʛ:qx`= ؓ/yFЃ š -Vt^BF> zt SEu9ijYɋjZ1LAY۱:)7U;Xu$]cV` wj @oJEh]E>e?jOslu92Pث5=Jov ;hEPNp{ |׀YB~&fc9 ^; jr@ϠudrҟZOŊfs|[E5A:>-qAomqA#; ਰFΜ}I?O՞؊6]pZ'pCE/$8q=UNɈ@BDػG;z`P~!h@a9#Q5jIv2 ʞu?oǦ,VƼr9 !D=/EQyMT|8 m"2([o`G{bRGGTiB@%1gd=vGL|0TTg(~w/ʧ'1(7tU@Hoi)kI$1~}NDaPj$ާtENUfh0-Uťٕ ?Igo|8GNWe(y]ky9Oݠv"h:U'A |INпa"q,eP: اMbQ6"UY'o"50lU-@ Ѓ=Ewb_*'ۓ?Oc֗_عE*uMygAOrk4E6T{uo%yhoR&I#A. ^圕y^ 2\RT%0a}yjMf`d>qC$ EQ<"[@uį Zh1 ~M;i U'yQkm"Ź,]۝JAH<4p(%V *Y( ]>0:~ qBTp@BgbA&${ = i% BJIDAT'5RO?76`mMr[:O'adnV` Nb; EJ![fo6j/P[J/-2hPЎUSQWp],^~;p8š-t.u'ڼr qMuT> 4µ}T%ay` ;ྜ^W]$ prݫ<=۫qf/~av￸d<rS}d/.-}zpT5>[޾EBG-*V5Yzпo(P8ɈdmžEQ-oPChQ Lj:F6r{oGR[ yvZIL*(܀.Z{S`V.R^iuaêx'cQU5]Nc-Q=-MrX՘3 ,ġ Ѓ_RPcFag0֜ζ27&湤PBZ{*_ B"f 7(ށh1vgZD5w#j0-OLVܿy IJgWy6r3g'02 oNvn-m9pez RI,MQjns8ܧ*"}ZK;_B%rf{#|ߍ^fqI|}B=fQFW ЧlW# pD'b~AI<*rw7%" ~NJ<ny'g\ݶ-#8+|/p5g5X6,Z٪:Ǎ(3ii̴oow)P4w >7>Eu{23vU'R禴˃[hn-ge.et0;X[(H5D~O\i'w^oo|Yα_qו K:XU7Z;vc7?0>W@٠%Pr8(oG^@tʯA!= c[oߦD^90GEAǩ7 w ܁7@p$x{n"%&/+. Kחˈ% ^U?y?Dh&(y7Í? Z CA֡/sO>CCe~0pA,[AtDpe^lk^/ M_)A V) 7{ME~"*0{=dN g}iS8m>"rZ[X F{ M`M*f~i8 [ڐk@Qd0wy{jAnt0NB3)D1x޵۶nR#fPoan~v^ձL2Ix+7 Xo͂^8pn/?c^uX˩5T;XYo=CZt8Z[z)6"Qa+X?) wH:ǾuWT0HD2!DUT *rWhFdp yu m!@8!B3S۱@CD8$|fUvߑo]80+ThA$M|Stb,b7y P 'c]B-2kS[QP`K1Um~~~,[?*S\g#9LDT|?>6A7n/"E7jێ߹7rp6;!R@zx氼ŵI]o& MDPx D&ϝiހQox^Eϯ@ȭrD 7",A8]ac:^>xPVAz5͝ V4/Y8g¸z {f~g,1~Ғ ݿ)LQj MXq/g6 æ 8 r6ҸV,[/фq W HԛHm c~ 8T3pܟZh|e50S>l%+X xi:O\ӛUE  gzך3h>C<U# `\])a_Q`oh[_Cε!QA?T6ϸ Ce"Ҫh#>RK\/S).e1*ܱǬ,UY"~udNߠ !  N`6X^<,dVqg~ۡuZܠ>Q~>EfyabŻ3,'=|8'%>yJ]l|h웖NGRA"9IU r=ss,'6z =f ~}Z io_-7!_F9$p+ȟ}[PegnKvHUi O\Dyor1..! E4bUd?/75]d$E' K.ն7-;0IZ\3MQo}X~T5& Wt\Jpӂ\1B]ntp3 po_я+.V1׸q7k7|*K9)ޖ,Ĭߡ%<0ُgNDȄ=Dġ)~aIכ5'¥_t ѯ%o5M]:!H{GnBSK8}`w,֣*Xhh9$(lHNG~h"~i4l,QZGDs-N i j;ҙjMrjN(S0DvcQxP! s#37IU1qix"}cbieIKUSirW \r=yUnCX@mïj^$Cϓ[zAw`Y1Zq[*09! X]WUA  xqGU2": 89ʪB'薌Zyr р Yp`M5k b;Vu>!Vʋcld'ĵk9Gb&u1E|" }mUxOe[36"RT$4+T*Q-$;i2V\Cv¼ *~1BY 1&'c9@/ġ';TAE6p:;~K4qoYa44D琺[z 8q $Prb9 ʻEfnP*2]|'爵_9ׂwaLE{X7*lF_z?UM&<2+Tڭ/i3<g{c^/3A\5̫<#Ng^n P@^CXiExǎL$]>&"SfA+ՓE0n>ő 'u݌wNE@ Xp㓘m*9XGCKe%x`lXbؿ]iiڔ`1F{AQ߆|iZpDۚr&) đgUC9l讫N؛J  m+.pz!/zi5,_< ѦpOhns+T9?qM8͝c& 'zertupWrPۀTFxsv)mS^$C҉f߁.U\c;V `:y~ivQ\jbUZ %)(1wv5a9Ǚj3DѤ6I2UcF_82fS͈l+y_Wʭ&'\DYlU(yItZ]8 cOD AuHG -2r՟׽}[F7*MCEդG/8u25QVAzDA%wp@m Gn%B"rСD4=: XR ,\tġ'~>pE| 0Zj\'isT;_Jz_;Mݳ(IM>DʿVq-,Z_JY`(Ǫp&p6:W"*o ,2Q(^."WjNBSc /ǩ̀('DġX́FDGʷ/:k.x_%Q$։ؐIk-Е3>?\V*t.9CP-moDq>û{sj p"(,w"W&JF+I"!WHŒ'ߏlz =V!#P W(CX7Zx6`$9GDiU|yxI,XuN;웭M=  П7N^(xU^QK4{GHHp4)2eմIq=f$yHPҽKG^N~~giu~`BV $*^7kvDh[$Ip3<$xl 9R%P?| 8Փs7(tx@g Ѓh!΀Qh})w`ڃy.@N P(TX4EU)$yy(_W7Z^sd{Q @2˱ă+-N}tc"H1+y|UO'|]Z a] X]Vnzg[gsڒ g(qq}>h:qY% Z榳e?̓L9T?|Ua%pWɻ,%8 墳+r/u {"6e @=_ma;C=Jdın>_Om%v BN`7,\w:6MJahO gNnɻB'f-#|zr*O0 m{ (1;gsQ| pe˥ЊeJ:A`pN<.q$!t9]s1p#]!|cM7YO'0BD&͘Ź}BM (@.q~.B937XUDbgO H3PS|[լu1g"Œe fѭڰS)NkfFv6 Qq<ഭl"Gb]@(*mUTw#PG+~BnGUDAAAz)(Wm#Ao\m)Pq#8F;ӵUz&;W;~( &D{`kNA`?-gN8J.w nNA%B?$C?\b1Qjep^k|d㇬ \ۧ5J򠎺8/qt\:HEAڅ;!([8SMéC2X'WY8ciN;Y| r W5b A?A9TMkGEՌUܯpK좗 ꋾFDIAս}>ʓ .DAۉb]Q0q <]yEld! D"TpXCB md8h* >hs!xP}40XŃ=ES1[bomҤ~'sPV(@6#mB,A( IE$I<;t;ǀ]Cv50Ww0|%5l;(xq؁x"w>m*!ΡXu)!HUh j$"R(w `_(Q D|* .,7$pKhqgsNBOA?\5G>#E* zQAvhhnRb;C}I"(&CWW:ji);|EQqy,7L ;98Ys "nwBA8G}xچ)ц;fiZqh(kqj\챥 Oy/׫3ؼ!Dָ#ޞۗHy<[My 4W;QȚ\{Qȯ~8$нql^l:XP0I<RGT'MjVmdDqf-M!If"hjJ7E NPMS?\WۗCx цL\Ql -rg3<Zb/$lBBU/&?D[y<[M>oI,l8c$a Ai~EkuDuSb*z"I`1p 9t"޾j\;Qvng#/K38 'rBB^2Q#(JL@yl}1#e8m&!IDqsR_WK.WUGωO n#Cx4Wx&y'%@p!քV9nY]LT)ĶSt } (z n\5"#-P4z? ܤ#q#_m()8OG#Cx${;\ *3d [P7zC'l(!Q8A=Σ q) 8[G֝C/}nvbˡ8du6iyQpПp;&AUj3sCt'Afܮ+!A 4ת.y mPtIMzN!o #{1v)Hmj$;(A;,d/Ft$@ @ @ @ @ @ @ @ @ @ @ @ @ @ fHo'ˆD!@mpؽ޾@ E޾@>x A{B@ =HzzΝ/&t ]C (@e 00p}yОD!@`!q@8@A@ Ё @ @@ t C :!q@8t/v@peb8B_5%>nm { vQT ]8Q&c\hp A@:c(|.ϟT`LUj@ Y(+'[}K|׀o=_z (!!=@"  :¤aAD+ @&z@+O ׷{(DJDc&(躾.dYCmꋟ1} A5'^yFn6pF7n2VFw]Y6qlEV@)c!p`K co_kZXhdhJOMnPTzlAl;ou/ˎGȪ#"<~`W`l(#ܹeV[Go!&7NTH2UDtDzFC2l#AK8[>%K]0x[]Z5BF!FypW`,3eCA8dXɭNJZIEA;鍝MN0B@bׁex'R,!p0ša=?۫k"^U/vGń}'UQa q ,$?gعħy "akJZe'zTD``7b+ sWS ,izcOG 5(B5^V׌p:m`{ʁ#`60q_(E98 8([g[ܤJPؓ`|c/M<X3 7] \#aQjKgϏ(?8؇ -g#`&sZ-TM`tKv߅=o/+h1$#J(ӰSמ>otaIn>﬊U:jؾahMM](Sj`n_]i_P Yr1p%ӡC8/J~H&F +فe'"*Dr+p })qg,3S .Eց|K47!.Faq Ϙ=#GcOJ|d`%(!i(VΈ󎓁G71%*d׵ "1gBA,Θ}iG%C rv-jsjƛ$K~%:~N>_p ~o_. CBzRvd8SK$j))W:ԕ.J]$q5d݁F^?g_2J_8 ,E8C VP0)5ǧ&^YJ⭁@c=!&rfjr(ڲ_/fa?'qgc!P:VW1vc%o׊b{t}F8;EWgY(p xJs޾@~IH%*zJW:d,& ,#VFUuUnmݍ9/cU`YUi0/1R (8+\)YTRrWO Q 0fuo೪ kEVCzfl˙MX ',-ð*[ \ۥXGws*c҇$M#GUA!Q"NۍV&592,Bc1=ŒXOiy(nq[?Uzml_} }WxD<:( pB2ny-Km蚏P{fTec[q BPPU^PHl'Ҳ j }e٩$@X/@ħY Ld##VލF(,FUSQ WƞU@ S(˭<=גUP|N>J+bKͮXtME^Zt+[{OSPV=< ŒQ맼i2 T ;{ϋ^i$3-o21Yl 5؜yL 2\*,ދ>I>ʲx@PePj%pL7W uZ * f5c*%>fvoEx?` WZ+ʟ-ki0qR;E9#خi  ռD  QyKX_p"4-a#2LD[g Y-FZ`KiU Լt=O^yahϘWrYmEBQ"^TY6j&Tw`8MzZvh82Nb% 6;J|:\TFa +6bw b#1*,<l/-s.ײ=,uٳ.*U8˲Q֙[?@#2[M$A}bPr:Ox/V0&PYbf]o_v ;m5P$C,;Cb R8`2ʲ2Cۼ,;Q+ B'`Пgo^/Ç˼}j 0T ^:c1163rBB<+N9S@AKXi達1mb_Y 09lAzeU_KCJ| Ĝ Fw Ksh1o-o|@Xw %?B =ҜD|_R,GMaXkKk0f8kUIi>zTMAQu_~9jOz3i{p΃CdcWirס@,CAC#xTC)}bu%0Yх+I_ݘeWsg>ﲭĵ_~UCJHMͺa7,]&"7=ʆǪ^= DO{r[hJ~v\}>4d9z H!r:Bn8ͯA0P B?\B C3Vp?>/+('o$V忀4*¨ھUj}sÆ@ K@ħp~P2o9ӹ KU4 5ى2֚h*XqMĹ~:3؈ Ad8H/A~ۀ^:65Q'%8k݀<ഀzQ rl4i?blsՉ_f>K8:xe?=All5"ۨ@'JtMD_OTSXص?i#M\͋Uꆗu} neU/a!w~F5Q7H"EE2 Oq'{8z CKp0HAV냈wyd97us 8B>򵑗)83u&9:&v܉u.5{z_ 'vF#]}=a-j1wB@"y5H_ au,'4u9,@0=+h Gmu Ч%Vg+$rM1y+VM!C]we`=Auoǀ4JnװXC]T'CGi#PAxǤszF,ua)ĈĭIi C^ =ġkt^{8iDq Slx)`SJM݋Á#D$tK}(t鯳xJ1 Q5 k%6 ׋&hg\{?ܓmIҵR )yMD8lf:Tնlqٖ]w%˪&~Unek{#*+T֦rʇ41z5@2 {b` tnM?xΕAKKy=Q+= 6a)EwF{Mn40vmS?@ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ .[cȫL%tEXtdate:create2022-05-20T13:22:18+00:00Kc%tEXtdate:modify2022-05-20T13:22:18+00:00:>tEXtSoftwarewww.inkscape.org<IENDB`zarr-developers-zarr-python-a2f2415/docs/_static/logo_bw.png000066400000000000000000001302301527014137600242000ustar00rootroot00000000000000PNG  IHDR;' pHYsodtEXtSoftwarewww.inkscape.org< IDATxweſ'*z/JE$XAPDP,"EcB'$T@swݛ[vw&a3;3LfO;Mm| i`S} p x #:~O\I@G>"lO}0&!3@҂fk-mhA$_9|\VOg;$D/V}8bʬےfOFO!\F?pgRFDaڡfm-mLhA=%\ '`U%m/nJNnCkafFDd~XAZgI Mmt IHpq&َdugr1{Jr "aljTN :5†>`;*02j;Hon* e{퇳6Npm' a n "h|mVCl-=cI>&96@4@((؜ WCIH&@5l?j{o"Ev Q5N~~ĹDk!=^3*ʮ6ʇ6 %]O [mOg֖t%Q}Av2;W$%%V"%Q{y=*Kfj-mm"#ͪ$|emDEp'HZ]߁ZnU ؾ gˀ%)ٽL'b oo!ۻ"@[ʹҖ6ʅ?֐4wh{vw%$b½;U*ؾvD@e"PbV:f2 83Q0hq@{D>eHn+(Ǡz? m_ߑ_J?Hپ6DK*%sA;rj>&V3O2ydžO؟HZS?Aw( 1nD CI%͞a?Dr$/i1= 4z=牂zPjm"zpJ9QEڶi; `" lntl?b{wo?ptw"42+ߵճr۷؜eETM3$ S/ؾ$G(AO$*r? կ/}H׏/IkBfD?{%}Նھd_9&m"A#]2T]?ARC ukcny J "69)#m"!ipJ o0gٞ i>I?&V'ja*y,~Ib:v[;+$ݚ ]}uG`:%]@^˒Z"`yg`I)'j$yo=IPҠt̵D#]@Ds"p /k$),mm" izY/CϘWTϰ$ 'V+'hc~MJTz?.bI;p*bg &TؖxOP^M KT<t0տhNopgyG%#iDWph.o V++$N;~&#|rVvtLt2MaDLpe6;F؏StES϶oYVi/b;p$mJ[=DPRHZJ9 @~JO耤Y$} s% |pyu o 2:Hŝ$4I ?Ib[mD^B- Fm@JT\+b`:rֿKo }ۯf;$C+)Ⱦ}SmN̾ W1"sm+>̹S#B[moDc}"%]/iVƴ&@b$*fWZgyߵRcV%ϲ}4%N_y8C$R;X81<.\I-%+QDTUA+mi>DP0rյ2W}nyIRbFU"N?+N+UO=Is~=O'dAV3Ոt笪*IҖ6Mv΢ !f f߰ tمh\7hi Y]7t 04/iY\Dp`%}w{,Dq&I:"rޱ}*q?!fAVgcۗ+ɃehA!iy>&WOf;$KQy=B&nm?퐴BfOw8-N9H!NeH*[ 矑SCp:jپ4.:E4 PY|&Բֵ}IHP-Ⱦ*nAu3q5?afrE~Y"|BY*ꨡ~IIJ*oG*})ؖO Dd(hv*Pwkjl͒6bfYf6{%%_TTz !OA%imD<.o"S(< 54ۯ8<*n &qDѽFJZ@[>hA 3N Oy\g{wۏdJP("fQ[logCIJ xhΘ,NE' }๞aOT֕ڣHGnKR"XI^X|xc$]*ilTM Fr  2 CII1nS6}[%ܓ&Ni$A[?7ڎО Bc%Yۇ߉zBB)7D 췉>V%JYhD (:l~ SkMmoo{tB=_̸Z@G Wтm*%%i۴ k 7\PC |pYΫm#^E:jHBkc&~B) άJ@eFD)^%T" M !.!(t"eKJ5諔 5tY!oßTu/Wo&{]^:"[.HL"X{I[iˌ6Gi?Jv9we*$,/ 'S>TN%~PJ@'!I>w:A#Jӻ7!ZRMg+K=sKWeu7qDr6;=̃B`8i:`=W#`SSvrhAp.!(#ђsmܷ%P5`:۟%rnǁnJDo%-J[g`P&1%1+#:e9~$&*yW};gY)G#. f$"iṱ7'Խ>tߊ|!;hSNJ#D $-H(OYҢt/-%rć>Q{q"B(C])TJ^7"gsٴHC]fqؾM[ vj*}o-DPI`s'$zìl?(WzxH\9,nN)ˣOaT%z.e90%eFHZ" "Chs._}:9꩒pE~S Y{nM~Wbu71* _}4B8f{Jfhfx"HѾ$\e†i|OFPzMbt<#neZuM&+TvqH좟P?]U1^F-!HE[Typ K"ݭ,%|@K ZLGei{gCxa.xT?R8%."bGS$+i4>3qJRDgÈ(eJex 8P҅` 3?N$mlPQɹ;AP:28G_ee|uv7&k]s~o)7DJw9s&ί׼8?~؏-DBND3E\ߧ{%Dc[1r+܀H F L,IS`Ԓu쾩A\~k4|D aʄAD /%GwbaG;P"ȝw!6uoRA-k5r]X!  M:"yJ6L%s(V x A5g3޹sOlݍ&1+wMO={I:G/4r_9KYryTk4r? {O:>"3/SYU/֋7r[d[Mt#PZd-ˀmSI;/gaBd-~,!iI)δ^?ƶwtgq%U( ?ۇI $!,m<{r:WRpG1Ht>BDږm_'ދ&+OL\h#>!ҫ>l$ \IyVm_6` ؾ3PjFk;X< ih"L~%:"s%[5Y~&GDҟȉKHc}oJ]_''EދU 7F%(HFҹEQBkI[H h[f\Omqu$]In`;ۛپ%PJ.%{ucI:D[\m\8{?LRJiU؅8#W9l]VW&4A%j5?$7D"<{K*~FHRҭ(`âJ`{}(icI!^M gWt9ک0z/CED ɾ}fu pOCtZ, jwf+R},WsM OHru4yG,%X,Nؤ;DNe Dζ>BvJqY~%IQnmqDKZFA(qf!tn8{};fBZ!T\߈Ⱦa´+˄2fDCgNn#'%f%𴤣LkD!"ZÍ&}O$]AUu=h`9z iDcLR fu(Akؾ̎tIK@p3>K<!\JX{.Ljb1$$͞Kt(n/iaIlߢY 8ʹ}+s7)`AW+Ժzί*cVLdD{&E%]Md{l_-2 >#-JA,nx=O$RJL ?hZ,nNIJ*itL^Kθ"~vI)ؾǡE|lWK!ym'lo)j=Jk $B^JURv,†:C;usI-" Db\L8Z/?S .$ t*TcnF[nVIrt1/?$zť&Ky#b<[%m^uDZZ] }ǒ]㊲eIJ2ɝ~'eUA@#'왻FZLda"PV;#[LdW{Zr_N IYG[#iBd:h"xh-Хg `{h85pAlе㬈:c5uL w ծHj=1pHزi8T=磓}˥:fC(Lvԋk4r2&\BhkHʒ.#L9S*/q%I3)>[cb0Ii qm_DpE |䟳klGZ%mYdI7B{~/͟*2UB.gyNI!{|^]={ A)wWqL(x8h=. KWZ 8/$-JjWqI!<9ϓl:"go]yk%_ #ϑ=i$ T4K Ot=N1ˊwslCʶ~:ܟ>W%~ؙP޳K*]v#N퓈GkRtL̾7&ۑeq{ *y({+{$"b#Enۛ1C}]!jW4&JS"f%ճY%}hA0.+'V)t=?FgCl`{ tD+m !(#IG*T;O"N DrJDs~$iUΑ6ȗ/r!TV9%2V9;꟱}-NI#"4HBuf3K2[IXYk}G@$DR@*)>'>O6t RN lL\OΚuaDjLz8I~? Srg5өr~Ba:۟#3)t4zv-(:SOu?Um i&I_"rEClf0Nˈ[0 %m-p|aM17PQ:D,_}<1PJJG[#cG ߓ4WZpH[!2&K:9V9W}} DȆڙ00m7DxbWvShy'$>Db++>" H :f5[oZhT6= z晴hڞh7}',>? +&1XpLmI~|'Jsg5 <漶fq0"+y6IwJƮDOHE`ݹyieOo_~'}@WD]'JodKysߚ56Zg!8Թ˛`3zB&C3N`ihV"?&ɇP"ȝwM\תڿ Y`>^D;`"po Rq{NuN5103dh/"ȝgh6TM8iM W\N'\RELxֶuV&*Eonp31{9q6!Te}Cۏ;J+8L~3=:Thzn3$m,?RXOT6 b87cx*]ew+AdRyλH9y%*LyDϨ$( ˕yqys\Ūp:4J-+܊cV:a74Օ$*1CCv%^Jc SWzHPBQ Yn];"M4.YP^ }9R{pZ"2ȶ=I+4 Rs&n]Lص59X0& qZxX`7,DpE'`00pTt]Ev'n&B܊ n~9T q*D68b%Qpc( Rg̻(6L+ad-DH-s/KhI;lHrnݒ=LqʥaP @{^L/Yhd!wEm>I?&ϣ e"|PZG?i(bvZ/QU#(:v71`ygvG5bBL8lEbQķ}Ʊ"=+Dͫ%DйfHHMUԂl?nv3i= Nq y@7=,L;&v-!r_&dų+ON :jM=r.^",?1X ݥhx>O_爕ڜ=vB't|&1AP,Nm9{4p^/128QO"s,O#]> gMwVMlε7]L'^%bKk7s bD!{ N3`f[ N3E`:캿h6Nvď0krߝ*HN&~'RKQh԰gX7Ib;[zY~DPu?q]]++,Wڅ{mH:^}^È`<1뙩EFg[3f|105A}Mgs YDP1nԵFsGtAdX>%zve.@ -vcܱ?=ǵFUe:fӯQ)MB'8f#edb%17L!zg\ubQG[#ۆ\4h~k$NU͆\DDo4Ca p"ȝkA |؟!QǞ@ gc{Ufn TV9 'ܹf%T͒/t̿DPtnLO lߋܗ` Um_`#I+zМԇs׃ eGPM 6l!n)s*kW_Q_},`."qay2^ j>ù4VAl%V;9̒ftBcۗ_%R鉖y(s = ¾+"Gv_xr:ۛ^dA瞠7DQ9WRthZVO>q% 'f?iel!yQ  > C:^s" /0ۥ?kFS`I$_z$a,TI :d@dJ+JXRf&'[IK%B;^IT(/uTմ!tI&=9".olWwt,A'K߲-b2I߂qC /)H^xך=Ol/ ʊw኶ t.}$>]B\GdVђ" `NIG+B DI?V`{Qkq _km7UgIZQ@x)Qu.|PHIe{'Idժi(FLntvwz>W~*l_:kBY~Mmb\fI߭oBY~M Sg[DTM>T;T(3|L ٶoT(3 a Sz/Qʍp '#M69aBCTbf<jԴA:_]a{q4J>;[8+w7Mss2NdqQzu]Q.KdAS?U)5@?u9Y"d QP]Ki-M@ɛteљޢB1^G#yrˆ wשLa2B7A9k<%sQDl IOV$Ot!\{x"#D;ih"EBT҃kvJkfp?,P]}D.yt&7FhOqy#e%[BƎZ J@c]1 {FJ}tB wluMt& WFjYQ!oP ʔ^ FD1yp][{z"f^9rT/QJ"CZHxM" "}gc#jU1JB`UޔsTִ 2?Ϫ +?nk%Gq{:z&jCOg[Dpף~nzw[XUOde:_`bN`/Oo z"'>/rͷy1Ι۟/T=矦fz?ـê_|%[_VAe5ZI\@=Wo$ eMKwUm bUѝy@ Ltܶ͏ "ȝcUBѩp4D;Dzw̞?o{3T=_7]>ӥEfoV5:& zkᏤmEQĤHVV 8b `abV]-x<F w]Od0y9=~WY"ȝkyCGD5~oQ]Jo"ȝkq"e!uκ5jk|BH>,"YNg\5qC7:$T1.:} `Qn`~^'O"ȝs}B*SJ7w.1~ȕkιדI~pyA>\aD;T@~d&uM4nΏxE "ةqzm5'J5[?Z-,I 0:BBk8Ν|jy>\D;jD|u~gM!{xA܇Cz mdê*2@snc{ wϫNmE969T ө}IKX-[ݗ́ԖHJxgJߦ5jsw4 7d!J'>p5BO5G~,7,imI"~!=.=PNy[v>ˁ+3Ǹ!١,$i sّ _eNIKH:Xz,[e%w_.6t qw_"N$eR[tm&P"᭪G( =D5&,#Fc$!IqoTWY<$ T m2TfϷe*4\Bvܳ燉fP8Uz =O$3VWW9tf" 2ysE3 /$af0 "$":[!&bf}-0YDP tlQ ;pG>*i?Ie2L< ih"'m:(PIݖq91fDǿy qZxN(z6N !A\!y!8 IDAT `DEVt1ޗ?loݠ0"0 . o"c1+cms{آ쭅1ߙ(+rKnj=Qk<熢D+Ξt 1{>(R)+'f$?{87 $ϿI9:-_4S."fx \H}C4^@}@t %DBI1#[UnVO娵ؚHIoM ~l!cdBY_1F/[XU0 B{BY_s|{I,JFmD3S2E8 Y*_o5Gޑ#eF bJS sܫJ^5sݺ e}ݺO7zof֙a6B# aof;(?O/h6LJ )]+k؝DDO=#VKec#OKI*]FÄi좣UTTmA~BZ7TB`53b?w+y?o@5.Oa6rhdT?3=ښA)isfk|@F#_k{pIQi\Wk{H\({y?y"x&}}z FʈX]^ :DE8Sh$UY9YzRtQo"x`ߺM(^~Z[)iK9맵M~MNj&,K6üt/iր؃Ӧt6X(x_] /&Hy\hT%]gJ>&ӆM,N*Mf?gI! {zaзs2xw#xxʢ);#ʻW#x2gd=}A4l27f@ۓ>&:~nPY7ǁTCJHRM|d2m -5p+in>I4R$~pt@%$i~G|D|йh8һ lIK8RkB FKR_[>)R~Zov{@'y'"TxLܾ<j˰&XH/X i*Yq^:E;yߟ2(v :k*U6:STabB,%F;'ǷI)\"xG;qsz-zswc4`*) z"e" j_0@TKgD0 eDK Bx2#!bpz a~&0"4%I}rNFgEm`"s/o,t!>مXd(+ؚh"ڳӾi Dp `U"]`7 9,A)t#UILt'r$\,SeR"ڲϚ??p21~p,J | NԎRks>AaSB7(C Y8Dp7}{XwsLEsL7<;.L2s?/*v=,&N=Jϋӿ@$7L/꼫Sk*g #˃ҾB|$W9ྡFu $D+KD:5sm_j" آ=FD{+n_ϋ:+OGׄ,0g6^%ēzyaU.'ܗYm`bJ‰ Ust],JGɈs0ߤVpK$ZIԸu込'[+KӞ=Okk%lFR{g$Q4DW Ù{q/˧OyA#.IzLuƔ/T4D;ZTzFZ,IrGS!.>AsSSAyl"K]ql"حt[F5aIyjBwZ/o36"@op(U}ʶٞDYqԮ 5Ν2lWHnݏ{7ΕuYc6o"ȝ+Br{AşרYDpU[+}=ښE}˳,pgϽݚI 2&Y5m I+{q/u>F?ݪ:6zAg"]cGt. t~(Џ5r׼Z@%*}w^z #:y< $L`:7?{5R{]^h"k칯[3` UAt;uW4dySIcmѽETdךsJW?_7dY*j:% ?Շk5rޜ jW.M^"ȝ{з^CCYeJ+h-HW2aL]H'u*I Ve mI`(Z`U@'Ś/ΡT$q>l= $]At_ݝRwMzYl^%/9&lY6s/wƃv)-TzL 2ۤ?$&ƏJb{סia18c gSi$=PzlKYg>_y=sDG@Ҽk2yJ)a0{R^D*qAlV HXҵ|(1F?q?.C""Ë;HF|#%]5LstmK̾ Hڱ$"i3=Ah <"iIKaDB\s}RB~|;۟LQ#h&nmON򜓉xh"C"Ck 9%};+"h]q;j" M-$HvGٞ؏kOW]߯;Ɂ$Mjo9E!p;w ,pK"OůIڭUN _-n8Q"]VdXC칖Z=pTLevwT%ixLjk/#k߱A&n WFzف HY# 9Ջńnm$NloDI[fmmlAL>C~Q "")kt_0ř8MdxuWqc{, B^X1-Cd$qCO}uqtvt7QXXk@{ppr#N%G4Y+/<!/#Dђ)it@ڊhY1K6"x=dyÞHIY͞>"xL9ED̯:î^A `8 ,\1ll`$Jخ$jIt^ ؙhi]4lHmeb2&D;u56x>:x;RK5+B6ğԓ_FDd܍I@1[=*I!VOa*fu]L`FM[ݷc+󋘚f/R2._:70ϡk.l5=Ap" N U){6 P sx@58 o)0 ~l=#Y}d*dFR'*'J\>Ͽ cVagc4E<4 :3`0D5?c &`6< 1 < )y .;PyK œ,`s@0ػK ˍ;Z ؤ뀍n;œxX`t5ɘ | $a>󆑰t X$1 F=s,my[cj/ONtṖ޽Ԓ 6sA"`YnJ/"7vJ/92>[XKD~h94~;ƹAֶzY0UʁԯQNu}#nKkf}Մ,]Ǒp,_31Kle^=j:smD`הv}#:N}M@45ihVe/2?^=EkWyj] O`klJQv`8aJ-hb;gǩ:5!'y)&(C ߺotsA=,TOP< /y. :i @$fǥzkT4OW`-&u%vR ?vExڤ>y'mf}t;m\xAAY|3{pNF193asDuw Sɳne"ۙ<&%o;0۸hV:ҙs5 - g]C+gx:y}@(J>w6LM{ֹWT\G61zbS*9oM^]=gL/8B3:  O1&/#4#? ZNYscҙ|ys]֍F;0Vݻ)ST6?7-4`j0aXI?갎4 Ob!m3ԬS/^#=G]7:( sݎZr}o^1X6 trfeq.|=:@pSDѨHhU`zQ0ؤ>o ;+>^ 1 @:]Jjd1;Q"j,-Ldqձt~ ̍#R{QI7Kq&@ib!&mOk`+fe;Sa%*I[Ž3hBE[?DFaoKuQO6>JR#mܗǗ 8gu9a69 I1cA9T{G W`v>F`D=A٪>`˔{hܾ@uNب~/1}7fg-O~\zc]V!,k㫐LcowJ!f7|)y? _i;$R΢nE=gi@0:à"9{U0wKv^~T*QGs F`;bQ\`#2T5&_FD.G9Xؚ1hζM{Rnrvt[\¾Cqrgk`:PW2ԹfeOUy{*jrƃ-UD-"sBrэk+NTuOq"rl?a^{Ն>{3|‹Yѭe[D#KAۯs&Qh-=ϰҾXBxDd]}Z(|9܋l*_ZU/P眘A8/S+38@SS|3hHYrܩDn*%"wzMXUw8KU.U<.66?!"r)zn**ZU$P~rm"%1-RED6aδ$lt݉7cYdLwE1)\Tum/"W#`"rl6U@w[q܄ƞ#!ߦ(.XTVU=;X*`5Uc?ueEp5~sY;rgv" ni1/ߡ\ȳ?i=owwb~uebϻ`ܤFN%VDn t$nW̑n&8^Un?_D\DDUoP 1­˼xN]YUx\DB2S~rFg)쎥- xՐ|ADnż?6M9R/bdI-`goϩOs`7"2n>_cQ;aEDVMSUט ,0Yȝ/̠5E&^D6(uv9`}DD>)"sPa}|wM%(*G!zݬ, "ce70eU4M\e@#܅%ZrۘzNcc.L $MKw*g`n^&'߉ )]Tu *PDd-8p,{f(c?J0cm\)HWs AD1[}6]FUG"2 S̕ee#pu,՘671ak~EGUC}ܔI_]T u~  evl?^DVO%"lmP< "r|"RDlVYA2yLD~#"PkUj"" @S_d[Dʧ> =L׶{3l"f8K\63JOv"2DD+6=װ~|XW1<9/~o\K)qvEhWU0я1{S"r,EO-`!"GF睱\[IQ~`m0PZ >z~3'beHeT("+O4e)X׃JYEd{؄˘}foL[lE-sgU}3b襃v&Y3!pt3TVeb_YeǕU|9H\-V|cq$b*Tɍdmjwϐ}^#矩\C>vI*bgEd]U)e"" w.qҹ)\*qy}XҒMV%ލx P;NúvMly4WI1gȍT`nآy'$aoߐˠy4r0xύ \qQlBI@-9FݱI:Wi1sN uR?Qad{L}r!v^VN^QUwW`XIQE= dłnR]"6KAv"2W`':tH n6뤝@NTջ6U[OSTS2Tf|TTuoˆW]DdV]}jrv^E)j#10+[z`p3EUvrz}ʹy`ێ}|$n;:X,=D9_Ի2 m ^.Ŏ آ2 p"o\i00[]9}p.R똼UC̰HNa+4ͨR'̻ظ`/lXjձNVP,yMj\s;E}iדL#yAFHXEΙSx9sxV`;˝u+C۫{؂Pl15$o܀3O`zsh#,0I|!IE^W@p-ٔ{X [=y@mn༅&܁}\+ar}Nd)kD̹ KFuٯp;k 4Բt- MTu];C+>,y&$oPU{> 9Ey. BF.%o XA?у+MF 0Y=7*uX Mx0ZتzNKDm|Té7OQO9vC:uWot#MQUb;ϰ8rswCU͘3N͂ǿ~8a݌:"Ra~2 6<xπ hV`? , r?c`m(Ẹ(Z(6^t5XTu[U}b~G_&h>wEdu+F?WakO~JTm(p<:FUl wajDdj9L|CDDU_R=1OIb-ȡEdg .vD(⤥'N 8Kڬ펠9E^I'N5d'oU^YUN1P``gJ-x+Q^:ק^MtN?vioo!ro4 MO;Q׻ uѹESzr1A ɓ1!hsGp=nEd-ڽ2N{iz~8UTې'w93|SUM7vqW/Wv [zW:7qK\P\0j(eiK64;9{K}}|@UǪ6,3"qM1 Q# q$)Fv&?[h/Fn T0z^O."b.rSnvgU3m "Ww0X-f;ޫSռs`;c/lFap(ˋHsyj0[곆'@UP/Ӏ47XGx6sǟR`1/'Ed)Xr%zX=9lHDf @p'nEd%6nC 9iyd|U V8]jUU&ȩy#TXt7{<^y ˅dn軬s jIrԵ1p$"u[0oTjx4-"2^FI}('@USo`;'.X)4 s~skw|&7`jرW 0o Tz_y>y `ҟJVkD珹X \=)~%UDvEDl3>&/aY䗘q<9DdI5 [ٕFx.ƀOl+sF ە螳{5<z9T:ф~Kla1IU6[?hX:)Zg|NyROƂ[^{ =Lu1Ot0p{6:^5F\07УgI:}~ |,8l6{ kȍ1GD +߂5vQl]L0Kñzt&S khAws܉cvu/K5fo+h {Sa4KNPLB6?\?l\ Surl)ob.g<+#'[@1[INԄi՗$FQ=/nK-~1u[QףwFZf>]ds),eXm^(II{ Q3˯BoM {~FybapM\6h'@?.t] |6z0u#>-> e,z 3FzTsyUGU_la6>-LR0{8\Px6Deչ$"si-i11]Tml?D$YHGQqXD|VEI*FN,kGLm7;9}+)G"UT]*"ct?V 0X%q6A{K>--duy|LVS3T#j5':?P|؊Y_DZX9]D84DX_lr/a RG2 SȢjRrIDV.P x8#U-."u.T}71/ae^EUwVXqrYr>I9CUn.+"{0U}GUD"" n^r5f=#DdUg`p8.LY׮=]Gz!s>*"8HߤH nd˫mEԧ4. {Kl% ],( 9:K/~#LQlS^#rN;܆IԅᣱLyDs fG#Ryck ωߝTuo#Hujap4?ʢ1x[US!9/v9`ivOȟEl@`YU=Z={ß\Vnc\DV.m֪QD61~M\ӛ&"},/d?Ezy9oZnݱ03"X7Tl2= MTLPvϊNhvAgbk8=slLLe~[UDd# PPڕU˪zQD>-/qY6U1CmY,$7q=bǼ\t9 S IVՋQՇׯ9^\馷st˥4J!~$aڙ<n}fC8zQWTul¿|VDK1j@oL?"r'N7>dk6:MD>+`zמCYV~0ض9O(716w6똧G9$uIT>`x&"ϩ.)Ɋ:U}BU=H0RDfqc;;T^xex+"?tиmr.3MdѠ51pR^JPD=svR¤sQ>aYLwbmq SsCuPNC_3SUbovtX;׀2WgfJjɧ'_׾Lm U%c !; mӷnem)cWx+܄ֈ!"TY[S;U}-眡RUB0]jn""C05f^4INS6j-tMIU?ݕcDj,] 'I_d8Xs, 9cމr v9ޙC,-7鐜er(0F51yWj4̟<<2feU㨝ȸΉ|SYpۤ 3֟V{=<1,dHm/? 4`]w`A;P}:u~$uӁ"  pNH9gL߶i㯍r0qNJir. ލǗvS4ߩ :ԟI5Mu[zo,lg|Pl UBH15)=g^UO)* nƶƹ V\UO:B?TZ5_ñ08*lz[t/sB+~4йn7")!/}AYdXqwMBé^ $cgqr/ȭ"RLwIZ_Hܷz r1Ꝫ%$d :YD@/GWxڀ^D vo^,:'uCOE%\l4>>4էM?|!A Ys<: e_U}.ώ-hבX,w:8F;@ΡC )p\."hy*ޯK&`1"7u">[rH,.'0yZON+,A͌&ǰTuFgYʭuj)瓱ޔ5ZrE`lpɄ1X`ap^9uZ?EUPʩzx( F(ux#1qsbtʐ\mOlGX~TzUY.`y[U g鿈ccٶDp,DYr5Xn!͈8fHBΠxHc SSLI^X<r>XY{,S0='ՠ«U=cmU'ۖ*P19oay޾'4i7ƿ77XkJLy`q 7zj,vu$q&yu~c 0IJu8Z/EA; kV A\=̉yDq) >|OEm}MlK\>SfsbiI?ckn6np$`; +^Ԓzr 'Q|R0Db'l0 s_%:$^H>W)gs5)@,F1kt"X NWUr Qm( x3Xvv SiFQ }04l낼>1sh:SǶz0{a/:gg6s"S!vf.@?ph4&n~w ͎Ss C7Lt/.54i=c%o Ixɩ_)6ű'0% IDAT#ð߷9KDFtoo<*m2N\QC0/'iJ}bf2"Iw}n@1lE=x8 :>'5wΙSo\ 7BHZϩՕ z~~} &aa 1ف`{ÛCvcx$n zFEmu.Y'OֹSN^vaʶv4ZM+1>w $fd{X`nI/1a_ѹ6z\@!]"_ ႣR8j2Ap:pF/-dHiܤGyyfǢ:̡~7mȦF<rG@$rǜXv\ 5 5 c-ژ` oC4,J)|sMQ/CY;@Jd=$U P{Y^_%\S:'!;z3ECbF= C15 9fR@1\sc7HMQ$@V֘,<@0ծ З75c5k>Π%I|L.FotYww5s}c1Liiw^?B-m"+(`a [?A,n|65L~ҹ/ >2Z)>$횢]8t3Z DɬVIiCRELHzy3x 7gӤ_+`oܗrcc 8cI$,]qZ`ie1q .'1:LWY {ݢmD|b>SkحY$b`6IrxqB .)=4X5vPQ>8rфB_\tB^aWz&$j+qG$Ĥ6{~puh|)F#Ra .V`I,tfkyXXӀM3ֹ%mQY$]{ S;7*Apl]ȭÝ2UJP j;"b~0H. ^baEmL&۸@5Z$DL8vyqژ sZ:>b65y Koy\Ӄ?P  B?@=9rXG$;X@\YƦcIj % ?6Hi|-L=MIYaܾo@d,%m"e;Sj<U5 9$zOw:9H7|AV}M6uj`Tu)ouHlG! ,/px@jn>iT u4Lw1f^wm8:>L.-,~8l܃vVvJN:=hɓm=cN'"{ xG=dԍ- RWMQ?ek{'M`.|a5Lډ]ʩS--!y@P }KMJJw\A$;Sˆ19HP5-Ap|V,CxX%^w-8m>8υ:otð#(v`y,l[خlF`WUB:b❽@ϳo ys`>-{*(?!a+lhuX1/i)5$wn׻ygLp/&M}raicrmh6ha,[Е^gWy[`m:סqAC,Emw_ G_m9^:ݍ:w  rP IR/,V (ՂI7Y8Y1?nM^M̨=_/? 5ޝ]ڣvA:5KD ߐѣ@6~l!ݮVGiIxpDM-9} z}"8>.T|0lgZ,+@zp7},Km &c9Buq\Fo`hK s Em#NX[ F) `.yjMٝ׷)yR}G;O`ޯr/ ׇc|,&^# nG6}- <|Z z3{%9v?IIkaPj>>R@пL@3f OM;? 1 <oos<\D~Zs!!q`nH1^,ݜl( hVƬ~BzSו/zN県̓y:3:h&Lr}g_H/6vBR[+C<isb>!ߙdd#g2j126y AҞ_f Y_ǼVe-BJs XYg~dm@e!5?;QU!$S>`*Ȯ>U%@azFP4x> n$6>4NmJC>rަ_Z=J8bǀp-s@01c19O]JNF] Jfyh a!Do>{H_:s!jc4S9(,Z ز< 4I=(|!gGa`~rg&0u,#V6`TZ/6&$/zޙUO$[a_Ƒn̍vRAb>B do8KӔ>G6>SMcr"`zL*azLͲ,ALAbAEa;es u,.MkŔTf\ A!HVvCj3!!OD,@pNJ/ 7a{|nCo2^V=a.|jAp4̮/OsChH2w yv1,khHNS1oq Kz4+azC?``M,㢟 _P}<^TY fd ?̉ޚ'R,Z5|<灎KSf !# q=H>΋r]Ӏ;jwoApFM27va`);%xka^eJ֠aA|\zţk8|Fie6*Qv>T[ Ga\tt}p;nL-92@;6DEXyTxhStY99jxtX.Z_CByBi˧LQbI`B"p0{%mPql}F?AÈ}% --,|6vgs`ѫ{NMoppZ@i ~>t ^< qtިu09xNT^d2ps6Ki'F@qz/3yM'ȟC;sQ-vu2o >8i"ӱE:E3=z..qYf@ Gb}xՑFN?kM>ϲkbrLnp*nhev`>1=qoQ;gE\Q7b0}z:D<`ǐFFB & eJ;S$ ZtNHN ]f%' xK6A#e 4gt}O_B}@`.҄:F?97~dVxT*RH8c'28=1w@lW+fk(ރ@")cev`]}e*KVb =PSDlQ;!Xj a@N, ac@1 WƖ@9gǂT[:~2 hLbhL=jҨ27;< \rkdBw2,12Ņg:0ƂI&%ȨQyѹ濍@@&!& PeX,Ѧ/{1Cw6jΞ#c">lr ziF*[r֜|g*6~,UtkjIb-^vϔl2Sk._ |AmO4_tlԇp|lkq >hp?fy5`v0%-t6i!@O (cpR=@:25c+Iks7_@sWB8JI2c0OxB!Ā:-݇Es  N>WjaG[߁ 8/$X880nA }pw_3Z<C~@ijjwyo$jv>N@1vEu~tWƂEOƈ3E`W/{9@?NM#rOGG%@pf<UQ09m)s\:q^-y1;X8]@p׬mv؏PSK6[78>;>wtaMLv,6rtk$ح*3۷g?뻡0+M>-aғw'ΦN [q*G,ƍgC;+'=Rv`tpͳ5@>NmꁏβԦ3V*2NR,Vt?g@ [ITs <BV Y3} iP(yD7l׌v(LG5yK,nX3p^2]F:'{hU-Bmd76= /|)&$ ?vAOꝌsd]"Va}7!}.{OTgKm +;^K4! IM 6vyAP$Aot\@#7V@KF3c}"ى@~#N,U>R)91a 3 \}ׇ(l}1JP{d1,˽-ޙ//:P( `/I7f 6 ˛Xg?U;^ ޻+5C؝Z`h6z/j =yn ~_ s+ *wk[!d Pl 3v6aX^wϩATOa:a0AKqS,yxUji@$p6qF?$벯0Vh&`.|Ϫt ,mz$Ir=GCXm0/,A=Ob*@+x޷q1_Un H*`T! 4eHLI!Vtd˫!z|ј`L;NU3NVꙪ111K;eTOٺY~ EjleyUN{+)LTUK2/𜈜'"KDOQ1ϜK#hY/\fT%JI|szVRQg),a/39QU uIh<!;}rKOva"܉YVSߧ]SI$T|U; C8]D5S^@0DtY "܅0|X^?TPtjS2\gCy3/gw#!Q pS},)hG!ZDҗ@/>!he(RzVq|+ y Vt; *i[T5U=R3b@p;`po@Iq*(?/ (zJ!)8$`vB@)Kt bK2Cy-þwA+ ,ͷ8@X<: NN=$cTPvvxȻ6^+)KΨ6B]@ N3T0@0l"2|::۵*/f1) 4 niPy_fm{.ʮUIi$ J @/ fxm&"bju#҃:K#YEd$XP([WI+a솒NC@_GD2&IDAT~R X5X[U.L%v"ҿ3ƅ6RտVc2>{b w; 1*X#C| u^x*K hPv"2qY\mCKf5+h/; u~pB.!zT obpv?Fp^U},B@DȞl =<:.`KU]WUt%YT9Uй8 /"NjȂSU"`a ~UX"p vc1PJ*UFc*aO3mG]2vn1PU{+lb6oC8@X;?=8A-F "sXʇ0Pn6QT2I`gsc0NDYP0k܉`B%=#p/f9F Tu3U_̅9Dl~R,╪Q͊PUoWխ1k01&+@t2 @Dv0Gr;u*-NuTuku<UZxsœJz. Dd9GAznU?*S 7Cpۋo2 l ,r=nw~% 9;/mMu+DdVU鸾<DdX{XD(U}+sL`?3ډ+[,%w"ӱ-Ѫd+d0 RJ ,?cT3^LDvf%S|FKDdj=ŗ\7uLmM4TvJ fߋaY!x@\)$7KDdML=}'h0ފKUu|vK?:qlT QI%^RK<`F; !"ñ~wL>p!pڳ+EDd>,oQ,ӁX;EERHWP3|ʇW(U}]#.}nc * _DrQYs6aLtǩޡJ:@m/LwSU})UR`pIT޾,Ӂ׈7V6 HW.J*) ORXU}ȾUҾ 1o1ATVI%e~h,2NRY%K534UPp*PIl>lhR* "" cʷJ*pR@,/".MTէ[%TRI#i1SnKSJJ(ITYZ2 ̃%G;M_%$J ^x g<C%TRI^2ˀ/m,Z-#_%TRI4#"]/ epl/sTRI%386 hY/\$:ZTR ^xiuPJ*?""CP c6 X2(K/ TT*rKGaia40܎!._TRIdf/3x |K>ZUo(cTRI2 /H۱ WֱJ*T Xe/ 8X\B% rVfj UX|'p){%T26`/qJ燀+@)4 R$c'PB% i` 4[aQT2U^T*R@gRA""|.\3w:URI%TНT@Ёpq*@¤`mH]l?~zET7iUhRAFqpN_EJz%K* (Xpׁ_;EJ )LDd5 ,Yv}nrK%Ys;.pTPɀfG* (*۩0(U}ȾURI#iW@I"",CNQ7[%x``JLDdla"pZd*y- P0}B̆Jq=dfppbG* ""KPT"V T@0HDDvf>.VЧ p`,w'?aǿA̴y!MN lH1/;knc bah ;*14mj166hC?{S:~哪[om7s {OK+`58$k}u 5pLw.ҤFJ>A @3( WM>D'.}smR] @(-af7w l".}smR @;)-cf7 Zw,$Bƀ3Xh r-A.E`o˹633b$3ES0\OI> vSZF!lz}FAx8Cؒsm; LG!I AX,w7d Lc@@J~5 Qw'd @fdfo<=nkr ̅B Rr4Vl IJx7 RdNl/*`8:ǁ!w+Z`I+2+ ̋ \ Ǥqk6 B R*4 Cx5׺N^R'R.'K .l k 3XG7Rd^)38C8>'ӴqF B }QpBN쩬 AIv]&H_]o6l5@E!(| ա.m(2h T!ci'^?wgG=(g dQV>Zv!ϸ&HV%w.4@HU(R ӽ6 UHLsxɶRt\ F_AxNAH9i  HR9+  uHm FQwߙk]S)R7 N (pI}ls?3KZRRwl3 L]'ĥ+@jA!3 =;AxUi 3;xbـ3z@F!1D.crtK\F 4B ef+G`6ϙ O( 4F Owuߛ Hk٩2J`wg!`RT6Ruqx0=`p~mD6 @L!2SN;gxH(zfv @?v[()[ IENDB`zarr-developers-zarr-python-a2f2415/docs/_static/logo_horizontal.svg000066400000000000000000000351061527014137600260020ustar00rootroot00000000000000 zarr-developers-zarr-python-a2f2415/docs/api/000077500000000000000000000000001527014137600211665ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/docs/api/zarr/000077500000000000000000000000001527014137600221445ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/docs/api/zarr/abc/000077500000000000000000000000001527014137600226715ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/docs/api/zarr/abc/buffer.md000066400000000000000000000002321527014137600244610ustar00rootroot00000000000000--- title: buffer --- ::: zarr.abc options: show_root_heading: true show_root_toc_entry: true members: false ::: zarr.abc.buffer zarr-developers-zarr-python-a2f2415/docs/api/zarr/abc/codec.md000066400000000000000000000000511527014137600242640ustar00rootroot00000000000000--- title: codec --- ::: zarr.abc.codec zarr-developers-zarr-python-a2f2415/docs/api/zarr/abc/index.md000066400000000000000000000012661527014137600243270ustar00rootroot00000000000000--- title: zarr.abc --- # zarr.abc Abstract base classes for extending Zarr-Python. - **[zarr.abc.buffer](./buffer.md)** - Providing access to underlying memory via [buffers](https://docs.python.org/3/c-api/buffer.html) - **[zarr.abc.codec](./codec.md)** - Expressing [zarr codecs](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-encoding) - **[zarr.abc.metadata](./metadata.md)** - Creating metadata classes compatible with the Zarr API - **[zarr.abc.numcodec](./numcodec.md)** - Protocols and classes for modeling codec interface used by numcodecs - **[zarr.abc.store](./store.md)** - ABC for implementing Zarr stores and managing getting and setting bytes in a store zarr-developers-zarr-python-a2f2415/docs/api/zarr/abc/metadata.md000066400000000000000000000000571527014137600247750ustar00rootroot00000000000000--- title: metadata --- ::: zarr.abc.metadata zarr-developers-zarr-python-a2f2415/docs/api/zarr/abc/numcodec.md000066400000000000000000000000571527014137600250120ustar00rootroot00000000000000--- title: numcodec --- ::: zarr.abc.numcodec zarr-developers-zarr-python-a2f2415/docs/api/zarr/abc/store.md000066400000000000000000000000511527014137600243430ustar00rootroot00000000000000--- title: store --- ::: zarr.abc.store zarr-developers-zarr-python-a2f2415/docs/api/zarr/api/000077500000000000000000000000001527014137600227155ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/docs/api/zarr/api/asynchronous.md000066400000000000000000000000661527014137600257740ustar00rootroot00000000000000--- title: asynchronous --- ::: zarr.api.asynchronouszarr-developers-zarr-python-a2f2415/docs/api/zarr/api/index.md000066400000000000000000000002341527014137600243450ustar00rootroot00000000000000--- title: zarr.api --- # zarr.api Zarr provides both an [async](./asynchronous.md) and a [sync](./synchronous.md) API. See those pages for more details. zarr-developers-zarr-python-a2f2415/docs/api/zarr/api/synchronous.md000066400000000000000000000002421527014137600256270ustar00rootroot00000000000000--- title: synchronous --- ::: zarr.api options: show_root_heading: true show_root_toc_entry: true members: false ::: zarr.api.synchronouszarr-developers-zarr-python-a2f2415/docs/api/zarr/array.md000066400000000000000000000000431527014137600236010ustar00rootroot00000000000000::: zarr.Array ::: zarr.AsyncArray zarr-developers-zarr-python-a2f2415/docs/api/zarr/buffer/000077500000000000000000000000001527014137600234155ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/docs/api/zarr/buffer/cpu.md000066400000000000000000000000241527014137600245220ustar00rootroot00000000000000::: zarr.buffer.cpu zarr-developers-zarr-python-a2f2415/docs/api/zarr/buffer/gpu.md000066400000000000000000000000241527014137600245260ustar00rootroot00000000000000::: zarr.buffer.gpu zarr-developers-zarr-python-a2f2415/docs/api/zarr/buffer/index.md000066400000000000000000000002611527014137600250450ustar00rootroot00000000000000--- title: zarr.buffer --- Zarr provides buffer classes for both the [cpu](./cpu.md) and [gpu](./gpu.md). Generic buffer functionality is also detailed below. ::: zarr.buffer zarr-developers-zarr-python-a2f2415/docs/api/zarr/codecs.md000066400000000000000000000000471527014137600237270ustar00rootroot00000000000000--- title: codecs --- ::: zarr.codecs zarr-developers-zarr-python-a2f2415/docs/api/zarr/codecs/000077500000000000000000000000001527014137600234045ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/docs/api/zarr/codecs/numcodecs.md000066400000000000000000000000641527014137600257060ustar00rootroot00000000000000--- title: numcodecs --- ::: zarr.codecs.numcodecs zarr-developers-zarr-python-a2f2415/docs/api/zarr/config.md000066400000000000000000000000471527014137600237340ustar00rootroot00000000000000--- title: config --- ::: zarr.config zarr-developers-zarr-python-a2f2415/docs/api/zarr/dtype.md000066400000000000000000000000451527014137600236120ustar00rootroot00000000000000--- title: dtype --- ::: zarr.dtype zarr-developers-zarr-python-a2f2415/docs/api/zarr/errors.md000066400000000000000000000000461527014137600240020ustar00rootroot00000000000000--- title: errors --- ::: zarr.errorszarr-developers-zarr-python-a2f2415/docs/api/zarr/experimental.md000066400000000000000000000002401527014137600251570ustar00rootroot00000000000000--- title: experimental --- Experimental functionality is not stable and may change or be removed at any point. ## Classes ::: zarr.experimental.cache_store zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/000077500000000000000000000000001527014137600241545ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/array.md000066400000000000000000000000521527014137600256110ustar00rootroot00000000000000--- title: zarr.array --- ::: zarr.array zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/consolidate_metadata.md000066400000000000000000000001101527014137600306320ustar00rootroot00000000000000--- title: zarr.consolidate_metadata --- ::: zarr.consolidate_metadata zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/create.md000066400000000000000000000000541527014137600257400ustar00rootroot00000000000000--- title: zarr.create --- ::: zarr.create zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/create_array.md000066400000000000000000000000701527014137600271340ustar00rootroot00000000000000--- title: zarr.create_array --- ::: zarr.create_array zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/create_group.md000066400000000000000000000000701527014137600271520ustar00rootroot00000000000000--- title: zarr.create_group --- ::: zarr.create_group zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/create_hierarchy.md000066400000000000000000000001001527014137600277660ustar00rootroot00000000000000--- title: zarr.create_hierarchy --- ::: zarr.create_hierarchy zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/empty.md000066400000000000000000000000521527014137600256310ustar00rootroot00000000000000--- title: zarr.empty --- ::: zarr.empty zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/empty_like.md000066400000000000000000000000641527014137600266400ustar00rootroot00000000000000--- title: zarr.empty_like --- ::: zarr.empty_like zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/from_array.md000066400000000000000000000000641527014137600266370ustar00rootroot00000000000000--- title: zarr.from_array --- ::: zarr.from_array zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/full.md000066400000000000000000000000501527014137600254330ustar00rootroot00000000000000--- title: zarr.full --- ::: zarr.full zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/full_like.md000066400000000000000000000000621527014137600264420ustar00rootroot00000000000000--- title: zarr.full_like --- ::: zarr.full_like zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/group.md000066400000000000000000000000521527014137600256270ustar00rootroot00000000000000--- title: zarr.group --- ::: zarr.group zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/load.md000066400000000000000000000000501527014137600254100ustar00rootroot00000000000000--- title: zarr.load --- ::: zarr.load zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/ones.md000066400000000000000000000000501527014137600254350ustar00rootroot00000000000000--- title: zarr.ones --- ::: zarr.ones zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/ones_like.md000066400000000000000000000000621527014137600264440ustar00rootroot00000000000000--- title: zarr.ones_like --- ::: zarr.ones_like zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/open.md000066400000000000000000000000501527014137600254320ustar00rootroot00000000000000--- title: zarr.open --- ::: zarr.open zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/open_array.md000066400000000000000000000000641527014137600266350ustar00rootroot00000000000000--- title: zarr.open_array --- ::: zarr.open_array zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/open_consolidated.md000066400000000000000000000001021527014137600301600ustar00rootroot00000000000000--- title: zarr.open_consolidated --- ::: zarr.open_consolidated zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/open_group.md000066400000000000000000000000641527014137600266530ustar00rootroot00000000000000--- title: zarr.open_group --- ::: zarr.open_group zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/open_like.md000066400000000000000000000000621527014137600264410ustar00rootroot00000000000000--- title: zarr.open_like --- ::: zarr.open_like zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/print_debug_info.md000066400000000000000000000001001527014137600300020ustar00rootroot00000000000000--- title: zarr.print_debug_info --- ::: zarr.print_debug_info zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/save.md000066400000000000000000000000501527014137600254270ustar00rootroot00000000000000--- title: zarr.save --- ::: zarr.save zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/save_array.md000066400000000000000000000000641527014137600266320ustar00rootroot00000000000000--- title: zarr.save_array --- ::: zarr.save_array zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/save_group.md000066400000000000000000000000641527014137600266500ustar00rootroot00000000000000--- title: zarr.save_group --- ::: zarr.save_group zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/zeros.md000066400000000000000000000000521527014137600256350ustar00rootroot00000000000000--- title: zarr.zeros --- ::: zarr.zeros zarr-developers-zarr-python-a2f2415/docs/api/zarr/functions/zeros_like.md000066400000000000000000000000641527014137600266440ustar00rootroot00000000000000--- title: zarr.zeros_like --- ::: zarr.zeros_like zarr-developers-zarr-python-a2f2415/docs/api/zarr/group.md000066400000000000000000000000431527014137600236170ustar00rootroot00000000000000::: zarr.Group ::: zarr.AsyncGroup zarr-developers-zarr-python-a2f2415/docs/api/zarr/index.md000066400000000000000000000043571527014137600236060ustar00rootroot00000000000000# API Reference Complete reference documentation for the Zarr-Python API. ::: zarr options: show_root_heading: true show_root_toc_entry: true members: false ## Core API ### Essential Classes and Functions - **[Array](array.md)** - The main Zarr array class for N-dimensional data - **[Group](group.md)** - Hierarchical organization of arrays and subgroups - **[create_array](functions/create_array.md)** and **[create_group](functions/create_group.md)** - Creating new arrays and groups - **[open](functions/open.md)**, **[open_array](functions/open_array.md)**, and **[open_group](functions/open_group.md)** - Opening existing Zarr stores, arrays, and groups ### Data Operations - **[load](functions/load.md)** - Loading data from Zarr stores - **[save](functions/save.md)** - Saving data to Zarr format ### Data Types and Configuration - **[Data Types](dtype.md)** - Supported NumPy data types and type handling - **[Configuration](config.md)** - Runtime configuration and settings ## Storage and Compression - **[Codecs](codecs.md)** - Compression and filtering codecs - **[Storage](storage.md)** - Storage backend implementations and interfaces - **[Registry](registry.md)** - Codec and storage backend registry ## API Variants Zarr-Python provides both synchronous and asynchronous APIs: - **[Async API](./api/asynchronous.md)** - Asynchronous operations for concurrent access - **[Sync API](./api/synchronous.md)** - Synchronous operations for simple usage ## Abstract Base Classes The ABC module defines interfaces for extending Zarr: - **[Codec ABC](abc/codec.md)** - Interface for custom compression codecs - **[Metadata ABC](abc/metadata.md)** - Interface for metadata handling - **[Store ABC](abc/store.md)** - Interface for custom storage backends ## Utilities - **[Errors](errors.md)** - Exception classes and error handling - **[Testing](testing/index.md)** - Utilities for testing Zarr-based code ## Getting Help - Check the [User Guide](../../user-guide/index.md) for tutorials and examples - Browse function signatures and docstrings in the API reference - Report issues on [GitHub](https://github.com/zarr-developers/zarr-python) - Join discussions on the [Zarr community forum](https://github.com/zarr-developers/community) zarr-developers-zarr-python-a2f2415/docs/api/zarr/metadata.md000066400000000000000000000001101527014137600242360ustar00rootroot00000000000000--- title: metadata --- ::: zarr.metadata ::: zarr.metadata.migrate_v3 zarr-developers-zarr-python-a2f2415/docs/api/zarr/registry.md000066400000000000000000000000521527014137600243330ustar00rootroot00000000000000--- title: registry --- ::: zarr.registryzarr-developers-zarr-python-a2f2415/docs/api/zarr/storage.md000066400000000000000000000001401527014137600241250ustar00rootroot00000000000000--- title: storage --- ## Attributes ::: zarr.storage.StoreLike ## Classes ::: zarr.storage zarr-developers-zarr-python-a2f2415/docs/api/zarr/testing/000077500000000000000000000000001527014137600236215ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/docs/api/zarr/testing/buffer.md000066400000000000000000000000431527014137600254110ustar00rootroot00000000000000## Buffer ::: zarr.testing.buffer zarr-developers-zarr-python-a2f2415/docs/api/zarr/testing/index.md000066400000000000000000000004161527014137600252530ustar00rootroot00000000000000--- title: zarr.testing --- # zarr.testing See the following sub-modules: - [zarr.testing.buffer](./buffer.md) - [zarr.testing.stateful](./stateful.md) - [zarr.testing.store](./store.md) - [zarr.testing.strategies](./strategies.md) - [zarr.testing.utils](./utils.md) zarr-developers-zarr-python-a2f2415/docs/api/zarr/testing/stateful.md000066400000000000000000000000471527014137600257730ustar00rootroot00000000000000## Stateful ::: zarr.testing.stateful zarr-developers-zarr-python-a2f2415/docs/api/zarr/testing/store.md000066400000000000000000000000421527014137600252730ustar00rootroot00000000000000 ## Store ::: zarr.testing.store zarr-developers-zarr-python-a2f2415/docs/api/zarr/testing/strategies.md000066400000000000000000000000541527014137600263140ustar00rootroot00000000000000 ## Strategies ::: zarr.testing.strategies zarr-developers-zarr-python-a2f2415/docs/api/zarr/testing/utils.md000066400000000000000000000000411527014137600252760ustar00rootroot00000000000000## Utils ::: zarr.testing.utils zarr-developers-zarr-python-a2f2415/docs/blog/000077500000000000000000000000001527014137600213405ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/docs/blog/.authors.yml000066400000000000000000000002251527014137600236250ustar00rootroot00000000000000authors: d-v-b: name: Davis Bennett description: Core developer avatar: https://github.com/d-v-b.png url: https://github.com/d-v-b zarr-developers-zarr-python-a2f2415/docs/blog/index.md000066400000000000000000000001241527014137600227660ustar00rootroot00000000000000# Blog News, release highlights, and design notes from the Zarr-Python developers. zarr-developers-zarr-python-a2f2415/docs/blog/posts/000077500000000000000000000000001527014137600225105ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/docs/blog/posts/3.3.0-release.md000066400000000000000000000307111527014137600251130ustar00rootroot00000000000000--- date: 2026-07-30 authors: - d-v-b categories: - Release --- # Zarr-Python 3.3.0 We're happy to announce the release of version 3.3.0 of Zarr-Python. It's been a while since our last release ([3.2.1](https://github.com/zarr-developers/zarr-python/releases/tag/v3.2.1) dropped in May of this year), and we're bringing some exciting additions to the latest version. For the full release notes, see the [3.3.0 release notes](../../release-notes.md), otherwise stick around for an overview of two performance-centric highlights of this release. ## Faster low-latency storage Relevant issues and pull requests: - [#3524](https://github.com/zarr-developers/zarr-python/issues/3524) -- the performance report that started this work - [#3885](https://github.com/zarr-developers/zarr-python/pull/3885) -- synchronous codec APIs and the `FusedCodecPipeline` ### The cost of async overhead Zarr-Python 3.x uses async routines for fetching data and decoding chunks. In terms of code, this means our store (data fetching) and codec (chunk decoding) APIs are both async. This makes I/O against high-latency storage backends like cloud object storage efficient. But for *low-latency* storage, like in-process memory or the file system, async routines add measurable overhead and offer no benefit. Async only adds value when there's work to be done while waiting for I/O to complete, but when I/O latency is low, it completes too quickly to run anything while waiting, and we are left paying the performance bill for obligatory async task scheduling that offered no value. This performance problem became acute when Zarr-Python users reported that in-memory array indexing workloads ran *slower* in Zarr-Python 3.1.3 relative to Zarr-Python 2.18.7 ([#3524](https://github.com/zarr-developers/zarr-python/issues/3524)). Fortunately this performance regression had a straightforward fix (I don't say "easy" because it was a lot of work). ### Synchronous execution restores performance If async overhead makes low-latency storage slow, does *removing* that overhead restore performance? Yes, it does! In [#3885](https://github.com/zarr-developers/zarr-python/pull/3885) we defined synchronous versions of our storage and codec APIs -- the `SyncByteGetter` and `SyncByteSetter` protocols, plus a `get_ranges_sync` method on the `Store` ABC -- and then combined them in a new codec orchestration class called `FusedCodecPipeline`. The `FusedCodecPipeline` is an opt-in alternative to the default (the `BatchedCodecPipeline`) that gives large speedups for low-latency storage. It is currently marked [experimental](../../user-guide/experimental.md), so we may change it as we learn more; the default pipeline is untouched, and existing code keeps working unless you opt in. The win here is *not* a faster compressor. It is the removal of async scheduling overhead (including some [nasty `asyncio.to_thread` overhead](https://github.com/python/cpython/issues/136084)), plus a few vectorized fast paths for dense, uncompressed shards. And we only expect this new pipeline to accelerate workloads targeting a subset of storage backends, namely any store with methods that advertise low latency. On this author's 10-core Apple M4 laptop, the `FusedCodecPipeline` delivers the following results against memory-backed arrays: - uncompressed writes are *~4 times faster* - uncompressed reads are *~5 times faster* - compressed writes are *~2 times faster* - compressed reads are *~2 times faster* These numbers came from a [runnable example](../../user-guide/examples/codec_pipeline_performance.md) that ships with the documentation. Run it yourself to get a sense of how the `FusedCodecPipeline` behaves on your system -- when and how you use it depends on your hardware, your array layout, and how your chunks are compressed. What's certain is that for in-memory arrays, and arrays saved to the local file system, the `FusedCodecPipeline` is worth a try. Getting good numbers requires choosing the right level of thread-based parallelism for your workload, which is part of the configuration of the `FusedCodecPipeline`. For uncompressed chunks there's no CPU-bound work to do after fetching a chunk and so thread-based parallelism is worse than useless and slows things down. But for compressed chunks, threading offers a substantial payoff. ### How to use it Select the pipeline through the [runtime configuration](../../user-guide/config.md) by setting `codec_pipeline.path`. Set it globally to affect every array created or opened afterwards: ```python exec="true" session="blog-330" source="above" import zarr zarr.config.set( {"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"} ) ``` Or scope it to a block of code by using `zarr.config.set` as a context manager, which is the safer choice if you only want the new pipeline for part of your program: ```python exec="true" session="blog-330" source="above" result="ansi" import numpy as np import zarr from zarr.storage import MemoryStore with zarr.config.set( {"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"} ): arr = zarr.create_array( store=MemoryStore(), shape=(1000, 1000), chunks=(100, 100), shards=(1000, 1000), dtype="float32", ) arr[:] = np.random.random((1000, 1000)).astype("float32") result = arr[:] print(result.shape) ``` Thread-based parallelism is configured separately, via `codec_pipeline.max_workers`. It defaults to `None`, meaning a pool sized to `os.cpu_count()`. Note that this setting is read *only* by the `FusedCodecPipeline` -- the default `BatchedCodecPipeline` ignores it, so tuning it without opting in above does nothing. As noted, memory-backed and uncompressed workloads often do better with a single worker, which runs everything inline on the calling thread: ```python exec="true" session="blog-330" source="above" import zarr # No thread pool: run codec compute inline. Often best for uncompressed, # memory-backed arrays, where there's no CPU-bound work to overlap. zarr.config.set({"codec_pipeline.max_workers": 1}) # A fixed-size thread pool, which pays off once compression is in play. zarr.config.set({"codec_pipeline.max_workers": 8}) # Or back to the default, sized to the number of CPUs. zarr.config.set({"codec_pipeline.max_workers": None}) ``` To return to the default pipeline, set `codec_pipeline.path` back to the batched implementation: ```python exec="true" session="blog-330" source="above" import zarr zarr.config.set( {"codec_pipeline.path": "zarr.core.codec_pipeline.BatchedCodecPipeline"} ) ``` ## Faster sharded reads Relevant issues and pull requests: - [#3004](https://github.com/zarr-developers/zarr-python/pull/3004) -- optimize partial shard reads - [#3925](https://github.com/zarr-developers/zarr-python/pull/3925) -- `Store.get_ranges` for concurrent, coalesced multi-range reads - [#3987](https://github.com/zarr-developers/zarr-python/pull/3987) -- control coalescing through `ArrayConfig` and the runtime config ### How sharding works Chunks encoded with the `sharding_indexed` codec contain a secondary level of chunking, called subchunks. For example, if the `chunk_grid` field of the array metadata declares an "outer chunk" size of, say `(10, 10)`, a `sharding_indexed` codec in the `codecs` field could declare an "inner chunk" size of `(5, 5)`. Readers accessing such a chunk will observe a stored object (a stream of bytes) that decodes to an array with size `(10, 10)` (the "outer chunk"), which is comprised of four separate, contiguous byte ranges that each decode to a `(5, 5)` inner chunk. Each inner chunk occupies its own byte range in the outer chunk. A reader can satisfy a request for all four inner chunks by issuing four separate byte-range requests, or by making a *single* request for a byte range that spans all four inner chunks. The latter option is nice because it cuts down on the number of requests we need. Historically Zarr-Python used this optimization when reading entire outer chunks; in 3.3.0, we use this optimization in more cases, resulting in more efficient I/O patterns for sharded reads. ### Interval equivalence Byte ranges, being intervals, obey some combination rules: the values in two half-open intervals `[a, b), [b, c)` can be captured by the single interval `[a, c)`. That means a reader can get multiple inner chunks with *one* byte-range request by requesting a range of bytes starting with the first byte of the first subchunk and ending with the last byte of the last subchunk. When individual requests are expensive, this kind of optimization is worth a lot. The requested inner chunks are not necessarily contiguous -- there might be a byte range gap between them. As long as that gap is not too big, its often efficient to fetch the entire byte range, gap included, and pick out the inner chunk byte ranges after I/O is done. ### Byte range coalescing We call this procedure -- merging adjacent byte ranges -- "byte range coalescing", and it's a new performance optimization shipping in Zarr-Python 3.3.0. Unlike the `FusedCodecPipeline`, this one is on by default with base settings we think are good, so most users won't need to tune anything. Two knobs control it, both documented in the [runtime configuration guide](../../user-guide/config.md). Nearby byte ranges in the same shard are merged into a single request when the gap between them is no larger than `array.sharding_coalesce_max_gap_bytes` (default 1 MiB) and the merged read stays within `array.sharding_coalesce_max_bytes` (default 16 MiB). The gap threshold is what trades wasted bytes against saved requests: raising it reads more data you didn't ask for, in exchange for fewer requests. For a runnable demonstration -- counting the store requests saved and timing them against a store with simulated latency -- see the [sharded read coalescing example](../../user-guide/examples/sharding_coalescing.md). You can set them globally, or per array by passing `config={...}` to [`zarr.create_array`][]: ```python exec="true" session="blog-330" source="above" result="ansi" import zarr from zarr.storage import MemoryStore arr = zarr.create_array( store=MemoryStore(), shape=(1000, 1000), chunks=(100, 100), shards=(1000, 1000), dtype="float32", config={ "sharding_coalesce_max_gap_bytes": 4 * 1024**2, # 4 MiB "sharding_coalesce_max_bytes": 64 * 1024**2, # 64 MiB }, ) print(arr.shape) ``` ## Tell us what you think We hope these new features are helpful, and we would appreciate any feedback that helps us improve them, or any other aspect of Zarr-Python. ## Going faster The updates in this release are just the first step of a larger performance-oriented direction for Zarr-Python. Landing these two enhancements taught us a *lot* about the performance-sensitive areas of the library. We can and will invest more time in performance tuning, e.g. by adding or changing abstractions, writing code for special cases, etc. We plan to consider including compiled code that should enable significant performance improvements. The [`zarrs`](https://zarrs.dev/) project is an ecosystem of Zarr tools written in Rust, with [extremely high performance](https://book.zarrs.dev/#-zarrs-is-fast-). Is there a `zarrs` binding in Zarr-Python's future? I hope so! We are keenly observing development of [`zarrista`](https://developmentseed.org/zarrista/latest/) as a proof-of-concept for what a Python-`zarrs` binding layer might look like. Stay tuned! ## Contributors Thanks to everyone who contributed to this release (`*` marks a first-time contributor to Zarr-Python): - [@aldenks](https://github.com/aldenks) - [@AMBRA7592](https://github.com/AMBRA7592) * - [@binggao1230](https://github.com/binggao1230) * - [@chuckwondo](https://github.com/chuckwondo) * - [@d-v-b](https://github.com/d-v-b) - [@DimitriPapadopoulos](https://github.com/DimitriPapadopoulos) - [@goutamadwant](https://github.com/goutamadwant) * - [@ilan-gold](https://github.com/ilan-gold) - [@jhamman](https://github.com/jhamman) - [@josh-ag2](https://github.com/josh-ag2) * - [@kabilar](https://github.com/kabilar) * - [@keewis](https://github.com/keewis) - [@lhoupert](https://github.com/lhoupert) * - [@maxrjones](https://github.com/maxrjones) - [@NIK-TIGER-BILL](https://github.com/NIK-TIGER-BILL) * - [@oldrobotdev](https://github.com/oldrobotdev) * - [@SAY-5](https://github.com/SAY-5) * - [@sehoffmann](https://github.com/sehoffmann) * - [@selmanozleyen](https://github.com/selmanozleyen) * - [@stibrew](https://github.com/stibrew) * - [@TomAugspurger](https://github.com/TomAugspurger) - [@zkoppert](https://github.com/zkoppert) * zarr-developers-zarr-python-a2f2415/docs/blog/posts/3.4.0-release.md000066400000000000000000000323351527014137600251200ustar00rootroot00000000000000--- date: 2026-09-10 authors: - d-v-b categories: - Release draft: false --- # `zarr` 3.4.0 `zarr` 3.4.0 is out! This post covers the highlights of the latest version of `zarr`. We will also introduce some new subpackages we are developing as part of an effort to make `zarr` more modular. See the full [release notes](../../release-notes.md) for the per-PR breakdown, or keep reading to catch the digested version. ## Stackification This first item is not a new `zarr` feature. Rather, it's a **Zarr-Python** feature: Zarr-Python (the project) is becoming a *stack* of related Python packages, foremost among them `zarr`, the Python library. In the last few months Zarr-Python gained the following stand-alone Python packages: - [`zarr-metadata`](https://zarr-metadata.readthedocs.io/), a library narrowly scoped to modelling Zarr metadata documents (e.g., `zarr.json`), with precise type definitions and validation routines. Designed for anyone who wants type safety for Zarr V2 or V3 metadata without re-implementing the Zarr specs. `zarr-metadata` is very lightweight -- it does not depend on `zarr` -- so it should be a very cheap dependency for projects to add, either as a runtime or test dependency. We hope to replace most of the metadata logic in `zarr` with the contents of `zarr-metadata`. - [`zarr-indexing`](https://zarr-indexing.readthedocs.io/), a library concerned with data structures and algorithms for lazy, chunked array indexing routines based on Google's trailblazing [TensorStore](https://google.github.io/tensorstore/) library. We hope to replace the array indexing logic in `zarr` with routines defined in `zarr-indexing`, and we also want to use `zarr-indexing` to support new lazy indexing functionality for arrays in `zarr`. - [`zarr-http-server`](https://zarr-http-server.readthedocs.io/), a library that serves a `zarr` `Store`, `Array` or `Group` over HTTP via an ASGI app. This allows any array-like data to be dynamically exposed as Zarr. For example, you can take TIFF stacks, access them with `tifffile`, and use `tifffile`'s Zarr layer with `zarr-http-server` to expose the TIFF data as Zarr over HTTP for consumption by clients that don't know how to read TIFF. There are no plans for `zarr` to depend on this package. These three packages are just a start. We have in-flight PRs for defining packages for Zarr chunk key encodings ([PR 4298](https://github.com/zarr-developers/zarr-python/pull/4298)), stores ([PR 4318](https://github.com/zarr-developers/zarr-python/pull/4318)), and codecs ([PR 4319](https://github.com/zarr-developers/zarr-python/pull/4319)). All of these packages are pre-1.0 and changing rapidly; `zarr` does not yet depend on them. When the relevant APIs stabilize, we plan to gradually replace functionality defined in `zarr` today for the equivalent functionality in a subpackage. For example, the [store PR](https://github.com/zarr-developers/zarr-python/pull/4318) includes a `legacy` module that contains a verbatim copy of `zarr`'s current `Store` API, so that when `zarr` depends on `zarr-storage`, we can swap out functionality defined in `zarr` for identical functionality in `zarr-storage`, and users won't notice any difference beyond an additional installed package. The strategy here is to identify the logically separate parts of a Zarr implementation, and build a separate package around each part. This is not an original idea: the Rust [`zarrs`](https://zarrs.dev/) project is composed of many separate packages and this architecture has worked well for that project. Why are we doing this? Factoring `zarr` into separate packages may seem like a lot of churn with little upside beyond more maintenance. It's true that there will be new kinds of maintenance, and publishing gets a bit more complicated. But we see the following advantages to organizing Zarr-Python into subpackages: - **Modularity:** Installing `zarr` brings in `numcodecs` and other dependencies required for the full Zarr format. If a Python user only needs to parse Zarr metadata documents, e.g. because they are using [`zarrista`](https://developmentseed.org/zarrista/latest/) or [`tensorstore`](https://google.github.io/tensorstore/) to do IO, then the full set of `zarr` dependencies is wasteful. We want to make it as easy as possible for people to use Zarr correctly. That means it should be easy to install a package that defines the syntax and semantics of the Zarr metadata documents, without committing consumers of that package to a full runtime. - **Agility:** We believe that separate packages will enable more effective development of the Zarr stack. For example, we want to develop a new version of our storage APIs that formally separate sync and async stores. We could carry out this development inside `zarr`, but `zarr` releases would mix storage API developments with ordinary `zarr` development, and so changes to the store API itself would not be transparent to consumers, e.g. implementers of third-party storage backends. If the storage API is defined entirely in a separate package, then we can version that package exactly when the storage API changes, which makes the design history much more visible to consumers. - **Focus:** Functionality in `zarr` has a tendency to be as good as it needs to be for `zarr`'s needs. This is reasonable, but it can complicate the decision about when to make internal APIs public. For example, the core metadata APIs in `zarr` are not clearly marked public, which is unfortunate because the metadata is a really important part of Zarr! We hope that devoting an entire package to, e.g., Zarr metadata, makes it easier to define a public API surface that `zarr` and any other consumer can depend on. We want these subpackages to treat `zarr` as the main, but not sole, consumer, which means we can implement useful features even if `zarr` doesn't need them today. - **Packaging:** Defining core APIs in separate packages solves a packaging problem for `zarr`. If `zarr` defines core APIs like storage and codec interfaces internally, third-party packages must depend on `zarr` to access those APIs. This in turn means `zarr` cannot offer those third-party packages as optional dependencies without incurring a circular dependency. Introducing a common dependency for `zarr` and `third-party-zarr-package` removes the circular dependency issue entirely. For the full list of subpackages, see the "Related Projects" [page in our docs](../../subprojects.md). ## Our Roadmap With the Zarr V2 -> V3 format transition largely settled, it's worth reflecting on how we want `zarr` to evolve. The previous section outlined one facet of that evolution -- defining `zarr` as the top of a stack of interlocking Zarr tools. There are other big changes ahead, foremost among them our goal of unlocking a vast performance improvement by wrapping the `zarrs` Rust crate via [PyO3](https://pyo3.rs/) bindings defined in [`zarrista`](https://developmentseed.org/zarrista/latest/). You can read more about this direction and other future plans in our new [roadmap document](../../roadmap.md), which is published as part of the 3.4.0 release. ## Rectilinear Chunk Grid Improvements The experimental rectilinear (irregular) chunk grid support (opt in with `zarr.config.set({"array.rectilinear_chunks": True})`) got several API improvements and bug fixes, mostly in [#4218](https://github.com/zarr-developers/zarr-python/pull/4218) and [#4272](https://github.com/zarr-developers/zarr-python/pull/4272): - Chunk grid flavor (regular vs rectilinear) now consistently follows input syntax: a nested per-dimension list always requests a rectilinear grid, even when the edges happen to be uniform. This fixes a regression in 3.3.0, where uniform-plus-short-tail lists requested a regular grid, which changed resize semantics. - `zarr.from_array(..., chunks="keep")` reproduces a rectilinear source grid exactly (in O(ndim)); sharding under a rectilinear shard grid is preserved instead of dropped. - `Array.chunks` is defined for any sharded array; `Array.info` and `Array.nchunks_initialized` work for rectilinear shard grids instead of raising (`info` shows the shard shape as ``). - Zero-length axes: `chunks=-1` and `chunks=False` resolve to chunk size 1 ([#4307](https://github.com/zarr-developers/zarr-python/pull/4307), [#4328](https://github.com/zarr-developers/zarr-python/pull/4328)) instead of `ZeroDivisionError` / invalid metadata. ## Useful Codec Errors `zarr` does not support every possible codec out of the box. Instead, we allow external packages to register their codec implementations with `zarr` at runtime, which broadens the palette of codecs we can support. When `zarr` reads an array with a codec it doesn't recognize, there's a good chance that the fix is to install the package that ships that codec implementation. But how should a user know which package they need? Previously, `zarr` handled an unknown codec with an opaque error, conveying only that `zarr` doesn't recognize the codec. Now, thanks to PRs [#4277](https://github.com/zarr-developers/zarr-python/pull/4277) and [#4351](https://github.com/zarr-developers/zarr-python/pull/4351), we maintain a table that relates the names of well-known codecs `zarr` doesn't ship with to packages that provide implementations. We use this table to provide a useful error message when an unknown codec is encountered. For example, opening an array whose metadata names the `imagecodecs_jpeg` codec, without any package that provides it installed: ```python exec="true" session="codec-errors" source="above" result="ansi" import json import zarr from zarr.buffer.cpu import buffer_prototype metadata = { "zarr_format": 3, "node_type": "array", "shape": [4, 4], "data_type": "uint8", "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4, 4]}}, "chunk_key_encoding": {"name": "default"}, "fill_value": 0, "codecs": [{"name": "bytes"}, {"name": "imagecodecs_jpeg"}], "attributes": {}, } store = {"zarr.json": buffer_prototype.buffer.from_bytes(json.dumps(metadata).encode())} try: zarr.open_array(store, mode="r") except zarr.errors.UnknownCodecError as e: print(e) ``` The table lives in [`zarr.registry`](../../api/zarr/registry.md); if you maintain a package that provides a codec, please add it there. ## New Dependency: `msgspec` Zarr metadata is composed of JSON documents that must comply with various specification documents, e.g. the Zarr V2 and V3 specs. `zarr` should model these JSON documents in a way that's correct, so that we generate interoperable data, and useful for developers, so they can also generate interoperable data. To do that we need to define two things: precise types, and routines for coercing or narrowing input values with unknown shapes into our precise types. Writing types for JSON data structures has gotten a lot easier in Python with the development of `TypedDict` utility types, but writing a validation function that checks an unknown input against an arbitrary `TypedDict` is arbitrary work. This isn't a hard programming task; rather, it's one that's very easy to mess up, e.g. by omitting an important check, or failing to model a field as optional. Runtime validation libraries like `pydantic`, `beartype`, and `msgspec` use type annotations to generate runtime type checking routines. Using one of these libraries could spare us a lot of hand-written type checking code. We have long discussed adding a runtime type checker to `zarr`, and as of 3.4.0 we are now using [`msgspec`](https://github.com/msgspec/msgspec) in this capacity ([#4063](https://github.com/zarr-developers/zarr-python/pull/4063), resolving [#3285](https://github.com/zarr-developers/zarr-python/issues/3285)). We chose `msgspec` because of its performance and minimal dependency count, and we expect it to deliver big value whenever we introduce new codecs, chunk grids, or data types to `zarr`, as we can just define the type for the metadata and get a validation routine for free. ## Tell us what you think As always, we would love any and all feedback about the work highlighted in this release! We would especially appreciate feedback from Python users who work with Zarr data and **don't** use the `zarr` package, since we are hoping to target their needs with our new family of subpackages. ## Contributors Thanks to everyone who contributed to this release (`*` marks a first-time contributor to Zarr-Python): - [@arcusbuilds](https://github.com/arcusbuilds) * - [@cucuwang](https://github.com/cucuwang) * - [@d-v-b](https://github.com/d-v-b) - [@dylanpulver](https://github.com/dylanpulver) * - [@glaziermag](https://github.com/glaziermag) * - [@jhamman](https://github.com/jhamman) - [@JOhnsonKC201](https://github.com/JOhnsonKC201) * - [@paraseba](https://github.com/paraseba) - [@selmanozleyen](https://github.com/selmanozleyen) - [@sheikhayaan](https://github.com/sheikhayaan) * - [@Tomatokeftes](https://github.com/Tomatokeftes) * - [@vup903](https://github.com/vup903) * __Our [previous blog post](3.3.0-release.md) omitted the contributor list, so we've updated that post to include one.__ !!! info "AI Usage Disclaimer" The outline for this post was generated by Claude Fable after prompting with the set of unreleased changes since 3.3.0. That outline was then heavily expanded by @d-v-b, introducing a large number of typographical errors. Those errors were removed, code snippets added, and hyperlinks resolved, by Claude. zarr-developers-zarr-python-a2f2415/docs/contributing.md000066400000000000000000001200211527014137600234420ustar00rootroot00000000000000# Contributing Zarr is a community maintained project. We welcome contributions in the form of bug reports, bug fixes, documentation, enhancement proposals and more. This page provides information on how best to contribute. ## Asking for help If you have a question about how to use Zarr, please post your question on StackOverflow using the ["zarr" tag](https://stackoverflow.com/questions/tagged/zarr). If you don't get a response within a day or two, feel free to raise a [GitHub issue](https://github.com/zarr-developers/zarr-python/issues/new) including a link to your StackOverflow question. We will try to respond to questions as quickly as possible, but please bear in mind that there may be periods where we have limited time to answer questions due to other commitments. ## Bug reports If you find a bug, please raise a [GitHub issue](https://github.com/zarr-developers/zarr-python/issues/new). Please include the following items in a bug report: 1. A minimal, self-contained snippet of Python code reproducing the problem. You can format the code nicely using markdown, e.g.: ```python exec="false" reason="illustrative pseudocode with a '# etc.' placeholder, not runnable" import zarr g = zarr.group() # etc. ``` 2. An explanation of why the current behavior is wrong/not desired, and what you expect instead. 3. Information about the version of Zarr, along with versions of dependencies and the Python interpreter, and installation information. The version of Zarr can be obtained from the `zarr.__version__` attribute. Please also state how Zarr was installed, e.g., "installed via pip into a virtual environment", or "installed using conda". Information about other packages installed can be obtained by executing `pip freeze` (if using pip to install packages) or `conda env export` (if using conda to install packages) from the operating system command prompt. The version of the Python interpreter can be obtained by running a Python interactive session, e.g.: ```console python ``` ```ansi Python 3.12.7 | packaged by conda-forge | (main, Oct 4 2024, 15:57:01) [Clang 17.0.6 ] on darwin ``` ## Enhancement proposals If you have an idea about a new feature or some other improvement to Zarr, please raise a [GitHub issue](https://github.com/zarr-developers/zarr-python/issues/new) first to discuss. We very much welcome ideas and suggestions for how to improve Zarr, but please bear in mind that we are likely to be conservative in accepting proposals for new features. The reasons for this are that we would like to keep the Zarr code base lean and focused on a core set of functionalities, and available time for development, review and maintenance of new features is limited. But if you have a great idea, please don't let that stop you from posting it on GitHub, just please don't be offended if we respond cautiously. ## AI-assisted contributions AI coding tools are increasingly common in open source development. These tools are welcome in Zarr-Python, but the same standards apply to all contributions regardless of how they were produced — whether written by hand, with AI assistance, or generated entirely by an AI tool. ### You are responsible for your changes If you submit a pull request, you are responsible for understanding and having fully reviewed the changes. You must be able to explain why each change is correct and how it fits into the project. ### Communication must be your own PR descriptions, issue comments, and review responses must be in your own words. The substance and reasoning must come from you. Using AI to polish grammar or phrasing is fine, but do not paste AI-generated text as comments or review responses. ### Review every line You must have personally reviewed and understood all changes before submitting. If you used AI to generate code, you are expected to have read it critically and tested it. The PR description should explain the approach and reasoning — do not leave it to reviewers to figure out what the code does and why. ### Keep PRs reviewable Generating code with AI is fast; reviewing it is not. A large diff shifts the burden from the contributor to the reviewer. PRs that cannot be reviewed in reasonable time with reasonable effort may be closed, regardless of their potential usefulness or correctness. Use AI tools not only to write code but to prepare better, more reviewable PRs — well-structured commits, clear descriptions, and minimal scope. If you are planning a large AI-assisted contribution (e.g., a significant refactor or a new subsystem), **open an issue first** to discuss the scope and approach with maintainers. Maintainers may also request that large changes be broken into smaller, reviewable pieces. ### Documentation The same principles apply to documentation. Zarr has domain-specific semantics (chunked storage, codec pipelines, Zarr v2/v3 format details) that AI tools frequently get wrong. Do not submit documentation that you haven't carefully read and verified. ## Contributing code and/or documentation ### Forking the repository The Zarr source code is hosted on GitHub at the following location: * [https://github.com/zarr-developers/zarr-python](https://github.com/zarr-developers/zarr-python) You will need your own fork to work on the code. Go to the link above and hit the ["Fork"](https://github.com/zarr-developers/zarr-python/fork) button. Then clone your fork to your local machine: ```bash git clone git@github.com:your-user-name/zarr-python.git cd zarr-python git remote add upstream git@github.com:zarr-developers/zarr-python.git ``` ### Creating a development environment The root `Justfile` defines development and CI commands. [just](https://just.systems/) runs these commands, while [Hatch](https://hatch.pypa.io/latest/index.html) manages the Python environments declared in `pyproject.toml`. Install [uv](https://docs.astral.sh/uv/getting-started/installation/) first, then the two task tools. `uv tool install` puts them in their own environments, which a plain `pip install` cannot do on a Python that marks itself externally managed (Debian, Ubuntu, Homebrew): ```bash uv tool install hatch==1.16.5 uv tool install rust-just==1.58.0 # or any option from https://just.systems/man/en/packages.html just # list available commands just envs # list Python environments just setup # create the default test environment just test ``` Test recipes default to `test.py3.12-optional`. Set `HATCH_ENV` to select a different interpreter or dependency set, just as CI does. `just gpu` reads `GPU_HATCH_ENV` instead, so an exported `HATCH_ENV` cannot silently send GPU tests to an environment built without the `gpu` feature, and `just doctest` always runs in the `doctest` environment. On Windows, run these commands in Git Bash. ```bash HATCH_ENV=test.py3.13-minimal just test HATCH_ENV=min_deps just coverage HATCH_ENV=upstream just coverage GPU_HATCH_ENV=gputest.py3.12 just gpu just test tests/test_array.py -k 'resize and not async' ``` Arguments after the recipe name are forwarded to the underlying tool. Use `just --show test` to inspect a command. Package-specific commands live in the `justfile` inside each package directory. The root recipes delegate to these files, so you can also run package commands from the repository root: ```bash just zarr-metadata # list this package's recipes just zarr-metadata test just zarr-indexing test-tensorstore just zarr-http-server docs-check ``` The package justfile sets the working directory and defines the command and its environment. Root recipes forward arguments without duplicating those definitions. ### Creating a branch Before you do any new work or submit a pull request, please open an issue on GitHub to report the bug or propose the feature you'd like to add. It's best to synchronize your fork with the upstream repository, then create a new, separate branch for each piece of work you want to do. E.g.: ```bash git checkout main git fetch upstream git checkout -b shiny-new-feature upstream/main git push -u origin shiny-new-feature ``` This changes your working directory to the 'shiny-new-feature' branch. Keep any changes in this branch specific to one bug or feature so it is clear what the branch brings to Zarr. To update this branch with latest code from Zarr, you can retrieve the changes from the main branch and perform a rebase: ```bash git fetch upstream git rebase upstream/main ``` This will replay your commits on top of the latest Zarr git main. If this leads to merge conflicts, these need to be resolved before submitting a pull request. Alternatively, you can merge the changes in from upstream/main instead of rebasing, which can be simpler: ```bash git pull upstream main ``` Again, any conflicts need to be resolved before submitting a pull request. ### Running the test suite Zarr includes a suite of unit tests. The simplest way to run the unit tests is to invoke: ```bash just test ``` All tests are automatically run via GitHub Actions for every pull request and must pass before code can be accepted. Test coverage is also collected automatically via the Codecov service. ### Code standards - using prek All code must conform to the PEP8 standard. Regarding line length, lines up to 100 characters are allowed, although please try to keep under 90 wherever possible. `Zarr` uses a set of git hooks managed by [`prek`](https://github.com/j178/prek), a fast, Rust-based pre-commit hook manager compatible with `.pre-commit-config.yaml`. The recipes pin the prek version: `just lint` and `just hooks` run it through `uvx`, and `just hooks-install` installs it as a persistent `uv tool` so the git hook can find it on later commits. The hooks can be installed locally by running: ```bash just hooks-install ``` This will run the checks every time a commit is created locally. The checks will by default only run on the files modified by a commit, but the checks can be triggered for all the files by running: ```bash just lint ``` You can also run hooks only for files in a specific directory: ```bash just hooks run --directory src/zarr ``` Or run hooks for files changed in the last commit: ```bash just hooks run --last-commit ``` To list all available hooks: ```bash just hooks list ``` If you would like to skip the failing checks and push the code for further discussion, use the `--no-verify` option with `git commit`. ### Test coverage > **Note:** Test coverage for Zarr-Python 3 is currently not at 100%. This is a known issue and help is welcome to bring test coverage back to 100%. See issue #2613 for more details. Zarr strives to maintain 100% test coverage under the latest Python stable release. Both unit tests and docstring doctests are included when computing coverage. Running: ```bash just coverage ``` will automatically run the test suite with coverage and produce an XML coverage report. This should be 100% before code can be accepted into the main code base. You can also generate an HTML coverage report by running: ```bash just coverage-html ``` When submitting a pull request, coverage will also be collected across all supported Python versions via the Codecov service, and will be reported back within the pull request. Codecov coverage must also be 100% before code can be accepted. ### Documentation Docstrings for user-facing classes and functions should follow the [numpydoc](https://numpydoc.readthedocs.io/en/stable/format.html#docstring-standard) standard, including sections for Parameters and Examples. All examples should run and pass as doctests under Python 3.12. Zarr uses mkdocs for documentation, hosted on readthedocs.org. Documentation is written in the Markdown markup language (.md files) in the `docs` folder. The documentation consists both of prose and API documentation. All user-facing classes and functions are included in the API documentation, under the `docs/api` folder using the [mkdocstrings](https://mkdocstrings.github.io/) extension. Add any new public functions or classes to the relevant markdown file in `docs/api/*.md`. Any new features or important usage information should be included in the user-guide (`docs/user-guide`). Any changes should also be included as a new file in the `changes` directory. The documentation can be built locally by running: ```bash just docs-build ``` `just docs-check` also runs the documentation source checks used in CI. The resulting built documentation will be available in the `site` folder. `just docs-serve` serves a continuously updating version of the documentation during development at [http://127.0.0.1:8000/](http://127.0.0.1:8000/). This can be done by running: ```bash just docs-serve ``` #### Sub-package documentation sites Each sub-package under `packages/` ships its own Material for MkDocs site, built from within the package directory: ```bash cd packages/zarr-metadata uv run --group docs mkdocs build --strict ``` Configuration shared by all package sites (theme, plugins, markdown extensions) lives in `packages/mkdocs-base.yml`, symlinked into each package directory; each package's `mkdocs.yml` pulls it in via `INHERIT` and defines only the site identity (name, description, repo and site URLs) and nav. `INHERIT` merges mappings but replaces lists, so a package that sets `theme.features` or `markdown_extensions` silently drops every shared entry; add extensions and features to the base instead. The header source widget's version fact is overridden by `packages/source-version.js`, symlinked into each package's `docs/_static/`, which replaces the monorepo's latest release with the latest tag matching the package's `zarr_-v` prefix (both derived from `repo_url` at runtime, so the file needs no per-package edits). On Windows, enable symlinks in git (`git config core.symlinks true`, with Developer Mode on) before cloning, or local builds will ship a file containing the link target instead of the script. To add a site for a new sub-package, copy `mkdocs.yml`, the `docs/` folder, `.readthedocs.yaml`, and the `docs` dependency group in `pyproject.toml` from an existing package, then adjust the site name, description, URLs, and nav. Recreate the `mkdocs-base.yml` and `docs/_static/source-version.js` symlinks rather than copying the files, and copy the sdist `exclude` and `force-include` entries from an existing package's `pyproject.toml`, which ship the linked files in place of the links. Add `packages/mkdocs-base.yml` and `packages/source-version.js` to the package workflow's `paths` filters and to the `git diff` in its `.readthedocs.yaml`, so changes to the shared files build the site. Each site is hosted as its own Read the Docs project. To set one up for a new sub-package: 1. Create a new project on [readthedocs.org](https://app.readthedocs.org) importing the `zarr-python` repository, named after the package (e.g. `zarr-metadata`). 2. In the project's admin settings, set the configuration file path to `packages//.readthedocs.yaml`. That file also cancels pull request builds that don't touch the package. 3. Add an automation rule matching the package's release tags (custom match `^zarr_-v`, note the underscore) with the action "Activate version". Automation rules only apply to versions detected after the rule is created, so activate any earlier release tags manually from the versions list. 4. When activating a version, edit its slug to the bare version number (`0.4.0`, not `v0.4.0`). RTD keeps a version record for every tag in the monorepo, including inactive ones not shown in the dashboard's versions list, and all of zarr-python's own release tags are `v`-prefixed — so a `v`-prefixed slug collides with zarr-python's release history ("A version with that slug already exists"), while bare version numbers cannot collide. #### Adding executable code blocks in the documentation Zarr uses [Markdown Exec](https://pawamoy.github.io/markdown-exec/usage/) to execute code blocks in Markdown files. Add `exec="true"` to a code block header for it to be executed when the docs are built. For example: ````md ```python exec="true" print("Hello world") ``` ```` Below are other useful options that can be added to the code block. See [Markdown Exec's documentation](https://pawamoy.github.io/markdown-exec/usage/#options-summary) for a full list: - `source="above"` makes sure the code within the code block is also rendered in the documentation (rather than just the output). - `session=""` executes code blocks in a named session reusing previously defined variables. - `result="ansi"` or `result="html"` to render the output. If the code does not produce output, you should leave off the `result` option to prevent an empty cell from rendering in the docs. For example: ````md ```python exec="true" session="contributing" source="above" result="ansi" print("Hello world") ``` ```` renders as: ```python exec="true" session="contributing" source="above" result="ansi" print("Hello world") ``` #### Validating code blocks: `exec` vs `test` Every Python code block in the documentation is checked by a test (`tests/test_docs.py`) so that examples cannot quietly rot — the bug that motivated this was an example calling `zarr.create_array(..., mode="w")`, an argument that does not exist, which went unnoticed because nothing ran it. A block declares *how* it is validated using one of two independent attributes: - **`exec="true"`** — Markdown Exec runs the block **at docs-build time to render its output** into the page. This is the attribute described above; it is also what the test suite executes. Use it for ordinary examples whose output should appear in the docs. - **`test="true"`** — the block is **run by the test suite only**, *not* at build time. Use this for an example that should be validated but cannot run in the docs-build environment — for example one that needs a GPU or a cloud backend. Markdown Exec leaves a `test="true"` block as a static, syntax-highlighted snippet (it never executes it), while the test suite still runs it (see the marker note below). A block may carry both (`exec="true" test="true"`), though in practice `exec="true"` already implies it is tested, so you rarely need `test="true"` alongside it. The two attributes are kept separate on purpose: `exec=` controls *build-time rendering* and `test=` controls *test-time validation*. Tagging a GPU/cloud example `exec="true"` would make `mkdocs build` try to run it on a machine without that infrastructure and fail the build; `test="true"` lets it be validated without being built. ##### Opting a block out of validation A handful of blocks genuinely cannot run and are not executable Python — a REPL transcript, a deliberately-incorrect "before" snippet, a `--8<--` file include. Mark these explicitly by opening the fence with `exec="false" reason="REPL output transcript, not executable source"` (supply a reason that fits the block). `exec="false"` with a non-empty `reason` is an explicit, greppable opt-out. A test (`test_no_unvalidated_blocks`) requires **every** Python block to be either `exec="true"`, `test="true"`, or `exec="false"` with a reason — so a block can never silently skip validation. A bare ` ```python ` fence, or a typo like `exec="on"`, fails that test. Markdown Exec only renders `exec="true"` fences; the `mkdocs_hooks.py` hook at the repository root makes `test="true"` and `exec="false"` fences render as ordinary highlighted code blocks. Without it, these fences would fail superfences parsing and their contents would spill into the page as raw markdown. ##### Marker-bound blocks (GPU, S3) A `test="true"` block that needs special infrastructure declares a pytest marker with `markers="..."`, which binds it to that infrastructure in the test suite: - `markers="gpu"` — run only under `pytest -m gpu` (the GPU CI environment); skipped elsewhere via `importorskip("cupy")`. - `markers="s3"` — run against a mock S3 (moto) backend supplied by a test fixture, so the example can use a bare `s3://…` URL with no test-only connection details on show. ##### Placement of `test="true"` blocks Because Markdown Exec does not execute a `test="true"` (or `exec="false"`) block, placing one *before* an `exec="true"` block on the same page can disrupt the build-time execution of that later block. Put `test="true"` blocks **after** all `exec="true"` blocks on the page (or on a page where they are the only Python block). The `test_test_only_blocks_come_last` test enforces this, and the CI docs build runs with `--strict` so any such breakage fails the build rather than passing as a warning. #### Building documentation without executing code blocks Sometimes, you may want the documentation to build quicker. You can disable code block execution by commenting out the [markdown-exec plugin](https://github.com/zarr-developers/zarr-python/blob/884a8c91afcc3efe28b3da952be3b85125c453cb/mkdocs.yml#L132) in the mkdocs configuration file. This will make code blocks and cross references render incorrectly (i.e., expect build warnings), but also reduces build time by ~3x. Be sure to undo the commenting out before opening your pull request. ### Changelog zarr-python uses [towncrier](https://towncrier.readthedocs.io/en/stable/tutorial.html) to manage release notes. Most pull requests should include at least one news fragment describing the changes. To add a release note, you'll need the GitHub issue or pull request number and the type of your change (`feature`, `bugfix`, `doc`, `removal`, `misc`). With that, run `just changelog`, which will prompt you for the issue number, change type, and the news text: ```bash just changelog ``` Alternatively, you can manually create the files in the `changes` directory using the naming convention `{issue-number}.{change-type}.md`. See the [towncrier](https://towncrier.readthedocs.io/en/stable/tutorial.html) docs for more. ## Project governance This section documents the processes that core developers follow to maintain the project. The current core developers are listed in [`TEAM.md`](https://github.com/zarr-developers/zarr-python/blob/main/TEAM.md). ### Merging pull requests Pull requests submitted by an external contributor should be reviewed and approved by at least one core developer before being merged. Ideally, pull requests submitted by a core developer should be reviewed and approved by at least one other core developer before being merged. Pull requests should not be merged until all CI checks have passed (GitHub Actions, Codecov) against code that has had the latest main merged in. Before merging, the milestone must be set to decide whether a PR will be in the next patch, minor, or major release. The next section explains which types of changes go in each release. ### Self-merging pull requests The default is that a pull request opened by a core developer is reviewed and approved by at least one other core developer before it is merged. We trust core developers to use their judgment, though, and we would rather bias toward action than make routine changes wait on review they do not really need. So a core developer may merge their own pull request whenever they judge the change to be low-risk, provided the standard merge requirements are met — CI is green against code that has had the latest `main` merged in, a changelog fragment has been added, and the milestone is set — and other core developers have had a fair chance to weigh in. As a rule of thumb, leave the pull request open for a few days before self-merging, unless it is genuinely trivial or time-sensitive. If you are confident a change is fine, merge it; if you have real doubts, ask for a review. It is generally advisable to ping another developer in the PR description for awareness about the direction, even if you choose not to request a formal review. Some changes warrant more caution, and a second reviewer is usually worth seeking even when you could self-merge: changes to the public API, anything touching data-format or on-disk compatibility, and performance-sensitive code. These are the most expensive to get wrong and the hardest to reverse. Reverts, by contrast, are cheap — if a self-merged change turns out to be a mistake, reverting it is itself a low-risk change that any core developer can make, and the reworked version can go through normal review. When something recently merged is actively causing harm — a broken `main`, a release blocker, or data corruption — fix it fast and request review after the fact rather than waiting. This policy exists to lower the cost of routine work and to help newer core developers grow comfortable merging changes. It is not a license to merge past an unresolved objection: if another core developer asks to review a change, give them that chance. ### Release procedure To give the release visibility and a single place to track progress, open an issue on GitHub announcing the release using the [release checklist template](https://github.com/zarr-developers/zarr-python/issues/new?template=release-checklist.md). The release checklist includes all steps necessary for the release. #### Read the Docs versions for subpackage releases Read the Docs creates a version for every tag in the repository, including tags for `zarr-python` and the subpackages. This can leave the desired public slug attached to the wrong tag. For example, the slug `v0.4.0` may already belong to the `zarr-python` tag `v0.4.0` when releasing `zarr-metadata` from `zarr_metadata-v0.4.0`. Before activating a subpackage release in Read the Docs: 1. Open **Admin > Versions** for the subpackage documentation project. 2. Find the version currently using the desired slug, such as `v0.4.0`, and change its slug to an unused value, such as `xv0.4.0`. 3. Find the version created from the subpackage tag, such as `zarr_metadata-v0.4.0`, and assign it the released slug, `v0.4.0`. 4. Verify that the release URL resolves to the subpackage tag, for example `https://zarr-metadata.readthedocs.io/en/v0.4.0/`. Use the same process for `zarr-indexing` tags such as `zarr_indexing-v0.2.0` and any other subpackage hosted as a separate Read the Docs project. ## Compatibility and versioning policies ### Versioning Versions of this library are identified by a triplet of integers with the form `..`, for example `3.0.4`. A release of `zarr-python` is associated with a new version identifier. That new identifier is generated by incrementing exactly one of the components of the previous version identifier by 1. When incrementing the `major` component of the version identifier, the `minor` and `patch` components are reset to 0. When incrementing the minor component, the patch component is reset to 0. Releases are classified by the library changes contained in that release. This classification determines which component of the version identifier is incremented on release. * **major** releases (for example, `2.18.0` -> `3.0.0`) are for changes that will require extensive adaptation efforts from many users and downstream projects. For example, breaking changes to widely-used user-facing APIs should only be applied in a major release. Users and downstream projects should carefully consider the impact of a major release before adopting it. In advance of a major release, developers should communicate the scope of the upcoming changes, and help users prepare for them. * **minor** releases (for example, `3.0.0` -> `3.1.0`) are for changes that do not require significant effort from most users or downstream projects to respond to. API changes are possible in minor releases if the burden on users imposed by those changes is sufficiently small. For example, a recently released API may need fixes or refinements that are breaking, but low impact due to the recency of the feature. Such API changes are permitted in a minor release. Minor releases are safe for most users and downstream projects to adopt. * **patch** releases (for example, `3.1.0` -> `3.1.1`) are for changes that contain no breaking or behavior changes for downstream projects or users. Examples of changes suitable for a patch release are bugfixes and documentation improvements. Users should always feel safe upgrading to the latest patch release. Note that this versioning scheme is not consistent with [Semantic Versioning](https://semver.org/). Contrary to SemVer, the Zarr library may release breaking changes in `minor` releases, or even `patch` releases under exceptional circumstances. But we should strive to avoid doing so. A better model for our versioning scheme is [Intended Effort Versioning](https://jacobtomlinson.dev/effver/), or "EffVer". The guiding principle of EffVer is to categorize releases based on the *expected effort required to upgrade to that release*. Zarr developers should make changes as smooth as possible for users. This means making backwards-compatible changes wherever possible. When a backwards-incompatible change is necessary, users should be notified well in advance, e.g. via informative deprecation warnings. ### Data format compatibility The Zarr library is an implementation of a file format standard defined externally -- see the [Zarr specifications website](https://zarr-specs.readthedocs.io) for the list of Zarr file format specifications. If an existing Zarr format version changes, or a new version of the Zarr format is released, then the Zarr library will generally require changes. It is very likely that a new Zarr format will require extensive breaking changes to the Zarr library, and so support for a new Zarr format in the Zarr library will almost certainly come in a new `major` release. When the Zarr library adds support for a new Zarr format, there may be a period of accelerated changes as developers refine newly added APIs and deprecate old APIs. In such a transitional phase breaking changes may be more frequent than usual. ### Deprecation policy Our versioning policy (above) commits us to minimizing the effort users spend upgrading. A deprecation cycle is the main tool we use to honor that commitment. Rather than removing or changing public behavior abruptly, we first ship a release that keeps the old behavior working while warning that it is going away, and only remove it in a later release. This section defines when a deprecation cycle is required, how to decide whether a removal is worth it, and the concrete steps a deprecation must follow. This policy governs the user-facing Python API of `zarr-python`: importable names, function and method signatures, the exceptions raised, and other observable runtime behavior. It does not govern the Zarr data format, whose compatibility is described under [Data format compatibility](#data-format-compatibility) above, nor the `zarr.experimental` namespace, which is governed by the [Experimental API policy](#experimental-api-policy) below. Private API (names prefixed with `_`, and anything not documented as public) may change at any time without a deprecation cycle. Any backwards-incompatible change to public API, including removing a name, changing a signature in a non-additive way, or changing observable behavior, requires a deprecation cycle unless it qualifies for one of the [exceptions](#exceptions) listed below. #### Deciding whether to deprecate A deprecation cycle has a real cost. Every user of the affected API must eventually migrate and we carry the deprecated code and its warning until removal. Before starting one, weigh the costs and benefits and record your reasoning in the pull request, so that proposals are evaluated on a shared scale. NumPy's [backwards compatibility policy (NEP 23)](https://numpy.org/neps/nep-0023-backwards-compatibility.html#general-principles) is a good guide to the trade-offs involved. The factors below follow the same principles. - **How many users are affected.** `zarr-python` is widely used, including by downstream libraries, so assume that any public API is used by someone unless there is concrete evidence otherwise. Most users do not follow our issue tracker and discover a removal only when their code breaks after upgrading, so absence of reported usage is not evidence of absence of usage. Base the estimate on usage data where possible, using code search across downstream projects, documented usage, or download statistics. - **The cost of migration to users.** Is there a drop-in replacement? Can the migration be performed mechanically, or does it require users to redesign their code? A change with a clear, easy migration path is much cheaper to justify than one without. - **The cost of keeping the API.** What do we pay by not removing it? Is there maintenance burden, bug surface, duplicated logic, or a worse design that we can't improve while the old API exists? A high carrying cost strengthens the case for removal. - **Who benefits.** Benefits can include improved functionality, usability, and performance for users, as well as lower maintenance cost and better future extensibility for developers. A change that gives users something they want is a stronger case than one that benefits only maintainers, since users pay the migration cost either way. - **Whether the goal can be met without removal.** Often a refactor, an alias, or an additive change achieves the same end without breaking anyone. Prefer those. As mentioned above, removing public API is never free. If, after weighing these factors, a removal is not clearly worthwhile, prefer to keep the API. A deprecation warning is a commitment to change or remove. If you only want to steer users away from an API without removing it, say so in its documentation rather than emitting a warning. Conversely, do not leave a decision to remove an API indefinitely deferred. A vague intention to "eventually" break something is itself a form of technical debt. Either commit to the deprecation and begin the cycle, or decide to keep the API and design around it. #### How to deprecate Once a deprecation is decided, it must follow these steps so that users get a consistent, actionable signal: 1. **Emit a warning at runtime.** Raise `zarr.errors.ZarrDeprecationWarning` for an API that will be *removed*. Use `zarr.errors.ZarrFutureWarning` instead when behavior will *change* rather than disappear (for example, a default value that will change). The distinction is about who needs to see the warning. Python hides `DeprecationWarning` by default except in `__main__` and under test runners, so a removal warning reaches the authors of the code that calls the deprecated API, whereas `FutureWarning` is shown to everyone, which a behavior change needs because it alters results without any change to the user's code. The [`typing_extensions.deprecated`](https://typing-extensions.readthedocs.io/en/latest/#deprecated) decorator is the preferred way to deprecate whole functions, methods, and classes. When calling `warnings.warn` directly, set `stacklevel` so the warning points at the caller's code rather than zarr internals. 2. **Write an actionable message.** The warning message must state the planned removal or change (a specific version if known, otherwise "a future release") and the migration path (i.e., what the user should do instead). A deprecation warning with no replacement guidance is incomplete. 3. **Document it.** Add a "Deprecated" admonition to the docstring, stating the same removal target and migration path as the warning, so the deprecation appears in the rendered API reference, and update any affected user-guide prose. 4. **Add a changelog entry.** Add a `removal` changelog fragment in `changes/` (for example, `changes/1234.removal.md`) so the deprecation is announced under "Deprecations and Removals" in the release notes. When the API is finally removed, add a second `removal` fragment for that release. When an API is being replaced rather than simply dropped, deprecate the old name and introduce its replacement in a single step, rather than repeatedly adjusting the same signature across releases. Users should have to migrate only once. #### How long a deprecation lasts A deprecated API must remain available, emitting its warning, for **at least 6 months and at least one minor release** before it is removed. The same minimum applies to a behavior change announced with `ZarrFutureWarning`. The old behavior stays the default, with the warning, for at least that long before the change takes effect. These are minimums rather than specific targets. For widely-used API, prefer a longer cycle, and communicate the upcoming removal beyond the warning itself (for example, in release notes or a migration guide). The removal itself is a backwards-incompatible change, and is classified under our [versioning policy](#versioning) by the upgrade effort it imposes. Use a `major` release for the removal of widely-used API versus a `minor` release when the impact is genuinely small. This differs from [Semantic Versioning](https://semver.org/), under which any removal would require a `major` release. Again, the [versioning policy](#versioning) above explains why. The deprecation period, not the release type, is the guarantee users rely on. A removal is only permitted once the minimum period has elapsed, regardless of which release it lands in. #### Exceptions A full deprecation cycle may be shortened or skipped in some cases. When it is, explain why in the pull request and the changelog entry: - **Security fixes and data-corruption bugs**, where continuing the old behavior actively harms users. - **Changes forced by the Zarr format specification**, which are governed by [Data format compatibility](#data-format-compatibility) and may move faster during a format transition. - **Recently introduced API**, which has had little time to be adopted. API that has not yet appeared in a release can be changed freely. API that appeared in a recent release may be changed in a `minor` release with a shortened cycle, as the [versioning policy](#versioning) allows, when the number of affected users is small because the API is new. - **Experimental API**, which is exempt and governed by the [Experimental API policy](#experimental-api-policy). This includes the deprecated re-export left in `zarr.experimental` when a feature is promoted to stable, which that policy keeps for one minor release. ## Experimental API policy The `zarr.experimental` namespace contains features that are under active development and may change without notice. When contributing to or depending on experimental features, please keep the following in mind: ### For contributors When adding a new feature to `zarr.experimental`: 1. Place the feature under `src/zarr/experimental/` and export it from `src/zarr/experimental/__init__.py`. 2. Document the feature in `docs/user-guide/experimental.md` and note clearly that it is experimental. 3. Add a changelog entry categorized as `feature`. We aim to either **promote** or **remove** experimental features within **6 months** of their addition. To promote a feature to stable: 1. Move it from `zarr.experimental` to the appropriate stable module. 2. Keep a deprecated re-export in `zarr.experimental` for one minor release. 3. Update the documentation to reflect the stable location. ### For users Features in `zarr.experimental` carry no stability guarantees. They may be changed or removed in any release, including patch releases. If you depend on an experimental feature, pin your `zarr-python` version accordingly. ## Benchmarks Zarr uses [pytest-benchmark](https://pytest-benchmark.readthedocs.io/en/latest/) for running performance benchmarks as part of our test suite. The benchmarks are found in `tests/benchmarks`. By default pytest is configured to run these benchmarks as plain tests (i.e., no benchmarking). To run a benchmark with timing measurements, run `just benchmark`. Pass pytest arguments to select benchmarks, for example `just benchmark -k test_morton_order`. The benchmarks are run as part of the continuous integration suite through [codspeed](https://app.codspeed.io/zarr-developers/zarr-python). ## Building distributions and maintaining dependencies `just just-check` verifies that the root `Justfile` and each package `justfile` are formatted the way CI expects; `just --fmt` rewrites them in place if it complains. Run `just build` to produce a source distribution and wheel in `dist/`. Use `just lock-check` to check the dependency lockfile, or `just lock` to update it. `just typecheck` runs the type checker independently of the other lint hooks. Preview release notes with `just changelog-draft`; use `just check-changelogs` to validate fragment names, optionally passing a package's `changes/` directory. zarr-developers-zarr-python-a2f2415/docs/index.md000066400000000000000000000061011527014137600220440ustar00rootroot00000000000000# Zarr-Python **Useful links**: [Source Repository](https://github.com/zarr-developers/zarr-python) | [Issue Tracker](https://github.com/zarr-developers/zarr-python/issues) | [Developer Chat](https://ossci.zulipchat.com/) | [Zarr specifications](https://zarr-specs.readthedocs.io) Zarr is a powerful library for storage of n-dimensional arrays, supporting chunking, compression, and various backends, making it a versatile choice for scientific and large-scale data. Zarr-Python is a Python library for reading and writing Zarr groups and arrays. Highlights include: * Specification support for both Zarr format 2 and 3. * Create and read from N-dimensional arrays using NumPy-like semantics. * Flexible storage enables reading and writing from local, cloud and in-memory stores. * High performance: Enables fast I/O with support for asynchronous I/O and multi-threading. * Extensible: Customizable with user-defined codecs and stores. ## Installation Zarr requires Python 3.12 or higher. You can install it via `pip`: ```bash pip install zarr ``` or `conda`: ```bash conda install -c conda-forge zarr ``` ## Navigating the documentation
- [:material-clock-fast:{ .lg .middle } __Quick start__](quick-start.md) --- New to Zarr? Check out the quick start guide. It contains a brief introduction to Zarr's main concepts and links to additional tutorials. - [:material-book-open:{ .lg .middle } __User guide__](user-guide/index.md) --- A detailed guide for how to use Zarr-Python. - [:material-api:{ .lg .middle } __API Reference__](api/zarr/index.md) --- The reference guide contains a detailed description of the functions, modules, and objects included in Zarr. The reference describes how the methods work and which parameters can be used. It assumes that you have an understanding of the key concepts. - [:material-package-variant:{ .lg .middle } __Related projects__](subprojects.md) --- Companion packages developed in the zarr-python repository and released independently, such as `zarr-metadata` and `zarr-indexing`, plus pointers to the wider Zarr ecosystem. - [:material-account-group:{ .lg .middle } __Contributor's Guide__](contributing.md) --- Want to contribute to Zarr? We welcome contributions in the form of bug reports, bug fixes, documentation, enhancement proposals and more. The contributing guidelines will guide you through the process of improving Zarr.
## Project Status More information about the Zarr format can be found on the [main website](https://zarr.dev). If you are using Zarr-Python, we would [love to hear about it](https://github.com/zarr-developers/community/issues/19). ### Funding and Support The project is fiscally sponsored by [NumFOCUS](https://numfocus.org/), a US 501(c)(3) public charity, and development has been supported by the [MRC Centre for Genomics and Global Health](https://github.com/cggh/) and the [Chan Zuckerberg Initiative](https://chanzuckerberg.com/). [Donate to Zarr](https://numfocus.org/donate-to-zarr) to support the project! zarr-developers-zarr-python-a2f2415/docs/overrides/000077500000000000000000000000001527014137600224175ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/docs/overrides/main.html000066400000000000000000000003051527014137600242270ustar00rootroot00000000000000 {% extends "base.html" %} {% block outdated %} You're not viewing the latest version. Click here to go to latest. {% endblock %} zarr-developers-zarr-python-a2f2415/docs/overrides/stylesheets/000077500000000000000000000000001527014137600247735ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/docs/overrides/stylesheets/extra.css000066400000000000000000000111711527014137600266310ustar00rootroot00000000000000:root { --gradient-start: #e58077; --gradient-mid-1: #e57a77; --gradient-mid-2: #e46876; --gradient-mid-3: #e34b75; --gradient-mid-4: #e12374; --gradient-mid-5: #e01073; --gradient-end: #bb1085; /* Primary theme colors --md-primary-fg-color: #e34b75; --md-primary-fg-color--light: #e57a77; --md-primary-fg-color--dark: #bb1085; /* Accent colors */ --md-accent-fg-color: #e01073; --md-accent-fg-color--transparent: rgba(224, 16, 115, 0.1); /* Text colors that work well with the palette */ --md-text-color: #333333; --md-text-color--light: #666666; } /* Dark mode color adjustments */ [data-md-color-scheme="slate"] { --md-primary-fg-color: #e57a77; --md-primary-fg-color--light: #e58077; --md-primary-fg-color--dark: #bb1085; --md-accent-fg-color: #e46876; --md-accent-fg-color--transparent: rgba(228, 104, 118, 0.1); } /* Header styling with gradient background */ .md-header { background: linear-gradient( 135deg, var(--gradient-start) 0%, var(--gradient-mid-1) 16.66%, var(--gradient-mid-2) 33.33%, var(--gradient-mid-3) 50%, var(--gradient-mid-4) 66.66%, var(--gradient-mid-5) 83.33%, var(--gradient-end) 100% ); box-shadow: 0 2px 8px rgba(187, 16, 133, 0.15); } /* Ensure header text is readable over gradient */ .md-header__title, .md-header__button, .md-header .md-icon { color: white; } /* Search box styling in the header */ .md-header .md-search__input { background-color: rgba(255, 255, 255, 0.15); border: 1px solid rgba(255, 255, 255, 0.2); } /* Navigation tabs */ .md-tabs { background: linear-gradient( 90deg, var(--gradient-mid-3) 0%, var(--gradient-mid-4) 50%, var(--gradient-mid-5) 100% ); } .md-tabs__link { color: rgba(255, 255, 255, 0.9); } .md-tabs__link--active, .md-tabs__link:hover { color: white; opacity: 1; } /* Sidebar navigation */ .md-nav__link--active { color: var(--md-primary-fg-color); font-weight: 500; } .md-nav__link:hover { color: var(--md-accent-fg-color); } /* Code blocks */ .highlight { border-left: 4px solid var(--md-accent-fg-color); background-color: rgba(228, 104, 118, 0.05); } /* Admonitions */ .md-typeset .admonition.note { border-color: var(--md-primary-fg-color); } .md-typeset .admonition.note > .admonition-title { background-color: rgba(227, 75, 117, 0.1); border-color: var(--md-primary-fg-color); } .md-typeset .admonition.tip { border-color: var(--gradient-mid-1); } .md-typeset .admonition.tip > .admonition-title { background-color: rgba(229, 122, 119, 0.1); border-color: var(--gradient-mid-1); } .md-typeset .admonition.warning { border-color: var(--gradient-end); } .md-typeset .admonition.warning > .admonition-title { background-color: rgba(187, 16, 133, 0.1); border-color: var(--gradient-end); } /* Links */ .md-content a { color: var(--md-accent-fg-color); } .md-content a:hover { color: var(--gradient-end); } /* Table of contents */ .md-nav--secondary .md-nav__link--active { color: var(--md-accent-fg-color); border-left: 2px solid var(--md-accent-fg-color); padding-left: calc(1rem - 2px); } /* Footer */ .md-footer { background-color: var(--gradient-end); } /* Buttons and interactive elements */ .md-button { background: linear-gradient(135deg, var(--md-primary-fg-color), var(--md-accent-fg-color)); border: none; color: white; transition: all 0.3s ease; } .md-button:hover { transform: translateY(-2px); box-shadow: 0 4px 12px rgba(187, 16, 133, 0.3); } /* Scrollbar styling */ ::-webkit-scrollbar { width: 8px; } ::-webkit-scrollbar-track { background: rgba(187, 16, 133, 0.1); } ::-webkit-scrollbar-thumb { background: linear-gradient( 180deg, var(--md-primary-fg-color), var(--md-accent-fg-color) ); border-radius: 4px; } ::-webkit-scrollbar-thumb:hover { background: linear-gradient( 180deg, var(--md-accent-fg-color), var(--gradient-end) ); } /* Search results highlighting */ .md-search-result__title { color: var(--md-primary-fg-color); } .md-search-result__teaser mark { background-color: rgba(224, 16, 115, 0.2); color: var(--gradient-end); } .md-header__button.md-logo img, .md-header__button.md-logo svg { height: 42px !important; /* Increase from default ~24px */ width: auto !important; max-height: none !important; padding: 0 0 0 16px !important; /* Keep left padding, remove others */ margin: 0 !important; /* Remove any margin */ } /* Also remove padding from the logo button container except left */ .md-header__button.md-logo { padding: 0 0 0 8px !important; /* Keep some left padding on container */ margin: 0 !important; min-width: auto !important; } zarr-developers-zarr-python-a2f2415/docs/quick-start.md000066400000000000000000000132051527014137600232070ustar00rootroot00000000000000# Quick start This page will help you get up and running with the Zarr library in Python to efficiently manage and analyze multi-dimensional arrays. Zarr must be installed first -- see the [installation guide](user-guide/installation.md) if you have not installed it yet. ## Creating an Array To get started, you can create a simple Zarr array: ```python exec="true" session="quickstart" import numpy as np from pprint import pprint import io import warnings warnings.filterwarnings( "ignore", message="Numcodecs codecs are not in the Zarr version 3 specification*", category=UserWarning ) np.random.seed(0) ``` ```python exec="true" session="quickstart" source="above" result="ansi" import zarr import numpy as np # Create a 2D Zarr array z = zarr.create_array( store="data/example-1.zarr", shape=(100, 100), chunks=(10, 10), dtype="f4", overwrite=True, ) # Assign data to the array z[:, :] = np.random.random((100, 100)) print(z.info) ``` Here, we created a 2D array of shape `(100, 100)`, chunked into blocks of `(10, 10)`, and filled it with random floating-point data. This array was written to a `LocalStore` in the `data/example-1.zarr` directory. ### Compression and Filters Zarr supports data compression and filters. For example, to use Blosc compression: ```python exec="true" session="quickstart" source="above" result="ansi" # Create a 2D Zarr array with Blosc compression z = zarr.create_array( store="data/example-2.zarr", shape=(100, 100), chunks=(10, 10), dtype="f4", overwrite=True, compressors=zarr.codecs.BloscCodec( cname="zstd", clevel=3, shuffle="shuffle" ) ) # Assign data to the array z[:, :] = np.random.random((100, 100)) print(z.info) ``` This compresses the data using the Blosc codec with shuffle enabled for better compression. ## Hierarchical Groups Zarr allows you to create hierarchical groups, similar to directories: ```python exec="true" session="quickstart" source="above" result="ansi" # Create nested groups and add arrays root = zarr.group("data/example-3.zarr", overwrite=True) foo = root.create_group(name="foo") bar = root.create_array( name="bar", shape=(100, 10), chunks=(10, 10), dtype="f4" ) spam = foo.create_array(name="spam", shape=(10,), dtype="i4") # Assign values bar[:, :] = np.random.random((100, 10)) spam[:] = np.arange(10) # print the hierarchy print(root.tree()) ``` This creates a group hierarchy with a group (`foo`) and two arrays (`bar` and `spam`). ### Batch Hierarchy Creation Zarr provides tools for creating a collection of arrays and groups with a single function call. Suppose we want to copy existing groups and arrays into a new storage backend: ```python exec="true" session="quickstart" source="above" result="code" # Create nested groups and add arrays root = zarr.group("data/example-4.zarr", attributes={'name': 'root'}, overwrite=True) foo = root.create_group(name="foo") bar = root.create_array( name="bar", shape=(100, 10), chunks=(10, 10), dtype="f4" ) nodes = {'': root.metadata} | {k: v.metadata for k,v in root.members()} # Report nodes output = io.StringIO() pprint(nodes, stream=output, width=60, depth=3) result = output.getvalue() print(result) # Create new hierarchy from nodes new_nodes = dict(zarr.create_hierarchy(store=zarr.storage.MemoryStore(), nodes=nodes)) new_root = new_nodes[''] assert new_root.attrs == root.attrs ``` Note that [`zarr.create_hierarchy`][] will only initialize arrays and groups -- copying array data must be done in a separate step. ## Persistent Storage Zarr supports persistent storage to disk or cloud-compatible backends. While examples above utilized a [`zarr.storage.LocalStore`][], a number of other storage options are available. A single-file store can also be created using the [`zarr.storage.ZipStore`][]: ```python exec="true" session="quickstart" source="above" # Store the array in a ZIP file store = zarr.storage.ZipStore("data/example-5.zip", mode="w") z = zarr.create_array( store=store, shape=(100, 100), chunks=(10, 10), dtype="f4" ) # write to the array z[:, :] = np.random.random((100, 100)) # the ZipStore must be explicitly closed store.close() ``` To open an existing array from a ZIP file: ```python exec="true" session="quickstart" source="above" result="ansi" # Open the ZipStore in read-only mode store = zarr.storage.ZipStore("data/example-5.zip", read_only=True) z = zarr.open_array(store, mode='r') # read the data as a NumPy Array print(z[:]) ``` Zarr also integrates seamlessly with cloud object storage such as Amazon S3 and Google Cloud Storage using external libraries like [s3fs](https://s3fs.readthedocs.io/en/latest/) or [gcsfs](https://gcsfs.readthedocs.io/en/latest/). Remote storage support requires the `remote` optional dependencies (`pip install "zarr[remote]"`) as well as a filesystem library for your storage service, such as `s3fs` for S3: ```python test="true" session="s3demo" markers="s3" source="above" import zarr import numpy as np z = zarr.create_array( "s3://example-bucket/foo", shape=(100, 100), chunks=(10, 10), dtype="f4" ) z[:, :] = np.random.random((100, 100)) ``` See the [Remote Store](user-guide/storage.md#remote-store) section of the storage guide for more detail, including how to configure the underlying filesystem with `storage_options`. ## Next steps This page only scratches the surface. Continue with the [User Guide](user-guide/index.md), in particular: - **[Arrays](user-guide/arrays.md)** - creating, reading, and writing arrays in depth - **[Groups](user-guide/groups.md)** - organizing arrays into hierarchies - **[Storage](user-guide/storage.md)** - the full range of local, remote, and in-memory storage options zarr-developers-zarr-python-a2f2415/docs/release-notes.md000066400000000000000000003603231527014137600235140ustar00rootroot00000000000000# Release notes ## 3.4.2 (2026-10-27) ### Features - Reading data encoded with the `pickle` codec now requires an explicit opt-in, because unpickling data from an untrusted store can execute arbitrary code ([GHSA-jqc8-qjfv-5m6r](https://github.com/zarr-developers/numcodecs/security/advisories/GHSA-jqc8-qjfv-5m6r)). Enable it with `zarr.config.set({"array.allow_unsafe_pickle_decode": True})` or the environment variable `ZARR_ARRAY__ALLOW_UNSAFE_PICKLE_DECODE=True`. Without the opt-in, reading such data raises `zarr.errors.PickleCodecDisabledError`. numcodecs has its own opt-in (`numcodecs.pickles.ALLOW_UNSAFE_PICKLE_DECODE`), which must also be enabled. Writing with the `pickle` codec is unchanged. ([#4465](https://github.com/zarr-developers/zarr-python/pull/4465)) ### Bugfixes - Resizing an array with `delete_outside_chunks=True` now generates only the out-of-bounds chunk coordinates and deletes them with a bounded pool of workers, instead of materializing the entire old and new grids and all deletion tasks. This bounds auxiliary memory during shrinking and mixed growing/shrinking resizes while preserving retained chunks. Growing, or shrinking within the last chunk of each axis, no longer schedules any deletion work. When a deletion fails, the remaining deletions are now canceled before the error is raised; previously they ran to completion. ([#4344](https://github.com/zarr-developers/zarr-python/pull/4344)) - `zarr.open_array` and `zarr.open` (sync and async) now apply the `config` argument when opening an existing array. Previously `config` was only used when a new array was created, so `zarr.open_array(store, config={"read_missing_chunks": False})` silently ran an existing array with the global defaults. `config` is now a named parameter of both functions and of `Array.open`/`AsyncArray.open`. `zarr.open` raises a `TypeError` naming the problem when `config` is given for a group. ([#4408](https://github.com/zarr-developers/zarr-python/pull/4408)) - Fixed several ways `ZipStore` could lose data or fail. A store created with `mode="w"` or `mode="x"` now keeps its entries when it is used again after `close()`, `move()`, or unpickling, instead of truncating the archive. Every method now opens the archive on first use, under a lock, so `get`, `get_partial_values`, `set_if_not_exists`, and `clear` no longer raise `AttributeError` on a store that was never opened, and threads sharing a store no longer open the archive twice. An unpickled store now opens its archive on first use rather than when it is unpickled, so writing a dask array to a `ZipStore` no longer produces an unreadable file. Where reopening an archive would silently drop its entries, the store now raises instead: `RuntimeError` after a failed `close()`, `zipfile.BadZipFile` if the file is not a complete zip archive, and `io.UnsupportedOperation` for a file object that is not readable and seekable. The type of the `mode` parameter now includes `"x"`. Builds on [#3593](https://github.com/zarr-developers/zarr-python/pull/3593) by [@oelhammouchi](https://github.com/oelhammouchi). ([#4450](https://github.com/zarr-developers/zarr-python/pull/4450)) - Reads through `FusedCodecPipeline` no longer hang in a process forked after an earlier read had started the codec thread pool. The pool is now reset in the child process after a fork. Previously the child inherited an executor whose threads did not exist, so small reads were queued on a pool nothing would run. ([#4483](https://github.com/zarr-developers/zarr-python/pull/4483)) - Structured data types now default string (`S`, `U`) and raw-bytes (`V`) fields to zero bytes. Before, an array with a raw-bytes field failed to create without an explicit fill value, and string fields defaulted to `"0"`. ([#4495](https://github.com/zarr-developers/zarr-python/pull/4495)) - The minimum supported numcodecs version is now 0.16.6, and 0.17.0 is excluded. Those versions leave `Pickle.decode` enabled by default, which allows arbitrary code execution when reading an untrusted Zarr v2 array ([GHSA-jqc8-qjfv-5m6r](https://github.com/zarr-developers/numcodecs/security/advisories/GHSA-jqc8-qjfv-5m6r)). ([#4465](https://github.com/zarr-developers/zarr-python/pull/4465)) - The `zarr.storage` module can now be serialized with `pickle` and `cloudpickle`. Previously, code that referenced the module (for example a Dask task that used `zarr.storage.ObjectStore`) failed with `TypeError: cannot pickle 'VerboseModule' object`. ([#4557](https://github.com/zarr-developers/zarr-python/pull/4557)) ### Improved Documentation - Document the removal of `LRUStoreCache` in the v3 migration guide, and point to the experimental `zarr.experimental.cache_store.CacheStore` as its replacement. ([#3850](https://github.com/zarr-developers/zarr-python/pull/3850)) - Added a deprecation policy to the contributing guide. It defines when a deprecation cycle is required for public API changes, a shared framework for deciding whether a removal is worthwhile, the mechanics a deprecation must follow (warning class, message contents, documentation, and changelog entry), and a minimum deprecation period of at least 6 months and at least one minor release before removal. ([#4051](https://github.com/zarr-developers/zarr-python/pull/4051)) - Clarify how Zarr format 3 codec pipeline roles map to `filters`, `serializer`, and `compressors` when creating arrays. ([#4166](https://github.com/zarr-developers/zarr-python/pull/4166)) - Correct the docstrings of `zarr.save`, `zarr.save_array` and `zarr.save_group` to say the functions write to any store. Previously the docstrings stated the functions wrote to only the local file system, which was inaccurate. ([#4520](https://github.com/zarr-developers/zarr-python/pull/4520)) ### Misc - [#4543](https://github.com/zarr-developers/zarr-python/pull/4543), [#4547](https://github.com/zarr-developers/zarr-python/pull/4547) ## 3.4.1 (2026-10-08) ### Bugfixes - Chunk sizes must now be at least 1, while an array extent may still be 0. On a zero-length axis, `chunks=-1`, `chunks=False`, `chunks="auto"`, `shards=-1` and `shards=False` all give chunk size 1, and a shard spanning an axis is rounded up to a multiple of the inner chunk size, so `shards=-1` works when the axis length is not a multiple of it. A rectilinear chunk grid can be declared on a zero-length dimension; its chunk sizes are kept for later growth. `FixedDimension(size=0, ...)` raises `ValueError`. `ArrayV2Metadata(chunks=(0,))` is still accepted and written as given, and is read back the way a stored 0 is read (below). `create_hierarchy` now builds every node before it deletes or stores anything, so a node that cannot be built leaves the store untouched. Regular chunk grid metadata reads a `bool` chunk size as the integer it equals and rejects a string or mapping as a chunk shape. `RectilinearChunkGridMetadata` (experimental) does the same for `bool` edges and rejects NumPy integers and floats such as `4.0` with `TypeError`; it used to store them as given, which produced JSON floats the specification does not allow. Stored metadata with a regular chunk size of 0 or JSON `false` (Zarr format 2 `chunks`, Zarr format 3 `regular` `chunk_shape`) is now read as chunk size 1, or the inner chunk size for sharded arrays, which previously could not be opened. The result does not depend on how far the axis has grown since. zarr-python wrote such sizes for arrays created with a zero-length axis until 3.4, and from 2.18.7 to 3.2.1 for an explicit `chunks=(0,)` or `False` on any axis; those arrays hold no data. On a zero-length axis the array opens silently, and appending stores chunks of size 1. On a longer axis it opens with a `ZarrUserWarning` that says the array holds only its fill value, how to recreate it with the wanted chunk shape (`zarr.from_array(array.store, name=array.path, data=array, chunks=..., overwrite=True, write_data=False)` keeps the data type, fill value, attributes and codecs), and how to keep it by storing valid metadata instead (`array.update_attributes({})`, then `zarr.consolidate_metadata` if the metadata is consolidated). JSON `true`, which zarr-python 3.0 and 3.2 wrote for `chunks=True`, is read as 1 without a warning, in regular and rectilinear chunk grids and in the inner chunk shape of any sharding codec. Rectilinear edges stored as integral JSON floats (`[[4.0, 2]]`), as 3.2 wrote them, are read as integers without a warning. A float anywhere else is rejected. Writing data to an array whose stored chunk size is 0 or `false` first re-reads the stored metadata and stores its repair, unless another writer has stored valid metadata since, so that other readers find the chunks written; in a `ZipStore` this adds a second entry for the metadata document, as every metadata update does. If the stored metadata now lays out chunks differently, the write raises `ValueError` asking to reopen the array and stores nothing. `update_attributes` and `resize` store the repair as before. No other operation does: a group's consolidated metadata keeps the array's metadata as it was stored until the array itself writes the repair. ([#4334](https://github.com/zarr-developers/zarr-python/pull/4334)) - Nested structured data types now round-trip through Zarr format 2 metadata. Converting a NumPy dtype with field titles or subarray fields raises `ValueError` naming the unsupported field, instead of producing duplicate or raw-byte fields; this is a limitation of the current zarr-python dtype implementation, not of the format. Padded layouts are still converted to packed layouts, now with a `ZarrUserWarning` that field values are preserved but offsets and itemsize may change. These checks apply to nested fields too. New `zdtypes` and `structured_dtypes` hypothesis strategies back property tests for dtype serialization and layout conversion. ([#4340](https://github.com/zarr-developers/zarr-python/pull/4340)) - `DateTime64` and `TimeDelta64` now normalize `μs` to `us` and keep generic-unit scale factors when converting NumPy dtypes and writing V2 and V3 metadata. V2 metadata spells a scaled generic unit explicitly, such as `[2generic]`, because NumPy's `dtype.str` drops the scale. Generic temporal arrays keep their dtype parameters when CPU buffers are allocated and convert byte order correctly, and generic datetime fill values can be read back, including in structured fields. The default NaT fill value carries the scale factor, so missing chunks of an array such as `m8[2us]` read back as NaT on NumPy < 2.2 instead of an arbitrary count. ([#4342](https://github.com/zarr-developers/zarr-python/pull/4342)) - `LocalStore` no longer blocks the event loop on filesystem calls. `open`, `clear`, `delete`, `delete_dir`, `list`, `list_prefix`, `list_dir`, `move` and `getsize` now run their disk I/O in a worker thread, as `get` and `set` already did. The `LocalStore` tests fail if any async store method touches the filesystem from the event loop thread. ([#4353](https://github.com/zarr-developers/zarr-python/pull/4353)) - Coordinate selections, including `vindex` masks, and orthogonal integer-array selections no longer allocate memory proportional to the total number of chunks, so selecting a few points from an array with billions of chunks no longer raises `MemoryError`. The indexers keep only the occupied chunk IDs and run-end offsets (`chunk_run_ends`); unsorted selections may still need a sort. The dense `chunk_nitems` and `chunk_nitems_cumsum` attributes of `CoordinateIndexer` and `IntArrayDimIndexer` are still available: each is computed on first access and cached, so its identity and in-place mutations persist, and `dataclasses.asdict()` still materializes them. `repr` omits them. Accessing one still allocates memory proportional to the full chunk grid. The new `zarr.core.common.ceildiv_int` computes chunk counts and slice bounds in exact integer arithmetic; `ceildiv` keeps its floating-point behaviour. ([#4356](https://github.com/zarr-developers/zarr-python/pull/4356)) - Runtime code no longer uses `assert`, and ruff's `S101` rule keeps it out. Asserts are stripped under `python -O`, and two were load-bearing: `GroupMetadata.from_dict` asserted on `node_type`, which the array-to-group fallback in `zarr.open` relied on, and `make_store` asserted on `mode`. Both now raise proper errors, `NodeTypeValidationError` and `ValueError`. ([#4363](https://github.com/zarr-developers/zarr-python/pull/4363)) - Arrays whose stored `regular` chunk grid mixes integers with lists of edge lengths, such as `"chunk_shape": [2, [5, 10, 5]]` (written by zarr 3.2.0 and 3.2.1 for `chunks=(2, (5, 10, 5))`, with float edges for float input), can be read again without enabling `array.rectilinear_chunks`. The grid is read as the rectilinear grid it describes, with a `ZarrUserWarning`. Re-saving the array's metadata, or writing chunks to it, stores that rectilinear grid and so requires the flag. A group's consolidated metadata keeps the array's metadata as stored. The `array.rectilinear_chunks` flag now gates reading and storing array metadata that declares a rectilinear chunk grid, including in consolidated metadata, rather than the construction of `RectilinearChunkGridMetadata`. An operation that would store such metadata with the flag off (creating an array, `Array.resize`, deleting a member of a group with consolidated metadata, `create_hierarchy`) raises before it deletes or writes anything, and the error names the array. ([#4375](https://github.com/zarr-developers/zarr-python/pull/4375)) - Chunk specifications are now judged by the chunk normalizer alone; the separate duck-typed check for rectilinear input is gone. - A chunk size is anything Python's integer protocol accepts, so NumPy integer scalars and 0-d integer arrays work wherever a chunk or shard shape is given, including `zarr.create(..., zarr_format=2)`, which no longer tests the truth value of a NumPy array. A non-integer scalar raises `TypeError`. - An explicit list of chunk edges always declares a rectilinear dimension, even when the edges are uniform, so the Zarr format 2 and sharding restrictions apply the same whether the grid is given as lists or as metadata, and `zarr.create` stores it as a rectilinear grid, as `create_array` does. - A `RectilinearChunkGridMetadata` made only of bare integers describes a regular grid and is accepted for Zarr format 2 and as the chunk shape of a sharded array. - The `chunks` and `chunk_shape` parameters of `zarr.create` and the `ShardsLike` type now declare every specification the normalizer accepts. ([#4376](https://github.com/zarr-developers/zarr-python/pull/4376)) - Zarr format 2 arrays with a `u1` data type can now be read as unsigned 8-bit integers. Serialized metadata continues to use the canonical `|u1` spelling. ([#4381](https://github.com/zarr-developers/zarr-python/pull/4381)) - Made the `fill_value` check on the write path substantially cheaper. `NDBuffer.all_equal` runs on every chunk write when `write_empty_chunks` is false, which is the default, and it dominated the cost of writing chunks. Two changes. The bit-pattern comparison used for a zero `fill_value`, which keeps `-0.0` distinct from `0.0`, now views the buffer as an unsigned integer of the same width instead of as a void dtype; NumPy has vectorised integer comparison loops but no void one, so the void form fell back to a generic elementwise path. Separately, the scan now walks slices of the leading axis, doubling in length, and returns on the first slice that differs, so a buffer that is not uniformly `fill_value` no longer has to be read in full. The leading axis is used rather than a flattened view because a chunk is normally a strided view into a larger array and cannot be flattened without a copy. A buffer that really is uniform is still covered in the same number of element comparisons. Results are unchanged, including for `-0.0`, NaN, and non-contiguous buffers. ([#4382](https://github.com/zarr-developers/zarr-python/pull/4382)) - Zarr format 2 arrays whose data type is a single-byte boolean or integer, or raw bytes, written with a `<` or `>` byte order (`b1`, `f8"` on a little-endian host. - Base64 fill values of structured data types in V3 metadata are read and written as little-endian bytes regardless of the in-memory byte order of the fields. - Tests no longer assume a little-endian host. ([#4435](https://github.com/zarr-developers/zarr-python/pull/4435)) - Fixed `FsspecStore.list_dir` and `FsspecStore.list_prefix` returning nothing below the root of a store whose path is `""` or `"/"`, such as a store opened from `"reference://"`. Nested groups in kerchunk-style references now list their members. ([#4452](https://github.com/zarr-developers/zarr-python/pull/4452)) - Fixed slow reads of 0-d arrays with large fixed-length string dtypes (e.g. `|S1000000`) caused by copying the chunk data multiple times. ([#4466](https://github.com/zarr-developers/zarr-python/pull/4466)) - A stored shard too short to hold its index, including an empty value, now raises `ValueError` reporting a truncated or corrupt shard on every read and partial-write path under either codec pipeline. Previously the asynchronous full-shard read and partial write treated an empty shard as missing, reading the fill value or overwriting it, while other paths failed with a checksum or decode error. zarrs and tensorstore reject such shards too. ([#4475](https://github.com/zarr-developers/zarr-python/pull/4475)) - A sharding codec nested inside another sharding codec is now validated like the outer one: its inner chunk shape must divide the chunk it encodes. Such an array used to be created and opened without error and then read back wrong data or the fill value. `ShardingCodec` also rejects an inner chunk size of 0 or `False` at construction with a `ValueError` naming the dimension, as other implementations do; it used to raise `ZeroDivisionError` when the array was created or opened, or on the first write when nested. ([#4480](https://github.com/zarr-developers/zarr-python/pull/4480)) ### Misc - [#4368](https://github.com/zarr-developers/zarr-python/pull/4368), [#4372](https://github.com/zarr-developers/zarr-python/pull/4372), [#4377](https://github.com/zarr-developers/zarr-python/pull/4377), [#4392](https://github.com/zarr-developers/zarr-python/pull/4392), [#4407](https://github.com/zarr-developers/zarr-python/pull/4407), [#4417](https://github.com/zarr-developers/zarr-python/pull/4417), [#4455](https://github.com/zarr-developers/zarr-python/pull/4455), [#4491](https://github.com/zarr-developers/zarr-python/pull/4491) ## 3.4.0 (2026-09-15) ### Features - JSON metadata validation now delegates to ``msgspec.convert`` for the type coercions it supports (``Literal`` membership, ``int`` / ``bool`` strictness, list-to-tuple), replacing the per-field hand-written ``parse_*`` logic. User-defined attributes retain their existing JSON handling. A latent generator-exhaustion bug in ``parse_storage_transformers`` is also fixed. See #3285. As a result some metadata inputs are now parsed more strictly. The previous per-field checks compared values with ``==``, which accepts any numerically equal object, so a float such as ``2.0`` was accepted as ``zarr_format``; it is now rejected because it is not an ``int``. Booleans are likewise no longer accepted where an ``int`` is expected, since ``bool`` is an ``int`` subclass. Metadata that conforms to the Zarr specification is unaffected. ([#4063](https://github.com/zarr-developers/zarr-python/pull/4063)) - `zarr.registry.get_codec_class` now raises `zarr.errors.UnknownCodecError` instead of `KeyError` when no implementation is registered for a codec, and `zarr.core.config.BadConfigError` instead of `KeyError` when the implementation named in `config["codecs"][name]` is not registered. `zarr.registry.get_numcodec` raises `UnknownCodecError` instead of the `ValueError` numcodecs raises for an unregistered Zarr format 2 codec id (`numcodecs.errors.UnknownCodecError` on numcodecs 0.15.1 and later). All of these are subclasses of `ValueError`, so `except ValueError` is unaffected, but `except KeyError` and `except numcodecs.errors.UnknownCodecError` are. These errors now name Python packages known to provide the codec, so that a user who cannot read an array learns what to install: ```text An implementation for codec 'wavpack' is not available. Register one explicitly using the codec registry (see ...), or install a Python package that registers a codec implementation with numcodecs. Known packages supporting this codec: wavpack-numcodecs. ``` The tables covering this live in `src/zarr/registry.py`, one per Zarr format, and include the codecs `numcodecs` gates behind its own optional dependencies (`zfpy`, `pcodec`, `crc32c`, `msgpack2`). Codec authors can add their published package to them. A codec whose `from_dict` raises `KeyError` on a malformed configuration now surfaces as `zarr.errors.MetadataValidationError` naming the codec and the missing key. Previously it was reported as `UnknownCodecError: Unknown codec: ''`, presenting a configuration key as though it were a codec name, and on the `zarr.open` path a bare `KeyError` could be swallowed by the array-then-group fallback and reported as an unrelated group error. `zarr.errors.UnknownCodecError` is now exported from `zarr.errors`. ([#4277](https://github.com/zarr-developers/zarr-python/pull/4277)) - `zarr.create_array`, `Group.create_array`, `zarr.from_array`, and the entry points built on them now accept a numpy array as the `chunks` or `shards` specification, alongside ints, tuples, and numpy integer scalars. This is new for that API: it has never accepted numpy arrays in any 3.x release, because each entry point compared the specification to the `"auto"` or `"keep"` sentinel string before normalizing it, and for a numpy array that comparison raised numpy's ambiguous-truth-value `ValueError`. Those sentinel checks are now guarded so array-like specifications reach the normalizer, bringing this API in line with the legacy `zarr.create` / `zarr.array` / `zarr.open_array` functions, which have accepted numpy arrays since 2.x. ([#4329](https://github.com/zarr-developers/zarr-python/pull/4329)) ### Bugfixes - Array creation is now O(1) in the number of chunks per dimension. Chunk normalization returns a `ChunkGrid` whose uniform dimensions are stored as a size + extent pair (`FixedDimension`) instead of being expanded to one entry per chunk, so creating arrays like `zarr.create_array(store, shape=(2**62,), chunks=(1,), dtype='int32')` succeeds instantly instead of raising `ValueError` or allocating gigabytes of memory. The intermediate `ChunksTuple` representation was removed in the process, and `ChunksLike` now admits per-dimension specs that mix a bare int (uniform chunk size) with explicit edge-length sequences, matching what the normalizer and the rectilinear grid spec already accepted. This fixes the array-creation half of #4174; the coordinate-selection allocation reported there is still tracked in that issue (#4172 fixed the related case of sorted 1-D coordinate selections). ([#4218](https://github.com/zarr-developers/zarr-python/pull/4218)) - Allow `Group.require_array` to accept a `ZDType` for `dtype`, matching the other array creation methods. Previously an existing array could only be required with a string or NumPy dtype. ([#4189](https://github.com/zarr-developers/zarr-python/pull/4189)) - Consolidated metadata is now reconstructed independently of the order the keys appear in on disk. Previously, sibling subtrees whose keys were not adjacent in the persisted mapping lost their children, which made nodes unreachable through consolidated metadata -- most visibly for sibling groups whose names differ only by case. ([#4227](https://github.com/zarr-developers/zarr-python/pull/4227)) - `FsspecStore.from_mapper` and `FsspecStore.from_url` no longer fail when converting a synchronous instance of an async-capable filesystem whose storage options contain objects that cannot be serialized to JSON (e.g. an `azure.identity.DefaultAzureCredential`). The async instance is now constructed from the original filesystem arguments instead of a JSON round-trip. ([#4239](https://github.com/zarr-developers/zarr-python/pull/4239)) - Numpy integers are accepted as chunk sizes again. Since 3.3.0 a per-dimension chunk size that was a numpy integer (e.g. `chunks=(np.int64(2), np.int64(2))`, as produced by any computed chunk shape) raised `TypeError: 'numpy.int64' object is not iterable`, because the scalar chunk path narrowed on `int` while its caller dispatched on `numbers.Integral`. The same regression had broken numpy arrays as chunk specifications through the legacy `zarr.create` / `zarr.array` / `zarr.open_array` functions, which accepted them in 2.x and 3.2.x; those work again. (`zarr.create_array` and the functions built on it gain numpy-array support separately, in #4329.) A chunk specification that is neither an integer nor iterable now reports the offending value instead of failing with an opaque iteration error. ([#4257](https://github.com/zarr-developers/zarr-python/pull/4257)) - The `cast_value` codec now requires `cast-value-rs>=0.4.2`. Earlier versions of that backend silently corrupted data when handed an array that was not row-major — the layout the `transpose` codec produces — so a `cast_value` codec next to a `transpose` codec would either write transposed values with no error or fail with `ValueError: Input array must be contiguous`. The minimum version is enforced at runtime as well as in the package metadata, so an environment that already has an older `cast-value-rs` installed now raises `ImportError` when the codec is used, instead of corrupting data. ([#4260](https://github.com/zarr-developers/zarr-python/pull/4260)) - Accept [universal-pathlib](https://github.com/fsspec/universal_pathlib) `UPath` objects wherever zarr accepts a `StoreLike` value. A remote `UPath` now creates an `FsspecStore` using the filesystem and storage options the `UPath` already carries, and a local `UPath` creates a `LocalStore`, so that `UPath('/data')` and `Path('/data')` behave the same. Previously this worked only by accident: in universal-pathlib < 0.3 every `UPath` subclassed `pathlib.Path` and implemented `__fspath__`, so remote paths were either converted to a URI string by the caller or wrapped in a `LocalStore` that happened to dispatch through fsspec. Since universal-pathlib 0.3 remote paths do neither, and passing one raised `TypeError: Unsupported type for store_like`. `FsspecStore.from_upath` also now converts the `UPath`'s filesystem to async mode, instead of raising `TypeError` for synchronous filesystems and warning for sync-mode instances of async ones. ([#4265](https://github.com/zarr-developers/zarr-python/pull/4265)) - Explicit per-chunk size lists now always produce a rectilinear chunk grid, even when the sizes happen to describe a regular grid (all equal, or all equal with a smaller trailing chunk). Previously such input was silently collapsed to a regular grid, which changed resize semantics: a regular grid grows by extending the uniform pattern, while a rectilinear grid appends a new edge chunk — the behavior an append-oriented layout like `(168,) * 13 + (24,)` relies on. The grid kind now follows the input syntax, matching 3.2.x: scalar chunk sizes (including numpy integers and the `-1` sentinel) produce a regular grid, nested sequences produce a rectilinear grid. Rectilinear grids remain gated behind `zarr.config.set({"array.rectilinear_chunks": True})`. See #4174 for the accompanying O(1) chunk normalization change. One consequence for users who never enable rectilinear chunks: because a nested sequence now always requests a rectilinear grid, a per-dimension sequence of edge lengths that happens to be uniform — for example the `((4,), (4,))` or `[[3, 3, 1]]` form that a dask array's `.chunks` attribute produces — is no longer quietly accepted as a regular grid when the `array.rectilinear_chunks` option is off. Such input raises `ValueError: Rectilinear chunk grids are experimental and disabled by default`, exactly as it did in 3.2.x; the silent acceptance existed only in 3.3.0. Pass one integer per dimension (e.g. `chunks=(4, 4)`, or a dask array's `.chunksize`) to request a regular grid. `zarr.from_array` with the default `chunks="keep"` / `shards="keep"` now reproduces the source's stored grid exactly: a rectilinear grid is passed through in O(number of dimensions), with uniform dimensions keeping their bare-int shorthand; sharding under a rectilinear shard grid is preserved instead of being silently dropped; and the default `write_data=True` copy works for every grid kind. `Array.chunks` is now defined for any sharded array (the inner chunks of a shard are always regular), and for sharded arrays with a rectilinear shard grid `Array.info` no longer raises — it reports the shard shape as `` — while `Array.nchunks_initialized` counts the chunks of each initialized shard individually instead of raising. Apart from the nested-sequence input form noted above, everything described here concerns rectilinear chunk grids, which remain an experimental feature gated behind `zarr.config.set({"array.rectilinear_chunks": True})`; arrays with regular chunk grids are unaffected. ([#4218](https://github.com/zarr-developers/zarr-python/pull/4218)) - A `scale_offset` codec configured with a string-valued zero scale is now rejected. `scale` accepts strings, and no string is ever equal to `0`, so `"0"`, `"0.0"` and the hex form `"0x0000000000000000"` skipped the "scale must be non-zero" check that the numeric `0` triggers. On float data types the array was created, every chunk was written as zero and read back as `nan` with no error, and the zero scale was persisted to the metadata so reopening the store reproduced it; on integer data types the codec raised `ZeroDivisionError` instead of `ValueError`. The check now runs on the parsed scalar rather than the value as supplied. ([#4279](https://github.com/zarr-developers/zarr-python/pull/4279)) - Fixed a `ValueError` when setting an orthogonal selection on a sharded array where more than one dimension is indexed by an array. The sharding codec re-derives an indexer from the chunk selection it is handed, which turns such a selection into a coordinate selection addressing the value buffer flat, so the write failed on a shape mismatch. Both partial-encode paths are fixed, so the write works under either codec pipeline. ([#4284](https://github.com/zarr-developers/zarr-python/pull/4284)) - Fixed integer array indexing with unsigned index dtypes. An unsorted index such as `np.array([3, 0], dtype="uint8")` spanning more than one chunk raised `IndexError`, because the order check used `np.diff`, which wraps on unsigned dtypes and misclassified a descending selection as increasing. Separately, a `uint64` index raised `IndexError` on both `array[...]` and `array.vindex[...]` — sorted or not — because `uint64` promotes to `float64` against a signed chunk offset. Index arrays are now cast to `intp`. Unsigned indices are bounds-checked before this conversion, so values such as `np.uint64(2**64 - 1)` are rejected rather than wrapping to a negative index and reading or overwriting an element at the end of the array. Negative-index normalization copies indices before modifying them, preserving caller-owned arrays and supporting read-only index arrays. Reusing one index array across axes of different lengths now normalizes each axis independently. ([#4286](https://github.com/zarr-developers/zarr-python/pull/4286)) - `zarr.from_array` now defaults to the fill value and the attributes of the source array. Previously both were silently discarded: the array was created with the data type's default scalar and no attributes. An explicit `fill_value=None` now selects the data type's default scalar (Zarr format 3) or a null fill value (Zarr format 2), consistently with `create_array`, and an empty `attributes` dict creates the array with no attributes. ([#4288](https://github.com/zarr-developers/zarr-python/pull/4288)) - Fixed an infinite loop when creating a 0-dimensional array with `shards="auto"` while the `array.target_shard_size_bytes` config option is set. Such arrays now resolve to `shards=()`, matching the behavior when no shard size target is configured. ([#4305](https://github.com/zarr-developers/zarr-python/pull/4305)) - Fixed `chunks=-1` on a zero-length axis resolving to an invalid chunk size of 0, which caused a `ValueError`, `ZeroDivisionError`, or infinite loop depending on the sharding configuration. Such axes now get chunk size 1, matching `chunks="auto"`. ([#4307](https://github.com/zarr-developers/zarr-python/pull/4307)) - Fixed a `ValueError` when setting an orthogonal selection on a sharded array that mixes an integer index with two or more array indices, such as `a.oindex[[3, 1, 2], 1, [0, 2]] = value`. The fix for the array-only case in #4284 reshaped the value only when its shape matched the coordinate selection exactly; the sharding codec now also ravels a value that is the selection's shape minus the integer-indexed axes. Values of any other rank are left alone, so a write that is invalid on an unsharded array fails the same way on a sharded one. Both partial-encode paths share one helper for this. ([#4316](https://github.com/zarr-developers/zarr-python/pull/4316)) - Opening a Zarr format 3 array whose codec's ``from_dict`` raised a ``KeyError`` with no arguments used to fail with an unrelated ``IndexError: tuple index out of range`` while formatting the error message. Because that ``IndexError`` is not a ``ValueError``, it also escaped the array-then-group fallback in ``zarr.open`` and broke group operations such as ``Group.members()`` and ``"child" in group`` when any child array used such a codec. The ``KeyError`` is now always reported as a ``MetadataValidationError`` naming the codec, with the offending key included only when the ``KeyError`` carried one. ([#4324](https://github.com/zarr-developers/zarr-python/pull/4324)) - `zarr.from_array` now deep-copies the source array's attributes instead of sharing nested dicts and lists between the source and the new array. Previously, mutating a nested attribute on the new array (for example ``dst.attrs["meta"]["tags"].append(...)``) silently changed the source array's in-memory attributes too. Deeply nested attributes can raise `RecursionError` during the copy even if they can be stored and reopened; the threshold depends on Python's recursion limit and call stack. Pass `attributes={}` to omit inherited attributes. ([#4325](https://github.com/zarr-developers/zarr-python/pull/4325)) - Fixed `chunks=False` on a zero-length axis resolving to a chunk size of 0, which raised a `ValueError` for Zarr format 3, raised a `ZeroDivisionError` with `shards="auto"`, and silently wrote invalid `chunks` metadata for Zarr format 2. `False` now takes the same path as `chunks=-1`, so such axes get chunk size 1, matching `chunks="auto"`. ([#4328](https://github.com/zarr-developers/zarr-python/pull/4328)) - `from_array` preserves a Zarr source's explicit data type instead of trying to infer it from its NumPy dtype, allowing variable-length bytes arrays to be copied. ([#4335](https://github.com/zarr-developers/zarr-python/pull/4335)) - The documentation build and the documentation test suite no longer delete a `data/` directory relative to the current working directory. Two executable docs sessions opened with `shutil.rmtree('data', ignore_errors=True)` to make their examples re-runnable; because executed docs blocks run in the process working directory rather than the docs tree, `mkdocs build -f /mkdocs.yml` or `pytest tests/test_docs.py` started from any directory containing a `data/` folder — a project checkout, or `/` — silently emptied it. The sdist ships `docs/` and `tests/` and `testpaths` collects `docs/user-guide`, so this reached anyone running the shipped test suite, not only contributors. The deletions are gone; the on-disk examples in the quick start, arrays, groups, storage and performance guides now create with `overwrite=True` (or `zarr.save_array(..., mode="w")`), which is also what a reader re-running an example needs, and a new docs test rejects any executed block that calls a filesystem deletion. ([#4339](https://github.com/zarr-developers/zarr-python/pull/4339)) - Missing Zarr format 3 imagecodecs now name `imagecodecs-zarr` as a known provider, alongside `virtual-tiff` where both packages register the codec. ([#4351](https://github.com/zarr-developers/zarr-python/pull/4351)) - `LocalStore` now retries the rename that publishes a written file when Windows reports the destination as transiently busy (`ERROR_ACCESS_DENIED` or `ERROR_SHARING_VIOLATION`). Replacing a name that was itself replaced moments earlier intermittently fails this way in a single process, which aborted otherwise ordinary writes. The retry is bounded to well under a second, is a no-op off Windows, and never retries the `FileExistsError` that the `exclusive` path uses to report an existing node. This mitigates but does not fully resolve #3522: a second process holding the destination open for longer than the retry budget will still fail. Zarr v2 had the equivalent retry from #698 and it was not carried over when atomic writes arrived in #3412. ([#4358](https://github.com/zarr-developers/zarr-python/pull/4358)) ### Improved Documentation - Added a Roadmap page to the documentation outlining future plans and intended changes to the library. ([#4149](https://github.com/zarr-developers/zarr-python/pull/4149)) - Converted remaining reStructuredText-style double-backtick markup to Markdown single backticks in the docstrings of `zarr.api.asynchronous`, `zarr.api.synchronous`, `zarr.core.array`, `zarr.registry`, and `zarr.storage._common`. No functional changes. ([#4193](https://github.com/zarr-developers/zarr-python/pull/4193)) - Document how to reassign Read the Docs version slugs when publishing a subpackage release. ([#4236](https://github.com/zarr-developers/zarr-python/pull/4236)) - Added a "Related Projects" page to the documentation listing the companion packages developed in this repository — `zarr-metadata` and `zarr-indexing` — and linked it from the landing page. Links to those packages now use the canonical `https://zarr.readthedocs.io/projects/...` URLs, and each companion package's documentation links back to the `zarr-python` docs. ([#4247](https://github.com/zarr-developers/zarr-python/pull/4247)) ### Misc - [#4213](https://github.com/zarr-developers/zarr-python/pull/4213), [#4261](https://github.com/zarr-developers/zarr-python/pull/4261), [#4326](https://github.com/zarr-developers/zarr-python/pull/4326), [#4331](https://github.com/zarr-developers/zarr-python/pull/4331) ## 3.3.0 (2026-07-30) ### Features - Optimizes reading multiple chunks from a shard. Serial calls to `Store.get()` in the sharding codec have been replaced with a single call to `Store.get_ranges()`, which coalesces nearby byte ranges and fetches them concurrently. ([#3004](https://github.com/zarr-developers/zarr-python/pull/3004)) - Added a `subchunk_write_order` option to `ShardingCodec` to control the physical order of subchunks within a shard. Supported values are `morton`, `unordered`, `lexicographic`, and `colexicographic`. `unordered` makes no guarantee about subchunk layout. This setting affects only on-disk layout, not the data read back, and is not persisted in array metadata: it applies per codec instance and is not recovered when reopening a sharded array. ([#3826](https://github.com/zarr-developers/zarr-python/pull/3826)) - Added `SyncByteGetter` and `SyncByteSetter` runtime-checkable protocols and a `get_ranges_sync` method on the `Store` ABC. These let custom byte getters/setters opt into the synchronous codec pipeline's fast path for in-memory IO, which the sharding codec uses for its inner chunks. ([#3885](https://github.com/zarr-developers/zarr-python/pull/3885)) - Added `FusedCodecPipeline`, an opt-in codec pipeline that runs codec compute synchronously and in bulk (avoiding the per-chunk async scheduling overhead of the default `BatchedCodecPipeline`), giving large speedups for sharded arrays. The default `BatchedCodecPipeline` is unchanged for standard configurations, so existing code keeps working unless you opt in; enable the new pipeline with `zarr.config.set({"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"})`. ([#3885](https://github.com/zarr-developers/zarr-python/pull/3885)) - Add `zarr.abc.store.Store.get_ranges` for concurrent, coalesced multi-range reads from a single key. The method is defined on the `Store` ABC with a default implementation built on `Store.get`, so every store inherits a working version; stores with native multi-range backends (e.g. `FsspecStore`) can override for efficiency. Coalescing knobs (`max_concurrency`, `max_gap_bytes`, `max_coalesced_bytes`) are passed as keyword arguments to `get_ranges`. Failures from underlying fetches surface as a `BaseExceptionGroup` (PEP 654); callers should use `except*` to filter for specific exception types such as `FileNotFoundError`. ([#3925](https://github.com/zarr-developers/zarr-python/pull/3925)) - Two new fields on `ArrayConfig` control how the sharding codec coalesces partial-shard reads: `sharding_coalesce_max_gap_bytes` (default 1 MiB) and `sharding_coalesce_max_bytes` (default 16 MiB). When reading multiple chunks from the same shard, nearby byte ranges are merged into a single request to the store if separated by no more than `sharding_coalesce_max_gap_bytes` and the merged read stays within `sharding_coalesce_max_bytes`. Defaults are seeded from the matching `array.sharding_coalesce_max_gap_bytes` / `array.sharding_coalesce_max_bytes` keys in [`zarr.config`][] at array-creation time, and can be overridden per array by passing `config={...}` to [`zarr.create_array`][]. ([#3987](https://github.com/zarr-developers/zarr-python/pull/3987)) - Added `Group.get_array`, `Group.get_group`, `AsyncGroup.get_array`, and `AsyncGroup.get_group`: type-safe accessors that return the child array or group at a given path, raising `ArrayNotFoundError` / `GroupNotFoundError` if no node exists there, and `ContainsGroupError` / `ContainsArrayError` if the node is not of the requested kind. Unlike `Group.__getitem__`, which returns `Array | Group`, these methods have precise return types. Nested paths like `"subgroup/subarray"` are supported. ([#4128](https://github.com/zarr-developers/zarr-python/pull/4128)) - `ZipStore` now accepts an open binary file-like object in place of a path, enabling zip archives on remote storage (e.g. a file opened with `fsspec` or an `obstore.ReadableFile`). Operations that require a filesystem location (`clear`, `move`) raise `NotImplementedError` for file-object-backed stores. ([#4187](https://github.com/zarr-developers/zarr-python/pull/4187)) ### Bugfixes - Stop emitting an `UnstableSpecificationWarning` when serializing the `struct` data type to Zarr V3 metadata. The `struct` data type now has a stable Zarr V3 specification. The legacy `structured` alias and the unspecified `null_terminated_bytes`, `raw_bytes`, and `variable_length_bytes` data types continue to warn. ([#4100](https://github.com/zarr-developers/zarr-python/pull/4100)) - Fix equality comparison of `ArrayV2Metadata` and `ArrayV3Metadata` objects with a `NaN` fill value. Such objects are now compared by their JSON-serialized form, so two otherwise-identical metadata objects with a `NaN` (or infinite) fill value compare equal. ([#2929](https://github.com/zarr-developers/zarr-python/issues/2929)) - Fixed `BytesCodec.from_dict` so that `BytesCodec` instances roundtrip to / from their dict representation. `BytesCodec.from_dict` now interprets a missing `endian` configuration as `endian=None` (matching what `BytesCodec.to_dict` emits), instead of falling back to the system's native byte order. ([#3417](https://github.com/zarr-developers/zarr-python/pull/3417)) - Fixed `save_array`, `Group.__setitem__`, and `load` for 0-dimensional arrays. ([#3469](https://github.com/zarr-developers/zarr-python/issues/3469)) - Fixed inner-codec spec evolution for sharded arrays. The sharding codec now threads the array spec through its inner codec chain when evolving codecs, so a codec that changes the dtype upstream of `BytesCodec` no longer leaves the inner chain evolved against the wrong spec (which previously failed at decode time). This runs on the default `BatchedCodecPipeline` as well. Standard inner chains (`[BytesCodec]`, `[BytesCodec, ZstdCodec]`, transpose + bytes) are byte-identical to before. Restores the behavior of #2179. ([#3885](https://github.com/zarr-developers/zarr-python/pull/3885)) - Make chunk normalization properly handle `-1` as a compact representation of the length of an entire axis. Reject several previously-accepted but ill-defined chunk specifications: `chunks=True` (previously silently produced size-1 chunks), chunk tuples shorter than the array's number of dimensions (previously padded to the array's shape), and `None` as a per-dimension chunk size. These all now raise informative errors. Also fix chunk handling for 0-length array dimensions, and add explicit rejection of 0-length chunks. ([#3899](https://github.com/zarr-developers/zarr-python/pull/3899)) - Handle missing consolidated metadata in leaf Group nodes. ([#3954](https://github.com/zarr-developers/zarr-python/issues/3954)) - Corrected the JSON type definitions for the `numpy.datetime64` and `numpy.timedelta64` data types in Zarr V3 metadata: the `configuration` object (holding `unit` and `scale_factor`) is now required, matching the published specifications for these data types. Also updated the specification links in the docstrings to point to the zarr-extensions repository. ([#3955](https://github.com/zarr-developers/zarr-python/pull/3955)) - Fixed writing to 0-dimensional arrays that use the sharding codec. Previously assigning to a 0-dimensional sharded array raised an error. ([#3966](https://github.com/zarr-developers/zarr-python/pull/3966)) - Fix flaky stateful test bookkeeping when `delete_dir` matches string prefixes instead of true directory descendants. Previously a path such as `6/faNT…` could be incorrectly removed when deleting `6/f`. (See [issue #3977](https://github.com/zarr-developers/zarr-python/issues/3977).) ([#3977](https://github.com/zarr-developers/zarr-python/issues/3977)) - `FsspecStore.close()` no longer closes the underlying fsspec filesystem or its network session. fsspec caches and shares filesystem instances across callers, so the store cannot know whether it is the only user, and closing a shared session would break other stores; the filesystem's lifecycle belongs to whoever created it. ([#4165](https://github.com/zarr-developers/zarr-python/pull/4165)) - Fixed an invalid `zarr.create_array` example in the quick-start documentation (it passed an unsupported `mode` argument) and made the cloud-storage example execute against a mock S3 backend in CI. Added a test ensuring every Python code block in the documentation is either executed or explicitly opted out with a documented reason, so an invalid example can no longer go untested. ([#4016](https://github.com/zarr-developers/zarr-python/issues/4016)) - Fixed `ObjectStore.list_dir` for object-store listings that include a directory-marker object matching the requested non-root prefix. ([#4032](https://github.com/zarr-developers/zarr-python/issues/4032)) - Prevents mutation of the attributes dict provided by the user by copying them instead of keeping the reference ([#4059](https://github.com/zarr-developers/zarr-python/issues/4059)) - Fixed several storage and codec bugs: - Reading a value with a `SuffixByteRequest` larger than the value now correctly returns the whole value (matching HTTP `bytes=-N` suffix-range semantics), instead of silently returning incorrect data for `MemoryStore`. - `LoggingStore.get_partial_values` and `FsspecStore.get_partial_values` no longer return empty results when `key_ranges` is passed as a one-shot iterable (e.g. a generator). - `Store.getsize_prefix` no longer over-counts sibling keys that merely share a string prefix (e.g. `getsize_prefix("foo")` no longer includes keys under `foobar/`). - `ZipStore.close()` no longer raises `AttributeError` when the store was created but never opened (including when used as a context manager without any I/O). - `codecs_from_list` now raises a descriptive `TypeError` when a `BytesBytesCodec` immediately follows an `ArrayArrayCodec`, instead of a misleading "Required ArrayBytesCodec was not found" `ValueError`. ([#4074](https://github.com/zarr-developers/zarr-python/pull/4074)) - Fixed writing Fortran-ordered (F-contiguous) arrays through the variable-length string and bytes codecs and through numcodecs array-array filters such as `Delta`, `FixedScaleOffset` and `PackBits`. Chunks are now passed to numcodecs as C-contiguous arrays, so elements are no longer stored in transposed order. ([#4116](https://github.com/zarr-developers/zarr-python/pull/4116)) - Fix silent byte-order corruption for structured dtypes with the `bytes` codec: multi-byte fields are now byte-swapped to the codec's configured `endian` on write and decoded honoring it on read, so non-native-endian structured data (e.g. big-endian fields, as produced by virtual references to external data) round-trips correctly. ([#4141](https://github.com/zarr-developers/zarr-python/issues/4141)) - Fix `zarr.api.asynchronous.open_like` so it can create a new array by default when the target path does not already exist. It now defaults to `mode="a"`; when using a read-only store to open an existing array, pass `mode="r"` explicitly. ([#3352](https://github.com/zarr-developers/zarr-python/pull/3352)) - `MemoryStore` now copies buffers as they are written, so it never retains the caller's memory. Previously an uncompressed write handed the store a zero-copy view of the user's array, and mutating that array afterwards would silently rewrite chunks already committed to the store. Only `MemoryStore` is affected: stores that serialize on write, such as `LocalStore` and `ZipStore`, never aliased the caller's memory. Uncompressed writes to a `MemoryStore` are correspondingly slower, since the copy that makes the stored data independent is now actually performed; compressed writes are unchanged. Buffers supplied through the `store_dict` argument remain the caller's responsibility and are stored as-is. ([#4157](https://github.com/zarr-developers/zarr-python/pull/4157)) - Fixed the opt-in `FusedCodecPipeline` for sharded arrays whose inner or index codec chain contains a codec implementing only the async codec interface (no `SupportsSyncCodec`). Such arrays previously raised `TypeError: All codecs must implement SupportsSyncCodec` on both read and write; the pipeline now declines its synchronous fast path for them and falls back to the async path, matching the behavior of the default `BatchedCodecPipeline`. Fully sync-capable codec chains keep the fast path unchanged. ([#4179](https://github.com/zarr-developers/zarr-python/pull/4179)) - Fixed `TypeError: unhashable type: 'writeable void-scalar'` when writing to sharded arrays whose fill value is a `np.void` scalar, e.g. arrays with a structured dtype. `ArraySpec` equality and hashing now compare the fill value by its byte representation rather than numeric equality. As a result, two specs with a `NaN` (or `NaT`) fill value now compare equal, while fill values of `-0.0` and `0.0` now compare unequal. This also restores the sharding codec's per-chunk spec cache, which had been disabled because of this bug. ([#4183](https://github.com/zarr-developers/zarr-python/pull/4183)) - `FusedCodecPipeline` no longer runs chunk IO and codec compute on the thread driving zarr's internal event loop. Previously each read/write executed its synchronous fast path inline on that loop thread, and because every sync-API call from every user thread is serviced by the same loop, concurrent operations serialized behind each other's codec work — reported as the fused pipeline being slower than `BatchedCodecPipeline` for zstd-compressed data under multi-threaded (e.g. dask) access. The synchronous batch now runs on a worker thread (one hop per batch, not per chunk), keeping the loop free. Multi-threaded single-chunk reads of compressed data now scale with reader threads; single-threaded performance is unchanged. ([#4194](https://github.com/zarr-developers/zarr-python/pull/4194)) - The end-to-end benchmarks no longer invoke `sudo` to drop the OS page cache during a regular `pytest` run. Cache clearing is now opt-in via the `ZARR_BENCHMARK_CLEAR_CACHE` environment variable, which the benchmark CI jobs set. ([#4199](https://github.com/zarr-developers/zarr-python/pull/4199)) - Fixed the opt-in `FusedCodecPipeline` for serializers that advertise the partial-decode/encode mixins with only the documented async partial methods: the partial dispatch previously asserted on the private `_decode_partial_sync`/`_encode_partial_sync` hooks (an `AssertionError`, or an `AttributeError` mid-IO under `python -O`); such codecs now take the full-chunk sync path. ([#4201](https://github.com/zarr-developers/zarr-python/pull/4201)) - Fixed `FusedCodecPipeline` (the opt-in synchronous pipeline) silently skipping array-array/bytes-bytes codecs placed outside a sharding serializer on its partial-decode/partial-encode fast paths. With an outer compressor (e.g. `compressors=[GzipCodec()]` around a `ShardingCodec` serializer), the fused pipeline wrote non-conforming stored bytes that `BatchedCodecPipeline` (and any other conforming reader) could not read, and could fail to read data that `BatchedCodecPipeline` had written. With an outer array-array codec (e.g. `TransposeCodec`), it silently returned wrong data in both directions with no error. Only the opt-in `FusedCodecPipeline` was affected; the default `BatchedCodecPipeline` was never impacted. ([#4202](https://github.com/zarr-developers/zarr-python/pull/4202)) - Fixed silent data corruption in the experimental `FusedCodecPipeline`: reordering or duplicating fancy-index reads (e.g. `arr[perm, :]`, `arr.oindex[[0, 0, 1], :]`) on uncompressed, crc-free sharded arrays could return the shard in natural order because the vectorized whole-shard decode accepted any selection whose output shape matched the shard shape. The bulk decode now fires only for identity full-shard reads, declines structured dtypes (whose byte-order handling it lacks), and requires shard-index offsets to exactly tile the data section, so corrupt indexes with overlapping or out-of-range offsets can no longer be served as array data. ([#4203](https://github.com/zarr-developers/zarr-python/pull/4203)) - `ManagedMemoryStore.get_sync`/`set_sync`/`delete_sync` now apply the store's `path` prefix, matching the async `get`/`set`/`delete` methods. Previously the sync methods were inherited unchanged from `MemoryStore` and used the raw key, so code that takes the sync fast path (e.g. `FusedCodecPipeline`) would read and write chunks outside the store's `path` prefix, silently returning fill values when the data was re-read through a fresh handle. `GpuMemoryStore.set_sync` now converts its value to a `gpu.Buffer`, matching `set`, so writes through the sync API keep the store's all-values-are-GPU invariant. Also fixed `ManagedMemoryStore.get_partial_values` applying its `path` prefix twice whenever `path` is non-empty, which made it always return `None` for every requested key. The shared store test suite (`zarr.testing.store.StoreTests`) gained sync/async parity checks — comparing sync and async observations of the same key on the same store instance, including with a `byte_range` — so every store subclass now exercises this invariant. The suite's former `test_get_bytes`/`test_get_json` methods (and their `_sync` variants) were folded into these parity tests and no longer exist as separate methods. ([#4204](https://github.com/zarr-developers/zarr-python/pull/4204)) - Fixed several small correctness issues from the codec-pipeline performance work: construction-time codec warnings (e.g. sharding's "disables partial reads" warning) no longer fire twice per array open — including for `FusedCodecPipeline`, which previously re-warned via its own codec-chain reconstruction and, on the async fallback path, on every decode/encode call; `concurrent_iter` now schedules its tasks eagerly, matching its documented contract; an invalid `codec_pipeline.max_workers` config/environment value now warns and falls back to the default instead of raising mid-read; and `FusedCodecPipeline`'s async fallback helpers now cancel already-spawned fetch/decode/write tasks instead of abandoning them in the background when one fails. ([#4205](https://github.com/zarr-developers/zarr-python/pull/4205)) - Fixed `FusedCodecPipeline`'s gating of its synchronous fast paths: stores exposing only part of the sync surface (e.g. `set_sync` without `get_sync`) now fall back cleanly to the async path instead of failing mid-write, and `WrapperStore` now forwards `get_sync`/`set_sync`/`delete_sync` to the wrapped store so wrapped sync-capable stores keep the fast path. The capability decision uses a private, interim convention (`zarr.abc.store._store_supports_sync_io`) rather than new public API, pending a formal sync/async store architecture. Also fixed `LatencyStore`: synchronous reads and writes now pay the configured latency, `get_ranges`/`get_partial_values` no longer bypass latency injection, and derived stores (e.g. from `with_read_only`) keep a stochastic `(loc, scale)` latency configuration instead of freezing a single sample. ([#4206](https://github.com/zarr-developers/zarr-python/pull/4206)) - `DefaultChunkKeyEncoding.decode_chunk_key` now validates that a chunk key starts with the configured `c` prefix and raises `ValueError` for malformed keys, instead of silently decoding them incorrectly. ([#4219](https://github.com/zarr-developers/zarr-python/pull/4219)) ### Improved Documentation - Document the changes to `zarr.errors` in the 3.0 migration guide, including the removal of v2 exception classes and the introduction of `NodeNotFoundError`. ([#3009](https://github.com/zarr-developers/zarr-python/issues/3009)) - Clarify the difference between `zarr.load` and `zarr.open` in their docstrings. `load` eagerly reads data into an in-memory array, while `open` returns a lazy `Array` or `Group` backed by the store, with `See Also` cross-references linking the two. ([31817c68](https://github.com/zarr-developers/zarr-python/commit/31817c68)) - Updated the custom dtype example in `examples/custom_dtype/custom_dtype.py` to use only the public API, eliminating all non-public imports, illustrating what users should do. To better support this, the following types and functions were made available from public modules: | Type/Function | Non-public module | Public module | | ------------------------- | ------------------------ | ------------- | | `DataTypeValidationError` | `zarr.core.dtype.common` | `zarr.errors` | | `JSON` | `zarr.core.common` | `zarr.types` | | `ZarrFormat` | `zarr.core.common` | `zarr.types` | | `DTypeConfig_V2` | `zarr.core.dtype.common` | `zarr.types` | | `DTypeJSON` | `zarr.core.dtype.common` | `zarr.types` | | `DTypeSpec_V2` | `zarr.core.dtype.common` | `zarr.dtype` | | `check_dtype_spec_v2` | `zarr.core.dtype.common` | `zarr.dtype` | `DataTypeValidationError` was *moved* to `zarr.errors`. Importing it from `zarr.core.dtype.common` (its original location), `zarr.core.dtype`, or `zarr.dtype` still works but now raises a `ZarrDeprecationWarning`. The remaining types and functions are simply re-exported from the listed public module. ([#4052](https://github.com/zarr-developers/zarr-python/pull/4052)) - Document a self-merge policy in the contributor guide, describing when a core developer may merge their own pull request without a second reviewer and which changes warrant more caution. ([#4053](https://github.com/zarr-developers/zarr-python/pull/4053)) - Fixed many documentation errors found in a full review of the user guide, including prose contradicted by rendered example output on the performance page, invisible code blocks, an incorrect S3 example, stale "not yet implemented" claims in the v3 migration guide, and undocumented optional dependency groups. Also improved navigation order, cross-linking between pages, and coverage of group member enumeration, bulk attribute updates, and the `use_consolidated` keyword. ([#4132](https://github.com/zarr-developers/zarr-python/pull/4132)) - Fixed the documented default of ``max_age_seconds`` in the ``CacheStore`` docstring: the default is ``"infinity"`` (no expiration), not ``None``, which is rejected. Also noted that ``cache_store`` must support deletes. ([#4133](https://github.com/zarr-developers/zarr-python/pull/4133)) - Added a blog section to the documentation, with a post covering two performance highlights of the 3.3.0 release: the opt-in `FusedCodecPipeline` and byte-range coalescing for partial reads of sharded arrays. Added two runnable examples that accompany the post: `examples/codec_pipeline_performance` compares the `BatchedCodecPipeline` and `FusedCodecPipeline` on a sharded array across two stores and two codec regimes, showing when the fused pipeline's thread pool helps and when it does not, and `examples/sharding_coalescing` demonstrates how read coalescing reduces the number of store requests when reading subregions of a sharded array. Also removed the hardware-specific speedup figures from the `FusedCodecPipeline` release note, since they depend on the array layout, codec, and machine. ([#4191](https://github.com/zarr-developers/zarr-python/pull/4191)) ### Deprecations and Removals - The ``BloscShuffle`` and ``BloscCname`` enums (``zarr.codecs.BloscShuffle``, ``zarr.codecs.BloscCname``) are now deprecated. Pass the equivalent literal string (e.g. ``"zstd"``, ``"bitshuffle"``) when constructing a ``BloscCodec``. The enum classes remain importable but emit ``DeprecationWarning`` on member access, and will be removed in a future release. They are no longer ``Enum`` subclasses: constructor calls (e.g. ``BloscCname("zstd")``), iteration, and ``.value`` access no longer work. ``BloscCodec.cname`` and ``BloscCodec.shuffle`` are now plain strings rather than enum members. Additional renames in ``zarr.codecs.blosc`` from the same change: the type aliases ``Shuffle`` and ``CName`` are now ``BloscShuffleLiteral`` and ``BloscCnameLiteral``, the constant ``SHUFFLE`` is now ``BLOSC_SHUFFLE`` (with a new ``BLOSC_CNAME`` alongside it), and ``BloscShuffle.from_int`` now returns a literal string rather than an enum member. ([#3963](https://github.com/zarr-developers/zarr-python/pull/3963)) - The ``Endian`` (``zarr.codecs.bytes.Endian``) and ``ShardingCodecIndexLocation`` (``zarr.codecs.ShardingCodecIndexLocation``) enums are now deprecated. Pass the equivalent literal string instead (e.g. ``"little"`` / ``"big"``, ``"start"`` / ``"end"``). The enum classes remain importable but emit ``DeprecationWarning`` on member access, and will be removed in a future release. ``BytesCodec.endian`` and ``ShardingCodec.index_location`` are now plain strings rather than enum members. Two follow-on changes from this deprecation: - ``NDBuffer.byteorder`` now returns a literal string (``"little"`` or ``"big"``) rather than an ``Endian`` member. Subclasses overriding this property should update their return type. - The module-level binding ``zarr.codecs.bytes.default_system_endian`` was removed. ``BytesCodec()`` continues to default to ``sys.byteorder``; external callers that imported ``default_system_endian`` should use ``sys.byteorder`` directly. Additionally, the module-level function ``zarr.codecs.sharding.parse_index_location`` was made private as part of this change. ([#3968](https://github.com/zarr-developers/zarr-python/pull/3968)) - Removed the NumPy 1.x implementation of the `VariableLengthUTF8` data type because NumPy 1.x is no longer supported under [SPEC0](https://scientific-python.org/specs/spec-0000/). ([#3973](https://github.com/zarr-developers/zarr-python/pull/3973)) ### Misc - [#4139](https://github.com/zarr-developers/zarr-python/pull/4139), [#4140](https://github.com/zarr-developers/zarr-python/pull/4140), [#3908](https://github.com/zarr-developers/zarr-python/pull/3908), [#3972](https://github.com/zarr-developers/zarr-python/pull/3972), [#3975](https://github.com/zarr-developers/zarr-python/pull/3975), [#3979](https://github.com/zarr-developers/zarr-python/pull/3979), [#3990](https://github.com/zarr-developers/zarr-python/pull/3990), [#3998](https://github.com/zarr-developers/zarr-python/pull/3998), [#4000](https://github.com/zarr-developers/zarr-python/pull/4000), [#4001](https://github.com/zarr-developers/zarr-python/pull/4001), [#4012](https://github.com/zarr-developers/zarr-python/pull/4012), [#4046](https://github.com/zarr-developers/zarr-python/pull/4046), [#4054](https://github.com/zarr-developers/zarr-python/pull/4054), [#4073](https://github.com/zarr-developers/zarr-python/issues/4073), [#4086](https://github.com/zarr-developers/zarr-python/issues/4086), [#4138](https://github.com/zarr-developers/zarr-python/pull/4138) - [#4172](https://github.com/zarr-developers/zarr-python/pull/4172) ## 3.2.1 (2026-05-05) ### Bugfixes - Fixed a `CastValue` validation bug where the "can we use an out-of-range mode" check inspected the source dtype instead of the target dtype. This meant arrays with a float source dtype and an integer target dtype incorrectly raised a `ValueError` when configured with a `wrap` out-of-range mode. ([#3938](https://github.com/zarr-developers/zarr-python/pull/3938)) - Fixed a bug where the codec pipeline evolved each codec against the original array spec instead of the spec produced by upstream array-to-array codecs. This caused failures whenever an upstream codec changed the dtype between codec boundaries — e.g. arrays using `CastValue` to convert a single-byte source dtype (`int8`) to a multi-byte target dtype (`int16`) raised a `ValueError` from `BytesCodec` about a missing `endian` configuration. ([#3941](https://github.com/zarr-developers/zarr-python/pull/3941)) - Fixed breakage in existing fsspec-dependent workflows caused by associating the "memory" URL scheme with instances of `ManagedMemoryStore` instead of fsspec's memory-backed store. After this change, store URLs with a "memory" scheme are handled differently when `fsspec` is installed: with `fsspec`, a `FsspecStore` backed by a `MemoryFileSystem` is used. Without `fsspec`, a `ManagedMemoryStore` is used. ([#3944](https://github.com/zarr-developers/zarr-python/pull/3944)) ## 3.2.0 (2026-04-30) ### Features - Adds a new in-memory storage backend called `ManagedMemoryStore`. Instances of `ManagedMemoryStore` function similarly to `MemoryStore`, but instances of `ManagedMemoryStore` can be constructed from a URL like `memory://store`. ([#3679](https://github.com/zarr-developers/zarr-python/pull/3679)) - Added `array.read_missing_chunks` configuration option. When set to `False`, reading missing chunks raises a `ChunkNotFoundError` instead of filling them with the array's fill value. ([#3748](https://github.com/zarr-developers/zarr-python/pull/3748)) - Added `Struct` class (subclass of `Structured`) implementing the zarr-extensions `struct` dtype spec. Uses object-style field format and dict fill values. Legacy `Structured` remains available for backward compatibility. ([#3781](https://github.com/zarr-developers/zarr-python/pull/3781)) - Add support for rectilinear (variable-sized) chunk grids. This feature is experimental and must be explicitly enabled via `zarr.config.set({'array.rectilinear_chunks': True})`. Rectilinear chunks can be used through: - **Creating arrays**: Pass nested sequences (e.g., `[[10, 20, 30], [50, 50]]`) to `chunks` in `zarr.create_array`, `zarr.from_array`, `zarr.zeros`, `zarr.ones`, `zarr.full`, `zarr.open`, and related functions, or to `chunk_shape` in `zarr.create`. - **Opening existing arrays**: Arrays stored with the `rectilinear` chunk grid are read transparently via `zarr.open` and `zarr.open_array`. - **Rectilinear sharding**: Shard boundaries can be rectilinear while inner chunks remain regular. **Breaking change**: The `validate` method on `BaseCodec` and `CodecPipeline` now receives a `ChunkGridMetadata` instance instead of a `ChunkGrid` instance for the `chunk_grid` parameter. Third-party codecs that override `validate` and inspect the chunk grid will need to update their type annotations. No known downstream packages were using this parameter. ([#3802](https://github.com/zarr-developers/zarr-python/pull/3802)) - Add `cast_value` and `scale_offset` codecs. ([#3874](https://github.com/zarr-developers/zarr-python/pull/3874)) ### Bugfixes - Fix `SyncError` raised when assigning a `zarr.Array` as the value in a `__setitem__` call (e.g. `dst[:] = src` where `src` is a zarr array). The source array is now converted to a NumPy array before entering the async codec pipeline. ([#3611](https://github.com/zarr-developers/zarr-python/issues/3611)) - Fix an issue that prevents the correct parsing of special NumPy `uint32` dtypes resulting e.g. from bit wise operations on `uint32` arrays on Windows. ([#3797](https://github.com/zarr-developers/zarr-python/pull/3797)) - Fix `ZipStore.list()`, `list_dir()`, and `exists()` to auto-open the zip file when called before `open()`, consistent with the existing behavior of `get()` and `set()`. ([#3846](https://github.com/zarr-developers/zarr-python/issues/3846)) - Fix handling of `NaT` default fill values for `datetime64` and `timedelta64` data types. Equality checks now use `numpy.isnat` so that the default fill value compares correctly against `NaT`. ([#3863](https://github.com/zarr-developers/zarr-python/pull/3863)) - Use the unit associated with the `Datetime64` data type when creating the default `Nat` scalar value. ([#3920](https://github.com/zarr-developers/zarr-python/pull/3920)) ### Improved Documentation - Document removal of `zarr.storage.init_group` in v3 migration guide, with replacement using `zarr.open_group`/`zarr.create_group`. ([#2720](https://github.com/zarr-developers/zarr-python/issues/2720)) - Document the `threading.max_workers` configuration option in the performance guide. ([#3492](https://github.com/zarr-developers/zarr-python/issues/3492)) - Corrects the type annotation reported for the `batch_info` parameter in the `CodecPipeline.write` method docstring. ([#3836](https://github.com/zarr-developers/zarr-python/pull/3836)) - Remove result="ansi" from code blocks in the user guide that were causing empty output cells in the rendered documentation. ([#3845](https://github.com/zarr-developers/zarr-python/pull/3845)) ### Deprecations and Removals - Remove deprecated `zarr.convenience` and `zarr.creation` modules. ([#3900](https://github.com/zarr-developers/zarr-python/pull/3900)) - Remove the deprecated `zarr_version` parameter from several functions and methods. That parameter is replaced with `zarr_format`. ([#3901](https://github.com/zarr-developers/zarr-python/pull/3901)) - Remove deprecated `Group` methods `array`, `require_dataset`, and `create_dataset`. ([#3902](https://github.com/zarr-developers/zarr-python/pull/3902)) - Remove deprecated `AsyncArray.create` and `Array.create` methods. ([#3903](https://github.com/zarr-developers/zarr-python/pull/3903)) ### Misc - [#3546](https://github.com/zarr-developers/zarr-python/issues/3546), [#3793](https://github.com/zarr-developers/zarr-python/pull/3793), [#3800](https://github.com/zarr-developers/zarr-python/pull/3800), [#3828](https://github.com/zarr-developers/zarr-python/pull/3828), [#3830](https://github.com/zarr-developers/zarr-python/pull/3830), [#3833](https://github.com/zarr-developers/zarr-python/pull/3833), [#3837](https://github.com/zarr-developers/zarr-python/pull/3837), [#3897](https://github.com/zarr-developers/zarr-python/pull/3897) ## 3.1.6 (2026-03-19) ### Features - Exposes the array runtime configuration as an attribute called `config` on the `Array` and `AsyncArray` classes. The previous `AsyncArray._config` attribute is now a deprecated alias for `AsyncArray.config`. ([#3668](https://github.com/zarr-developers/zarr-python/pull/3668)) - Adds a method for creating a new `Array` / `AsyncArray` instance with a new runtime configuration, and fixes inaccurate documentation about the `write_empty_chunks` configuration parameter. ([#3668](https://github.com/zarr-developers/zarr-python/pull/3668)) - Adds synchronous methods to stores that do not benefit from an async event loop. The shape of these methods is defined by protocol classes to support structural subtyping. ([#3725](https://github.com/zarr-developers/zarr-python/pull/3725)) - Fix near-miss penalty in `_morton_order` with hybrid ceiling+argsort strategy. ([#3718](https://github.com/zarr-developers/zarr-python/pull/3718)) ### Bugfixes - Correct the target bytes number for auto-chunking when auto-sharding. ([#3603](https://github.com/zarr-developers/zarr-python/pull/3603)) - Fixed a bug in the sharding codec that prevented nested shard reads in certain cases. ([#3655](https://github.com/zarr-developers/zarr-python/pull/3655)) - Fix obstore `_transform_list_dir` implementation to correctly relativize paths (removing `lstrip` usage). ([#3657](https://github.com/zarr-developers/zarr-python/pull/3657)) - Raise error when trying to encode `numpy.dtypes.StringDType` with `na_object` set. ([#3695](https://github.com/zarr-developers/zarr-python/pull/3695)) - `CacheStore`, `LoggingStore` and `LatencyStore` now support with_read_only. ([#3700](https://github.com/zarr-developers/zarr-python/pull/3700)) - Skip chunk coordinate enumeration in resize when the array is only growing, avoiding unbounded memory usage for large arrays. ([#3702](https://github.com/zarr-developers/zarr-python/pull/3702)) - Fix a performance bug in morton curve generation. ([#3705](https://github.com/zarr-developers/zarr-python/pull/3705)) - Add a dedicated in-memory cache for byte-range requests to the experimental `CacheStore`. ([#3710](https://github.com/zarr-developers/zarr-python/pull/3710)) - `BaseFloat._check_scalar` rejects invalid string values. ([#3586](https://github.com/zarr-developers/zarr-python/issues/3586)) - Apply drop_axes squeeze in partial decode path for sharding. ([#3763](https://github.com/zarr-developers/zarr-python/pull/3763)) - Set `copy=False` in reshape operation. ([#3649](https://github.com/zarr-developers/zarr-python/pull/3649)) - Validate that dask-style chunks have regular shapes. ([#3779](https://github.com/zarr-developers/zarr-python/pull/3779)) ### Improved Documentation - Add documentation example for creating uncompressed arrays in the Compression section of the user guide. ([#3464](https://github.com/zarr-developers/zarr-python/issues/3464)) - Add AI-assisted code policy to the contributing guide. ([#3769](https://github.com/zarr-developers/zarr-python/pull/3769)) - Added a glossary. ([#3767](https://github.com/zarr-developers/zarr-python/pull/3767)) ### Misc - [#3562](https://github.com/zarr-developers/zarr-python/pull/3562), [#3605](https://github.com/zarr-developers/zarr-python/pull/3605), [#3619](https://github.com/zarr-developers/zarr-python/pull/3619), [#3623](https://github.com/zarr-developers/zarr-python/pull/3623), [#3636](https://github.com/zarr-developers/zarr-python/pull/3636), [#3648](https://github.com/zarr-developers/zarr-python/pull/3648), [#3656](https://github.com/zarr-developers/zarr-python/pull/3656), [#3658](https://github.com/zarr-developers/zarr-python/pull/3658), [#3673](https://github.com/zarr-developers/zarr-python/pull/3673), [#3704](https://github.com/zarr-developers/zarr-python/pull/3704), [#3706](https://github.com/zarr-developers/zarr-python/pull/3706), [#3708](https://github.com/zarr-developers/zarr-python/pull/3708), [#3712](https://github.com/zarr-developers/zarr-python/pull/3712), [#3713](https://github.com/zarr-developers/zarr-python/pull/3713), [#3717](https://github.com/zarr-developers/zarr-python/pull/3717), [#3721](https://github.com/zarr-developers/zarr-python/pull/3721), [#3728](https://github.com/zarr-developers/zarr-python/pull/3728), [#3778](https://github.com/zarr-developers/zarr-python/pull/3778) ## 3.1.5 (2025-11-21) ### Bugfixes - Fix formatting errors in the release notes section of the docs. ([#3594](https://github.com/zarr-developers/zarr-python/pull/3594)) ## 3.1.4 (2025-11-20) ### Features - The `Array` class can now also be parametrized in the same manner as the `AsyncArray` class, allowing Zarr format v2 and v3 `Array`s to be distinguished. New types have been added to `zarr.types` to help with this. ([#3304](https://github.com/zarr-developers/zarr-python/pull/3304)) - Adds `zarr.experimental.cache_store.CacheStore`, a `Store` that implements caching by combining two other `Store` instances. See the [docs page](https://zarr.readthedocs.io/en/latest/user-guide/experimental#cachestore) for more information about this feature. ([#3366](https://github.com/zarr-developers/zarr-python/pull/3366)) - Adds a `zarr.experimental` module for unstable user-facing features. ([#3490](https://github.com/zarr-developers/zarr-python/pull/3490)) - Add a `array.target_shard_size_bytes` to [`zarr.config`][] to allow users to set a maximum number of bytes per-shard when `shards="auto"` in, for example, [`zarr.create_array`][]. ([#3547](https://github.com/zarr-developers/zarr-python/pull/3547)) - Make `async_array` on the [`zarr.Array`][] class public (`_async_array` will remain untouched, but its stability is not guaranteed). ([#3556](https://github.com/zarr-developers/zarr-python/pull/3556)) ### Bugfixes - Fix a bug that prevented `PCodec` from being properly resolved when loading arrays using that compressor. ([#3483](https://github.com/zarr-developers/zarr-python/pull/3483)) - Fixed a bug that prevented Zarr Python from opening Zarr V3 array metadata documents that contained extra keys with permissible values (dicts with a `"must_understand"` key set to `"false"`). ([#3530](https://github.com/zarr-developers/zarr-python/pull/3530)) - Fixed a bug where the `"consolidated_metadata"` key was written to metadata documents even when consolidated metadata was not used, resulting in invalid metadata documents. ([#3535](https://github.com/zarr-developers/zarr-python/pull/3535)) - Improve write performance to large shards by up to 10x. ([#3560](https://github.com/zarr-developers/zarr-python/issues/3560)) ### Improved Documentation - Use mkdocs-material for Zarr-Python documentation ([#3118](https://github.com/zarr-developers/zarr-python/pull/3118)) - Document different values of StoreLike with examples in the user guide. ([#3303](https://github.com/zarr-developers/zarr-python/issues/3303)) - Reorganize the top-level `examples` directory to give each example its own sub-directory. Adds content to the docs for each example. ([#3502](https://github.com/zarr-developers/zarr-python/pull/3502)) - Updated 3.0 Migration Guide to include function signature change to zarr.Array.resize function. ([#3536](https://github.com/zarr-developers/zarr-python/issues/3536)) ### Misc - [#3515](https://github.com/zarr-developers/zarr-python/pull/3515), [#3532](https://github.com/zarr-developers/zarr-python/pull/3532), [#3533](https://github.com/zarr-developers/zarr-python/pull/3533), [#3553](https://github.com/zarr-developers/zarr-python/pull/3553) ## 3.1.3 (2025-09-18) ### Features - Add a command-line interface to migrate v2 Zarr metadata to v3. Corresponding functions are also provided under zarr.metadata. ([#1798](https://github.com/zarr-developers/zarr-python/issues/1798)) - Add obstore implementation of delete_dir. ([#3310](https://github.com/zarr-developers/zarr-python/pull/3310)) - Adds a registry for chunk key encodings for extensibility. This allows users to implement a custom `ChunkKeyEncoding`, which can be registered via `register_chunk_key_encoding` or as an entry point under `zarr.chunk_key_encoding`. ([#3436](https://github.com/zarr-developers/zarr-python/pull/3436)) - Trying to open a group at a path where an array already exists now raises a helpful error. ([#3444](https://github.com/zarr-developers/zarr-python/pull/3444)) ### Bugfixes - Prevents creation of groups (.create_group) or arrays (.create_array) as children of an existing array. ([#2582](https://github.com/zarr-developers/zarr-python/issues/2582)) - Fix a bug preventing `ones_like`, `full_like`, `empty_like`, `zeros_like` and `open_like` functions from accepting an explicit specification of array attributes like shape, dtype, chunks etc. The functions `full_like`, `empty_like`, and `open_like` now also more consistently infer a `fill_value` parameter from the provided array. ([#2992](https://github.com/zarr-developers/zarr-python/pull/2992)) - LocalStore now uses atomic writes, which should prevent some cases of corrupted data. ([#3411](https://github.com/zarr-developers/zarr-python/issues/3411)) - Fix a potential race condition when using `zarr.create_array` with the `data` parameter set to a NumPy array. Previously Zarr was iterating over the newly created array with a granularity that was too low. Now Zarr chooses a granularity that matches the size of the stored objects for that array. ([#3422](https://github.com/zarr-developers/zarr-python/pull/3422)) - Fix ChunkGrid definition (broken in 3.1.2) ([#3425](https://github.com/zarr-developers/zarr-python/pull/3425)) - Ensure syntax like `root['/subgroup']` works equivalently to `root['subgroup']` when using consolidated metadata. ([#3428](https://github.com/zarr-developers/zarr-python/pull/3428)) - Creating a new group with `zarr.group` no longer errors. This fixes a regression introduced in version 3.1.2. ([#3431](https://github.com/zarr-developers/zarr-python/pull/3431)) - Setting `fill_value` to a float like `0.0` when the data type of the array is an integer is a common mistake. This change lets Zarr Python read arrays with this erroneous metadata, although Zarr Python will not create such arrays. ([#3448](https://github.com/zarr-developers/zarr-python/pull/3448)) ### Deprecations and Removals - The `Store.set_partial_writes` method, which was not used by Zarr-Python, has been removed. `store.supports_partial_writes` is now always `False`. ([#2859](https://github.com/zarr-developers/zarr-python/issues/2859)) ### Misc - [#3376](https://github.com/zarr-developers/zarr-python/pull/3376), [#3390](https://github.com/zarr-developers/zarr-python/pull/3390), [#3403](https://github.com/zarr-developers/zarr-python/pull/3403), [#3449](https://github.com/zarr-developers/zarr-python/pull/3449) ## 3.1.2 (2025-08-25) ### Features - Added support for async vectorized and orthogonal indexing. ([#3083](https://github.com/zarr-developers/zarr-python/pull/3083)) - Make config param optional in init_array ([#3391](https://github.com/zarr-developers/zarr-python/pull/3391)) ### Bugfixes - Ensure that -0.0 is not considered equal to 0.0 when checking if all the values in a chunk are equal to an array's fill value. ([#3144](https://github.com/zarr-developers/zarr-python/issues/3144)) - Fix a bug in `create_array` caused by iterating over chunk-aligned regions instead of shard-aligned regions when writing data. Additionally, the behavior of `nchunks_initialized` has been adjusted. This function consistently reports the number of chunks present in stored objects, even when the array uses the sharding codec. ([#3299](https://github.com/zarr-developers/zarr-python/pull/3299)) - Opening an array or group with `mode="r+"` will no longer create new arrays or groups. ([#3307](https://github.com/zarr-developers/zarr-python/pull/3307)) - Added `zarr.errors.ArrayNotFoundError`, which is raised when attempting to open a zarr array that does not exist, and `zarr.errors.NodeNotFoundError`, which is raised when failing to open an array or a group in a context where either an array or a group was expected. ([#3367](https://github.com/zarr-developers/zarr-python/pull/3367)) - Ensure passing `config` is handled properly when `open`ing an existing array. ([#3378](https://github.com/zarr-developers/zarr-python/pull/3378)) - Raise a Zarr-specific error class when a codec can't be found by name when deserializing the given codecs. This avoids hiding this error behind a "not part of a zarr hierarchy" warning. ([#3395](https://github.com/zarr-developers/zarr-python/pull/3395)) ### Misc - [#3098](https://github.com/zarr-developers/zarr-python/pull/3098), [#3288](https://github.com/zarr-developers/zarr-python/pull/3288), [#3318](https://github.com/zarr-developers/zarr-python/pull/3318), [#3368](https://github.com/zarr-developers/zarr-python/issues/3368), [#3371](https://github.com/zarr-developers/zarr-python/pull/3371), [#3372](https://github.com/zarr-developers/zarr-python/pull/3372), [#3374](https://github.com/zarr-developers/zarr-python/pull/3374) ## 3.1.1 (2025-07-28) ### Features - Add lightweight implementations of `.getsize()` and `.getsize_prefix()` for ObjectStore. ([#3227](https://github.com/zarr-developers/zarr-python/pull/3227)) ### Bugfixes - Creating a Zarr format 2 array with the `order` keyword argument no longer raises a warning. ([#3112](https://github.com/zarr-developers/zarr-python/pull/3112)) - Fixed the error message when passing both `config` and `write_empty_chunks` arguments to reflect the current behaviour (`write_empty_chunks` takes precedence). ([#3112](https://github.com/zarr-developers/zarr-python/pull/3112)) - Creating a Zarr format 3 array with the `order` argument now consistently ignores this argument and raises a warning. ([#3112](https://github.com/zarr-developers/zarr-python/pull/3112)) - When using [`from_array`][zarr.api.asynchronous.from_array] to copy a Zarr format 2 array to a Zarr format 3 array, if the memory order of the input array is `"F"` a warning is raised and the order ignored. This is because Zarr format 3 arrays are always stored in "C" order. ([#3112](https://github.com/zarr-developers/zarr-python/pull/3112)) - The `config` argument to [`zarr.create`][zarr.create] (and functions that create arrays) is now used - previously it had no effect. ([#3112](https://github.com/zarr-developers/zarr-python/pull/3112)) - Ensure that all abstract methods of [`ZDType`][zarr.core.dtype.ZDType] raise a `NotImplementedError` when invoked. ([#3251](https://github.com/zarr-developers/zarr-python/pull/3251)) - Register 'gpu' marker with pytest for downstream StoreTests. ([#3258](https://github.com/zarr-developers/zarr-python/pull/3258)) - Expand the range of types accepted by `parse_data_type` to include strings and Sequences. - Move the functionality of `zarr.core.dtype.parse_data_type` to a new function called `zarr.dtype.parse_dtype`. This change ensures that nomenclature is consistent across the codebase. `zarr.core.dtype.parse_data_type` remains, so this change is not breaking. ([#3264](https://github.com/zarr-developers/zarr-python/pull/3264)) - Fix a regression introduced in 3.1.0 that prevented `inf`, `-inf`, and `nan` values from being stored in `attributes`. ([#3280](https://github.com/zarr-developers/zarr-python/pull/3280)) - Fixes [`Group.nmembers()`][zarr.Group.nmembers] ignoring depth when using consolidated metadata. ([#3287](https://github.com/zarr-developers/zarr-python/pull/3287)) ### Improved Documentation - Expand the data type docs to include a demonstration of the `parse_data_type` function. Expand the docstring for the `parse_data_type` function. ([#3249](https://github.com/zarr-developers/zarr-python/pull/3249)) - Add a section on codecs to the migration guide. ([#3273](https://github.com/zarr-developers/zarr-python/pull/3273)) ### Misc - Remove warnings about vlen-utf8 and vlen-bytes codecs ([#3268](https://github.com/zarr-developers/zarr-python/pull/3268)) ## 3.1.0 (2025-07-14) ### Features - Ensure that invocations of `create_array` use consistent keyword arguments, with consistent defaults. [`zarr.api.synchronous.create_array`][] now takes a `write_data` keyword argument The `Group.create_array` method takes `data` and `write_data` keyword arguments. The functions [`zarr.api.asynchronous.create`][], [`zarr.api.asynchronous.create_array`] and the methods `Group.create_array`, `Group.array`, had the default `fill_value` changed from `0` to the `DEFAULT_FILL_VALUE` value, which instructs Zarr to use the default scalar value associated with the array's data type as the fill value. These are all functions or methods for array creation that mirror, wrap or are wrapped by, another function that already has a default `fill_value` set to `DEFAULT_FILL_VALUE`. This change is necessary to make these functions consistent across the entire codebase, but as this changes default values, new data might have a different fill value than expected after this change. For data types where 0 is meaningful, like integers or floats, the default scalar is 0, so this change should not be noticeable. For data types where 0 is ambiguous, like fixed-length unicode strings, the default fill value might be different after this change. Users who were relying on how Zarr interpreted `0` as a non-numeric scalar value should set their desired fill value explicitly after this change. - Added public API for Buffer ABCs and implementations. Use `zarr.buffer` to access buffer implementations, and `zarr.abc.buffer` for the interface to implement new buffer types. Users previously importing buffer from `zarr.core.buffer` should update their imports to use `zarr.buffer`. As a reminder, all of `zarr.core` is considered a private API that's not covered by zarr-python's versioning policy. ([#2871](https://github.com/zarr-developers/zarr-python/issues/2871)) - Adds zarr-specific data type classes. This change adds a `ZDType` base class for Zarr V2 and Zarr V3 data types. Child classes are defined for each NumPy data type. Each child class defines routines for `JSON` serialization. New data types can be created and registered dynamically. Prior to this change, Zarr Python had two streams for handling data types. For Zarr V2 arrays, we used NumPy data type identifiers. For Zarr V3 arrays, we used a fixed set of string enums. Both of these systems proved hard to extend. This change is largely internal, but it does change the type of the `dtype` and `data_type` fields on the `ArrayV2Metadata` and `ArrayV3Metadata` classes. Previously, `ArrayV2Metadata.dtype` was a NumPy `dtype` object, and `ArrayV3Metadata.data_type` was an internally-defined `enum`. After this change, both `ArrayV2Metadata.dtype` and `ArrayV3Metadata.data_type` are instances of `ZDType`. A NumPy data type can be generated from a `ZDType` via the `ZDType.to_native_dtype()` method. The internally-defined Zarr V3 `enum` class is gone entirely, but the `ZDType.to_json(zarr_format=3)` method can be used to generate either a string, or dictionary that has a string `name` field, that represents the string value previously associated with that `enum`. For more on this new feature, see the [documentation](user-guide/data_types.md) ([#2874](https://github.com/zarr-developers/zarr-python/pull/2874)) - Added `NDBuffer.empty` method for faster ndbuffer initialization. ([#3191](https://github.com/zarr-developers/zarr-python/pull/3191)) - The minimum version of NumPy has increased to 1.26. ([#3226](https://github.com/zarr-developers/zarr-python/pull/3226)) - Add an alternate `from_array_metadata_and_store` constructor to `CodecPipeline`. ([#3233](https://github.com/zarr-developers/zarr-python/pull/3233)) ### Bugfixes - Fixes a variety of issues related to string data types. - Brings the `VariableLengthUTF8` data type Zarr V3 identifier in alignment with Zarr Python 3.0.8 - Disallows creation of 0-length fixed-length data types - Adds a regression test for the `VariableLengthUTF8` data type that checks against version 3.0.8 - Allows users to request the `VariableLengthUTF8` data type with `str`, `"str"`, or `"string"`. ([#3170](https://github.com/zarr-developers/zarr-python/pull/3170)) - Add human readable size for No. bytes stored to `info_complete` ([#3190](https://github.com/zarr-developers/zarr-python/pull/3190)) - Restores the ability to create a Zarr V2 array with a `null` fill value by introducing a new class `DefaultFillValue`, and setting the default value of the `fill_value` parameter in array creation routines to an instance of `DefaultFillValue`. For Zarr V3 arrays, `None` will act as an alias for a `DefaultFillValue` instance, thus preserving compatibility with existing code. ([#3198](https://github.com/zarr-developers/zarr-python/pull/3198)) - Fix the type of `ArrayV2Metadata.codec` to constrain it to `numcodecs.abc.Codec | None`. Previously the type was more permissive, allowing objects that can be parsed into Codecs (e.g., the codec name). The constructor of `ArrayV2Metadata` still allows the permissive input when creating new objects. ([#3232](https://github.com/zarr-developers/zarr-python/pull/3232)) ### Improved Documentation - Add a self-contained example of data type extension to the `examples` directory, and expanded the documentation for data types. ([#3157](https://github.com/zarr-developers/zarr-python/pull/3157)) - Add a description on how to create a RemoteStore of a specific filesystem to the `Remote Store` section in `docs/user-guide/storage.md`. State in the docstring of `FsspecStore.from_url` that the filesystem type is inferred from the URL scheme. It should help a user handling the case when the type of FsspecStore doesn't match the URL scheme. ([#3212](https://github.com/zarr-developers/zarr-python/pull/3212)) ### Deprecations and Removals - Removes default chunk encoding settings (filters, serializer, compressors) from the global configuration object. This removal is justified on the basis that storing chunk encoding settings in the config required a brittle, confusing, and inaccurate categorization of array data types, which was particularly unsuitable after the recent addition of new data types that didn't fit naturally into the pre-existing categories. The default chunk encoding is the same (Zstandard compression, and the required object codecs for variable length data types), but the chunk encoding is now generated by functions that cannot be reconfigured at runtime. Users who relied on setting the default chunk encoding via the global configuration object should instead specify the desired chunk encoding explicitly when creating an array. This change also adds an extra validation step to the creation of Zarr V2 arrays, which ensures that arrays with a `VariableLengthUTF8` or `VariableLengthBytes` data type cannot be created without the correct "object codec". ([#3228](https://github.com/zarr-developers/zarr-python/pull/3228)) - Removes support for passing keyword-only arguments positionally to the following functions and methods: `save_array`, `open`, `group`, `open_group`, `create`, `get_basic_selection`, `set_basic_selection`, `get_orthogonal_selection`, `set_orthogonal_selection`, `get_mask_selection`, `set_mask_selection`, `get_coordinate_selection`, `set_coordinate_selection`, `get_block_selection`, `set_block_selection`, `Group.create_array`, `Group.empty`, `Group.zeroes`, `Group.ones`, `Group.empty_like`, `Group.full`, `Group.zeros_like`, `Group.ones_like`, `Group.full_like`, `Group.array`. Prior to this change, passing a keyword-only argument positionally to one of these functions or methods would raise a deprecation warning. That warning is now gone. Passing keyword-only arguments to these functions and methods positionally is now an error. ## 3.0.10 (2025-07-03) ### Bugfixes - Removed an unnecessary check from `_fsspec._make_async` that would raise an exception when creating a read-only store backed by a local file system with `auto_mkdir` set to `False`. ([#3193](https://github.com/zarr-developers/zarr-python/pull/3193)) - Add missing import for AsyncFileSystemWrapper for _make_async in _fsspec.py ([#3195](https://github.com/zarr-developers/zarr-python/pull/3195)) ## 3.0.9 (2025-06-30) ### Features - Add `zarr.storage.FsspecStore.from_mapper()` so that `zarr.open()` supports stores of type `fsspec.mapping.FSMap`. ([#2774](https://github.com/zarr-developers/zarr-python/pull/2774)) - Implemented `move` for `LocalStore` and `ZipStore`. This allows users to move the store to a different root path. ([#3021](https://github.com/zarr-developers/zarr-python/pull/3021)) - Added `zarr.errors.GroupNotFoundError`, which is raised when attempting to open a group that does not exist. ([#3066](https://github.com/zarr-developers/zarr-python/pull/3066)) - Adds `fill_value` to the list of attributes displayed in the output of the `AsyncArray.info()` method. ([#3081](https://github.com/zarr-developers/zarr-python/pull/3081)) - Use `numpy.zeros` instead of `np.full` for a performance speedup when creating a `zarr.core.buffer.NDBuffer` with `fill_value=0`. ([#3082](https://github.com/zarr-developers/zarr-python/pull/3082)) - Port more stateful testing actions from [Icechunk](https://icechunk.io/en/stable/). ([#3130](https://github.com/zarr-developers/zarr-python/pull/3130)) - Adds a `with_read_only` convenience method to the `Store` abstract base class (raises `NotImplementedError`) and implementations to the `MemoryStore`, `ObjectStore`, `LocalStore`, and `FsspecStore` classes. ([#3138](https://github.com/zarr-developers/zarr-python/pull/3138)) ### Bugfixes - Ignore stale child metadata when reconsolidating metadata. ([#2921](https://github.com/zarr-developers/zarr-python/issues/2921)) - For Zarr format 2, allow fixed-length string arrays to be created without automatically inserting a `Vlen-UT8` codec in the array of filters. Fixed-length string arrays do not need this codec. This change fixes a regression where fixed-length string arrays created with Zarr Python 3 could not be read with Zarr Python 2.18. ([#3100](https://github.com/zarr-developers/zarr-python/pull/3100)) - When creating arrays without explicitly specifying a chunk size using `zarr.create` and other array creation routines, the chunk size will now set automatically instead of defaulting to the data shape. For large arrays this will result in smaller default chunk sizes. To retain previous behaviour, explicitly set the chunk shape to the data shape. This fix matches the existing chunking behaviour of `zarr.save_array` and `zarr.api.asynchronous.AsyncArray.create`. ([#3103](https://github.com/zarr-developers/zarr-python/pull/3103)) - When `zarr.save` has an argument `path=some/path/` and multiple arrays in `args`, the path resulted in `some/path/some/path` due to using the `path` argument twice while building the array path. This is now fixed. ([#3127](https://github.com/zarr-developers/zarr-python/pull/3127)) - Fix `zarr.open` default for argument `mode` when `store` is `read_only` ([#3128](https://github.com/zarr-developers/zarr-python/pull/3128)) - Suppress `FileNotFoundError` when deleting non-existent keys in the `obstore` adapter. When writing empty chunks (i.e. chunks where all values are equal to the array's fill value) to a zarr array, zarr will delete those chunks from the underlying store. For zarr arrays backed by the `obstore` adapter, this will potentially raise a `FileNotFoundError` if the chunk doesn't already exist. Since whether or not a delete of a non-existing object raises an error depends on the behavior of the underlying store, suppressing the error in all cases results in consistent behavior across stores, and is also what `zarr` seems to expect from the store. ([#3140](https://github.com/zarr-developers/zarr-python/pull/3140)) - Trying to open a StorePath/Array with `mode='r'` when the store is not read-only creates a read-only copy of the store. ([#3156](https://github.com/zarr-developers/zarr-python/pull/3156)) ## 3.0.8 (2025-05-19) !!! warning In versions 3.0.0 to 3.0.7 opening arrays or groups with `mode='a'` (the default for many builtin functions) would cause any existing paths in the store to be deleted. This is fixed in 3.0.8, and we recommend all users upgrade to avoid this bug that could cause unintentional data loss. ### Features - Added a `print_debug_info` function for bug reports. ([#2913](https://github.com/zarr-developers/zarr-python/pull/2913)) ### Bugfixes - Fix a bug that prevented the number of initialized chunks being counted properly. ([#2862](https://github.com/zarr-developers/zarr-python/pull/2862)) - Fixed sharding with GPU buffers. ([#2978](https://github.com/zarr-developers/zarr-python/pull/2978)) - Fix structured `dtype` fill value serialization for consolidated metadata ([#2998](https://github.com/zarr-developers/zarr-python/issues/2998)) - It is now possible to specify no compressor when creating a zarr format 2 array. This can be done by passing `compressor=None` to the various array creation routines. The default behaviour of automatically choosing a suitable default compressor remains if the compressor argument is not given. To reproduce the behaviour in previous zarr-python versions when `compressor=None` was passed, pass `compressor='auto'` instead. ([#3039](https://github.com/zarr-developers/zarr-python/pull/3039)) - Fixed the typing of `dimension_names` arguments throughout so that it now accepts iterables that contain `None` alongside `str`. ([#3045](https://github.com/zarr-developers/zarr-python/pull/3045)) - Using various functions to open data with `mode='a'` no longer deletes existing data in the store. ([#3062](https://github.com/zarr-developers/zarr-python/pull/3062)) - Internally use `typesize` constructor parameter for `numcodecs.blosc.Blosc` to improve compression ratios back to the v2-package levels. ([#2962](https://github.com/zarr-developers/zarr-python/pull/2962)) - Specifying the memory order of Zarr format 2 arrays using the `order` keyword argument has been fixed. ([#2950](https://github.com/zarr-developers/zarr-python/issues/2950)) ### Misc - [#2972](https://github.com/zarr-developers/zarr-python/pull/2972), [#3027](https://github.com/zarr-developers/zarr-python/pull/3027), [#3049](https://github.com/zarr-developers/zarr-python/pull/3049) ## 3.0.7 (2025-04-22) ### Features - Add experimental ObjectStore storage class based on obstore. ([#1661](https://github.com/zarr-developers/zarr-python/pull/1661)) - Add `zarr.from_array` using concurrent streaming of source data ([#2622](https://github.com/zarr-developers/zarr-python/pull/2622)) ### Bugfixes - 0-dimensional arrays are now returning a scalar. Therefore, the return type of `__getitem__` changed to NDArrayLikeOrScalar. This change is to make the behavior of 0-dimensional arrays consistent with `numpy` scalars. ([#2718](https://github.com/zarr-developers/zarr-python/pull/2718)) - Fix `fill_value` serialization for `NaN` in `ArrayV2Metadata` and add property-based testing of round-trip serialization ([#2802](https://github.com/zarr-developers/zarr-python/pull/2802)) - Fixes `ConsolidatedMetadata` serialization of `nan`, `inf`, and `-inf` to be consistent with the behavior of `ArrayMetadata`. ([#2996](https://github.com/zarr-developers/zarr-python/pull/2996)) ### Improved Documentation - Updated the 3.0 migration guide to include the removal of "." syntax for getting group members. ([#2991](https://github.com/zarr-developers/zarr-python/issues/2991), [#2997](https://github.com/zarr-developers/zarr-python/pull/2997)) ### Misc - Define a new versioning policy based on Effective Effort Versioning. This replaces the old Semantic Versioning-based policy. ([#2924](https://github.com/zarr-developers/zarr-python/issues/2924), [#2910](https://github.com/zarr-developers/zarr-python/pull/2910)) - Make warning filters in the tests more specific, so warnings emitted by tests added in the future are more likely to be caught instead of ignored. ([#2714](https://github.com/zarr-developers/zarr-python/pull/2714)) - Avoid an unnecessary memory copy when writing Zarr to a local file ([#2944](https://github.com/zarr-developers/zarr-python/pull/2944)) ## 3.0.6 (2025-03-20) ### Bugfixes - Restore functionality of `del z.attrs['key']` to actually delete the key. ([#2908](https://github.com/zarr-developers/zarr-python/pull/2908)) ## 3.0.5 (2025-03-07) ### Bugfixes - Fixed a bug where `StorePath` creation would not apply standard path normalization to the `path` parameter, which led to the creation of arrays and groups with invalid keys. ([#2850](https://github.com/zarr-developers/zarr-python/pull/2850)) - Prevent update_attributes calls from deleting old attributes ([#2870](https://github.com/zarr-developers/zarr-python/pull/2870)) ### Misc - [#2796](https://github.com/zarr-developers/zarr-python/pull/2796) ## 3.0.4 (2025-02-23) ### Features - Adds functions for concurrently creating multiple arrays and groups. ([#2665](https://github.com/zarr-developers/zarr-python/pull/2665)) ### Bugfixes - Fixed a bug where `ArrayV2Metadata` could save `filters` as an empty array. ([#2847](https://github.com/zarr-developers/zarr-python/pull/2847)) - Fix a bug when setting values of a smaller last chunk. ([#2851](https://github.com/zarr-developers/zarr-python/pull/2851)) ### Misc - [#2828](https://github.com/zarr-developers/zarr-python/pull/2828) ## 3.0.3 (2025-02-14) ### Features - Improves performance of FsspecStore.delete_dir for remote filesystems supporting concurrent/batched deletes, e.g., s3fs. ([#2661](https://github.com/zarr-developers/zarr-python/pull/2661)) - Added `zarr.config.enable_gpu` to update Zarr's configuration to use GPUs. ([#2751](https://github.com/zarr-developers/zarr-python/pull/2751)) - Avoid reading chunks during writes where possible. [#757](https://github.com/zarr-developers/zarr-python/issues/757) ([#2784](https://github.com/zarr-developers/zarr-python/pull/2784)) - `LocalStore` learned to `delete_dir`. This makes array and group deletes more efficient. ([#2804](https://github.com/zarr-developers/zarr-python/pull/2804)) - Add `zarr.testing.strategies.array_metadata` to generate ArrayV2Metadata and ArrayV3Metadata instances. ([#2813](https://github.com/zarr-developers/zarr-python/pull/2813)) - Add arbitrary `shards` to Hypothesis strategy for generating arrays. ([#2822](https://github.com/zarr-developers/zarr-python/pull/2822)) ### Bugfixes - Fixed bug with Zarr using device memory, instead of host memory, for storing metadata when using GPUs. ([#2751](https://github.com/zarr-developers/zarr-python/pull/2751)) - The array returned by `zarr.empty` and an empty `zarr.core.buffer.cpu.NDBuffer` will now be filled with the specified fill value, or with zeros if no fill value is provided. This fixes a bug where Zarr format 2 data with no fill value was written with un-predictable chunk sizes. ([#2755](https://github.com/zarr-developers/zarr-python/pull/2755)) - Fix zip-store path checking for stores with directories listed as files. ([#2758](https://github.com/zarr-developers/zarr-python/pull/2758)) - Use removeprefix rather than replace when removing filename prefixes in `FsspecStore.list` ([#2778](https://github.com/zarr-developers/zarr-python/pull/2778)) - Enable automatic removal of `needs release notes` with labeler action ([#2781](https://github.com/zarr-developers/zarr-python/pull/2781)) - Use the proper label config ([#2785](https://github.com/zarr-developers/zarr-python/pull/2785)) - Alters the behavior of `create_array` to ensure that any groups implied by the array's name are created if they do not already exist. Also simplifies the type signature for any function that takes an ArrayConfig-like object. ([#2795](https://github.com/zarr-developers/zarr-python/pull/2795)) - Enitialise empty chunks to the default fill value during writing and add default fill values for datetime, timedelta, structured, and other (void* fixed size) data types ([#2799](https://github.com/zarr-developers/zarr-python/pull/2799)) - Ensure utf8 compliant strings are used to construct numpy arrays in property-based tests ([#2801](https://github.com/zarr-developers/zarr-python/pull/2801)) - Fix pickling for ZipStore ([#2807](https://github.com/zarr-developers/zarr-python/pull/2807)) - Update numcodecs to not overwrite codec configuration ever. Closes [#2800](https://github.com/zarr-developers/zarr-python/issues/2800). ([#2811](https://github.com/zarr-developers/zarr-python/pull/2811)) - Fix fancy indexing (e.g. arr[5, [0, 1]]) with the sharding codec ([#2817](https://github.com/zarr-developers/zarr-python/pull/2817)) ### Improved Documentation - Added new user guide on GPU. ([#2751](https://github.com/zarr-developers/zarr-python/pull/2751)) ## 3.0.2 (2025-01-31) ### Features - Test `getsize()` and `getsize_prefix()` in `StoreTests`. ([#2693](https://github.com/zarr-developers/zarr-python/pull/2693)) - Test that a `ValueError` is raised for invalid byte range syntax in `StoreTests`. ([#2693](https://github.com/zarr-developers/zarr-python/pull/2693)) - Separate instantiating and opening a store in `StoreTests`. ([#2693](https://github.com/zarr-developers/zarr-python/pull/2693)) - Add a test for using Stores as context managers in `StoreTests`. ([#2693](https://github.com/zarr-developers/zarr-python/pull/2693)) - Implemented `LoggingStore.open()`. ([#2693](https://github.com/zarr-developers/zarr-python/pull/2693)) - `LoggingStore` is now a generic class. ([#2693](https://github.com/zarr-developers/zarr-python/pull/2693)) - Change StoreTest's `test_store_repr`, `test_store_supports_writes`, `test_store_supports_partial_writes`, and `test_store_supports_listing` to be implemented using `@abstractmethod`, rather than raising `NotImplementedError`. ([#2693](https://github.com/zarr-developers/zarr-python/pull/2693)) - Test the error raised for invalid buffer arguments in `StoreTests`. ([#2693](https://github.com/zarr-developers/zarr-python/pull/2693)) - Test that data can be written to a store that's not yet open using the store.set method in `StoreTests`. ([#2693](https://github.com/zarr-developers/zarr-python/pull/2693)) - Adds a new function `init_array` for initializing an array in storage, and refactors `create_array` to use `init_array`. `create_array` takes two new parameters: `data`, an optional array-like object, and `write_data`, a bool which defaults to `True`. If `data` is given to `create_array`, then the `dtype` and `shape` attributes of `data` are used to define the corresponding attributes of the resulting Zarr array. Additionally, if `data` is given and `write_data` is `True`, then the values in `data` will be written to the newly created array. ([#2761](https://github.com/zarr-developers/zarr-python/pull/2761)) ### Bugfixes - Wrap sync fsspec filesystems with `AsyncFileSystemWrapper`. ([#2533](https://github.com/zarr-developers/zarr-python/pull/2533)) - Added backwards compatibility for Zarr format 2 structured arrays. ([#2681](https://github.com/zarr-developers/zarr-python/pull/2681)) - Update equality for `LoggingStore` and `WrapperStore` such that 'other' must also be a `LoggingStore` or `WrapperStore` respectively, rather than only checking the types of the stores they wrap. ([#2693](https://github.com/zarr-developers/zarr-python/pull/2693)) - Ensure that `ZipStore` is open before getting or setting any values. ([#2693](https://github.com/zarr-developers/zarr-python/pull/2693)) - Use stdout rather than stderr as the default stream for `LoggingStore`. ([#2693](https://github.com/zarr-developers/zarr-python/pull/2693)) - Match the errors raised by read only stores in `StoreTests`. ([#2693](https://github.com/zarr-developers/zarr-python/pull/2693)) - Fixed `ZipStore` to make sure the correct attributes are saved when instances are pickled. This fixes a previous bug that prevented using `ZipStore` with a `ProcessPoolExecutor`. ([#2762](https://github.com/zarr-developers/zarr-python/pull/2762)) - Updated the optional test dependencies to include `botocore` and `fsspec`. ([#2768](https://github.com/zarr-developers/zarr-python/pull/2768)) - Fixed the fsspec tests to skip if `botocore` is not installed. Previously they would have failed with an import error. ([#2768](https://github.com/zarr-developers/zarr-python/pull/2768)) - Optimize full chunk writes. ([#2782](https://github.com/zarr-developers/zarr-python/pull/2782)) ### Improved Documentation - Changed the machinery for creating changelog entries. Now individual entries should be added as files to the `changes` directory in the `zarr-python` repository, instead of directly to the changelog file. ([#2736](https://github.com/zarr-developers/zarr-python/pull/2736)) ### Other - Created a type alias `ChunkKeyEncodingLike` to model the union of `ChunkKeyEncoding` instances and the dict form of the parameters of those instances. `ChunkKeyEncodingLike` should be used by high-level functions to provide a convenient way for creating `ChunkKeyEncoding` objects. ([#2763](https://github.com/zarr-developers/zarr-python/pull/2763)) ## 3.0.1 (2025-01-17) * Implement `zarr.from_array` using concurrent streaming ([#2622](https://github.com/zarr-developers/zarr-python/pull/2622)). ### Bug fixes * Fixes `order` argument for Zarr format 2 arrays ([#2679](https://github.com/zarr-developers/zarr-python/pull/2679)). * Fixes a bug that prevented reading Zarr format 2 data with consolidated metadata written using `zarr-python` version 2 ([#2694](https://github.com/zarr-developers/zarr-python/issues/2694)). * Ensure that compressor=None results in no compression when writing Zarr format 2 data ([#2708](https://github.com/zarr-developers/zarr-python/issues/2708)). * Fix for empty consolidated metadata dataset: backwards compatibility with Zarr-Python 2 ([#2695](https://github.com/zarr-developers/zarr-python/pull/2695)). ### Documentation * Add v3.0.0 release announcement banner ([#2677](https://github.com/zarr-developers/zarr-python/pull/2677)). * Quickstart guide alignment with V3 API ([#2697](https://github.com/zarr-developers/zarr-python/pull/2697)). * Fix doctest failures related to numcodecs 0.15 ([#2727](https://github.com/zarr-developers/zarr-python/pull/2727)). ### Other * Removed some unnecessary files from the source distribution to reduce its size. ([#2686](https://github.com/zarr-developers/zarr-python/pull/2686)). * Enable codecov in GitHub actions ([#2682](https://github.com/zarr-developers/zarr-python/pull/2682)). * Speed up hypothesis tests ([#2650](https://github.com/zarr-developers/zarr-python/pull/2650)). * Remove multiple imports for an import name ([#2723](https://github.com/zarr-developers/zarr-python/pull/2723)). ## 3.0.0 (2025-01-09) 3.0.0 is a new major release of Zarr-Python, with many breaking changes. See the [v3 migration guide](user-guide/v3_migration.md) for a listing of what's changed. Normal release note service will resume with further releases in the 3.0.0 series. Release notes for the zarr-python 2.x and 1.x releases can be found here: https://zarr.readthedocs.io/en/support-v2/release.html zarr-developers-zarr-python-a2f2415/docs/roadmap.md000066400000000000000000000257271527014137600223770ustar00rootroot00000000000000# Roadmap This page describes where Zarr-Python is headed: the goals for the next major cycle of work, the changes we intend to make, and how those changes will be released. It is a living document; discussion and counter-proposals are welcome on the [Zarr-Python issue tracker](https://github.com/zarr-developers/zarr-python/issues). *The history of this roadmap, including the detailed technical proposals it was distilled from, can be traced in the [zarr-python-planning](https://github.com/zarr-developers/zarr-python-planning) repository.* !!! note This roadmap reflects the current thinking of the core developers. It is a statement of direction, not a schedule. We don't know how long these changes will take, only that we are committed to moving the project in the direction outlined here. ## Where we are The [3.0 release](https://github.com/zarr-developers/zarr-python/releases/tag/v3.0.0) was a total redesign of the library's internals, with three goals: full support for the Zarr V2 and V3 storage formats, storage APIs that are ergonomic for high-latency storage (such as cloud storage), and backwards compatibility with Zarr-Python 2.x where possible. Those goals were largely achieved! Going by the content of issues and pull requests submitted to the library, few users are grappling with 2.x → 3.x migration issues. Instead, we see users asking for things like better APIs, where "better" usually means faster. The 3.x redesign was carried out under hard backwards-compatibility constraints, and it inherited many structural patterns from the 2.x implementation it replaced. The library has never had a release cycle whose primary goal was the *shape* of the internals. The next body of work — which we call **"v4"** — is that overdue investment. We think iterating on the internals of the library will make it *much* easier to bring faster, more expressive APIs to Zarr-Python users. ## Goals If the 3.0 goals could be sloganized as "migrate to Zarr V3, and improve cloud storage support", the slogan for the v4 goals is: **"a frictionless Zarr-based Python ecosystem for chunked arrays"**. Zarr-Python should be *foundational* for the growing number of Python packages that work with data in the Zarr format. Concretely, that means pushing in these directions: - Deliver excellent performance, out of the box, while retaining maintainability. - Make Zarr-Python APIs ergonomic and useful for developers. - Expand our scope to cover vital quality-of-life routines like data copying, rechunking, and the like. - Ease the growth of Python tools across all levels of the Zarr stack. - Accelerate the implementation of new codecs, chunk grids, chunk key encodings, etc. An important design input: [`zarrs`](https://github.com/zarrs/zarrs) (Rust) and [TensorStore](https://github.com/google/tensorstore) (C++) are two independent Zarr implementations that use architectural patterns we want to learn from. We see them as complementary rather than competitive. !!! note Many of the features in this roadmap will not require breaking public 3.x APIs. We can and will ship those features in 3.x releases; at the same time, we consider it clarifying to frame the coherent development direction as vectored at a 4.0 milestone. ## The Zarr stack Different applications need different levels of Zarr support: a convention validator only needs to read metadata documents; a visualization tool may only need read-only array access; other tools need everything. We think of this as a "Zarr stack", from most abstract to most concrete: 1. **Conventions** — application and/or domain-specific schemas built on top of Zarr (OME-NGFF, GeoZarr, anndata-zarr, multiscales). 2. **Groups** — Zarr hierarchies, traversal, group-level attributes. 3. **Arrays** — the user-facing array object, plus indexing and slicing. 4. **Chunk decoding** — the codec pipeline. 5. **Chunk addressing** — chunk grids and key encodings that map array coordinates to store keys. 6. **Stores** — the key-value layer. 7. **Metadata** — pure data documents describing arrays and groups. Today, Zarr-Python is a monolith that serves every level: a consumer who only needs metadata handling has to install the full dependency footprint of the whole library, and a faster chunk-decoding implementation cannot plug in without re-implementing the layers above it. The v4 direction is to re-shape Zarr-Python around the stack, so that each level is something you can depend on, conform to, or replace, without buying every other level. We plan to "stackify" Zarr-Python by spinning core functionality out into separate Python packages, e.g. `zarr-metadata`, `zarr-indexing`, `zarr-storage`, `zarr-codec`, `zarr-dtype`, each with narrow scope, all composed in the `zarr` package. The Rust `zarrs` library successfully uses a structure like this, and we are keen to share the benefits of a more modular, maintainable codebase. Two of these subpackages, [`zarr-metadata`](https://zarr.readthedocs.io/projects/zarr-metadata/en/latest/) and [`zarr-indexing`](https://zarr.readthedocs.io/projects/zarr-indexing/en/latest/), are already published. ## What we intend to change The following section details how we want to evolve the internal logic that drives Zarr-Python. ### Foundation: swappable backends We propose to refactor Zarr-Python internals around a *swappable engine* — a protocol, or protocols, that define the core routines a Zarr implementation must support. Zarr-Python becomes one user-facing API that can be driven by multiple backends, including externally defined backends. We think this will allow users on many different platforms to get the best performance for their particular environment while retaining a familiar API. #### Rust bindings We want a Python backend (i.e., the status quo), but also a Rust-based backend, via bindings to the [`zarrs`](https://docs.rs/zarrs/latest/zarrs/) crate. The [zarrs-python](https://zarrs-python.readthedocs.io/en/latest/) project demonstrates that bridging `zarrs` and Zarr-Python buys a *lot* of performance in the specific case of chunk encoding. But zarrs-python is constrained today by limited modularity in Zarr-Python internals. Refactoring our internals around swappable backends should address this limitation. Any Python package that interfaces with `zarrs` will need Pythonic bindings to the Rust library. So we are *very* excited about the [zarrista](https://developmentseed.org/zarrista/latest/) package, which aims to provide complete Python bindings for `zarrs`. #### Sync / Async partitioning Internally we will branch over two kinds of backends: synchronous and asynchronous. The synchronous backend is suitable for arrays and groups persisted to low-latency storage like in-memory stores or local file systems, where async scheduling is pure friction. The asynchronous backend will use Python's `async` support and will provide concurrent APIs where it helps: for arrays and groups persisted to high-latency storage. ### Lazy indexing The Zarr-Python Array API was initially designed to mirror NumPy, with eager array indexing syntax. `Array.__getitem__` performs IO eagerly and returns a NumPy array. That was helpful to the dominant use-case at the time of its creation, but it means deferred IO and computation currently require an external library such as Dask. It means there is no built-in support for representing multi-step reads as a single deferred plan. Further, it means that every chained selection round-trips to storage independently. We can fix this by introducing an API for lazy indexing. Under this model, an array indexing operation like `array[::2]` desugars to a declarative state like `(array, selection)`. Chained selections like `array[10:100][::2]` are fused immediately, and we defer actual IO for the time when the result of indexing is needed. [TensorStore](https://google.github.io/tensorstore/) is an excellent role model for Zarr-Python here, and we can deliver this functionality without breaking ordinary indexing behavior. See this [discussion](https://github.com/zarr-developers/zarr-python/discussions/1603) for more background. ### Data types First-class support for ML-specific dtypes — `bfloat16`, the `float8` variants, packed `int4`/`uint4` — via [`ml_dtypes`](https://github.com/jax-ml/ml_dtypes). These data types have specifications written up in `zarr-extensions`, but there's no simple to get them integrated in Zarr-Python today. ### Device-agnostic IO Make Zarr-Python's IO surfaces device-agnostic rather than adding GPU support as a bolted-on feature: stores and codecs grow APIs for writing into a caller-provided buffer (`read_into`, `decode_into`), and the `Array` facade returns array-like objects in the user's chosen Array API namespace. GPU support falls out once the assumption of CPU destinations is removed, and CPU paths get faster too, because pre-allocated output buffers eliminate per-chunk allocation. ### Configuration, registries, and plugins Move configuration from "global mutable state read implicitly" to "typed data passed explicitly": a typed config object replacing the untyped global `donfig` dict, array-scoped runtime config passed at open time, a registry redesign that addresses implementations by stable identity and resolves plugin name-conflicts deliberately, and named profiles replacing global mutators. ### Coordinated and distributed writes This area is actually an unfinished aspect of the 2.x → 3.0 migration: Zarr-Python 2.x supported synchronization logic via file-based locks, and we have not implemented equivalent functionality in 3.x. We don't have *concrete* plans for closing this gap. Re-implementing simple object-based locking, for backends that support it, is a direct solution we should consider. But a transactional storage model, where a sequence of basic storage operations like reading and writing could be submitted in a batch and executed serially, with rollbacks under failure, is also quite appealing. As with array indexing, TensorStore is the trailblazer here, and we can learn from its example. We can also avoid the need for synchronization mechanisms entirely with better planning. Many users of the 2.x synchronization tooling needed to simply write values from one chunked source to another, without worrying about chunk alignment. This can be addressed e.g. by creating a write plan that partitions the input chunks into batches within which writes cannot race. ## How to get involved - **Discuss the plans.** Comments and counter-proposals on any of the themes above are welcome on the [issue tracker](https://github.com/zarr-developers/zarr-python/issues) and in the [developer chat](https://ossci.zulipchat.com/). - **Review in-flight work.** The `IndexTransform` algebra that lazy indexing is built on is in review at [#3906](https://github.com/zarr-developers/zarr-python/pull/3906). - **Weigh in as a downstream maintainer.** If your project's use of Zarr-Python would be affected by the codec API rewrite, the stores rewrite, or the lazy-indexing work, the planning phase is the time to surface workloads or patterns that don't fit. zarr-developers-zarr-python-a2f2415/docs/subprojects.md000066400000000000000000000025661527014137600233130ustar00rootroot00000000000000# Subprojects Alongside `zarr` itself, the [zarr-python repository](https://github.com/zarr-developers/zarr-python) hosts a small number of companion packages. Each one is developed in the same repository but versioned, released, and documented independently, so you can depend on it without taking on `zarr` as a dependency.
- [:material-code-json:{ .lg .middle } __zarr-metadata__](https://zarr.readthedocs.io/projects/zarr-metadata/) --- Spec-defined metadata types, models, and validators for Zarr v2 and v3, with minimal dependencies. Useful if your software reads or writes Zarr metadata documents but does not need a full Zarr implementation. ```bash pip install zarr-metadata ``` - [:material-vector-polyline:{ .lg .middle } __zarr-indexing__](https://zarr.readthedocs.io/projects/zarr-indexing/) --- Composable, lazy coordinate transforms for Zarr array indexing. Makes the mapping from requested coordinates to stored coordinates a first-class, composable value, and resolves which chunks a selection touches. ```bash pip install zarr-indexing ``` - [:material-server:{ .lg .middle } __zarr-http-server__](https://zarr.readthedocs.io/projects/zarr-http-server/) --- HTTP server for Zarr stores, arrays, and groups. ```bash pip install zarr-http-server ```
zarr-developers-zarr-python-a2f2415/docs/user-guide/000077500000000000000000000000001527014137600224665ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/docs/user-guide/arrays.md000066400000000000000000000776741527014137600243360ustar00rootroot00000000000000# Working with arrays ## Creating an array Zarr has several functions for creating arrays. For example: ```python exec="true" session="arrays" import numpy as np ``` ```python exec="true" session="arrays" source="above" result="ansi" import zarr z = zarr.create_array(store="memory://arrays-demo", shape=(10000, 10000), chunks=(1000, 1000), dtype='int32') print(z) ``` The code above creates a 2-dimensional array of 32-bit integers with 10000 rows and 10000 columns, divided into chunks where each chunk has 1000 rows and 1000 columns (and so there will be 100 chunks in total). The data is written to an in-memory store: when `fsspec` is installed, a `memory://` URL resolves to a [`zarr.storage.FsspecStore`][] backed by fsspec's in-memory filesystem; otherwise a [`zarr.storage.ManagedMemoryStore`][] is used. See the [Storage guide](storage.md) for more details on stores, and [Persistent arrays](#persistent-arrays) for details on storing arrays in other stores. See [Data types](data_types.md) for an in-depth look at the data types supported by Zarr, and [Chunk size and shape](performance.md#chunk-size-and-shape) in the performance guide for guidance on choosing chunk shapes. See the [`zarr.create_array`][] API documentation for more detailed information about creating arrays. ## Reading and writing data Zarr arrays support a similar interface to [NumPy](https://numpy.org/doc/stable/) arrays for reading and writing data. For example, the entire array can be filled with a scalar value: ```python exec="true" session="arrays" source="above" z[:] = 42 ``` Regions of the array can also be written to, e.g.: ```python exec="true" session="arrays" source="above" import numpy as np z[0, :] = np.arange(10000) z[:, 0] = np.arange(10000) ``` The contents of the array can be retrieved by slicing, which will load the requested region into memory as a NumPy array, e.g.: ```python exec="true" session="arrays" source="above" result="ansi" print(z[0, 0]) ``` ```python exec="true" session="arrays" source="above" result="ansi" print(z[-1, -1]) ``` ```python exec="true" session="arrays" source="above" result="ansi" print(z[0, :]) ``` ```python exec="true" session="arrays" source="above" result="ansi" print(z[:, 0]) ``` ```python exec="true" session="arrays" source="above" result="ansi" print(z[:]) ``` More information about NumPy-style indexing can be found in the [NumPy documentation](https://numpy.org/doc/stable/user/basics.indexing.html). ## Persistent arrays In the examples above, compressed data for each chunk of the array was stored in main memory. Zarr arrays can also be stored on a file system, enabling persistence of data between sessions. To do this, we can change the store argument to point to a filesystem path: ```python exec="true" session="arrays" source="above" z1 = zarr.create_array(store='data/example-1.zarr', shape=(10000, 10000), chunks=(1000, 1000), dtype='int32', overwrite=True) ``` The array above will store its configuration metadata and all compressed chunk data in a directory called `'data/example-1.zarr'` relative to the current working directory. The [`zarr.create_array`][] function provides a convenient way to create a new persistent array or continue working with an existing array. Note, there is no need to close an array: data are automatically flushed to disk, and files are automatically closed whenever an array is modified. Persistent arrays support the same interface for reading and writing data, e.g.: ```python exec="true" session="arrays" source="above" z1[:] = 42 z1[0, :] = np.arange(10000) z1[:, 0] = np.arange(10000) ``` Check that the data have been written and can be read again: ```python exec="true" session="arrays" source="above" result="ansi" z2 = zarr.open_array('data/example-1.zarr', mode='r') print(np.all(z1[:] == z2[:])) ``` If you are just looking for a fast and convenient way to save NumPy arrays to disk then load back into memory later, the functions [`zarr.save`][], [`zarr.save_array`][] and [`zarr.load`][] may be useful. `zarr.save` refuses to replace an array that already exists at the path; `zarr.save_array` accepts `mode="w"` to do so. E.g.: ```python exec="true" session="arrays" source="above" result="ansi" a = np.arange(10) zarr.save_array('data/example-2.zarr', a, mode="w") print(zarr.load('data/example-2.zarr')) ``` Please note that there are a number of other options for persistent array storage, see the [Storage Guide](storage.md) for more details. ## Resizing and appending A Zarr array can be resized, which means that any of its dimensions can be increased or decreased in length. For example: ```python exec="true" session="arrays" source="above" result="ansi" z = zarr.create_array(store='data/example-3.zarr', shape=(10000, 10000), dtype='int32', chunks=(1000, 1000), overwrite=True) z[:] = 42 print(f"Original shape: {z.shape}") z.resize((20000, 10000)) print(f"New shape: {z.shape}") ``` Note that when an array is resized, the underlying data are not rearranged in any way. If one or more dimensions are shrunk, any chunks falling outside the new array shape will be deleted from the underlying store. [`zarr.Array.append`][] is provided as a convenience function, which can be used to append data to any axis. E.g.: ```python exec="true" session="arrays" source="above" result="ansi" a = np.arange(10000000, dtype='int32').reshape(10000, 1000) z = zarr.create_array(store='data/example-4.zarr', shape=a.shape, dtype=a.dtype, chunks=(1000, 100), overwrite=True) z[:] = a print(f"Original shape: {z.shape}") z.append(a) print(f"Shape after first append: {z.shape}") z.append(np.vstack([a, a]), axis=1) print(f"Shape after second append: {z.shape}") ``` ## Runtime configuration Zarr arrays are parametrized with a configuration that determines certain aspects of array behavior. We currently support five configuration options for arrays: `order`, `write_empty_chunks`, `read_missing_chunks`, `sharding_coalesce_max_gap_bytes`, and `sharding_coalesce_max_bytes`. | field | type | default | description | | - | - | - | - | | `order` | `Literal["C", "F"]` | `"C"` | The memory layout of arrays returned when reading data from the store. | `write_empty_chunks` | `bool` | `False` | Controls whether empty chunks are written to storage. See [Empty chunks](performance.md#empty-chunks). | `read_missing_chunks` | `bool` | `True` | Controls whether missing chunks are filled with the array's fill value on read. If `False`, reading missing chunks raises a [`ChunkNotFoundError`][zarr.errors.ChunkNotFoundError]. | `sharding_coalesce_max_gap_bytes` | `int` | `1048576` (1 MiB) | When reading multiple chunks from the same shard, nearby byte ranges separated by no more than this many bytes are coalesced into a single request to the store. | `sharding_coalesce_max_bytes` | `int` | `16777216` (16 MiB) | Requests will not be coalesced if doing so would exceed this byte size. !!! info The Zarr V3 spec states that readers should interpret an uninitialized chunk as containing the array's `fill_value`. By default, Zarr-Python follows this behavior: a missing chunk is treated as uninitialized and filled with the array's `fill_value`. However, if you know that all chunks have been written (i.e., are initialized), you may want to treat a missing chunk as an error. Set `read_missing_chunks=False` to raise a [`ChunkNotFoundError`][zarr.errors.ChunkNotFoundError] instead. !!! note `write_empty_chunks=False` skips writing chunks that are entirely the array's fill value. If `read_missing_chunks=False`, attempting to read these missing chunks will raise a [`ChunkNotFoundError`][zarr.errors.ChunkNotFoundError]. You can specify the configuration when you create an array with the `config` keyword argument. `config` can be passed as either a `dict` or an `ArrayConfig` object. ```python exec="true" session="arrays" source="above" result="ansi" arr = zarr.create_array({}, shape=(10,), dtype='int8', config={"write_empty_chunks": True}) print(arr.config) ``` To get an array view with a different config, use the `with_config` method. ```python exec="true" session="arrays" source="above" result="ansi" arr_f = arr.with_config({"order": "F"}) print(arr_f.config) ``` ## Zarr format 3 codec pipeline Zarr format 3 stores a single ordered list of codecs in the [`codecs`](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#array-metadata-codecs) field of array metadata. The [Zarr format 3 specification](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-encoding) classifies each codec by the input and output of its encode transform, as "array -> array", "array -> bytes", or "bytes -> bytes", and requires the list to contain zero or more array -> array codecs, followed by exactly one array -> bytes codec, followed by zero or more bytes -> bytes codecs. Zarr-Python's array creation functions expose that list through three parameters: `filters`, `serializer`, and `compressors`. These names are not defined in the Zarr format 3 specification. They are Zarr-Python aliases for the specification's codec kinds: - `filters`: the array -> array codecs, given as [`zarr.abc.codec.ArrayArrayCodec`][] instances. These transform chunk arrays into chunk arrays before serialization. - `serializer`: the array -> bytes codec, given as one [`zarr.abc.codec.ArrayBytesCodec`][] instance. This transforms a chunk array into bytes. Every Zarr format 3 array needs exactly one array -> bytes codec, either supplied explicitly or chosen by default. - `compressors`: the bytes -> bytes codecs, given as [`zarr.abc.codec.BytesBytesCodec`][] instances. These transform bytes into bytes after serialization. The `compressors` parameter is only for bytes-to-bytes codecs. If a codec is an `ArrayBytesCodec`, pass it with `serializer`, not `compressors`. For example, the built-in [`zarr.codecs.BytesCodec`][] can be supplied explicitly as the serializer: ```python exec="true" session="arrays" source="above" result="ansi" serializer = zarr.codecs.BytesCodec(endian="little") z_explicit_serializer = zarr.create_array( store="data/example-explicit-serializer.zarr", shape=(100,), chunks=(10,), dtype="int32", serializer=serializer, compressors=None, overwrite=True, ) print(z_explicit_serializer.serializer) print(f"Compressors: {z_explicit_serializer.compressors}") ``` The same rule applies to third-party Zarr format 3 codecs: if the codec is documented as an `ArrayBytesCodec`, provide an instance as `serializer=...`. ## Compressors A number of different compressors can be used with Zarr. Zarr includes Blosc, Zstandard and Gzip compressors. Additional compressors are available through a separate package called [NumCodecs](https://numcodecs.readthedocs.io/en/stable/) which provides various compressor libraries including LZ4, Zlib, BZ2 and LZMA. Different bytes-to-bytes compressors can be provided via the `compressors` keyword argument accepted by all array creation functions. For example: ```python exec="true" session="arrays" source="above" result="ansi" compressors = zarr.codecs.BloscCodec(cname='zstd', clevel=3, shuffle='bitshuffle') data = np.arange(100000000, dtype='int32').reshape(10000, 10000) z = zarr.create_array(store='data/example-5.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=compressors, overwrite=True) z[:] = data print(z.compressors) ``` This array above will use Blosc as the primary compressor, using the Zstandard algorithm (compression level 3) internally within Blosc, and with the bit-shuffle filter applied. When using a compressor, it can be useful to get some diagnostics on the compression ratio. Zarr arrays provide the [`zarr.Array.info`][] property which can be used to print useful diagnostics, e.g.: ```python exec="true" session="arrays" source="above" result="ansi" print(z.info) ``` The [`zarr.Array.info_complete`][] method inspects the underlying store and prints additional diagnostics, e.g.: ```python exec="true" session="arrays" source="above" result="ansi" print(z.info_complete()) ``` !!! note [`zarr.Array.info_complete`][] will inspect the underlying store and may be slow for large arrays. Use [`zarr.Array.info`][] if detailed storage statistics are not needed. If you don't specify a compressor, by default Zarr uses the Zstandard compressor. To create an array without any compression, set `compressors=None`: ```python exec="true" session="arrays" source="above" result="ansi" z_no_compress = zarr.create_array(store='data/example-uncompressed.zarr', shape=(10000, 10000), chunks=(1000, 1000), dtype='int32', compressors=None, overwrite=True) print(f"Compressors: {z_no_compress.compressors}") ``` In addition to Blosc and Zstandard, other compression libraries can also be used. For example, here is an array using Gzip compression, level 1: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(100000000, dtype='int32').reshape(10000, 10000) z = zarr.create_array(store='data/example-6.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=zarr.codecs.GzipCodec(level=1), overwrite=True) z[:] = data print(f"Compressors: {z.compressors}") ``` Here is an example using LZMA from [NumCodecs](https://numcodecs.readthedocs.io/en/stable/) with a custom filter pipeline including LZMA's built-in delta filter: ```python exec="true" session="arrays" source="above" result="ansi" import lzma from zarr.codecs.numcodecs import LZMA lzma_filters = [dict(id=lzma.FILTER_DELTA, dist=4), dict(id=lzma.FILTER_LZMA2, preset=1)] compressors = LZMA(filters=lzma_filters) data = np.arange(100000000, dtype='int32').reshape(10000, 10000) z = zarr.create_array(store='data/example-7.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=compressors, overwrite=True) print(f"Compressors: {z.compressors}") ``` ## Filters In some cases, compression can be improved by transforming the data in some way. For example, if nearby values tend to be correlated, then shuffling the bytes within each numerical value or storing the difference between adjacent values may increase compression ratio. Some compressors provide built-in filters that apply transformations to the data prior to compression. For example, the Blosc compressor has built-in implementations of byte- and bit-shuffle filters, and the LZMA compressor has a built-in implementation of a delta filter. However, to provide additional flexibility for implementing and using filters in combination with different compressors, Zarr also provides a mechanism for configuring filters outside of the primary compressor. Here is an example using a delta filter with the Blosc compressor: ```python exec="true" session="arrays" source="above" result="ansi" from zarr.codecs.numcodecs import Delta filters = [Delta(dtype='int32')] compressors = zarr.codecs.BloscCodec(cname='zstd', clevel=1, shuffle='shuffle') data = np.arange(100000000, dtype='int32').reshape(10000, 10000) z = zarr.create_array(store='data/example-9.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), filters=filters, compressors=compressors, overwrite=True) print(z.info_complete()) ``` For more information about available filter codecs, see the [Numcodecs](https://numcodecs.readthedocs.io/en/stable/) documentation. ## Advanced indexing Zarr arrays support several methods for advanced or "fancy" indexing, which enable a subset of data items to be extracted or updated in an array without loading the entire array into memory. Note that although this functionality is similar to some of the advanced indexing capabilities available on NumPy arrays and on h5py datasets, **the Zarr API for advanced indexing is different from both NumPy and h5py**, so please read this section carefully. For a complete description of the indexing API, see the documentation for the [`zarr.Array`][] class. ### Indexing with coordinate arrays Items from a Zarr array can be extracted by providing an integer array of coordinates. E.g.: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(10) ** 2 z = zarr.create_array(store='data/example-10.zarr', shape=data.shape, dtype=data.dtype, overwrite=True) z[:] = data print(z[:]) print(z.get_coordinate_selection([2, 5])) ``` Coordinate arrays can also be used to update data, e.g.: ```python exec="true" session="arrays" source="above" result="ansi" z.set_coordinate_selection([2, 5], [-1, -2]) print(z[:]) ``` For multidimensional arrays, coordinates must be provided for each dimension, e.g.: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(15).reshape(3, 5) z = zarr.create_array(store='data/example-11.zarr', shape=data.shape, dtype=data.dtype, overwrite=True) z[:] = data print(z[:]) ``` ```python exec="true" session="arrays" source="above" result="ansi" print(z.get_coordinate_selection(([0, 2], [1, 3]))) ``` ```python exec="true" session="arrays" source="above" result="ansi" z.set_coordinate_selection(([0, 2], [1, 3]), [-1, -2]) print(z[:]) ``` For convenience, coordinate indexing is also available via the `vindex` property, as well as the square bracket operator, e.g.: ```python exec="true" session="arrays" source="above" result="ansi" print(z.vindex[[0, 2], [1, 3]]) z.vindex[[0, 2], [1, 3]] = [-3, -4] ``` ```python exec="true" session="arrays" source="above" result="ansi" print(z[:]) ``` ```python exec="true" session="arrays" source="above" result="ansi" print(z[[0, 2], [1, 3]]) ``` When the indexing arrays have different shapes, they are broadcast together. That is, the following two calls are equivalent: ```python exec="true" session="arrays" source="above" result="ansi" print(z[1, [1, 3]]) print(z[[1, 1], [1, 3]]) ``` ### Indexing with a mask array Items can also be extracted by providing a Boolean mask. E.g.: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(10) ** 2 z = zarr.create_array(store='data/example-12.zarr', shape=data.shape, dtype=data.dtype, overwrite=True) z[:] = data print(z[:]) ``` ```python exec="true" session="arrays" source="above" result="ansi" sel = np.zeros_like(z, dtype=bool) sel[2] = True sel[5] = True print(z.get_mask_selection(sel)) ``` ```python exec="true" session="arrays" source="above" result="ansi" z.set_mask_selection(sel, [-1, -2]) print(z[:]) ``` Here's a multidimensional example: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(15).reshape(3, 5) z = zarr.create_array(store='data/example-13.zarr', shape=data.shape, dtype=data.dtype, overwrite=True) z[:] = data print(z[:]) ``` ```python exec="true" session="arrays" source="above" result="ansi" sel = np.zeros_like(z, dtype=bool) sel[0, 1] = True sel[2, 3] = True print(z.get_mask_selection(sel)) ``` ```python exec="true" session="arrays" source="above" result="ansi" z.set_mask_selection(sel, [-1, -2]) print(z[:]) ``` For convenience, mask indexing is also available via the `vindex` property, e.g.: ```python exec="true" session="arrays" source="above" result="ansi" print(z.vindex[sel]) ``` ```python exec="true" session="arrays" source="above" result="ansi" z.vindex[sel] = [-3, -4] print(z[:]) ``` Mask indexing is conceptually the same as coordinate indexing, and is implemented internally via the same machinery. Both styles of indexing allow selecting arbitrary items from an array, also known as point selection. ### Orthogonal indexing Zarr arrays also support methods for orthogonal indexing, which allows selections to be made along each dimension of an array independently. For example, this allows selecting a subset of rows and/or columns from a 2-dimensional array. E.g.: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(15).reshape(3, 5) z = zarr.create_array(store='data/example-14.zarr', shape=data.shape, dtype=data.dtype, overwrite=True) z[:] = data print(z[:]) ``` ```python exec="true" session="arrays" source="above" result="ansi" print(z.get_orthogonal_selection(([0, 2], slice(None)))) # select first and third rows ``` ```python exec="true" session="arrays" source="above" result="ansi" print(z.get_orthogonal_selection((slice(None), [1, 3]))) # select second and fourth columns ``` ```python exec="true" session="arrays" source="above" result="ansi" print(z.get_orthogonal_selection(([0, 2], [1, 3]))) # select rows [0, 2] and columns [1, 3] ``` Data can also be modified, e.g.: ```python exec="true" session="arrays" source="above" z.set_orthogonal_selection(([0, 2], [1, 3]), [[-1, -2], [-3, -4]]) ``` For convenience, the orthogonal indexing functionality is also available via the `oindex` property, e.g.: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(15).reshape(3, 5) z = zarr.create_array(store='data/example-15.zarr', shape=data.shape, dtype=data.dtype, overwrite=True) z[:] = data print(z.oindex[[0, 2], :]) # select first and third rows ``` ```python exec="true" session="arrays" source="above" result="ansi" print(z.oindex[:, [1, 3]]) # select second and fourth columns ``` ```python exec="true" session="arrays" source="above" result="ansi" print(z.oindex[[0, 2], [1, 3]]) # select rows [0, 2] and columns [1, 3] ``` ```python exec="true" session="arrays" source="above" result="ansi" z.oindex[[0, 2], [1, 3]] = [[-1, -2], [-3, -4]] print(z[:]) ``` Any combination of integer, slice, 1D integer array and/or 1D Boolean array can be used for orthogonal indexing. If the index contains at most one iterable, and otherwise contains only slices and integers, orthogonal indexing is also available directly on the array: ```python exec="true" session="arrays" source="above" result="ansi" data = np.arange(15).reshape(3, 5) z = zarr.create_array(store='data/example-16.zarr', shape=data.shape, dtype=data.dtype, overwrite=True) z[:] = data print(np.all(z.oindex[[0, 2], :] == z[[0, 2], :])) ``` ### Block indexing Zarr also supports block indexing, which allows selections of whole chunks based on their logical indices along each dimension of an array. For example, this allows selecting a subset of chunk aligned rows and/or columns from a 2-dimensional array. E.g.: ```python exec="true" session="arrays" source="above" data = np.arange(100).reshape(10, 10) z = zarr.create_array(store='data/example-17.zarr', shape=data.shape, dtype=data.dtype, chunks=(3, 3), overwrite=True) z[:] = data ``` Retrieve items by specifying their block coordinates: ```python exec="true" session="arrays" source="above" result="ansi" print(z.get_block_selection(1)) ``` Equivalent slicing: ```python exec="true" session="arrays" source="above" result="ansi" print(z[3:6]) ``` For convenience, the block selection functionality is also available via the `blocks` property, e.g.: ```python exec="true" session="arrays" source="above" result="ansi" print(z.blocks[1]) ``` Block index arrays may be multidimensional to index multidimensional arrays. For example: ```python exec="true" session="arrays" source="above" result="ansi" print(z.blocks[0, 1:3]) ``` Data can also be modified. Let's start by a simple 2D array: ```python exec="true" session="arrays" source="above" z = zarr.create_array(store='data/example-18.zarr', shape=(6, 6), dtype=int, chunks=(2, 2), overwrite=True) ``` Set data for a selection of items: ```python exec="true" session="arrays" source="above" result="ansi" z.set_block_selection((1, 0), 1) print(z[...]) ``` For convenience, this functionality is also available via the `blocks` property. E.g.: ```python exec="true" session="arrays" source="above" result="ansi" z.blocks[:, 2] = 7 print(z[...]) ``` Any combination of integer and slice can be used for block indexing: ```python exec="true" session="arrays" source="above" result="ansi" print(z.blocks[2, 1:3]) ``` ## Sharding Using small chunk shapes in very large arrays can lead to a very large number of chunks. This can become a performance issue for file systems and object storage. With Zarr format 3, a new sharding feature has been added to address this issue. With sharding, multiple chunks can be stored in a single storage object (e.g. a file). Within a shard, chunks are compressed and serialized separately. This allows individual chunks to be read independently. However, when writing data, a full shard must be written in one go for optimal performance and to avoid concurrency issues. That means that shards are the units of writing and chunks are the units of reading. Users need to configure the chunk and shard shapes accordingly. For guidance on choosing chunk and shard shapes, see [Sharding](performance.md#sharding) in the performance guide. Sharded arrays can be created by providing the `shards` parameter to [`zarr.create_array`][]. ```python exec="true" session="arrays" source="above" result="ansi" a = zarr.create_array('data/example-20.zarr', shape=(10000, 10000), shards=(1000, 1000), chunks=(100, 100), dtype='uint8', overwrite=True) a[:] = (np.arange(10000 * 10000) % 256).astype('uint8').reshape(10000, 10000) print(a.info_complete()) ``` In this example a shard shape of (1000, 1000) and a chunk shape of (100, 100) is used. This means that `10*10` chunks are stored in each shard, and there are `10*10` shards in total. Without the `shards` argument, there would be 10,000 chunks stored as individual files. ## Rectilinear (variable) chunk grids !!! warning "Experimental" Rectilinear chunk grids are an experimental feature and may change in future releases. This feature is expected to stabilize in Zarr version 3.3. Because the feature is still stabilizing, it is disabled by default and must be explicitly enabled: ```python exec="true" session="arrays" source="above" import zarr zarr.config.set({"array.rectilinear_chunks": True}) ``` Or via the environment variable `ZARR_ARRAY__RECTILINEAR_CHUNKS=True`. The examples below assume this config has been set. By default, Zarr arrays use a regular chunk grid where every chunk along a given dimension has the same size (except possibly the final boundary chunk). Rectilinear chunk grids allow each chunk along a dimension to have a different size. This is useful when the natural partitioning of the data is not uniform — for example, satellite swaths of varying width, time series with irregular intervals, or spatial tiles of different extents. ### Creating arrays with rectilinear chunks To create an array with rectilinear chunks, pass a nested list to the `chunks` parameter where each inner list gives the chunk sizes along one dimension: ```python exec="true" session="arrays" source="above" result="ansi" z = zarr.create_array( store=zarr.storage.MemoryStore(), shape=(60, 100), chunks=[[10, 20, 30], [50, 50]], dtype='int32', ) print(z.info) ``` In this example the first dimension is split into three chunks of sizes 10, 20, and 30, while the second dimension is split into two equal chunks of size 50. ### Reading and writing data Rectilinear arrays support the same indexing interface as regular arrays. Reads and writes that cross chunk boundaries of different sizes are handled automatically: ```python exec="true" session="arrays" source="above" result="ansi" import numpy as np data = np.arange(60 * 100, dtype='int32').reshape(60, 100) z[:] = data # Read a slice that spans the first two chunks (sizes 10 and 20) along axis 0 print(z[5:25, 0:5]) ``` ### Inspecting chunk sizes The `.write_chunk_sizes` property returns the actual data size of each storage chunk along every dimension. It works for both regular and rectilinear arrays and returns a tuple of tuples (matching the dask `Array.chunks` convention). When sharding is used, `.read_chunk_sizes` returns the inner chunk sizes instead: ```python exec="true" session="arrays" source="above" result="ansi" print(z.write_chunk_sizes) ``` For regular arrays, this includes the boundary chunk: ```python exec="true" session="arrays" source="above" result="ansi" z_regular = zarr.create_array( store=zarr.storage.MemoryStore(), shape=(100, 80), chunks=(30, 40), dtype='int32', ) print(z_regular.write_chunk_sizes) ``` Note that the `.chunks` property is not available for non-sharded rectilinear arrays, since there is no single uniform chunk shape — use `.write_chunk_sizes` (or `.read_chunk_sizes`) instead. Sharded arrays always have `.chunks`: it returns the inner chunk shape, which is always regular — the sharding codec requires a single uniform inner chunk shape, so only the shard boundaries can be rectilinear (see [Rectilinear shard boundaries](#rectilinear-shard-boundaries)). ### Resizing and appending Rectilinear arrays can be resized. When growing past the current edge sum, a new chunk is appended covering the additional extent. When shrinking, the chunk edges are preserved and the extent is re-bound (chunks beyond the new extent simply become inactive): ```python exec="true" session="arrays" source="above" result="ansi" z = zarr.create_array( store=zarr.storage.MemoryStore(), shape=(30,), chunks=[[10, 20]], dtype='float64', ) z[:] = np.arange(30, dtype='float64') print(f"Before resize: chunk_sizes={z.write_chunk_sizes}") z.resize((50,)) print(f"After resize: chunk_sizes={z.write_chunk_sizes}") ``` The `append` method also works with rectilinear arrays: ```python exec="true" session="arrays" source="above" result="ansi" z.append(np.arange(10, dtype='float64')) print(f"After append: shape={z.shape}, chunk_sizes={z.write_chunk_sizes}") ``` A rectilinear array can also be created with a zero-length dimension: because no non-empty list of positive chunk sizes can sum to 0, the chunk sizes given for such a dimension are stored as-is and describe the chunks the dimension will grow into on `append` or `resize` — the same state as resizing an existing rectilinear dimension down to 0. ### Compressors and filters Rectilinear arrays work with all codecs — compressors, filters, and checksums. Since each chunk may have a different size, the codec pipeline processes each chunk independently: ```python exec="true" session="arrays" source="above" result="ansi" z = zarr.create_array( store=zarr.storage.MemoryStore(), shape=(60, 100), chunks=[[10, 20, 30], [50, 50]], dtype='float64', filters=[zarr.codecs.TransposeCodec(order=(1, 0))], compressors=[zarr.codecs.BloscCodec(cname='zstd', clevel=3)], ) z[:] = np.arange(60 * 100, dtype='float64').reshape(60, 100) np.testing.assert_array_equal(z[:], np.arange(60 * 100, dtype='float64').reshape(60, 100)) print("Roundtrip OK") ``` ### Rectilinear shard boundaries Rectilinear chunk grids can also be used for shard boundaries when combined with sharding. In this case, the outer grid (shards) is rectilinear while the inner chunks remain regular. Each shard dimension must be divisible by the corresponding inner chunk size: ```python exec="true" session="arrays" source="above" result="ansi" z = zarr.create_array( store=zarr.storage.MemoryStore(), shape=(120, 100), chunks=(10, 10), shards=[[60, 40, 20], [50, 50]], dtype='int32', ) z[:] = np.arange(120 * 100, dtype='int32').reshape(120, 100) print(z[50:70, 40:60]) ``` Note that rectilinear inner chunks with sharding are not supported — only the shard boundaries can be rectilinear. For such arrays, `.chunks` returns the (regular) inner chunk shape, while `.shards` raises `NotImplementedError` since there is no single uniform shard shape — use `.write_chunk_sizes` for the per-dimension shard sizes. `.info` reports the shard shape as ``: ```python exec="true" session="arrays" source="above" result="ansi" print(f"chunks={z.chunks}") print(f"shard sizes={z.write_chunk_sizes}") print(z.info) ``` ### Metadata format Rectilinear chunk grid metadata uses run-length encoding (RLE) for compact serialization. When reading metadata, both bare integers and `[value, count]` pairs are accepted: - `[10, 20, 30]` — three chunks with explicit sizes - `[[10, 3]]` — three chunks of size 10 (RLE shorthand) - `[[10, 3], 5]` — three chunks of size 10, then one chunk of size 5 When writing, Zarr automatically compresses repeated values into RLE format. ## Features not yet ported to Zarr-Python 3 The following Zarr-Python 2 features are not yet available in Zarr-Python 3. ### Copying and migrating data See the Zarr-Python 2 documentation on [Copying and migrating data](https://zarr.readthedocs.io/en/support-v2/tutorial.html#copying-migrating-data) for more details. zarr-developers-zarr-python-a2f2415/docs/user-guide/attributes.md000066400000000000000000000050521527014137600252000ustar00rootroot00000000000000# Working with attributes Zarr arrays and groups support custom key/value attributes, which can be useful for storing application-specific metadata. For example: ```python exec="true" session="attributes" source="above" result="ansi" import zarr root = zarr.create_group(store="memory://attributes-demo") root.attrs['foo'] = 'bar' z = root.create_array(name='zzz', shape=(10000, 10000), dtype='int32') z.attrs['baz'] = 42 z.attrs['qux'] = [1, 4, 7, 12] print(sorted(root.attrs)) ``` ```python exec="true" session="attributes" source="above" result="ansi" print('foo' in root.attrs) ``` ```python exec="true" session="attributes" source="above" result="ansi" print(root.attrs['foo']) ``` ```python exec="true" session="attributes" source="above" result="ansi" print(sorted(z.attrs)) ``` ```python exec="true" session="attributes" source="above" result="ansi" print(z.attrs['baz']) ``` ```python exec="true" session="attributes" source="above" result="ansi" print(z.attrs['qux']) ``` Attributes can be deleted with the `del` operator: ```python exec="true" session="attributes" source="above" result="ansi" del z.attrs['baz'] print(sorted(z.attrs)) ``` Note that each attribute assignment or deletion writes the node's metadata document back to the store. To change several attributes in a single write, use [`zarr.Array.update_attributes`][] (or [`zarr.Group.update_attributes`][] for groups), which merges the given dict into the existing attributes and returns the updated array or group: ```python exec="true" session="attributes" source="above" result="ansi" z = z.update_attributes({'baz': 43, 'quux': True}) print(sorted(z.attrs)) ``` Internally Zarr uses JSON to store array and group attributes, so attribute values must be JSON serializable. When copying a Zarr array with [`zarr.from_array`][], its attributes are deep-copied by default so nested dictionaries and lists are independent of the source. Deeply nested attributes can raise `RecursionError` during this copy, even when the source array can be stored and reopened successfully. The threshold depends on Python's recursion limit and the current call stack; it is not a fixed Zarr nesting limit. Pass `attributes={}` if the copy should omit attributes, or provide an explicit attribute dictionary to replace the inherited attributes. When working with hierarchies that contain many arrays and groups, reading the attributes of each node separately can be slow. See [Consolidated metadata](consolidated_metadata.md) for a way to store the metadata (including attributes) of all nodes in a hierarchy in a single document. zarr-developers-zarr-python-a2f2415/docs/user-guide/cli.md000066400000000000000000000065551527014137600235720ustar00rootroot00000000000000# Command-line interface Zarr-Python provides a command-line interface that enables: - migration of Zarr v2 metadata to v3 (see the [3.0 Migration Guide](v3_migration.md) for migrating your *code* from the Zarr-Python 2 API to the Zarr-Python 3 API) - removal of v2 or v3 metadata ## Installation The command-line interface requires the `cli` optional dependencies. Install them with: ```bash pip install "zarr[cli]" ``` Without this extra, running `zarr` in a terminal will fail with `ModuleNotFoundError`. ## Getting help To see available commands run the following in a terminal: ```bash zarr --help ``` or to get help on individual commands: ```bash zarr migrate --help zarr remove-metadata --help ``` ## Migrate metadata from v2 to v3 ### Migrate to a separate location To migrate a Zarr array/group's metadata from v2 to v3 run: ```bash zarr migrate v3 path/to/input.zarr path/to/output.zarr ``` This will write new `zarr.json` files to `output.zarr`, leaving `input.zarr` un-touched. Note - this will migrate the entire Zarr hierarchy, so if `input.zarr` contains multiple groups/arrays, new `zarr.json` will be made for all of them. ### Migrate in-place If you'd prefer to migrate the metadata in-place run: ```bash zarr migrate v3 path/to/input.zarr ``` This will write new `zarr.json` files to `input.zarr`, leaving the existing v2 metadata un-touched. To open the array/group using the new metadata use: ```python exec="true" session="cli-open" source="above" import zarr # create a small array to open (stands in for the migrated store) zarr.create_array("data/cli-demo.zarr", shape=(4, 4), chunks=(2, 2), dtype="i4", overwrite=True) zarr_with_v3_metadata = zarr.open("data/cli-demo.zarr", zarr_format=3) ``` Once you are happy with the conversion, you can run the following to remove the old v2 metadata: ```bash zarr remove-metadata v2 path/to/input.zarr ``` Note there is also a shortcut to migrate and remove v2 metadata in one step: ```bash zarr migrate v3 path/to/input.zarr --remove-v2-metadata ``` ## Remove metadata Remove v2 metadata using: ```bash zarr remove-metadata v2 path/to/input.zarr ``` or v3 with: ```bash zarr remove-metadata v3 path/to/input.zarr ``` By default, this will only allow removal of metadata if a valid alternative exists. For example, you can't remove v2 metadata unless v3 metadata exists at that location. To override this behavior use `--force`: ```bash zarr remove-metadata v3 path/to/input.zarr --force ``` ## Dry run All commands provide a `--dry-run` option that will log changes that would be made on a real run, without creating or modifying any files. ```bash zarr migrate v3 path/to/input.zarr --dry-run Dry run enabled - no new files will be created or changed. Log of files that would be created on a real run: Saving metadata to file://path/to/input.zarr/zarr.json ``` ## Verbose You can also add `--verbose` **before** any command, to see a full log of its actions: ```bash zarr --verbose migrate v3 path/to/input.zarr zarr --verbose remove-metadata v2 path/to/input.zarr ``` ## Equivalent functions All features of the command-line interface are also available as functions in the `zarr.metadata.migrate_v3` module: [`migrate_v2_to_v3`][zarr.metadata.migrate_v3.migrate_v2_to_v3] and [`remove_metadata`][zarr.metadata.migrate_v3.remove_metadata]. See the [`zarr.metadata` API reference](../api/zarr/metadata.md) for details. zarr-developers-zarr-python-a2f2415/docs/user-guide/config.md000066400000000000000000000066561527014137600242720ustar00rootroot00000000000000# Runtime configuration [`zarr.config`][] is responsible for managing the configuration of zarr and is based on the [donfig](https://github.com/pytroll/donfig) Python library. Configuration values can be set using code like the following: ```python exec="true" session="config" source="above" result="ansi" import zarr zarr.config.set({'array.order': 'F'}) print(zarr.config.get('array.order')) ``` `zarr.config.set` can also be used as a context manager, which restores the previous configuration on exit, and `zarr.config.reset` restores the default configuration: ```python exec="true" session="config" source="above" result="ansi" zarr.config.reset() with zarr.config.set({'array.order': 'F'}): print(zarr.config.get('array.order')) print(zarr.config.get('array.order')) ``` Alternatively, configuration values can be set using environment variables, e.g. `ZARR_ARRAY__ORDER=F`. The configuration can also be read from a YAML file in standard locations. For more information, see the [donfig documentation](https://donfig.readthedocs.io/en/latest/). Configuration options include the following: - Default Zarr format `default_zarr_format` - Default array order in memory `array.order` - Whether empty chunks are written to storage `array.write_empty_chunks` - Enable experimental rectilinear chunks `array.rectilinear_chunks` - Whether missing chunks are filled with the array's fill value on read `array.read_missing_chunks` (default `True`). Set to `False` to raise a [`ChunkNotFoundError`][zarr.errors.ChunkNotFoundError] instead. - Whether data encoded with the `pickle` codec can be read `array.allow_unsafe_pickle_decode` (default `False`). Unpickling can execute arbitrary code, so only enable this for data from a source you trust, e.g. with `zarr.config.set({'array.allow_unsafe_pickle_decode': True})` or the environment variable `ZARR_ARRAY__ALLOW_UNSAFE_PICKLE_DECODE=True`. While it is disabled, reading such data raises a [`PickleCodecDisabledError`][zarr.errors.PickleCodecDisabledError]. Writing with the `pickle` codec is not affected. numcodecs has a separate opt-in for the same reason (`numcodecs.pickles.ALLOW_UNSAFE_PICKLE_DECODE = True`, or `NUMCODECS_ALLOW_UNSAFE_PICKLE_DECODE=1` before importing numcodecs); both must be enabled to read pickled data. - Async and threading options, e.g. `async.concurrency` and `threading.max_workers` - Selections of implementations of codecs, codec pipelines and buffers - Enabling GPU support with `zarr.config.enable_gpu()`. See [GPU support](gpu.md) for more. - Control request merging when reading multiple chunks from the same shard with `array.sharding_coalesce_max_gap_bytes` and `array.sharding_coalesce_max_bytes`. Reads of nearby chunks are coalesced into a single request to the store when separated by at most `sharding_coalesce_max_gap_bytes` and the resulting merged read is no larger than `sharding_coalesce_max_bytes`. For selecting custom implementations of codecs, pipelines, buffers and ndbuffers, first register the implementations in the registry and then select them in the config. For example, an implementation of the bytes codec in a class `'custompackage.NewBytesCodec'`, requires the value of `codecs.bytes` to be `'custompackage.NewBytesCodec'`. This is the current default configuration: ```python exec="true" session="config" source="above" result="ansi" from pprint import pprint import io output = io.StringIO() zarr.config.pprint(stream=output, width=60) print(output.getvalue()) ``` zarr-developers-zarr-python-a2f2415/docs/user-guide/consolidated_metadata.md000066400000000000000000000140501527014137600273200ustar00rootroot00000000000000# Consolidated metadata !!! warning The Consolidated Metadata feature in Zarr-Python is considered experimental for v3 stores. [zarr-specs#309](https://github.com/zarr-developers/zarr-specs/pull/309) has proposed a formal extension to the v3 specification to support consolidated metadata. Zarr-Python implements the Consolidated Metadata feature for both the v2 and v3 formats. Consolidated metadata can reduce the time needed to load the metadata for an entire hierarchy, especially when the metadata is being served over a network. Consolidated metadata essentially stores all the metadata for a hierarchy in the metadata of the root Group. ## Usage If consolidated metadata is present in a Zarr Group's metadata then it is used by default. The initial read to open the group will need to communicate with the store (reading from a file for a [`zarr.storage.LocalStore`][], making a network request for a [`zarr.storage.FsspecStore`][]). After that, any subsequent metadata reads to get child Group or Array nodes will *not* require reads from the store. In Python, the consolidated metadata is available on the `.consolidated_metadata` attribute of the `GroupMetadata` object. ```python exec="true" session="consolidated_metadata" source="above" result="ansi" import zarr import warnings warnings.filterwarnings("ignore", category=UserWarning) group = zarr.create_group(store="memory://consolidated-metadata-demo") print(group) array = group.create_array(shape=(1,), name='a', dtype='float64') print(array) ``` ```python exec="true" session="consolidated_metadata" source="above" result="ansi" array = group.create_array(shape=(2, 2), name='b', dtype='float64') print(array) ``` ```python exec="true" session="consolidated_metadata" source="above" result="ansi" array = group.create_array(shape=(3, 3, 3), name='c', dtype='float64') print(array) ``` ```python exec="true" session="consolidated_metadata" source="above" result="ansi" result = zarr.consolidate_metadata("memory://consolidated-metadata-demo") print(result) ``` If we open that group, the Group's metadata includes a `ConsolidatedMetadata` object holding the metadata for every child node, which can be used: ```python exec="true" session="consolidated_metadata" source="above" result="ansi" from pprint import pprint import io consolidated = zarr.open_group(store="memory://consolidated-metadata-demo") consolidated_metadata = consolidated.metadata.consolidated_metadata.metadata output = io.StringIO() pprint(dict(sorted(consolidated_metadata.items())), stream=output, width=60) print(output.getvalue()) ``` Operations on the group to get children automatically use the consolidated metadata: ```python exec="true" session="consolidated_metadata" source="above" result="ansi" print(consolidated['a']) # no read / HTTP request to the Store is required ``` With nested groups, the consolidated metadata is available on the children, recursively: ```python exec="true" session="consolidated_metadata" source="above" result="ansi" child = group.create_group('child', attributes={'kind': 'child'}) grandchild = child.create_group('grandchild', attributes={'kind': 'grandchild'}) consolidated = zarr.consolidate_metadata("memory://consolidated-metadata-demo") output = io.StringIO() pprint(consolidated['child'].metadata.consolidated_metadata, stream=output, width=60) print(output.getvalue()) ``` !!! info "Added in version 3.1.1" The keys in the consolidated metadata are sorted prior to writing. Keys are sorted in ascending order by path depth, where a path is defined as a sequence of strings joined by `"/"`. For keys with the same path length, lexicographic order is used to break the tie. This behavior ensures deterministic metadata output for a given group. ### Controlling the use of consolidated metadata By default, [`zarr.open_group`][] uses consolidated metadata if it is present, and falls back to reading metadata from the store otherwise. This behavior can be controlled with the `use_consolidated` keyword. Pass `use_consolidated=False` to ignore consolidated metadata and always read the metadata of child nodes directly from the store: ```python exec="true" session="consolidated_metadata" source="above" result="ansi" group = zarr.open_group(store="memory://consolidated-metadata-demo", use_consolidated=False) print(group.metadata.consolidated_metadata) ``` Passing `use_consolidated=True` instead raises an error if consolidated metadata is not found, which is useful when reading over a network, where relying on many per-node metadata requests would be slow. ## Synchronization and Concurrency Consolidated metadata is intended for read-heavy use cases on slowly changing hierarchies. For hierarchies where new nodes are constantly being added, removed, or modified, consolidated metadata may not be desirable. 1. It will add some overhead to each update operation, since the metadata would need to be re-consolidated to keep it in sync with the store. 2. Readers using consolidated metadata will regularly see a "past" version of the metadata, at the time they read the root node with its consolidated metadata. Readers who need the latest view of a changing hierarchy can pass `use_consolidated=False` to [`zarr.open_group`][] to always read child metadata directly from the store. ## Stores Without Support for Consolidated Metadata Some stores may want to opt out of the consolidated metadata mechanism. This may be for several reasons like: * They want to maintain read-write consistency, which is challenging with consolidated metadata. * They have their own consolidated metadata mechanism. * They offer good enough performance without need for consolidation. This type of store can declare it doesn't want consolidation by implementing `Store.supports_consolidated_metadata` and returning `False`. For stores that don't support consolidation, Zarr will: * Raise an error on `consolidate_metadata` calls, maintaining the store in its unconsolidated state. * Raise an error in `AsyncGroup.open(..., use_consolidated=True)` * Not use consolidated metadata in `AsyncGroup.open(..., use_consolidated=None)` zarr-developers-zarr-python-a2f2415/docs/user-guide/data_types.md000066400000000000000000000500431527014137600251470ustar00rootroot00000000000000# Array data types ## Zarr's data type model Zarr is designed for interoperability with NumPy, so if you are familiar with NumPy or any other N-dimensional array library, Zarr's model for array data types should seem familiar. However, Zarr data types have some unique features that are described in this document. Zarr arrays operate under an essential design constraint: unlike NumPy arrays, Zarr arrays are designed to be stored and accessed by other Zarr implementations. This means that, among other things, Zarr data types must be serializable to metadata documents in accordance with the Zarr specifications, which adds some unique aspects to the Zarr data type model. The following sections explain Zarr's data type model in greater detail and demonstrate the Zarr Python APIs for working with Zarr data types. ### Array data types Every Zarr array has a data type, which defines the meaning of the array's elements. An array's data type is encoded in the JSON metadata for the array. This means that the data type of an array must be JSON-serializable. In Zarr V2, the data type of an array is stored in the `dtype` field in array metadata. Zarr V3 changed the name of this field to `data_type` and also defined new rules for the values that can be assigned to the `data_type` field. For example, in Zarr V2, the boolean array data type was represented in array metadata as the string `"|b1"`. In Zarr V3, the same type is represented as the string `"bool"`. ### Scalars Zarr also specifies how array elements, i.e., scalars, are encoded in array metadata. This is necessary because Zarr uses a field in array metadata to define a default value for chunks that are not stored. This field, called `fill_value` in both Zarr V2 and Zarr V3 metadata documents, contains a JSON value that can be decoded to a scalar value compatible with the array's data type. For the boolean data type, the scalar encoding is simple—booleans are natively supported by JSON, so Zarr saves booleans as JSON booleans. Other scalars, like floats or raw bytes, have more elaborate encoding schemes, and in some cases, this scheme depends on the Zarr format version. ## Data types in Zarr version 2 Version 2 of the Zarr format defined its data types relative to [NumPy's data types](https://numpy.org/doc/stable/reference/arrays.dtypes.html#data-type-objects-dtype), and added a few non-NumPy data types as well. With one exception ([structured data types](#structured-data-type)), the Zarr V2 JSON identifier for a data type is just the NumPy `str` attribute of that data type: ```python exec="true" session="data_types" source="above" result="ansi" import zarr import numpy as np import json store = {} np_dtype = np.dtype('int64') print(np_dtype.str) ``` ```python exec="true" session="data_types" source="above" result="ansi" z = zarr.create_array(store=store, shape=(1,), dtype=np_dtype, zarr_format=2) dtype_meta = json.loads(store['.zarray'].to_bytes())["dtype"] print(dtype_meta) ``` !!! note The `<` character in the data type metadata encodes the [endianness](https://numpy.org/doc/stable/reference/generated/numpy.dtype.byteorder.html), or "byte order," of the data type. As per the NumPy model, in Zarr version 2 each data type has an endianness where applicable. However, Zarr version 3 data types do not store endianness information. There are two special cases to consider: ["structured" data types](#structured-data-type), and ["object"](#object-data-type) data types. ### Structured data type NumPy allows the construction of a so-called "structured" data types comprised of ordered collections of named fields, where each field is itself a distinct NumPy data type. See the NumPy documentation [here](https://numpy.org/doc/stable/user/basics.rec.html). Crucially, NumPy does not use a special data type for structured data types—instead, NumPy implements structured data types as an optional feature of the so-called "Void" data type, which models arbitrary fixed-size byte strings. The `str` attribute of a regular NumPy void data type is the same as the `str` of a NumPy structured data type. This means that the `str` attribute does not convey information about the fields contained in a structured data type. For these reasons, Zarr V2 uses a special data type encoding for structured data types. The [V2 specification](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html#data-type-encoding) represents fields as `[fieldname, datatype]` or `[fieldname, datatype, shape]`, where the optional shape describes a subarray field. Field data types can themselves be structured. For example, `[["position", "i2'), ('field_b', [('subfield_c', '>f4'), ('subfield_d', 'i2')])]) print(np_dtype.str) ``` ```python exec="true" session="data_types" source="above" result="ansi" z = zarr.create_array(store=store, shape=(1,), dtype=np_dtype, zarr_format=2) dtype_meta = json.loads(store['.zarray'].to_bytes())["dtype"] print(dtype_meta) ``` ### Object data type The NumPy "object" type is essentially an array of references to arbitrary Python objects. It can model arrays of variable-length UTF-8 strings, arrays of variable-length byte strings, or even arrays of variable-length arrays, each with a distinct data type. This makes the "object" data type expressive, but also complicated to store. Zarr Python cannot persistently store references to arbitrary Python objects. But if each of those Python objects has a consistent type, then we can use a special encoding procedure to store the array. This is how Zarr Python stores variable-length UTF-8 strings, or variable-length byte strings. Although these are separate data types in this library, they are both "object" arrays in NumPy, which means they have the *same* Zarr V2 string representation: `"|O"`. So for Zarr V2 we have to disambiguate different "object" data type arrays on the basis of their encoding procedure, i.e., the codecs declared in the `filters` and `compressor` attributes of array metadata. If an array with data type "object" used the `"vlen-utf8"` codec, then it was interpreted as an array of variable-length strings. If an array with data type "object" used the `"vlen-bytes"` codec, then it was interpreted as an array of variable-length byte strings. This all means that the `dtype` field alone does not fully specify a data type in Zarr V2. The name of the object codec used, if one was used, is also required. Although this fact can be ignored for many simple numeric data types, any comprehensive approach to Zarr V2 data types must either reject the "object" data types or include the "object codec" identifier in the JSON form of the basic data type model. ## Data types in Zarr version 3 The NumPy-based Zarr V2 data type representation was effective for simple data types but struggled with more complex data types, like "object" and "structured" data types. To address these limitations, Zarr V3 introduced several key changes to how data types are represented: - Instead of copying NumPy character codecs, Zarr V3 defines an identifier for each data type. The basic data types are identified by strings like `"int8"`, `"int16"`, etc., and data types that require a configuration can be identified by a JSON object. For example, this JSON object declares a datetime data type: ```json { "name": "numpy.datetime64", "configuration": { "unit": "s", "scale_factor": 10 } } ``` - Zarr V3 data types do not have endianness. This is a departure from Zarr V2, where multi-byte data types are defined with endianness information. Instead, Zarr V3 requires that the endianness of encoded array chunks is specified in the `codecs` attribute of array metadata. The Zarr V3 specification leaves the in-memory endianness of decoded array chunks as an implementation detail. For more about data types in Zarr V3, see the [V3 specification](https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html). ## Data types in Zarr Python The two Zarr formats that Zarr Python supports specify data types in different ways: data types in Zarr version 2 are encoded as NumPy-compatible strings (or lists, in the case of structured data types), while data types in Zarr V3 are encoded as either strings or JSON objects. Zarr V3 data types do not have any associated endianness information, unlike Zarr V2 data types. Zarr Python needs to support both Zarr V2 and V3, which means we need to abstract over these differences. We do this with an abstract Zarr data type class: [ZDType][zarr.dtype.ZDType] which provides Zarr V2 and Zarr V3 compatibility routines for "native" data types. In this context, a "native" data type is a Python class, typically defined in another library, that models an array's data type. For example, [`numpy.dtypes.UInt8DType`][] is a native data type defined in NumPy. Zarr Python wraps the NumPy `uint8` with a [ZDType][zarr.dtype.ZDType] instance called [UInt8][zarr.dtype.UInt8]. As of this writing, the only native data types Zarr Python supports are NumPy data types. We could avoid the "native data type" jargon and just say "NumPy data type," but we do not want to rule out the possibility of using non-NumPy array backends in the future. Each data type supported by Zarr Python is modeled by a [ZDType][zarr.dtype.ZDType] subclass, which provides an API for the following operations: - Encoding and decoding a native data type - Encoding and decoding a data type to and from Zarr V2 and Zarr V3 array metadata - Encoding and decoding a scalar value to and from Zarr V2 and Zarr V3 array metadata - Casting a Python object to a scalar value consistent with the data type ### List of data types The following section lists the data types built in to Zarr Python. With a few exceptions, Zarr Python supports nearly all of the data types in NumPy. If you need a data type that is not listed here, it's possible to create it yourself: see [Adding new data types](#adding-new-data-types). #### Boolean - [Boolean][zarr.dtype.Bool] #### Integral - [Signed 8-bit integer][zarr.dtype.Int8] - [Signed 16-bit integer][zarr.dtype.Int16] - [Signed 32-bit integer][zarr.dtype.Int32] - [Signed 64-bit integer][zarr.dtype.Int64] - [Unsigned 8-bit integer][zarr.dtype.UInt8] - [Unsigned 16-bit integer][zarr.dtype.UInt16] - [Unsigned 32-bit integer][zarr.dtype.UInt32] - [Unsigned 64-bit integer][zarr.dtype.UInt64] #### Floating-point - [16-bit floating-point][zarr.dtype.Float16] - [32-bit floating-point][zarr.dtype.Float32] - [64-bit floating-point][zarr.dtype.Float64] - [64-bit complex floating-point][zarr.dtype.Complex64] - [128-bit complex floating-point][zarr.dtype.Complex128] #### String - [Fixed-length UTF-32 string][zarr.dtype.FixedLengthUTF32] - [Variable-length UTF-8 string][zarr.dtype.VariableLengthUTF8] #### Bytes - [Fixed-length null-terminated bytes][zarr.dtype.NullTerminatedBytes] - [Fixed-length raw bytes][zarr.dtype.RawBytes] - [Variable-length bytes][zarr.dtype.VariableLengthBytes] #### Temporal - [DateTime64][zarr.dtype.DateTime64] - [TimeDelta64][zarr.dtype.TimeDelta64] #### Struct-like - [Structured][zarr.dtype.Structured] !!! note "Zarr V3 Structured Data Types" In Zarr V3, structured data types are specified using the `struct` extension defined in the [zarr-extensions repository](https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/struct). The JSON representation uses an object format for fields: ```json { "name": "struct", "configuration": { "fields": [ {"name": "x", "data_type": "float32"}, {"name": "y", "data_type": "int64"} ] } } ``` For backward compatibility, Zarr Python also accepts the legacy `structured` name with tuple-format fields when reading existing data. Fill values for structured types are represented as JSON objects mapping field names to values: ```json {"x": 1.5, "y": 42} ``` When using structured types with multi-byte fields, the `bytes` codec must specify an explicit `endian` parameter. If omitted, Zarr Python assumes little-endian for legacy compatibility but emits a warning. ### Example usage This section will demonstrate the basic usage of Zarr data types. Create a `ZDType` from a native data type: ```python exec="true" session="data_types" source="above" from zarr.dtype import Int8 import numpy as np int8 = Int8.from_native_dtype(np.dtype('int8')) ``` Convert back to a native data type: ```python exec="true" session="data_types" source="above" native_dtype = int8.to_native_dtype() assert native_dtype == np.dtype('int8') ``` Get the default scalar value for the data type: ```python exec="true" session="data_types" source="above" default_value = int8.default_scalar() assert default_value == np.int8(0) ``` Serialize to JSON for Zarr V2: ```python exec="true" session="data_types" source="above" result="ansi" json_v2 = int8.to_json(zarr_format=2) print(json_v2) ``` !!! note The representation returned by `to_json(zarr_format=2)` is more abstract than the literal contents of Zarr V2 array metadata, because the JSON representation used by the `ZDType` classes must be distinct across different data types. As noted [earlier](#object-data-type), Zarr V2 identifies multiple distinct data types with the "object" data type identifier `"|O"`. Extra information is needed to disambiguate these data types from one another. That's the reason for the `object_codec_id` field you see here. And for V3: ```python exec="true" session="data_types" source="above" result="ansi" json_v3 = int8.to_json(zarr_format=3) print(json_v3) ``` Serialize a scalar value to JSON: ```python exec="true" session="data_types" source="above" result="ansi" json_value = int8.to_json_scalar(42, zarr_format=3) print(json_value) ``` Deserialize a scalar value from JSON: ```python exec="true" session="data_types" source="above" scalar_value = int8.from_json_scalar(42, zarr_format=3) assert scalar_value == np.int8(42) ``` ### Adding new data types Each Zarr data type is a separate Python class that inherits from [ZDType][zarr.dtype.ZDType]. You can define a custom data type by writing your own subclass of [ZDType][zarr.dtype.ZDType] and adding your data type to the data type registry. To see an executable demonstration of this process, see the [`custom_dtype` example](../user-guide/examples/custom_dtype.md). ### Data type resolution Although Zarr Python uses a different data type model from NumPy, you can still define a Zarr array with a NumPy data type object: ```python exec="true" session="data_types" source="above" result="ansi" from zarr import create_array import numpy as np a = create_array({}, shape=(10,), dtype=np.dtype('int')) print(a) ``` Or a string representation of a NumPy data type: ```python exec="true" session="data_types" source="above" result="ansi" a = create_array({}, shape=(10,), dtype=' ``` This example illustrates a general problem Zarr Python has to solve: how can we allow users to specify a data type as a string or a NumPy `dtype` object, and produce the right Zarr data type from that input? We call this process "data type resolution." Zarr Python also performs data type resolution when reading stored arrays, although in this case the input is a JSON value instead of a NumPy data type. For simple data types like `int`, the solution could be extremely simple: just maintain a lookup table that maps a NumPy data type to the Zarr data type equivalent. But not all data types are so simple. Consider this case: ```python exec="true" session="data_types" source="above" result="ansi" from zarr import create_array import numpy as np a = create_array({}, shape=(10,), dtype=[('a', 'f8'), ('b', 'i8')]) print(a.dtype) # this is the NumPy data type ``` ```python exec="true" session="data_types" source="above" result="ansi" print(a.metadata.data_type) # this is the Zarr data type ``` In this example, we created a [NumPy structured data type](https://numpy.org/doc/stable/user/basics.rec.html#structured-datatypes). This data type is a container that can hold any NumPy data type, which makes it recursive. It is not possible to make a lookup table that relates all NumPy structured data types to their Zarr equivalents, as there is a nearly unbounded number of different structured data types. So instead of a static lookup table, Zarr Python relies on a dynamic approach to data type resolution. Zarr Python defines a collection of Zarr data types. This collection, called a "data type registry," is essentially a dictionary where the keys are strings (a canonical name for each data type), and the values are the data type classes themselves. Dynamic data type resolution entails iterating over these data type classes, invoking that class' [from_native_dtype][zarr.dtype.ZDType.from_native_dtype] method, and returning a concrete data type instance if and only if exactly one of those constructor invocations is successful. In plain language, we take some user input, like a NumPy data type, offer it to all the known data type classes, and return an instance of the one data type class that can accept that user input. We want to avoid a situation where the same native data type matches multiple Zarr data types; that is, a NumPy data type should *uniquely* specify a single Zarr data type. But data type resolution is dynamic, so it's not possible to statically guarantee this uniqueness constraint. Therefore, we attempt data type resolution against *every* data type class, and if, for some reason, a native data type matches multiple Zarr data types, we treat this as an error and raise an exception. If you have a NumPy data type and you want to get the corresponding `ZDType` instance, you can use the `parse_dtype` function, which will use the dynamic resolution described above. `parse_dtype` handles a range of input types: - NumPy data types: ```python exec="true" session="data_types" source="above" result="ansi" import numpy as np from zarr.dtype import parse_dtype my_dtype = np.dtype('>M8[10s]') print(parse_dtype(my_dtype, zarr_format=2)) ``` - NumPy data type-compatible strings: ```python exec="true" session="data_types" source="above" result="ansi" dtype_str = '>M8[10s]' print(parse_dtype(dtype_str, zarr_format=2)) ``` - `ZDType` instances: ```python exec="true" session="data_types" source="above" result="ansi" from zarr.dtype import DateTime64 zdt = DateTime64(endianness='big', scale_factor=10, unit='s') print(parse_dtype(zdt, zarr_format=2)) # Use a ZDType (this is a no-op) ``` - Python dictionaries (requires `zarr_format=3`). These dictionaries must be consistent with the `JSON` form of the data type: ```python exec="true" session="data_types" source="above" result="ansi" dt_dict = {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 10}} print(parse_dtype(dt_dict, zarr_format=3)) ``` ```python exec="true" session="data_types" source="above" result="ansi" print(parse_dtype(dt_dict, zarr_format=3).to_json(zarr_format=3)) ``` zarr-developers-zarr-python-a2f2415/docs/user-guide/examples/000077500000000000000000000000001527014137600243045ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/docs/user-guide/examples/codec_pipeline_performance.md000066400000000000000000000003551527014137600321540ustar00rootroot00000000000000--8<-- "examples/codec_pipeline_performance/README.md" ## Source Code ```python exec="false" reason="pymdownx snippet include directive, not python source" --8<-- "examples/codec_pipeline_performance/codec_pipeline_performance.py" ``` zarr-developers-zarr-python-a2f2415/docs/user-guide/examples/custom_dtype.md000066400000000000000000000003031527014137600273410ustar00rootroot00000000000000--8<-- "examples/custom_dtype/README.md" ## Source Code ```python exec="false" reason="pymdownx snippet include directive, not python source" --8<-- "examples/custom_dtype/custom_dtype.py" ``` zarr-developers-zarr-python-a2f2415/docs/user-guide/examples/rectilinear_chunks.md000066400000000000000000000175731527014137600305170ustar00rootroot00000000000000# Rectilinear Chunk Grids This example demonstrates rectilinear (variable-sized) chunk grids, introduced in [#3802](https://github.com/zarr-developers/zarr-python/pull/3802). Rectilinear grids allow different chunk sizes along each dimension, which is useful for data that doesn't partition evenly — for example, sparse HEALPix cells grouped by parent tile, boundary-padded HPC arrays, or ingesting existing variable-chunked datasets via VirtualiZarr. See [Rectilinear (variable) chunk grids](../arrays.md#rectilinear-variable-chunk-grids) in the arrays guide for an introduction to the feature. The example chunks a HEALPix dataset by parent tile, writes it as a Zarr v3 array with a rectilinear chunk grid, and verifies the round trip through [Xarray](https://xarray.dev). !!! warning "Experimental" Rectilinear chunk grids are an experimental feature and may change in future releases. In addition, this example currently requires [a fork of Xarray](https://github.com/maxrjones/xarray/tree/poc/unified-zarr-chunk-grid) with rectilinear chunk grid support (this will ideally be incorporated into a future Xarray release), as well as the `dask`, `healpix-geo`, and `obstore` packages, and it reads an example dataset from a remote server. For these reasons the code on this page is not executed when the documentation is built; the outputs shown were captured from a live run. ## Setup Rectilinear chunk grids are disabled by default and must be explicitly enabled via the `array.rectilinear_chunks` configuration option: ```python exec="false" reason="requires an xarray fork with rectilinear chunk grid support and remote example data" import json import tempfile from pathlib import Path import numpy as np import xarray as xr from healpix_geo import nested from obstore.store import HTTPStore import zarr from zarr.storage import ObjectStore # Increase concurrency for better performance with obstore zarr.config.set({'async.concurrency': 128}) # Opt in to rectilinear chunks zarr.config.set({'array.rectilinear_chunks': True}) ``` ## Inspect the HEALPix dataset Load the remote Zarr store to understand the data structure before chunking it: ```python exec="false" reason="requires an xarray fork with rectilinear chunk grid support and remote example data" ob_store = HTTPStore.from_url("https://data-taos.ifremer.fr/GRID4EARTH/no_chunk_healpix.zarr") store = ObjectStore(ob_store) g = zarr.open_group(store, mode="r", zarr_format=2, use_consolidated=True) arr = g['da'] print("Members:", list(g.members())) print("Attrs:", dict(g.attrs)) print("Write chunk sizes:", arr.write_chunk_sizes) ``` ```text Members: [('cell_ids', ), ('da', )] Attrs: {} Write chunk sizes: ((55611, 55611, 55611, 55609),) ``` ## HEALPix-style variable chunking Inspired by [this use case](https://github.com/zarr-developers/zarr-python/pull/3534#issuecomment-3848669859): HEALPix grids where cells are grouped by parent tile at a coarser resolution level, producing variable-sized chunks along the cell dimension when accounting for sparsity. ```python exec="false" reason="requires an xarray fork with rectilinear chunk grid support and remote example data" da = xr.open_zarr( store, zarr_format=2, consolidated=True, ) depth = da.cell_ids.attrs['level'] new_depth = depth - 6 parents = nested.zoom_to(da.cell_ids, depth=depth, new_depth=new_depth) _, chunk_sizes = np.unique(parents, return_counts=True) print(chunk_sizes) ``` ```text [ 25 645 1510 2363 3203 74 769 3963 4096 233 1603 2450 4096 4096 3327 4047 4096 4096 1278 2113 4096 3879 4096 3842 2173 983 4046 2187 4095 1369 4096 4096 4096 4096 3515 1395 4096 3622 4096 4096 3875 4096 4096 4096 4096 4096 2034 4096 358 3991 4096 4096 4096 4096 2714 1210 4096 4096 4096 4096 92 3826 4096 2629 4096 1438 4096 353 4078 3410 2407 226 132 2738 1223 23] ``` Rechunk the dataset with these variable-sized chunks: ```python exec="false" reason="requires an xarray fork with rectilinear chunk grid support and remote example data" da = da.chunk({"cell_ids": tuple(chunk_sizes.tolist())}) print(da.chunks) ``` ```text Frozen({'cell_ids': (25, 645, 1510, 2363, 3203, 74, 769, 3963, 4096, 233, 1603, 2450, 4096, 4096, 3327, 4047, 4096, 4096, 1278, 2113, 4096, 3879, 4096, 3842, 2173, 983, 4046, 2187, 4095, 1369, 4096, 4096, 4096, 4096, 3515, 1395, 4096, 3622, 4096, 4096, 3875, 4096, 4096, 4096, 4096, 4096, 2034, 4096, 358, 3991, 4096, 4096, 4096, 4096, 2714, 1210, 4096, 4096, 4096, 4096, 92, 3826, 4096, 2629, 4096, 1438, 4096, 353, 4078, 3410, 2407, 226, 132, 2738, 1223, 23)}) ``` ## Write as rectilinear Zarr v3 Write the variable-chunked dataset to a local Zarr v3 store with rectilinear chunk grids enabled: ```python exec="false" reason="requires an xarray fork with rectilinear chunk grid support and remote example data" output_path = Path(tempfile.mkdtemp()) / "healpix_rectilinear.zarr" encoding = { "da": {"chunks": [chunk_sizes.tolist()]}, "cell_ids": {"chunks": [chunk_sizes.tolist()]}, } da.to_zarr(output_path, zarr_format=3, mode="w", encoding=encoding, consolidated=False) print(f"Written to: {output_path}") ``` ```text Written to: /var/folders/.../T/tmp6dibcrho/healpix_rectilinear.zarr ``` ## Verify the rectilinear metadata Inspect the output store to confirm the chunk grid is serialized as `"rectilinear"` in `zarr.json`, following the [rectilinear chunk grid extension spec](https://github.com/zarr-developers/zarr-extensions/tree/main/chunk-grids/rectilinear). Key things to look for in `chunk_grid`: - **`name`**: `"rectilinear"` (the extension identifier) - **`configuration.kind`**: `"inline"` (edge lengths stored directly in metadata) - **`configuration.chunk_shapes`**: one entry per dimension — here a single list for the 1D `cell_ids` axis. Each element is either: - a **bare integer** for a unique edge length (e.g., `25`, `645`) - a **`[value, count]` array** using [run-length encoding](https://github.com/zarr-developers/zarr-extensions/tree/main/chunk-grids/rectilinear#run-length-encoding) for consecutive repeated sizes (e.g., `[4096, 4]` means four consecutive chunks of size 4096) ```python exec="false" reason="requires an xarray fork with rectilinear chunk grid support and remote example data" # Read the zarr.json for the 'da' array da_meta_path = output_path / "da" / "zarr.json" meta = json.loads(da_meta_path.read_text()) print(meta['chunk_grid']) ``` ```text {'name': 'rectilinear', 'configuration': {'kind': 'inline', 'chunk_shapes': [[25, 645, 1510, 2363, 3203, 74, 769, 3963, 4096, 233, 1603, 2450, [4096, 2], 3327, 4047, [4096, 2], 1278, 2113, 4096, 3879, 4096, 3842, 2173, 983, 4046, 2187, 4095, 1369, [4096, 4], 3515, 1395, 4096, 3622, [4096, 2], 3875, [4096, 5], 2034, 4096, 358, 3991, [4096, 4], 2714, 1210, [4096, 4], 92, 3826, 4096, 2629, 4096, 1438, 4096, 353, 4078, 3410, 2407, 226, 132, 2738, 1223, 23]]}} ``` ## Round-trip verification Read the rectilinear store back and confirm the chunk sizes are preserved: ```python exec="false" reason="requires an xarray fork with rectilinear chunk grid support and remote example data" roundtrip = xr.open_zarr(output_path, zarr_format=3, consolidated=False) print("Round-trip chunk sizes:", roundtrip.chunks) ``` ```text Round-trip chunk sizes: Frozen({'cell_ids': (25, 645, 1510, 2363, 3203, 74, 769, 3963, 4096, 233, 1603, 2450, 4096, 4096, 3327, 4047, 4096, 4096, 1278, 2113, 4096, 3879, 4096, 3842, 2173, 983, 4046, 2187, 4095, 1369, 4096, 4096, 4096, 4096, 3515, 1395, 4096, 3622, 4096, 4096, 3875, 4096, 4096, 4096, 4096, 4096, 2034, 4096, 358, 3991, 4096, 4096, 4096, 4096, 2714, 1210, 4096, 4096, 4096, 4096, 92, 3826, 4096, 2629, 4096, 1438, 4096, 353, 4078, 3410, 2407, 226, 132, 2738, 1223, 23)}) ``` zarr-developers-zarr-python-a2f2415/docs/user-guide/examples/sharding_coalescing.md000066400000000000000000000003301527014137600306100ustar00rootroot00000000000000--8<-- "examples/sharding_coalescing/README.md" ## Source Code ```python exec="false" reason="pymdownx snippet include directive, not python source" --8<-- "examples/sharding_coalescing/sharding_coalescing.py" ``` zarr-developers-zarr-python-a2f2415/docs/user-guide/experimental.md000066400000000000000000000321171527014137600255110ustar00rootroot00000000000000# Experimental features This section contains documentation for experimental Zarr Python features. The features described here are exciting and potentially useful, but also volatile -- we might change them at any time. Take this into account if you consider depending on these features. See the [experimental API policy](../contributing.md#experimental-api-policy) for the stability guarantees (or lack thereof) that apply to everything documented on this page. ## `FusedCodecPipeline` A *codec pipeline* is the machinery that turns chunks of array data into stored bytes and back, by running the configured codecs (filters, serializer, compressors) and performing the storage IO. The default pipeline, `BatchedCodecPipeline`, schedules both the IO and codec work asynchronously -- roughly one coroutine per chunk operation. `FusedCodecPipeline` is an experimental alternative that runs codec compute and synchronous IO *synchronously*, avoiding that per-chunk async scheduling overhead and nasty [`asyncio.to_thread` overhead](https://github.com/python/cpython/issues/136084). On real workloads the scheduling cost can dominate the actual codec work, so removing it is a significant speedup -- especially for **sharded arrays**, where a single shard read or write involves many inner chunks. > **Note:** The win is *not* a faster compressor or a different on-disk format -- the bytes written are > identical. It is purely the removal of async scheduling overhead, plus a few vectorized fast paths > for dense, uncompressed shards i.e., removing compute where it is not needed. ### When it helps There are two main benefits in this new pipeline: 1. When storage IO is fast enough that the *scheduling* overhead, not the IO itself, is the bottleneck. That means **low-latency stores** that are themselves synchronous -- in particular [`zarr.storage.MemoryStore`][] and [`zarr.storage.LocalStore`][]. 2. Whenever codec work that is truly synchronous will not need the overhead of `async` scheduling i.e., inner-chunk codec work in sharding using something like `zstd`. We also now make use of `asyncio.as_completed` so that IO from asynchronous sources can begin decompression immediately. ### Opting in `FusedCodecPipeline` is opt-in: the default pipeline is unchanged, so existing code behaves exactly as before. Select it through the [runtime configuration](config.md), by setting `codec_pipeline.path`: ```python exec="true" session="experimental-fused" source="above" result="ansi" import zarr zarr.config.set( {"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"} ) ``` You can set this globally as above (affecting every array created or opened afterwards), or scope it to a block of code using `zarr.config.set` as a context manager: ```python exec="true" session="experimental-fused" source="above" import numpy as np import zarr from zarr.storage import MemoryStore with zarr.config.set( {"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"} ): # A sharded array on an in-memory store -- the low-latency case the # synchronous pipeline targets. arr = zarr.create_array( store=MemoryStore(), shape=(1000, 1000), chunks=(100, 100), shards=(1000, 1000), dtype="float32", ) arr[:] = np.random.random((1000, 1000)).astype("float32") result = arr[:] print(result.shape) ``` To return to the default pipeline, set `codec_pipeline.path` back to the batched implementation: ```python exec="true" session="experimental-fused" source="above" import zarr zarr.config.set( {"codec_pipeline.path": "zarr.core.codec_pipeline.BatchedCodecPipeline"} ) ``` ### Threading By default the synchronous pipeline runs fully threaded i.e., `os.cpu_count()`. For memory-backed workflows, you may find that setting `max_workers` to 1 helps (since requests for data from the store are GIL-locked, unlike, say, file-backed i/o). ```python exec="true" session="experimental-fused" source="above" import zarr # Use a fixed-size thread pool for codec compute. zarr.config.set({"codec_pipeline.max_workers": 8}) # Or "auto", sized to the number of CPUs. zarr.config.set({"codec_pipeline.max_workers": None}) ``` On many-core nodes a pool sized to `cpu_count` can oversubscribe workloads that already parallelize at a higher level (e.g. Dask). `codec_pipeline.max_workers` only affects `FusedCodecPipeline`; the default `BatchedCodecPipeline` ignores it. ## `CacheStore` Zarr Python 3.1.4 adds [`zarr.experimental.cache_store.CacheStore`][], which provides a dual-store caching implementation that can be wrapped around any Zarr store to improve performance for repeated data access. This is particularly useful when working with remote stores (e.g., S3, HTTP) where network latency can significantly impact data access speed. The CacheStore implements a cache that uses a separate Store instance as the cache backend, providing persistent caching capabilities with time-based expiration, size-based eviction, and flexible cache storage options. It automatically evicts the least recently used items when the cache reaches its maximum size. Because the `CacheStore` uses an ordinary Zarr `Store` object as the caching layer, you can reuse the data stored in the cache later. !!! note The CacheStore is a wrapper store that maintains compatibility with the full `zarr.abc.store.Store` API while adding transparent caching functionality. ### Basic Usage Creating a CacheStore requires both a source store and a cache store. The cache store can be any Store implementation, providing flexibility in cache persistence: ```python exec="true" session="experimental" source="above" import zarr from zarr.storage import LocalStore import numpy as np from tempfile import mkdtemp from zarr.experimental.cache_store import CacheStore # Create a local store and a separate cache store local_store_path = mkdtemp(suffix='.zarr') source_store = LocalStore(local_store_path) cache_store = zarr.storage.MemoryStore() # In-memory cache cached_store = CacheStore( store=source_store, cache_store=cache_store, max_size=256*1024*1024 # 256MB cache ) # Create an array using the cached store zarr_array = zarr.zeros((100, 100), chunks=(10, 10), dtype='f8', store=cached_store, mode='w') # Write some data to force chunk creation zarr_array[:] = np.random.random((100, 100)) ``` The dual-store architecture allows you to use different store types for source and cache, such as a remote store for source data and a local store for persistent caching. ### Performance Benefits The CacheStore provides significant performance improvements for repeated data access: ```python exec="true" session="experimental" source="above" import time # Benchmark reading with cache start = time.time() for _ in range(100): _ = zarr_array[:] elapsed_cache = time.time() - start # Compare with direct store access (without cache) zarr_array_nocache = zarr.open(local_store_path, mode='r') start = time.time() for _ in range(100): _ = zarr_array_nocache[:] elapsed_nocache = time.time() - start # Cache provides speedup for repeated access speedup = elapsed_nocache / elapsed_cache print(f"Speedup is {speedup}") ``` Cache effectiveness is particularly pronounced with repeated access to the same data chunks. ### Cache Configuration The CacheStore can be configured with several parameters: **max_size**: Controls the maximum size of cached data in bytes. The [Basic Usage](#basic-usage) example above sets a 256MB limit with `max_size=256*1024*1024`: ```python exec="true" session="experimental" source="above" # Unlimited cache size (use with caution) cache = CacheStore( store=source_store, cache_store=cache_store, max_size=None ) ``` **max_age_seconds**: Controls time-based cache expiration ```python exec="true" session="experimental" source="above" # Cache expires after 1 hour cache = CacheStore( store=source_store, cache_store=cache_store, max_age_seconds=3600 ) # Cache never expires cache = CacheStore( store=source_store, cache_store=cache_store, max_age_seconds="infinity" ) ``` **cache_set_data**: Controls whether written data is cached ```python exec="true" session="experimental" source="above" # Cache data when writing (default) cache = CacheStore( store=source_store, cache_store=cache_store, cache_set_data=True ) # Don't cache written data (read-only cache) cache = CacheStore( store=source_store, cache_store=cache_store, cache_set_data=False ) ``` ### Cache Statistics The CacheStore provides statistics to monitor cache performance and state: ```python exec="true" session="experimental" source="above" # Access some data to generate cache activity # (these chunks were already cached by the reads above, so both accesses are cache hits) data = zarr_array[0:50, 0:50] data = zarr_array[0:50, 0:50] # Get comprehensive cache information info = cached_store.cache_info() print(info['cache_store_type']) # e.g., 'MemoryStore' print(info['max_age_seconds']) print(info['max_size']) print(info['current_size']) print(info['tracked_keys']) print(info['cached_keys']) print(info['cache_set_data']) ``` The `cache_info()` method returns a dictionary with detailed information about the cache state. ### Cache Management The CacheStore provides methods for manual cache management: ```python exec="true" session="experimental" source="above" # Clear all cached data and tracking information import asyncio asyncio.run(cached_store.clear_cache()) # Check cache info after clearing info = cached_store.cache_info() assert info['tracked_keys'] == 0 assert info['current_size'] == 0 ``` The `clear_cache()` method is an async method that clears both the cache store (if it supports the `clear` method) and all internal tracking data. ### Best Practices 1. **Choose appropriate cache store**: Use MemoryStore for fast temporary caching or LocalStore for persistent caching 2. **Size the cache appropriately**: Set `max_size` based on available storage and expected data access patterns 3. **Use with remote stores**: The cache provides the most benefit when wrapping slow remote stores 4. **Monitor cache statistics**: Use `cache_info()` to tune cache size and access patterns 5. **Consider data locality**: Group related data accesses together to improve cache efficiency 6. **Set appropriate expiration**: Use `max_age_seconds` for time-sensitive data or "infinity" for static data ### Working with Different Store Types The CacheStore can wrap any store that implements the `zarr.abc.store.Store` interface and use any store type for the cache backend: #### Local Store with Memory Cache ```python exec="true" session="experimental-memory-cache" source="above" from zarr.storage import LocalStore, MemoryStore from zarr.experimental.cache_store import CacheStore from tempfile import mkdtemp local_store_path = mkdtemp(suffix='.zarr') source_store = LocalStore(local_store_path) cache_store = MemoryStore() cached_store = CacheStore( store=source_store, cache_store=cache_store, max_size=128*1024*1024 ) ``` #### Memory Store with Persistent Cache ```python exec="true" session="experimental-local-cache" source="above" from tempfile import mkdtemp from zarr.storage import MemoryStore, LocalStore from zarr.experimental.cache_store import CacheStore memory_store = MemoryStore() local_store_path = mkdtemp(suffix='.zarr') persistent_cache = LocalStore(local_store_path) cached_store = CacheStore( store=memory_store, cache_store=persistent_cache, max_size=256*1024*1024 ) ``` The dual-store architecture provides flexibility in choosing the best combination of source and cache stores for your specific use case. ### Examples from Real Usage Here's a complete example demonstrating cache effectiveness: ```python exec="true" session="experimental-final" source="above" import numpy as np import time from tempfile import mkdtemp import zarr import zarr.storage from zarr.experimental.cache_store import CacheStore # Create test data with dual-store cache local_store_path = mkdtemp(suffix='.zarr') source_store = zarr.storage.LocalStore(local_store_path) cache_store = zarr.storage.MemoryStore() cached_store = CacheStore( store=source_store, cache_store=cache_store, max_size=256*1024*1024 ) zarr_array = zarr.zeros((100, 100), chunks=(10, 10), dtype='f8', store=cached_store, mode='w') zarr_array[:] = np.random.random((100, 100)) # Demonstrate cache effectiveness with repeated access start = time.time() data = zarr_array[20:30, 20:30] # First access (cache miss) first_access = time.time() - start print(f"First access took {first_access}") start = time.time() data = zarr_array[20:30, 20:30] # Second access (cache hit) second_access = time.time() - start print(f"Second access took {second_access}") # Check cache statistics info = cached_store.cache_info() assert info['cached_keys'] > 0 # Should have cached keys assert info['current_size'] > 0 # Should have cached data print(f"Cache contains {info['cached_keys']} keys with {info['current_size']} bytes") ``` This example shows how the CacheStore can significantly reduce access times for repeated data reads, particularly important when working with remote data sources. The dual-store architecture allows for flexible cache persistence and management. zarr-developers-zarr-python-a2f2415/docs/user-guide/extending.md000066400000000000000000000124241527014137600250000ustar00rootroot00000000000000# Extending Zarr Zarr-Python 3 was designed to be extensible. This means that you can extend the library by writing custom classes and plugins. Currently, Zarr can be extended in the following ways: ## Custom codecs !!! note This section explains how custom codecs can be created for Zarr format 3 arrays. For Zarr format 2, codecs should subclass the [numcodecs.abc.Codec](https://numcodecs.readthedocs.io/en/stable/abc.html#numcodecs.abc.Codec) base class and register through [numcodecs.registry.register_codec](https://numcodecs.readthedocs.io/en/stable/registry.html#numcodecs.registry.register_codec). There are three types of codecs in Zarr: - array-to-array - array-to-bytes - bytes-to-bytes Array-to-array codecs are used to transform the array data before serializing to bytes. Examples include delta encoding or scaling codecs. Array-to-bytes codecs are used for serializing the array data to bytes. In Zarr, the main codec to use for numeric arrays is the [`zarr.codecs.BytesCodec`][]. Bytes-to-bytes codecs transform the serialized bytestreams of the array data. Examples include compression codecs, such as [`zarr.codecs.GzipCodec`][], [`zarr.codecs.BloscCodec`][] or [`zarr.codecs.ZstdCodec`][], and codecs that add a checksum to the bytestream, such as [`zarr.codecs.Crc32cCodec`][]. Custom codecs for Zarr are implemented by subclassing the relevant base class, see [`zarr.abc.codec.ArrayArrayCodec`][], [`zarr.abc.codec.ArrayBytesCodec`][] and [`zarr.abc.codec.BytesBytesCodec`][]. Most custom codecs should implement the `_encode_single` and `_decode_single` methods. These methods operate on single chunks of the array data. Alternatively, custom codecs can implement the `encode` and `decode` methods, which operate on batches of chunks, in case the codec is intended to implement its own batch processing. Custom codecs should also implement the following methods: - `compute_encoded_size`, which returns the byte size of the encoded data given the byte size of the original data. It should raise `NotImplementedError` for codecs with variable-sized outputs, such as compression codecs. - `validate` (optional), which can be used to check that the codec metadata is compatible with the array metadata. It should raise errors if not. - `resolve_metadata` (optional), which is important for codecs that change the shape, dtype or fill value of a chunk. - `evolve_from_array_spec` (optional), which can be useful for automatically filling in codec configuration metadata from the array metadata. To use custom codecs in Zarr, they need to be registered using the [entrypoint mechanism](https://packaging.python.org/en/latest/specifications/entry-points/). Commonly, entrypoints are declared in the `pyproject.toml` of your package under the `[project.entry-points."zarr.codecs"]` section. Zarr will automatically discover all codecs registered via the entrypoint mechanism in installed packages. ```toml [project.entry-points."zarr.codecs"] "custompackage.fancy_codec" = "custompackage:FancyCodec" ``` New codecs need to have their own unique identifier. To avoid naming collisions, it is strongly recommended to prefix the codec identifier with a unique name. For example, the codecs from `numcodecs` are prefixed with `numcodecs.`, e.g. `numcodecs.delta`. If someone opens an array that uses your codec without your package installed, Zarr raises [`zarr.errors.UnknownCodecError`][] explaining how to register an implementation. Zarr also keeps a small table of codec names and the published packages that provide them, and names those packages in that error. Once your package is on PyPI, please open a pull request adding it to the codec-package tables in `src/zarr/registry.py`, so that users get a message telling them exactly what to install. !!! note Note that the extension mechanism for the Zarr format 3 is still under development. Requirements for custom codecs including the choice of codec identifiers might change in the future. It is also possible to register codecs as replacements for existing codecs. This might be useful for providing specialized implementations, such as GPU-based codecs. In case of multiple codecs, the [`zarr.config`][] mechanism can be used to select the preferred implementation. ## Custom stores Custom stores can be created by implementing the [`zarr.abc.store.Store`][] interface. See [developing custom stores](storage.md#developing-custom-stores) for more information. ## Custom array buffers Zarr-python provides control over where and how arrays are stored in memory through [`zarr.abc.buffer.Buffer`][]. Currently both CPU (the default) and GPU implementations are provided (see [Using GPUs with Zarr](gpu.md) for more information). You can implement your own buffer classes by implementing the interface defined in [`zarr.abc.buffer.BufferPrototype`][]. Like codecs, custom buffer implementations can be registered via entrypoints, using the `zarr.buffer` and `zarr.ndbuffer` entrypoint groups. ## Custom data types Zarr supports user-defined data types. See the [data types documentation](data_types.md) for an explanation of how Zarr Python models data types and how to write your own, and the [custom data type example](examples/custom_dtype.md) for a complete worked example. ## Other extensions In the future, Zarr will support writing custom chunk grids. zarr-developers-zarr-python-a2f2415/docs/user-guide/glossary.md000066400000000000000000000145121527014137600246560ustar00rootroot00000000000000# Glossary This page defines key terms used throughout the zarr-python documentation and API. ## Array Structure ### Array An N-dimensional typed array stored in a Zarr [store](#store). An array's [metadata](#metadata) defines its shape, data type, chunk layout, and codecs. ### Group A container for [arrays](#array) and other groups, enabling hierarchical organization of data — similar to directories in a file system, or groups in HDF5. Like arrays, each group has its own [metadata](#metadata) and [attributes](#attributes). See the [groups documentation](groups.md). ### Chunk The fundamental unit of data in a Zarr array. An array is divided into chunks along each dimension according to the [chunk grid](#chunk-grid). Each chunk is independently compressed and encoded through the array's [codec](#codec) pipeline. When [sharding](#shard) is used, "chunk" refers to the inner chunks within each shard, because those are the compressible units. The chunks are the smallest units that can be read independently. !!! warning "Convention specific to zarr-python" The use of "chunk" to mean the inner sub-chunk within a shard is a convention adopted by zarr-python's `Array` API. In the Zarr V3 specification and in other Zarr implementations, "chunk" may refer to the top-level grid cells (which zarr-python calls "shards" when the sharding codec is used). Be aware of this distinction when working across libraries. **API**: [`Array.chunks`][zarr.Array.chunks] returns the chunk shape. When sharding is used, this is the inner chunk shape. ### Chunk Grid The partitioning of an array's elements into [chunks](#chunk). In Zarr V3, the chunk grid is defined in the array [metadata](#metadata) and determines the boundaries of each storage object. Zarr V3 supports two chunk grid types: - **Regular**: All chunks have the same shape (the last chunk along each dimension may be smaller than the declared size). - **Rectilinear** *(experimental)*: Each dimension can have different chunk sizes, specified as a list of edge lengths per dimension. Enable with `zarr.config.set({'array.rectilinear_chunks': True})`. When sharding is used, the chunk grid defines the [shard](#shard) boundaries, not the inner chunk boundaries. The inner chunk shape is defined within the [sharding codec](#shard). **API**: The `chunk_grid` field in array metadata contains the storage-level grid. [`Array.chunks`][zarr.Array.chunks] returns the chunk shape for regular grids. For all grid types, `Array.read_chunk_sizes` and `Array.write_chunk_sizes` return the per-dimension chunk sizes in dask-style `tuple[tuple[int, ...], ...]` format. Note that while the chunk grid is a public concept of the Zarr format, the classes zarr-python uses to model chunk grids are currently private API. ### Shard A storage object that contains one or more [chunks](#chunk). Sharding reduces the number of objects in a [store](#store) by grouping chunks together, which improves performance on file systems and object storage. Within each shard, chunks are compressed independently and can be read individually. However, writing requires updating the full shard for consistency, making shards the unit of writing and chunks the unit of reading. Sharding is implemented as a [codec](#codec) (the sharding indexed codec). When sharding is used: - The [chunk grid](#chunk-grid) in metadata defines the shard boundaries - The sharding codec's `chunk_shape` defines the inner chunk size - Each shard contains `shard_shape / chunk_shape` chunks per dimension **API**: [`Array.shards`][zarr.Array.shards] returns the shard shape, or `None` if sharding is not used. [`Array.chunks`][zarr.Array.chunks] returns the inner chunk shape. ## Storage ### Store A key-value storage backend that holds Zarr data and metadata. Stores implement the [`zarr.abc.store.Store`][] interface. Examples include local file systems, cloud object storage (S3, GCS, Azure), zip files, and in-memory dictionaries. Each [chunk](#chunk) or [shard](#shard) is stored as a single value (object or file) in the store, addressed by a key derived from its grid coordinates. ### Metadata The JSON document that describes an [array](#array) or [group](#group). In Zarr format 3 this is a single `zarr.json` document; Zarr format 2 stores the equivalent information in separate `.zarray`, `.zgroup`, and `.zattrs` documents. For arrays, metadata includes the shape, data type, [chunk grid](#chunk-grid), fill value, and [codec](#codec) pipeline. Metadata is stored alongside the data in the [store](#store). Zarr-Python does not yet expose its internal metadata representation as part of its public API. ### Attributes User-defined key-value pairs (any JSON-serializable values) attached to an [array](#array) or [group](#group). Attributes are stored in the [metadata](#metadata) document. See the [attributes documentation](attributes.md). ### Consolidated Metadata A copy of the [metadata](#metadata) of every array and group in a hierarchy, stored in the metadata of the root group so that the entire hierarchy can be inspected with a single read from the [store](#store). See the [consolidated metadata documentation](consolidated_metadata.md). ## Codecs ### Codec A transformation applied to array data during reading and writing. Codecs are chained into a pipeline and come in three types: - **Array-to-array**: Transforms like transpose that rearrange array elements - **Array-to-bytes**: Serialization that converts an array to a byte sequence (exactly one required) - **Bytes-to-bytes**: Compression or checksums applied to the serialized bytes The [sharding indexed codec](#shard) is a special array-to-bytes codec that groups multiple [chunks](#chunk) into a single storage object. ## API Properties The following properties are available on [`zarr.Array`][]: | Property | Description | |----------|-------------| | `.chunks` | Chunk shape — the inner chunk shape when sharding is used. Raises for rectilinear grids | | `.shards` | Shard shape, or `None` if no sharding | | `.read_chunk_sizes` | Per-dimension chunk data sizes (`tuple[tuple[int, ...], ...]`). Works for all grid types | | `.write_chunk_sizes` | Per-dimension storage chunk sizes (`tuple[tuple[int, ...], ...]`). Works for all grid types | | `.nchunks` | Total number of independently compressible units across the array | | `.cdata_shape` | Number of independently compressible units per dimension | zarr-developers-zarr-python-a2f2415/docs/user-guide/gpu.md000066400000000000000000000043201527014137600236020ustar00rootroot00000000000000# Using GPUs with Zarr Zarr can use GPUs to accelerate your workload by running `zarr.config.enable_gpu()`. !!! note `zarr-python` currently supports reading the ndarray data into device (GPU) memory as the final stage of the codec pipeline. Data will still be read into or copied to host (CPU) memory for encoding and decoding. In the future, codecs will be available for compressing and decompressing data on the GPU, avoiding the need to move data between the host and device for compression and decompression. ## Installation Zarr's GPU support requires [CuPy](https://cupy.dev), which in turn requires a CUDA-compatible NVIDIA GPU. CuPy can be installed alongside Zarr with the `gpu` extra (see [Installation](installation.md) for the other optional dependency groups): ```console pip install "zarr[gpu]" ``` This installs the `cupy-cuda12x` package. If you need a CuPy build for a different CUDA version, see the [CuPy installation guide](https://docs.cupy.dev/en/stable/install.html) and install the appropriate package yourself. ## Reading data into device memory Calling `zarr.config.enable_gpu()` configures Zarr to use GPU memory for the data buffers used internally by Zarr: ```python test="true" session="gpu-demo" markers="gpu" source="above" import zarr import cupy as cp zarr.config.enable_gpu() z = zarr.create_array( store="memory://gpu-demo", shape=(100, 100), chunks=(10, 10), dtype="float32", ) assert isinstance(z[:10, :10], cp.ndarray) ``` Note that the arrays returned by reads are of type `cupy.ndarray` rather than NumPy arrays. `zarr.config.enable_gpu()` returns a [donfig](https://donfig.readthedocs.io/en/latest/) `ConfigSet`, which can be used as a context manager to enable GPU support for a limited scope: ```python test="true" session="gpu-demo" markers="gpu" source="above" with zarr.config.enable_gpu(): data = z[:10, :10] assert isinstance(data, cp.ndarray) ``` Under the hood, `enable_gpu()` selects the GPU-backed buffer classes `zarr.buffer.gpu.Buffer` and `zarr.buffer.gpu.NDBuffer` via the `buffer` and `ndbuffer` configuration keys. See [Custom array buffers](extending.md#custom-array-buffers) for more on Zarr's buffer classes, including how to implement your own. zarr-developers-zarr-python-a2f2415/docs/user-guide/groups.md000066400000000000000000000154631527014137600243400ustar00rootroot00000000000000# Working with groups Zarr supports hierarchical organization of arrays via groups. As with arrays, groups can be stored in memory, on disk, or via other storage systems that support a similar interface. To create a group, use the [`zarr.create_group`][] function: ```python exec="true" session="groups" source="above" result="ansi" import zarr root = zarr.create_group(store="memory://groups-demo") print(root) ``` Zarr-Python provides three related functions for making groups: [`zarr.create_group`][] creates a new group; [`zarr.open_group`][] creates or re-opens a group depending on its `mode` argument (see below); and [`zarr.group`][], which is kept for compatibility with Zarr-Python 2, is equivalent to calling [`zarr.open_group`][] with `mode='a'` (or `mode='w'` when `overwrite=True`). Groups have a similar API to the Group class from [h5py](https://www.h5py.org/). For example, groups can contain other groups: ```python exec="true" session="groups" source="above" foo = root.create_group('foo') bar = foo.create_group('bar') ``` Groups can also contain arrays, e.g.: ```python exec="true" session="groups" source="above" result="ansi" z1 = bar.create_array(name='baz', shape=(10000, 10000), chunks=(1000, 1000), dtype='int32') print(z1) ``` Members of a group can be accessed with square-bracket item access, like a Python `dict`, e.g.: ```python exec="true" session="groups" source="above" result="ansi" print(root['foo']) ``` The '/' character can be used to access multiple levels of the hierarchy in one call, e.g.: ```python exec="true" session="groups" source="above" result="ansi" print(root['foo/bar']) ``` ```python exec="true" session="groups" source="above" result="ansi" print(root['foo/bar/baz']) ``` Accessing a member with `[]` returns either an [`zarr.Array`][] or a [`zarr.Group`][], depending on what is stored at the given path. When you expect a node of a particular kind, use [`zarr.Group.get_array`][] or [`zarr.Group.get_group`][] instead. These methods accept the same paths as `[]`, but they have precise return types and raise an error if no node exists at the given path, or if the node is not of the expected kind: ```python exec="true" session="groups" source="above" result="ansi" print(root.get_group('foo')) ``` ```python exec="true" session="groups" source="above" result="ansi" print(root.get_array('foo/bar/baz')) ``` ```python exec="true" session="groups" source="above" result="ansi" from zarr.errors import ContainsGroupError try: root.get_array('foo') except ContainsGroupError as e: print(e) ``` The [`zarr.Group.tree`][] method can be used to print a tree representation of the hierarchy, e.g.: ```python exec="true" session="groups" source="above" result="ansi" print(root.tree()) ``` The [`zarr.open_group`][] function provides a convenient way to create or re-open a group stored in a directory on the file-system, with sub-groups stored in sub-directories, e.g.: ```python exec="true" session="groups" source="above" result="ansi" root = zarr.open_group('data/group.zarr', mode='w') print(root) ``` ```python exec="true" session="groups" source="above" result="ansi" z = root.create_array(name='foo/bar/baz', shape=(10000, 10000), chunks=(1000, 1000), dtype='int32') print(z) ``` For more information on groups see the [`zarr.Group` API docs](../api/zarr/group.md). ## Exploring group contents Groups also support a dict-like interface for enumerating their contents. The [`zarr.Group.keys`][] method iterates over member names, and the `in` operator tests for membership: ```python exec="true" session="groups" source="above" result="ansi" print(list(root.keys())) print('foo' in root) ``` The [`zarr.Group.members`][] method returns `(name, member)` pairs for the arrays and groups contained in a group: ```python exec="true" session="groups" source="above" result="ansi" for name, member in root.members(): print(name, member) ``` By default only immediate members are returned. Pass `max_depth=None` to recursively traverse the whole hierarchy below a group: ```python exec="true" session="groups" source="above" result="ansi" for name, member in root.members(max_depth=None): print(name, member) ``` Members can be deleted with the `del` operator, which removes the member's metadata and data from the store: ```python exec="true" session="groups" source="above" result="ansi" del root['foo/bar/baz'] for name, member in root.members(max_depth=None): print(name, member) ``` ## Batch group creation You can also create multiple groups concurrently with a single function call. [`zarr.create_hierarchy`][] takes a [`Store`](../api/zarr/storage.md) instance and a dict of `key : metadata` pairs, parses that dict, and writes metadata documents to storage: ```python exec="true" session="groups" source="above" result="ansi" from zarr import create_hierarchy from zarr.core.group import GroupMetadata from zarr.storage import LocalStore from pprint import pprint import io node_spec = {'a/b/c': GroupMetadata()} nodes_created = dict(create_hierarchy(store=LocalStore(root='data'), nodes=node_spec, overwrite=True)) # Report nodes (pprint is used for cleaner rendering in the docs) output = io.StringIO() pprint(nodes_created, stream=output, width=60) print(output.getvalue()) ``` Note that we only specified a single group named `a/b/c`, but 4 groups were created. These additional groups were created to ensure that the desired node `a/b/c` is connected to the root group `''` by a sequence of intermediate groups. [`zarr.create_hierarchy`][] normalizes the `nodes` keyword argument to ensure that the resulting hierarchy is complete, i.e. all groups or arrays are connected to the root of the hierarchy via intermediate groups. Because [`zarr.create_hierarchy`][] concurrently creates metadata documents, it's more efficient than repeated calls to [`create_group`][zarr.create_group] or [`create_array`][zarr.create_array], provided you can statically define the metadata for the groups and arrays you want to create. ## Array and group diagnostics Diagnostic information about arrays and groups is available via the `info` property. E.g.: ```python exec="true" session="groups" source="above" result="ansi" root = zarr.group(store="memory://diagnostics-demo") foo = root.create_group('foo') bar = foo.create_array(name='bar', shape=1000000, chunks=100000, dtype='int64') bar[:] = 42 baz = foo.create_array(name='baz', shape=(1000, 1000), chunks=(100, 100), dtype='float32') baz[:] = 4.2 print(root.info) ``` ```python exec="true" session="groups" source="above" result="ansi" print(foo.info) ``` ```python exec="true" session="groups" source="above" result="ansi" print(bar.info_complete()) ``` ```python exec="true" session="groups" source="above" result="ansi" print(baz.info) ``` Groups also have the [`zarr.Group.tree`][] method, e.g.: ```python exec="true" session="groups" source="above" result="ansi" print(root.tree()) ``` zarr-developers-zarr-python-a2f2415/docs/user-guide/index.md000066400000000000000000000037561527014137600241320ustar00rootroot00000000000000# User Guide Welcome to the user guide, where you can learn more about using Zarr-Python! ## Getting Started New to Zarr-Python? Start here: - **[Installation](installation.md)** - Install Zarr-Python - **[Quick start](../quick-start.md)** - Quick overview of core functionality ## Core Concepts Learn the essential building blocks: - **[Arrays](arrays.md)** - Learn the fundamentals of working with arrays - **[Groups](groups.md)** - Organize your data with groups - **[Attributes](attributes.md)** - Attach metadata to your arrays and groups - **[Storage](storage.md)** - Learn how data is stored and accessed ## Configuration & Setup Customize your experience: - **[Runtime Configuration](config.md)** - Configure Zarr-Python for your needs - **[Command-Line Interface](cli.md)** - Migrate and manage Zarr metadata from the terminal - **[V3 Migration](v3_migration.md)** - Upgrading from version 2 to version 3 ## Advanced Topics Take your skills to the next level: - **[Data Types](data_types.md)** - Learn about supported and extensible data types - **[Performance](performance.md)** - Optimize for speed and efficiency - **[GPU](gpu.md)** - Leverage GPU acceleration - **[Extending](extending.md)** - Extend functionality with custom code - **[Consolidated Metadata](consolidated_metadata.md)** - Advanced metadata management - **[Experimental Features](experimental.md)** - Preview features that may change at any time ## Examples Worked, end-to-end examples: - **[Custom Data Type](examples/custom_dtype.md)** - Extend Zarr-Python with a user-defined data type - **[Rectilinear Chunk Grids](examples/rectilinear_chunks.md)** - Use variable-sized chunks along each dimension ## Reference - **[Glossary](glossary.md)** - Definitions of key terms (chunks, shards, codecs, etc.) ## Need Help? - Browse the [API Reference](../api/zarr/index.md) for detailed function documentation - Report issues on [GitHub](https://github.com/zarr-developers/zarr-python/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen) zarr-developers-zarr-python-a2f2415/docs/user-guide/installation.md000066400000000000000000000044271527014137600255200ustar00rootroot00000000000000# Installation ## Required dependencies Required dependencies include: - [Python](https://docs.python.org/3/) (3.12 or later) - [packaging](https://packaging.pypa.io/en/stable/) (22.0 or later) - [numpy](https://numpy.org) (2.0 or later) - [numcodecs](https://numcodecs.readthedocs.io/en/stable/) (0.14 or later) - [google-crc32c](https://github.com/googleapis/python-crc32c) (1.5 or later) - [typing_extensions](https://typing-extensions.readthedocs.io/en/latest/) (4.14 or later) - [donfig](https://donfig.readthedocs.io/en/latest/) (0.8 or later) ## pip Zarr is available on [PyPI](https://pypi.org/project/zarr/). Install it using `pip`: ```console pip install zarr ``` There are a number of optional dependency groups you can install for extra functionality. These can be installed using `pip install "zarr[]"`, e.g. `pip install "zarr[gpu]"` - `remote`: support for reading/writing to remote data stores (fsspec, obstore) - `gpu`: support for GPUs (cupy) - `cli`: support for the `zarr` [command-line interface](cli.md) (typer) - `optional`: support for path-like access to local and remote stores (universal-pathlib) - `cast-value-rs`: support for the `cast_value` codec (cast-value-rs) ## conda Zarr is also published to [conda-forge](https://conda-forge.org). Install it using `conda`: ```console conda install -c conda-forge zarr ``` Conda does not support optional dependencies, so you will have to manually install any packages needed to enable extra functionality. ## Nightly wheels Development wheels are built nightly and published to the [scientific-python-nightly-wheels](https://anaconda.org/scientific-python-nightly-wheels) index. To install the latest nightly build: ```console pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple zarr ``` Note that nightly wheels may be unstable and are intended for testing purposes. ## Dependency support Zarr has endorsed [Scientific-Python SPEC 0](https://scientific-python.org/specs/spec-0000/) and now follows the version support window as outlined below: - Python: 36 months after initial release - Core package dependencies (e.g. NumPy): 24 months after initial release ## Development To install the latest development version of Zarr, see the [contributing guide](../contributing.md). zarr-developers-zarr-python-a2f2415/docs/user-guide/performance.md000066400000000000000000000363131527014137600253170ustar00rootroot00000000000000# Optimizing performance ## Chunk optimizations ### Chunk size and shape In general, chunks of at least 1 megabyte (1M) uncompressed size seem to provide better performance, at least when using the Blosc compression library. The optimal chunk shape will depend on how you want to access the data. E.g., for a 2-dimensional array, if you only ever take slices along the first dimension, then chunk across the second dimension. If you know you want to chunk across an entire dimension you can use the full size of that dimension within the `chunks` argument, e.g.: ```python exec="true" session="performance" source="above" result="ansi" import zarr z1 = zarr.create_array(store={}, shape=(10000, 10000), chunks=(100, 10000), dtype='int32') print(z1.chunks) ``` Alternatively, if you only ever take slices along the second dimension, then chunk across the first dimension, e.g.: ```python exec="true" session="performance" source="above" result="ansi" z2 = zarr.create_array(store={}, shape=(10000, 10000), chunks=(10000, 100), dtype='int32') print(z2.chunks) ``` If you require reasonable performance for both access patterns then you need to find a compromise, e.g.: ```python exec="true" session="performance" source="above" result="ansi" z3 = zarr.create_array(store={}, shape=(10000, 10000), chunks=(1000, 1000), dtype='int32') print(z3.chunks) ``` If you are feeling lazy, you can let Zarr guess a chunk shape for your data by providing `chunks='auto'`, although please note that the algorithm for guessing a chunk shape is based on simple heuristics and may be far from optimal. E.g.: ```python exec="true" session="performance" source="above" result="ansi" z4 = zarr.create_array(store={}, shape=(10000, 10000), chunks='auto', dtype='int32') print(z4.chunks) ``` If you know you are always going to be loading the entire array into memory, you can turn off chunks by providing `chunks` equal to `shape`, in which case there will be one single chunk for the array: ```python exec="true" session="performance" source="above" result="ansi" z5 = zarr.create_array(store={}, shape=(10000, 10000), chunks=(10000, 10000), dtype='int32') print(z5.chunks) ``` ### Sharding If you have large arrays but need small chunks to efficiently access the data, you can use sharding. Sharding provides a mechanism to store multiple chunks in a single storage object or file. This can be useful because traditional file systems and object storage systems may have performance issues storing and accessing many files. Additionally, small files can be inefficient to store if they are smaller than the block size of the file system. Picking a good combination of chunk shape and shard shape is important for performance. The chunk shape determines what unit of your data can be read independently, while the shard shape determines what unit of your data can be written efficiently. For an example, consider you have a 100 GB array and need to read small chunks of 1 MB. Without sharding, each chunk would be one file resulting in 100,000 files. That can already cause performance issues on some file systems. With sharding, you could use a shard size of 1 GB. This would result in 1000 chunks per file and 100 files in total, which seems manageable for most storage systems. You would still be able to read each 1 MB chunk independently, but you would need to write your data in 1 GB increments. To use sharding, you need to specify the `shards` parameter when creating the array. ```python exec="true" session="performance" source="above" result="ansi" z6 = zarr.create_array(store={}, shape=(10000, 10000, 1000), shards=(1000, 1000, 1000), chunks=(100, 100, 100), dtype='uint8') print(z6.info) ``` `shards` can be `"auto"` as well, in which case Zarr chooses a shard shape for you. The `array.target_shard_size_bytes` configuration setting controls this choice: the cumulative uncompressed size of the chunks within each shard will be as close as possible to, without exceeding, that target. If the setting is `None` (the default), Zarr falls back to a built-in heuristic for choosing the shard shape. ### Chunk memory layout The memory layout of the in-memory arrays that Zarr produces and consumes can be changed via the `order` config option, to use either C or Fortran layout. This can matter for performance when the data is passed to other libraries that expect a particular memory layout. E.g.: ```python exec="true" session="performance" source="above" result="ansi" import numpy as np c = zarr.create_array(store={}, shape=(10000, 10000), chunks=(1000, 1000), dtype='int32', config={'order': 'C'}) print(c[:100, :100].flags.c_contiguous) ``` ```python exec="true" session="performance" source="above" result="ansi" with zarr.config.set({'array.order': 'F'}): f = zarr.create_array(store={}, shape=(10000, 10000), chunks=(1000, 1000), dtype='int32') print(f[:100, :100].flags.f_contiguous) ``` Note that for Zarr format 3 arrays the `order` option only affects the in-memory layout: the bytes written to storage are identical for both settings. The layout of the serialized data is instead determined by the array's codecs (e.g. the transpose codec), which can change how well the data compresses depending on the correlation structure within the data and which compression filters (e.g., byte-shuffle) have been applied. ### Subchunk memory layout The order of chunks **within each shard** can be changed via the `subchunk_write_order` parameter of the `ShardingCodec`. That parameter is a string which must be one of `["morton", "unordered", "lexicographic", "colexicographic"]`. By default [`morton`](https://en.wikipedia.org/wiki/Z-order_curve) order provides good spatial locality. [`lexicographic` (i.e., row-major)](https://en.wikipedia.org/wiki/Row-_and_column-major_order), for example, may be better suited to "batched" workflows where some form of sequential reading through a fixed number of outer dimensions is desired, and `colexicographic` is its reverse. `unordered` makes no guarantee about the order in which subchunks are laid out within a shard. ### Empty chunks It is possible to configure how Zarr handles the storage of chunks that are "empty" (i.e., every element in the chunk is equal to the array's fill value). When creating an array with `write_empty_chunks=False`, Zarr will check whether a chunk is empty before compression and storage. If a chunk is empty, then Zarr does not store it, and instead deletes the chunk from storage if the chunk had been previously stored. This optimization prevents storing redundant objects and can speed up reads, but the cost is added computation during array writes, since the contents of each chunk must be compared to the fill value, and these advantages are contingent on the content of the array. If you know that your data will form chunks that are almost always non-empty, then there is no advantage to the optimization described above. In this case, creating an array with `write_empty_chunks=True` will instruct Zarr to write every chunk without checking for emptiness. The default value of `write_empty_chunks` is `False`: ```python exec="true" session="performance" source="above" result="ansi" arr = zarr.create_array(store={}, shape=(1,), dtype='uint8') assert arr.config.write_empty_chunks == False ``` The following example illustrates the effect of the `write_empty_chunks` flag on the time required to write an array with different values: ```python exec="true" session="performance" source="above" result="ansi" import zarr import numpy as np import time def timed_write(write_empty_chunks): """ Measure the time required and number of objects created when writing to a Zarr array with random ints or fill value. """ chunks = (8192,) shape = (chunks[0] * 1024,) data = np.random.randint(0, 255, shape) dtype = 'uint8' arr = zarr.create_array( f'data/example-{write_empty_chunks}.zarr', shape=shape, chunks=chunks, dtype=dtype, fill_value=0, overwrite=True, config={'write_empty_chunks': write_empty_chunks} ) # initialize all chunks arr[:] = 100 result = [] for value in (data, arr.fill_value): start = time.time() arr[:] = value elapsed = time.time() - start result.append((elapsed, arr.nchunks_initialized)) return result # log results for write_empty_chunks in (True, False): full, empty = timed_write(write_empty_chunks) print(f'\nwrite_empty_chunks={write_empty_chunks}:\n\tRandom Data: {full[0]:.4f}s, {full[1]} objects stored\n\t Empty Data: {empty[0]:.4f}s, {empty[1]} objects stored\n') ``` In this example, writing random data is slightly slower with `write_empty_chunks=False`, because every chunk must be checked for emptiness before it is stored. Writing empty data with `write_empty_chunks=False` is substantially faster, however, and stores no objects at all. ### Changing chunk shapes (rechunking) Zarr-Python does not yet provide a built-in way to change the chunk shape of an existing array in place. Arrays can, however, be resized and appended to along any dimension — see [Resizing and appending](arrays.md#resizing-and-appending) — and data can be copied to a new array created with the desired chunk shape. ## Parallel computing and synchronization Zarr is designed to support parallel computing and enables concurrent reads and writes to arrays. This section covers how to optimize Zarr's concurrency settings for different parallel computing scenarios. ### Concurrent I/O operations Zarr uses asynchronous I/O internally to enable concurrent reads and writes across multiple chunks. The level of concurrency is controlled by the `async.concurrency` configuration setting, which determines the maximum number of concurrent I/O operations. The default value is 10, which is a conservative value. You may get improved performance by tuning the concurrency limit. You can adjust this value based on your specific needs: ```python exec="true" session="perf-concurrency" source="above" import zarr # Set concurrency for the current session zarr.config.set({'async.concurrency': 128}) # Or use environment variable # export ZARR_ASYNC_CONCURRENCY=128 ``` Higher concurrency values can improve throughput when: - Working with remote storage (e.g., S3, GCS) where network latency is high - Reading/writing many small chunks in parallel - The storage backend can handle many concurrent requests Lower concurrency values may be beneficial when: - Working with local storage with limited I/O bandwidth - Memory is constrained (each concurrent operation requires buffer space) - Using Zarr within a parallel computing framework (see below) ### Thread pool size (`threading.max_workers`) When synchronous Zarr code calls async operations internally, Zarr uses a `ThreadPoolExecutor` to run those coroutines. The `threading.max_workers` configuration option controls the maximum number of worker threads in that pool. By default it is `None`, which lets Python choose the pool size (typically `min(32, os.cpu_count() + 4)`). You can set it explicitly when you want more predictable resource usage: ```python exec="true" session="perf-workers" source="above" import zarr zarr.config.set({'threading.max_workers': 8}) ``` Reducing this value can help avoid overloading the event loop when Zarr is used inside a parallel computing framework such as Dask that already manages its own thread pool (see the Dask section below). Increasing it may improve throughput in CPU-bound workloads where many synchronous-to-async dispatches happen concurrently. ### Using Zarr with Dask [Dask](https://www.dask.org/) is a popular parallel computing library that works well with Zarr for processing large arrays. When using Zarr with Dask, it's important to consider the interaction between Dask's thread pool and Zarr's concurrency settings. **Important**: When using many Dask threads, you may need to reduce both Zarr's `async.concurrency` and `threading.max_workers` settings to avoid creating too many concurrent operations. The total number of concurrent I/O operations can be roughly estimated as: ```text total_concurrency ≈ dask_threads × zarr_async_concurrency ``` For example, if you're running Dask with 10 threads and Zarr's default concurrency of 10, you could potentially have up to 100 concurrent operations, which may overwhelm your storage system or cause memory issues. **Recommendation**: When using Dask with many threads, configure Zarr's concurrency settings: ```python exec="false" reason="requires dask, which is not in the docs test environment" import zarr import dask.array as da # If using Dask with many threads (e.g., 8-16), reduce Zarr's concurrency settings zarr.config.set({ 'async.concurrency': 4, # Limit concurrent async operations 'threading.max_workers': 4, # Limit Zarr's internal thread pool }) # Open Zarr array z = zarr.open_array('data/large_array.zarr', mode='r') # Create Dask array from Zarr array arr = da.from_array(z, chunks=z.chunks) # Process with Dask result = arr.mean(axis=0).compute() ``` **Configuration guidelines for Dask workloads**: - `async.concurrency`: Controls the maximum number of concurrent async I/O operations. Start with a lower value (e.g., 4-8) when using many Dask threads. - `threading.max_workers`: Controls Zarr's internal thread pool size for blocking operations (defaults to `None`, letting Python choose the pool size). Reduce this to avoid thread contention with Dask's scheduler. You may need to experiment with different values to find the optimal balance for your workload. Monitor your system's resource usage and adjust these settings based on whether your storage system or CPU is the bottleneck. ### Thread safety and process safety Zarr arrays are designed to be thread-safe for concurrent reads and writes from multiple threads within the same process. However, proper synchronization is required when writing to overlapping regions from multiple threads. For multi-process parallelism, Zarr provides safe concurrent writes as long as: - Different processes write to different chunks - The storage backend supports atomic writes (most do) When writing to the same chunks from multiple processes, you should use external synchronization mechanisms or ensure that writes are coordinated to avoid race conditions. ## Pickle support Zarr arrays and groups can be pickled, as long as the underlying store object can be pickled. All of the storage classes provided in the `zarr.storage` module can be pickled. If an array or group is backed by a persistent store such as a `zarr.storage.LocalStore`, `zarr.storage.ZipStore` or `zarr.storage.FsspecStore` then the store data **are not** pickled. The only thing that is pickled is the necessary parameters to allow the store to re-open any underlying files or databases upon being unpickled. Note that pickling a `zarr.storage.MemoryStore` copies the data it holds into the pickle stream: unpickling produces an independent in-memory copy, so a `MemoryStore` cannot be used to share data between processes. E.g., pickle/unpickle a local store array: ```python exec="true" session="performance" source="above" result="ansi" import pickle data = np.arange(100000) z1 = zarr.create_array(store='data/perf-example-2.zarr', shape=data.shape, chunks=data.shape, dtype=data.dtype, overwrite=True) z1[:] = data s = pickle.dumps(z1) z2 = pickle.loads(s) assert z1 == z2 print(np.all(z1[:] == z2[:])) ``` zarr-developers-zarr-python-a2f2415/docs/user-guide/storage.md000066400000000000000000000214601527014137600244570ustar00rootroot00000000000000# Storage guide Zarr-Python supports multiple storage backends, including: local file systems, Zip files, remote stores via [fsspec](https://filesystem-spec.readthedocs.io/en/latest/) (S3, HTTP, etc.), and in-memory stores. In Zarr-Python 3, stores must implement the abstract store API from [`zarr.abc.store.Store`][]. !!! note Unlike Zarr-Python 2 where the store interface was built around a generic `MutableMapping` API, Zarr-Python 3 utilizes a custom store API that utilizes Python's AsyncIO library. ## Implicit Store Creation In most cases, it is not required to create a `Store` object explicitly. Passing a string (or other [StoreLike value](#user-guide-store-like)) to Zarr's top level API will result in the store being created automatically: ```python exec="true" session="storage" source="above" result="ansi" import zarr # Implicitly creates a writable LocalStore group = zarr.create_group(store='data/foo/bar', overwrite=True) print(group) ``` ```python exec="true" session="storage" source="above" result="ansi" # Implicitly creates a read-only FsspecStore # Note: requires s3fs to be installed group = zarr.open_group( store='s3://noaa-nwm-retro-v2-zarr-pds', mode='r', storage_options={'anon': True} ) print(group) ``` ```python exec="true" session="storage" source="above" result="ansi" # Implicitly creates a MemoryStore data = {} group = zarr.create_group(store=data) print(group) ``` ### StoreLike {#user-guide-store-like} `StoreLike` values can be: - a `Path` or string indicating a location on the local file system. This will create a [local store](#local-store): ```python exec="true" session="storage" source="above" result="ansi" group = zarr.open_group(store='data/foo/bar') print(group) ``` ```python exec="true" session="storage" source="above" result="ansi" from pathlib import Path group = zarr.open_group(store=Path('data/foo/bar')) print(group) ``` - an FSSpec URI string, indicating a [remote store](#remote-store) location: ```python exec="true" session="storage" source="above" result="ansi" # Note: requires s3fs to be installed group = zarr.open_group( store='s3://noaa-nwm-retro-v2-zarr-pds', mode='r', storage_options={'anon': True} ) print(group) ``` - an empty dictionary or None, which will create a new [memory store](#memory-store): ```python exec="true" session="storage" source="above" result="ansi" group = zarr.create_group(store={}) print(group) ``` ```python exec="true" session="storage" source="above" result="ansi" group = zarr.create_group(store=None) print(group) ``` - a dictionary of string to [`Buffer`][zarr.abc.buffer.Buffer] mappings. This will create a [memory store](#memory-store), using this dictionary as the [`store_dict` argument][zarr.storage.MemoryStore]. - an FSSpec [FSMap object](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.FSMap), which will create an [FsspecStore](#remote-store). - a [universal-pathlib](https://github.com/fsspec/universal_pathlib) `UPath`, which will create an [FsspecStore](#remote-store), or a [local store](#local-store) if the `UPath` is local. Put your storage options on the `UPath` itself; passing a separate `storage_options` argument alongside one raises `TypeError`. ```python exec="false" reason="requires universal-pathlib, which is not in the docs environment" from upath import UPath group = zarr.open_group(UPath('s3://noaa-nwm-retro-v2-zarr-pds', anon=True), mode='r') ``` - a [`Store`][zarr.abc.store.Store] or [`StorePath`][zarr.storage.StorePath] - see explicit store creation below. ## Explicit Store Creation In some cases, it may be helpful to create a store instance directly. Zarr-Python offers built-in stores: [`zarr.storage.LocalStore`][], [`zarr.storage.FsspecStore`][], [`zarr.storage.ZipStore`][], [`zarr.storage.MemoryStore`][], and [`zarr.storage.ObjectStore`][]. ### Local Store The [`zarr.storage.LocalStore`][] stores data in a nested set of directories on a local filesystem: ```python exec="true" session="storage" source="above" result="ansi" store = zarr.storage.LocalStore('data/foo/bar', read_only=True) group = zarr.open_group(store=store, mode='r') print(group) ``` ### Zip Store The [`zarr.storage.ZipStore`][] stores the contents of a Zarr hierarchy in a single Zip file. The [Zip Store specification](https://github.com/zarr-developers/zarr-specs/pull/311) is currently in draft form: ```python exec="true" session="storage" source="above" result="ansi" store = zarr.storage.ZipStore('data.zip', mode='w') array = zarr.create_array(store=store, shape=(2,), dtype='float64') print(array) ``` In place of a path, `ZipStore` also accepts an open binary file object (for example a file opened with `fsspec`, or an `obstore` reader), enabling zip archives on remote storage. The file must stay open for as long as the store is in use: ```python exec="true" session="storage" source="above" result="ansi" store.close() f = open('data.zip', mode='rb') # must stay open while the store is used array = zarr.open_array(store=zarr.storage.ZipStore(f), mode='r') print(array[:]) f.close() ``` ### Remote Store The [`zarr.storage.FsspecStore`][] stores the contents of a Zarr hierarchy following the same logical layout as the [`LocalStore`][zarr.storage.LocalStore], except the store is assumed to be on a remote storage system such as cloud object storage (e.g. AWS S3, Google Cloud Storage, Azure Blob Store). The [`zarr.storage.FsspecStore`][] is backed by [fsspec](https://filesystem-spec.readthedocs.io/en/latest/) and can support any backend that implements the [AbstractFileSystem](https://filesystem-spec.readthedocs.io/en/stable/api.html#fsspec.spec.AbstractFileSystem) API. `storage_options` can be used to configure the fsspec backend: ```python exec="true" session="storage" source="above" result="ansi" # Note: requires s3fs to be installed store = zarr.storage.FsspecStore.from_url( 's3://noaa-nwm-retro-v2-zarr-pds', read_only=True, storage_options={'anon': True} ) group = zarr.open_group(store=store, mode='r') print(group) ``` The type of filesystem (e.g. S3, https, etc..) is inferred from the scheme of the url (e.g. s3 for "**s3**://noaa-nwm-retro-v2-zarr-pds"). In case a specific filesystem is needed, one can explicitly create it. For example to create an S3 filesystem (note that the filesystem must be created with `asynchronous=True`): ```python exec="true" session="storage" source="above" result="ansi" # Note: requires s3fs to be installed import fsspec fs = fsspec.filesystem('s3', anon=True, asynchronous=True) store = zarr.storage.FsspecStore(fs, path='noaa-nwm-retro-v2-zarr-pds', read_only=True) print(store) ``` When using an S3-compatible service other than AWS, pass the service endpoint to the filesystem via `client_kwargs={'endpoint_url': 'https://...'}`. ### Memory Store The [`zarr.storage.MemoryStore`][] stores Zarr data (metadata and chunks) in an in-memory dictionary: ```python exec="true" session="storage" source="above" result="ansi" data = {} store = zarr.storage.MemoryStore(data) array = zarr.create_array(store=store, shape=(2,), dtype='float64') print(array) ``` ### Object Store [`zarr.storage.ObjectStore`][] stores the contents of the Zarr hierarchy using any ObjectStore [storage implementation](https://developmentseed.org/obstore/latest/api/store/), including AWS S3 ([`obstore.store.S3Store`][]), Google Cloud Storage ([`obstore.store.GCSStore`][]), and Azure Blob Storage ([`obstore.store.AzureStore`][]). This store is backed by [obstore](https://developmentseed.org/obstore/latest/), which builds on the production quality Rust library [object_store](https://docs.rs/object_store/latest/object_store/). ```python exec="true" session="storage" source="above" result="ansi" from zarr.storage import ObjectStore from obstore.store import MemoryStore store = ObjectStore(MemoryStore()) array = zarr.create_array(store=store, shape=(2,), dtype='float64') print(array) ``` Here's an example of using ObjectStore for accessing remote data: ```python exec="true" session="storage" source="above" result="ansi" from zarr.storage import ObjectStore from obstore.store import S3Store s3_store = S3Store('noaa-nwm-retro-v2-zarr-pds', skip_signature=True, region="us-west-2") store = zarr.storage.ObjectStore(store=s3_store, read_only=True) group = zarr.open_group(store=store, mode='r') print(group.info) ``` !!! warning The [`zarr.storage.ObjectStore`][] class is experimental. ## Developing custom stores Zarr-Python [`zarr.abc.store.Store`][] API is meant to be extended. The Store Abstract Base Class includes all of the methods needed to be a fully operational store in Zarr Python. Zarr also provides a test harness for custom stores: [`zarr.testing.store.StoreTests`][]. See the [Custom stores](extending.md#custom-stores) section of the extending guide for more on implementing your own store. zarr-developers-zarr-python-a2f2415/docs/user-guide/v3_migration.md000066400000000000000000000275451527014137600254260ustar00rootroot00000000000000# 3.0 Migration Guide Zarr-Python 3 represents a major refactor of the Zarr-Python codebase. Some of the goals motivating this refactor included: * adding support for the Zarr format 3 specification (along with the Zarr format 2 specification) * cleaning up internal and user facing APIs * improving performance (particularly in high latency storage environments like cloud object stores) To accommodate this, Zarr-Python 3 introduces a number of changes to the API, including a number of significant breaking changes and deprecations. This page provides a guide explaining breaking changes and deprecations to help you migrate your code from version 2 to version 3. If we have missed anything, please open a [GitHub issue](https://github.com/zarr-developers/zarr-python/issues/new) so we can improve this guide. !!! tip This page is about migrating your *code* from the Zarr-Python 2 API to the Zarr-Python 3 API. If you want to migrate the *metadata* of stored data from Zarr format 2 to Zarr format 3, see the `zarr migrate` command described in the [command-line interface documentation](cli.md). ## Compatibility target The goals described above necessitated some breaking changes to the API (hence the major version update), but where possible we have maintained backwards compatibility in the most widely used parts of the API. This includes the [`zarr.Array`][] and [`zarr.Group`][] classes and the "top-level API" (e.g. [`zarr.open_array`][] and [`zarr.open_group`][]). ## Getting ready for 3.0 Before migrating to Zarr-Python 3, we suggest projects that depend on Zarr-Python take the following actions in order: 1. Pin the supported Zarr-Python version to `zarr>=2,<3`. This is a best practice and will protect your users from any incompatibilities that may arise during the release of Zarr-Python 3. This pin can be removed after migrating to Zarr-Python 3. 2. Limit your imports from the Zarr-Python package. Most of the primary API `zarr.*` will be compatible in Zarr-Python 3. However, the following breaking API changes are planned: - `numcodecs.*` will no longer be available in `zarr.*`. To migrate, import codecs directly from `numcodecs`: ```python exec="false" reason="intentionally shows the old/incorrect import for contrast" from numcodecs import Blosc # instead of: # from zarr import Blosc ``` - The `zarr.v3_api_available` feature flag is being removed. In Zarr-Python 3 the v3 API is always available, so you shouldn't need to use this flag. - `zarr.errors` has been consolidated. Several exception classes from Zarr-Python 2 (such as `zarr.errors.PathNotFoundError`) have been removed or replaced. For example, missing nodes now raise `zarr.errors.NodeNotFoundError` (which subclasses both `BaseZarrError` and `FileNotFoundError`) instead of `zarr.errors.PathNotFoundError`. Review any code that catches exceptions from `zarr.errors` after migrating. - The following internal modules are being removed or significantly changed. If your application relies on imports from any of the below modules, you will need to either a) modify your application to no longer rely on these imports or b) vendor the parts of the specific modules that you need. * `zarr.attrs` has gone, with no replacement * `zarr.codecs` has changed, see "Codecs" section below for more information * `zarr.context` has gone, with no replacement * `zarr.core` remains but should be considered private API * `zarr.hierarchy` has gone, with no replacement (use `zarr.Group` in place of `zarr.hierarchy.Group`) * `zarr.indexing` has gone, with no replacement * `zarr.meta` has gone, with no replacement * `zarr.meta_v1` has gone, with no replacement * `zarr.sync` has gone, with no replacement * `zarr.types` has gone, with no replacement * `zarr.util` has gone, with no replacement * `zarr.n5` has gone, see below for an alternative N5 option 3. Test that your package works with version 3. 4. Update the pin to include `zarr>=3,<4`. ## Zarr-Python 2 support window Zarr-Python 2.x is still available, though we recommend migrating to Zarr-Python 3 for its performance improvements and new features. Security and bug fixes were made to the 2.x series for six months following the first Zarr-Python 3 release (January 2025); the 2.x series is no longer actively maintained. If you need to use the latest Zarr-Python 2 release, you can install it with: ```console $ pip install "zarr==2.*" ``` !!! note Development and maintenance of the 2.x release series has moved to the [support/v2](https://github.com/zarr-developers/zarr-python/tree/support/v2) branch. Issues and pull requests related to this branch are tagged with the [V2](https://github.com/zarr-developers/zarr-python/labels/V2) label. ## Migrating to Zarr-Python 3 The following sections provide details on breaking changes in Zarr-Python 3. ### The Array class 1. Disallow direct construction - the signature for initializing the `Array` class has changed significantly. Please use [`zarr.create_array`][] or [`zarr.open_array`][] instead of directly constructing the [`zarr.Array`][] class. 2. Defaulting to `zarr_format=3` - newly created arrays will use the version 3 of the Zarr specification. To continue using version 2, set `zarr_format=2` when creating arrays or set `default_zarr_format=2` in Zarr's runtime configuration. 3. Function signature change to [`zarr.Array.resize`][] - the `resize` function now takes a `zarr.core.common.ShapeLike` input rather than separate arguments for each dimension. Use `resize((10,10))` in place of `resize(10,10)`. ### The Group class 1. Disallow direct construction - use [`zarr.open_group`][] or [`zarr.create_group`][] instead of directly constructing the `zarr.Group` class. 2. The h5py compatibility methods `create_dataset` and `require_dataset` have been removed. Use the following replacements: - [`zarr.Group.create_array`][] in place of `Group.create_dataset` - [`zarr.Group.require_array`][] in place of `Group.require_dataset` 3. Disallow "." syntax for getting group members. To get a member of a group named `foo`, use `group["foo"]` in place of `group.foo`. 4. The `zarr.storage.init_group` low-level helper function has been removed. Use [`zarr.open_group`][] or [`zarr.create_group`][] instead: ```diff - from zarr.storage import init_group - init_group(store, overwrite=True, path="my/path") + import zarr + zarr.open_group(store, mode="w", path="my/path") ``` ### The Store class The Store API has changed significantly in Zarr-Python 3. #### The base store class The `MutableMapping` base class has been replaced in favor of a custom abstract base class ([`zarr.abc.store.Store`][]). An asynchronous interface is used for all store methods that use I/O. This change ensures that these store methods are non-blocking and are as performant as possible. #### Store implementations Store implementations have moved from the top-level module to `zarr.storage`: ```diff title="Store import changes from v2 to v3" # Before (v2) - from zarr import MemoryStore + from zarr.storage import MemoryStore ``` The following stores have been renamed or changed: | v2 | v3 | |------------------------|------------------------------------| | `DirectoryStore` | [`zarr.storage.LocalStore`][] | | `FSStore` | [`zarr.storage.FsspecStore`][] | | `TempStore` | Use [`tempfile.TemporaryDirectory`][] with [`LocalStore`][zarr.storage.LocalStore] | | `LRUStoreCache` | [`zarr.experimental.cache_store.CacheStore`][] (experimental) | `LRUStoreCache` has no stable equivalent in Zarr-Python 3. The experimental [`CacheStore`](experimental.md#cachestore) wraps a store and caches its contents in a second store, evicting the least recently used entries once `max_size` is reached. For remote data read through [`FsspecStore`][zarr.storage.FsspecStore], fsspec's caching filesystems (`fsspec.implementations.cached`, e.g. a `simplecache::s3://...` URL) are another option. A number of deprecated stores were also removed. See [issue #1274](https://github.com/zarr-developers/zarr-python/issues/1274) for more details on the removal of these stores. - `N5Store` - see https://github.com/zarr-developers/n5py for an alternative interface to N5 formatted data. - `ABSStore` - use the [`zarr.storage.FsspecStore`][] instead along with fsspec's [adlfs backend](https://github.com/fsspec/adlfs). - `DBMStore` - `LMDBStore` - `SQLiteStore` - `MongoDBStore` - `RedisStore` The latter five stores in this list do not have an equivalent in Zarr-Python 3. If you are interested in developing a custom store that targets these backends, see [developing custom stores](storage.md#developing-custom-stores) or open an [issue](https://github.com/zarr-developers/zarr-python/issues) to discuss your use case. ### Codecs Codecs defined in ``numcodecs`` (and also imported into the ``zarr.codecs`` namespace in Zarr-Python 2) should still be used when creating Zarr format 2 arrays. Codecs for creating Zarr format 3 arrays are available in two locations: - `zarr.codecs` contains Zarr format 3 codecs that are defined in the [codecs section of the Zarr format 3 specification](https://zarr-specs.readthedocs.io/en/latest/v3/codecs/index.html). - `numcodecs.zarr3` contains codecs from `numcodecs` that can be used to create Zarr format 3 arrays, but are not necessarily part of the Zarr format 3 specification. ### Dependencies When installing using `pip`: - The new `remote` dependency group can be used to install a supported version of `fsspec`, required for remote data access. - The new `gpu` dependency group can be used to install a supported version of `cuda`, required for GPU functionality. - The `jupyter` optional dependency group has been removed, since v3 contains no jupyter specific functionality. ### Miscellaneous - The keyword argument `zarr_version` in most creation functions in `zarr` (e.g. [`zarr.create`][], [`zarr.open`][], [`zarr.group`][], [`zarr.array`][]) has been removed. Use `zarr_format` instead. ## Unimplemented Zarr-Python 2 features A few features of Zarr-Python 2 remain unimplemented in Zarr-Python 3. If any of the features listed below are important to your use case of Zarr-Python, please open (or comment on) a [GitHub issue](https://github.com/zarr-developers/zarr-python/issues/new). The following functions / methods have not been ported to Zarr-Python 3: - `zarr.copy` ([issue #2407](https://github.com/zarr-developers/zarr-python/issues/2407)) - `zarr.copy_all` ([issue #2407](https://github.com/zarr-developers/zarr-python/issues/2407)) - `zarr.copy_store` ([issue #2407](https://github.com/zarr-developers/zarr-python/issues/2407)) The following features (corresponding to function arguments to functions in `zarr`) have not been ported to Zarr-Python 3. Using these features will raise a warning or a `NotImplementedError`: - `cache_attrs` - `cache_metadata` - `chunk_store` ([issue #2495](https://github.com/zarr-developers/zarr-python/issues/2495)) - `meta_array` - `object_codec` ([issue #2617](https://github.com/zarr-developers/zarr-python/issues/2617)) - `synchronizer` ([issue #1596](https://github.com/zarr-developers/zarr-python/issues/1596)) The following features that were supported by Zarr-Python 2 have not been ported to Zarr-Python 3: - Object dtypes ([issue #2616](https://github.com/zarr-developers/zarr-python/issues/2616)) - Ragged arrays ([issue #2618](https://github.com/zarr-developers/zarr-python/issues/2618)) - Groups and Arrays do not implement `__enter__` and `__exit__` protocols ([issue #2619](https://github.com/zarr-developers/zarr-python/issues/2619)) - Default filters for object dtypes for Zarr format 2 arrays ([issue #2627](https://github.com/zarr-developers/zarr-python/issues/2627)) zarr-developers-zarr-python-a2f2415/examples/000077500000000000000000000000001527014137600213035ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/examples/README.md000066400000000000000000000023411527014137600225620ustar00rootroot00000000000000# Zarr Python Examples This directory contains complete, runnable examples demonstrating various features and use cases of Zarr Python. ## Directory Structure Each example is organized in its own subdirectory with the following structure: ``` examples/ ├── example_name/ │ ├── README.md # Documentation for the example │ └── example_name.py # Python source code └── ... ``` ## Adding New Examples To add a new example: 1. Create a new subdirectory: `examples/my_example/` 2. Add your Python code: `examples/my_example/my_example.py` 3. Create documentation: `examples/my_example/README.md` 4. Create a documentation page at `docs/user-guide/examples/my_example.md`. The documentation page should simply link to the `README.md` and the source code, e.g.: ```` # docs/user-guide/examples/my_example.md --8<-- "examples/my_example/README.md" ## Source Code ```python --8<-- "examples/my_example/my_example.py" ``` ```` 5. Update `mkdocs.yml` to include the new example in the navigation. ### Example README.md Format Your README.md should include: - A title (`# Example Name`) - Description of what the example demonstrates - Instructions for running the example zarr-developers-zarr-python-a2f2415/examples/codec_pipeline_performance/000077500000000000000000000000001527014137600266265ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/examples/codec_pipeline_performance/README.md000066400000000000000000000055421527014137600301130ustar00rootroot00000000000000# Codec Pipeline Performance This example compares the default `BatchedCodecPipeline` against the opt-in `FusedCodecPipeline` on a sharded array, across two stores (memory and local) and two codec regimes (uncompressed and gzip), at one worker and at `cpu_count`. A *codec pipeline* turns chunks of array data into stored bytes and back, running the configured codecs and performing the storage IO. The default `BatchedCodecPipeline` schedules both asynchronously -- roughly one coroutine per chunk operation. That model pays off for high-latency stores, where there is useful work to do while waiting on IO. For low-latency stores (in-process memory, the local filesystem) the IO completes too quickly for the overlap to be worth its cost, and the async scheduling becomes pure overhead. `FusedCodecPipeline` runs codec compute and synchronous IO synchronously, removing that overhead. It is [experimental](https://zarr.readthedocs.io/en/stable/user-guide/experimental/) and opt-in; the default pipeline is unchanged. ## What it shows - How to select a pipeline with `zarr.config.set`, and why the array must be *created* inside the config block: the pipeline class is resolved at array construction time and then travels with the array. - That the benefit depends strongly on layout and on whether compression is in play. Some configurations are slower under the fused pipeline -- the script reports speedups below 1.00x rather than hiding them. - That `codec_pipeline.max_workers` is read only by `FusedCodecPipeline`; the default pipeline ignores it entirely. ## Running ```bash uv run codec_pipeline_performance.py ``` The script has no arguments and writes only to an in-memory store. ## Interpreting the output The numbers are specific to your CPU, your Python build, and the workload chosen here. They are a measurement of your machine, not a published benchmark -- treat a single run as indicative and re-measure against your own data and store before switching pipelines in production. Two effects are worth watching for: - **The two codec regimes tell opposite stories about `max_workers`.** Uncompressed IO is dominated by per-chunk *scheduling*, so `Fused (1 worker)` is already fastest and a thread pool only adds overhead. gzip is genuinely CPU-bound: a single worker compresses chunk after chunk sequentially and can be *slower than the default*, while a thread pool spreads that compression over cores and reclaims the win. That flip is why the fused pipeline is threaded by default, and why pinning `max_workers=1` is worth it for memory-backed uncompressed data. - **Chunk size decides whether threading can help at all.** The 64×64 inner chunks here are small enough that per-chunk scheduling dominates uncompressed IO, yet large enough that per-chunk gzip is real work to parallelize. Much coarser chunks leave the pool with too few items to spread. codec_pipeline_performance.py000066400000000000000000000203711527014137600344470ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/examples/codec_pipeline_performance# /// script # requires-python = ">=3.11" # dependencies = [ # "zarr @ git+https://github.com/zarr-developers/zarr-python.git@main", # "numpy", # ] # /// """ Compare the `BatchedCodecPipeline` and the `FusedCodecPipeline`. The default `BatchedCodecPipeline` schedules storage IO and codec compute asynchronously -- roughly one coroutine per chunk operation. For a *sharded* array that means one coroutine per inner chunk inside every shard. That is the right model for high-latency stores, where there is useful work to do while waiting for IO. For low-latency stores (in-process memory, the local filesystem) the IO completes too quickly for the overlap to pay for itself, and the scheduling becomes pure overhead. The `FusedCodecPipeline` runs codec compute and synchronous IO synchronously, removing that overhead. Whether it wins, and whether its thread pool helps, depends on which resource is actually scarce: * Uncompressed IO is dominated by per-chunk *scheduling*, not compute. There is nothing for a thread pool to parallelize, so a single worker is already fastest and extra workers only add overhead. * gzip is genuinely CPU-bound. A single worker compresses every chunk sequentially and can be *slower than the default*, while a thread pool spreads that compression across cores and reclaims the win. This is when `max_workers > 1` earns its keep. Run it with: uv run codec_pipeline_performance.py Numbers are hardware-, layout-, and codec-dependent. Treat the output as a measurement of *your* machine, not as a published benchmark. """ from __future__ import annotations import operator import os import statistics import tempfile import timeit from pathlib import Path from typing import TYPE_CHECKING import numpy as np import zarr from zarr.storage import LocalStore, MemoryStore if TYPE_CHECKING: from collections.abc import Callable from zarr.abc.store import Store BATCHED = "zarr.core.codec_pipeline.BatchedCodecPipeline" FUSED = "zarr.core.codec_pipeline.FusedCodecPipeline" # gzip is CPU-bound to encode, which is exactly the regime where the thread # pool matters. Level 6 is gzip's own default. GZIP = {"name": "gzip", "configuration": {"level": 6}} # 4096x4096 int32 = 64 MiB, split into 16 shards of 1024x1024, each holding # 16x16 = 256 inner chunks of 64x64 -> 4096 inner chunks in total. The chunks # are small enough that per-chunk coroutine scheduling dominates uncompressed # IO, yet large enough that per-chunk gzip is real work to spread over cores. SHAPE = (4096, 4096) SHARDS = (1024, 1024) CHUNKS = (64, 64) DTYPE = "int32" CONFIGS: tuple[tuple[str, dict[str, object]], ...] = ( ("Batched (default)", {"codec_pipeline.path": BATCHED}), ("Fused (1 worker)", {"codec_pipeline.path": FUSED, "codec_pipeline.max_workers": 1}), ("Fused (cpu_count)", {"codec_pipeline.path": FUSED, "codec_pipeline.max_workers": None}), ) def time_call(fn: Callable[[], object], repeat: int = 3) -> float: """Median wall-clock seconds for one call to `fn`. `timeit.Timer` supplies `perf_counter` and disables the cyclic garbage collector during each run, so a collection triggered by earlier work cannot land inside a measurement. `number=1` because a single call here already moves 64 MiB -- the per-call overhead `timeit` amortizes is irrelevant at this scale. """ return statistics.median(timeit.Timer(fn).repeat(repeat=repeat, number=1)) def measure( settings: dict[str, object], store: Store, data: np.ndarray, compressors: object, ) -> tuple[float, float]: """Time one full write and one full read of `data` under `settings`. The whole operation runs inside `zarr.config.set`, not just the array construction. The pipeline class is resolved when the array is built, but `codec_pipeline.max_workers` is read *per operation*, so a timed call made outside the config block would silently use whatever worker count was globally in effect -- which makes every configuration look identical. """ everything = slice(None) def write_once() -> None: with zarr.config.set(settings): array = zarr.create_array( store=store, shape=SHAPE, chunks=CHUNKS, shards=SHARDS, dtype=DTYPE, compressors=compressors, fill_value=0, overwrite=True, ) operator.setitem(array, everything, data) write = time_call(write_once) # The bytes on disk are identical whichever pipeline wrote them, so reading # back what we just wrote isolates read performance on the same data. def read_once() -> object: with zarr.config.set(settings): return zarr.open_array(store=store, mode="r")[everything] read = time_call(read_once) if not np.array_equal(read_once(), data): raise AssertionError("round trip mismatch") return write, read def make_store(kind: str, tmp: Path) -> Store: if kind == "memory": return MemoryStore() return LocalStore(tmp / f"demo_{kind}_{os.getpid()}.zarr") def main() -> None: n_cpu = os.cpu_count() or 1 # Each regime gets the data that actually exercises it. `arange` is # trivially compressible, which is fine when nothing compresses it, but it # would make gzip finish almost instantly and hide the CPU-bound behavior # this example is about. The noisy array keeps gzip genuinely busy. n = int(np.prod(SHAPE)) plain_data = np.arange(n, dtype=DTYPE).reshape(SHAPE) noisy_data = np.random.default_rng(0).integers(0, 2**24, size=SHAPE, dtype=DTYPE) n_shards = int(np.prod([s // c for s, c in zip(SHAPE, SHARDS, strict=True)])) per_shard = int(np.prod([s // c for s, c in zip(SHARDS, CHUNKS, strict=True)])) print(f"zarr {zarr.__version__} | {n_cpu} CPUs") print( f"array {SHAPE} {DTYPE} = {plain_data.nbytes / 2**20:.0f} MiB | " f"{n_shards} shards x {per_shard} inner chunks = {n_shards * per_shard} chunks\n" ) with tempfile.TemporaryDirectory() as tmp: for store_kind in ("memory", "local"): for codec_label, compressors, data in ( ("uncompressed", None, plain_data), ("gzip-6 (CPU-bound)", GZIP, noisy_data), ): print(f"=== {store_kind} store / {codec_label} ===") print( f"{'pipeline':<22}{'write (s)':>11}{'vs base':>10}" f"{'read (s)':>12}{'vs base':>10}" ) results: dict[str, tuple[float, float]] = {} for label, settings in CONFIGS: store = make_store(store_kind, Path(tmp)) results[label] = measure(settings, store, data, compressors) base_write, base_read = results[CONFIGS[0][0]] for label, (write, read) in results.items(): print( f"{label:<22}{write:>10.3f}{base_write / write:>9.1f}x" f"{read:>11.3f}{base_read / read:>9.1f}x" ) # The headline comparison: does the thread pool earn its keep? single_write, single_read = results["Fused (1 worker)"] pool_write, pool_read = results["Fused (cpu_count)"] print( f" workers (cpu_count vs 1 worker): " f"write {single_write / pool_write:.1f}x " f"read {single_read / pool_read:.1f}x" ) print() print( "Reading it:\n" " * Uncompressed IO is scheduling-bound, so Fused (1 worker) is already\n" " fastest -- a thread pool has nothing to parallelize and only adds\n" " overhead.\n" " * gzip is CPU-bound, so Fused (1 worker) can be *slower* than the\n" " default, while Fused (cpu_count) spreads compression across cores\n" " and reclaims the win. That flip is why the fused pipeline is\n" " threaded by default, and why pinning max_workers=1 is worth it for\n" " memory-backed uncompressed data.\n" " * `codec_pipeline.max_workers` is read only by the FusedCodecPipeline;\n" " the default BatchedCodecPipeline ignores it." ) if __name__ == "__main__": main() zarr-developers-zarr-python-a2f2415/examples/custom_dtype/000077500000000000000000000000001527014137600240225ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/examples/custom_dtype/README.md000066400000000000000000000016161527014137600253050ustar00rootroot00000000000000# Custom Data Type This example demonstrates how to extend Zarr Python by defining a new data type. The example shows how to: - Define a custom `ZDType` class for the `int2` data type from [`ml_dtypes`](https://pypi.org/project/ml-dtypes/) - Implement all required methods for serialization and deserialization - Register the custom data type with Zarr's registry - Create and use arrays with the custom data type in both Zarr v2 and v3 formats ## Running the Example The script declares its dependencies inline ([PEP 723](https://peps.python.org/pep-0723/)), so the easiest way to run it is with [uv](https://docs.astral.sh/uv/), which installs them automatically: ```bash uv run examples/custom_dtype/custom_dtype.py ``` Alternatively, run it with plain Python, in which case you must first install `zarr`, `ml_dtypes`, and `pytest` yourself: ```bash python examples/custom_dtype/custom_dtype.py ``` zarr-developers-zarr-python-a2f2415/examples/custom_dtype/custom_dtype.py000066400000000000000000000217651527014137600271260ustar00rootroot00000000000000# /// script # requires-python = ">=3.12" # dependencies = [ # "zarr @ git+https://github.com/zarr-developers/zarr-python.git@main", # "ml_dtypes==0.5.4", # "pytest==8.4.1" # ] # /// # """ Demonstrate how to extend Zarr Python by defining a new data type """ import json import sys from pathlib import Path from typing import ClassVar, Literal, Self, TypeGuard, overload import ml_dtypes # necessary to add extra dtypes to NumPy import numpy as np import pytest import zarr from zarr.dtype import ZDType, check_dtype_spec_no_object_codec_v2, data_type_registry from zarr.errors import DataTypeValidationError from zarr.types import JSON, DTypeConfig_V2, DTypeJSON, ZarrFormat # This is the int2 array data type int2_dtype_cls = type(np.dtype("int2")) # This is the int2 scalar type int2_scalar_cls = ml_dtypes.int2 class Int2(ZDType[int2_dtype_cls, int2_scalar_cls]): """ This class provides a Zarr compatibility layer around the int2 data type (the ``dtype`` of a NumPy array of type int2) and the int2 scalar type (the ``dtype`` of the scalar value inside an int2 array). """ # This field is as the key for the data type in the internal data type registry, and also # as the identifier for the data type when serializaing the data type to disk for zarr v3 _zarr_v3_name: ClassVar[Literal["int2"]] = "int2" # this field will be used internally _zarr_v2_name: ClassVar[Literal["int2"]] = "int2" # we bind a class variable to the native data type class so we can create instances of it dtype_cls = int2_dtype_cls @classmethod def from_native_dtype(cls, dtype: np.dtype) -> Self: """Create an instance of this ZDType from a native dtype.""" if cls._check_native_dtype(dtype): return cls() raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) def to_native_dtype(self: Self) -> int2_dtype_cls: """Create an int2 dtype instance from this ZDType""" return self.dtype_cls() @classmethod def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[Literal["|b1"], None]]: """ Type check for Zarr v2-flavored JSON. This will check that the input is a dict like this: .. code-block:: json { "name": "int2", "object_codec_id": None } Note that this representation differs from the ``dtype`` field looks like in zarr v2 metadata. Specifically, whatever goes into the ``dtype`` field in metadata is assigned to the ``name`` field here. See the Zarr docs for more information about the JSON encoding for data types. """ return check_dtype_spec_no_object_codec_v2(data) and data["name"] == "int2" @classmethod def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[Literal["int2"]]: """ Type check for Zarr V3-flavored JSON. Checks that the input is the string "int2". """ return data == cls._zarr_v3_name @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create an instance of this ZDType from Zarr V3-flavored JSON. """ if cls._check_json_v2(data): return cls() # This first does a type check on the input, and if that passes we create an instance of the ZDType. msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v2_name!r}" raise DataTypeValidationError(msg) @classmethod def _from_json_v3(cls: type[Self], data: DTypeJSON) -> Self: """ Create an instance of this ZDType from Zarr V3-flavored JSON. This first does a type check on the input, and if that passes we create an instance of the ZDType. """ if cls._check_json_v3(data): return cls() msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal["int2"], None]: ... @overload def to_json(self, zarr_format: Literal[3]) -> Literal["int2"]: ... def to_json( self, zarr_format: ZarrFormat ) -> DTypeConfig_V2[Literal["int2"], None] | Literal["int2"]: """ Serialize this ZDType to v2- or v3-flavored JSON If the zarr_format is 2, then return a dict like this: .. code-block:: json { "name": "int2", "object_codec_id": None } If the zarr_format is 3, then return the string "int2" """ if zarr_format == 2: return {"name": "int2", "object_codec_id": None} if zarr_format == 3: return self._zarr_v3_name raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover def _check_scalar(self, data: object) -> TypeGuard[int | ml_dtypes.int2]: """ Check if a python object is a valid int2-compatible scalar The strictness of this type check is an implementation degree of freedom. You could be strict here, and only accept int2 values, or be open and accept any integer or any object and rely on exceptions from the int2 constructor that will be called in cast_scalar. """ return isinstance(data, (int, int2_scalar_cls)) def cast_scalar(self, data: object) -> ml_dtypes.int2: """ Attempt to cast a python object to an int2. We first perform a type check to ensure that the input type is appropriate, and if that passes we call the int2 scalar constructor. """ if self._check_scalar(data): return ml_dtypes.int2(data) msg = ( f"Cannot convert object {data!r} with type {type(data)} to a scalar compatible with the " f"data type {self}." ) raise TypeError(msg) def default_scalar(self) -> ml_dtypes.int2: """ Get the default scalar value. This will be used when automatically selecting a fill value. """ return ml_dtypes.int2(0) def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> int: """ Convert a python object to a JSON representation of an int2 scalar. This is necessary for taking user input for the ``fill_value`` attribute in array metadata. In this implementation, we optimistically convert the input to an int, and then check that it lies in the acceptable range for this data type. """ # We could add a type check here, but we don't need to for this example val: int = int(data) # type: ignore[call-overload] if val in {-2, -1, 0, 1}: return val raise ValueError("Invalid value. Expected -2, -1, 0, or 1.") def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> ml_dtypes.int2: """ Read a JSON-serializable value as an int2 scalar. We first perform a type check to ensure that the JSON value is well-formed, then call the int2 scalar constructor. The base definition of this method requires that it take a zarr_format parameter because other data types serialize scalars differently in zarr v2 and v3, but we don't use this here. """ if self._check_scalar(data): return ml_dtypes.int2(data) raise TypeError(f"Invalid type: {data}. Expected an int.") # after defining dtype class, it must be registered with the data type registry so zarr can use it data_type_registry.register(Int2._zarr_v3_name, Int2) # this parametrized function will create arrays in zarr v2 and v3 using our new data type @pytest.mark.parametrize("zarr_format", [2, 3]) def test_custom_dtype(tmp_path: Path, zarr_format: ZarrFormat) -> None: # create array and write values z_w = zarr.create_array( store=tmp_path, shape=(4,), dtype="int2", zarr_format=zarr_format, compressors=None, ) z_w[:] = [-1, -2, 0, 1] # open the array z_r = zarr.open_array(tmp_path, mode="r") print(z_r.info_complete()) # look at the array metadata meta_file = tmp_path / (".zarray" if zarr_format == 2 else "zarr.json") print(json.dumps(json.loads(meta_file.read_text()), indent=2)) if __name__ == "__main__": # Run the example with printed output, and a dummy pytest configuration file specified. # Without the dummy configuration file, at test time pytest will attempt to use the # configuration file in the project root, which will error because Zarr is using some # plugins that are not installed in this example. sys.exit( pytest.main( [ "-s", __file__, f"-c {__file__}", # Suppress: "PytestAssertRewriteWarning: Module already imported so # cannot be rewritten; zarr" "-W", "ignore::pytest.PytestAssertRewriteWarning", ] ) ) zarr-developers-zarr-python-a2f2415/examples/sharding_coalescing/000077500000000000000000000000001527014137600252715ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/examples/sharding_coalescing/README.md000066400000000000000000000057021527014137600265540ustar00rootroot00000000000000# Sharded Read Coalescing This example demonstrates byte-range coalescing for partial reads of sharded arrays, a performance optimization added in Zarr-Python 3.3.0 and enabled by default. A shard is one stored object containing many inner chunks, each occupying its own byte range. Reading N inner chunks could mean N separate byte-range requests. Because byte ranges are intervals, nearby ranges can be merged: `[a, b)` and `[b, c)` together cover `[a, c)`, so a single request can serve both. Merging trades reading some bytes you did not ask for against issuing fewer requests -- worthwhile whenever a request is expensive, as with object storage. ## What it shows - Reading scattered inner chunks from one shard with coalescing **off** issues one store request per inner chunk; with the **default** settings the same read collapses to a single request. - The resulting wall-clock difference against a store with simulated latency. - That coalescing changes only *how* data is fetched, never *what* is returned -- the script asserts both configurations produce identical arrays. - A case where coalescing changes nothing: a contiguous selection already has adjacent byte ranges, so it merges under any setting. ## Running ```bash uv run sharding_coalescing.py ``` ## How the comparison is set up Two details make the effect observable, and both are worth understanding if you adapt this script: - **The selection must have gaps.** A contiguous read produces adjacent byte ranges that merge regardless of configuration. The strided selections skip inner chunks, creating the gaps that the `sharding_coalesce_max_gap_bytes` budget decides whether to bridge. - **Latency must be charged per merged fetch.** The example defines a small `WrapperStore` subclass that sleeps in `get`. It deliberately does *not* keep `WrapperStore.get_ranges`, which forwards straight to the wrapped store and would bypass the latency entirely; inheriting the `Store` ABC's `get_ranges` instead runs the coalescer over its own `get`, so each merged fetch pays once. `zarr.testing.store` ships a ready-made `LatencyStore`, but importing it pulls in `pytest`. Defining the wrapper inline keeps the example runnable with only `zarr` and `numpy` installed. ## Configuration Two settings control the behavior, both settable globally via `zarr.config` or per array via `config=` on `zarr.create_array` / `Array.with_config`: | Setting | Default | Meaning | | --- | --- | --- | | `sharding_coalesce_max_gap_bytes` | 1 MiB | Merge two ranges only if the gap between them is no larger than this | | `sharding_coalesce_max_bytes` | 16 MiB | Never let a merged read exceed this size | Setting the gap to `0` merges only exactly-adjacent ranges, which approximates the pre-3.3.0 behavior; that is how the example emulates the old path. Raising the gap reads more unwanted bytes in exchange for fewer round trips -- the right value depends on how expensive a request is against how fast your link is. zarr-developers-zarr-python-a2f2415/examples/sharding_coalescing/sharding_coalescing.py000066400000000000000000000205211527014137600316310ustar00rootroot00000000000000# /// script # requires-python = ">=3.11" # dependencies = [ # "zarr @ git+https://github.com/zarr-developers/zarr-python.git@main", # "numpy", # ] # /// """ Demonstrate byte-range coalescing for partial reads of sharded arrays. A shard is a single stored object holding many inner chunks, each occupying its own byte range. Reading N inner chunks could mean issuing N separate byte-range requests to the store. Because byte ranges are intervals, a reader can instead merge nearby ranges: `[a, b)` and `[b, c)` together cover `[a, c)`, so one request can serve both. Merging trades reading some bytes you did not ask for against issuing fewer requests -- a good trade whenever a request is expensive, which is the normal case for object storage. Zarr-Python 3.3.0 does this automatically. Two settings control it: * `sharding_coalesce_max_gap_bytes` (default 1 MiB) -- merge two ranges only if the gap between them is no larger than this. * `sharding_coalesce_max_bytes` (default 16 MiB) -- never let a merged read exceed this size. Setting the gap to 0 disables merging of non-adjacent ranges, which approximates the pre-3.3.0 behavior. This script compares the two, counting store requests and measuring wall-clock time against a store with simulated latency. Run it with: uv run sharding_coalescing.py """ from __future__ import annotations import asyncio import operator import statistics import timeit from contextlib import contextmanager from functools import partial from typing import TYPE_CHECKING import numpy as np import zarr import zarr.core._coalesce as coalesce_module from zarr.abc.store import ByteRequest, RangeByteRequest, Store from zarr.storage import MemoryStore, WrapperStore if TYPE_CHECKING: from collections.abc import Callable, Iterator, Sequence from zarr.core.buffer import Buffer, BufferPrototype # Emulates the pre-3.3.0 behavior: with a zero gap budget, only ranges that are # exactly adjacent get merged, so scattered inner chunks are fetched one by one. NO_COALESCING = {"sharding_coalesce_max_gap_bytes": 0} # The shipped defaults. Spelled out here so the comparison is explicit rather # than relying on whatever the global config happens to be. DEFAULT_COALESCING = { "sharding_coalesce_max_gap_bytes": 1 << 20, # 1 MiB "sharding_coalesce_max_bytes": 16 << 20, # 16 MiB } GET_LATENCY_S = 0.005 # 5 ms per request, a modest stand-in for object storage class PerRequestLatencyStore(WrapperStore[Store]): """Wraps a store, charging a fixed latency per byte-range fetch. `zarr.testing.store` ships a `LatencyStore`, but importing it pulls in `pytest`; defining the wrapper here keeps this example runnable with only zarr and numpy installed. Two details matter for the measurement: * The latency is applied in `get`, which is what an individual fetch costs. * `get_ranges` is explicitly *not* overridden to forward to the wrapped store. `WrapperStore.get_ranges` does forward, which would skip this class's `get` entirely and make every configuration look identical. Inheriting the `Store` ABC's implementation instead runs the coalescer over `self.get`, so each *merged* fetch pays the latency once -- which is exactly the cost coalescing exists to reduce. """ get_ranges = Store.get_ranges def __init__(self, store: Store, *, get_latency: float) -> None: super().__init__(store) self.get_latency = get_latency def _with_store(self, store: Store) -> PerRequestLatencyStore: # `WrapperStore` rebuilds the wrapper when opening read-only, so the # latency setting has to be carried across. return type(self)(store, get_latency=self.get_latency) async def get( self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None, ) -> Buffer | None: await asyncio.sleep(self.get_latency) return await self._store.get(key, prototype, byte_range) @contextmanager def counting_requests() -> Iterator[Callable[[], int]]: """Count the store fetches issued inside the block. Wraps the coalescing planner rather than the store: every merged group it returns, plus every range it declined to merge, becomes exactly one fetch. Counting here rather than at the store means the number reported is the planner's decision, which is precisely what the settings control. """ original = coalesce_module.coalesce_ranges total = 0 def counting_coalesce_ranges( byte_ranges: Sequence[ByteRequest | None], *, max_gap_bytes: int, max_coalesced_bytes: int, ) -> tuple[ list[list[tuple[int, RangeByteRequest]]], list[tuple[int, ByteRequest | None]], ]: nonlocal total groups, uncoalescable = original( byte_ranges, max_gap_bytes=max_gap_bytes, max_coalesced_bytes=max_coalesced_bytes, ) total += len(groups) + len(uncoalescable) return groups, uncoalescable coalesce_module.coalesce_ranges = counting_coalesce_ranges try: yield lambda: total finally: coalesce_module.coalesce_ranges = original def measure_read(array: zarr.Array, selection: slice) -> tuple[int, float]: """Return (store fetches, median seconds) for reading `selection`.""" read = partial(operator.getitem, array, selection) with counting_requests() as fetches: result = read() requests = fetches() # `timeit.Timer` supplies the loop, `perf_counter`, and GC handling. The # median of several runs keeps one unlucky run from dominating. elapsed = statistics.median(timeit.Timer(read).repeat(repeat=5, number=1)) assert result.size > 0 # a read that returned nothing would time as "fast" return requests, elapsed def main() -> None: n = 8192 chunk = 64 inner_chunks = n // chunk base = MemoryStore() source = (np.arange(n, dtype="uint64") % 251).astype("uint8") # One shard holding every inner chunk, uncompressed so inner-chunk byte # offsets stay predictable and the demonstration is easy to reason about. writable = zarr.create_array( store=base, shape=(n,), chunks=(chunk,), shards=(n,), dtype="uint8", compressors=None ) writable[:] = source store = PerRequestLatencyStore(base, get_latency=GET_LATENCY_S) print(f"zarr {zarr.__version__}") print(f"array: {n} uint8 values, {inner_chunks} inner chunks of {chunk} in a single shard") print(f"store: MemoryStore wrapped with {GET_LATENCY_S * 1000:.0f} ms of latency per request\n") # A strided selection touches inner chunks with unread chunks in between, # so there are real gaps for the coalescer to bridge. A contiguous # selection would merge under any setting, since its ranges are adjacent. selections = { "every 2nd inner chunk": slice(None, None, chunk * 2), "every 4th inner chunk": slice(None, None, chunk * 4), "contiguous quarter": slice(0, n // 4), } header = f"{'selection':<24} {'coalescing':<12} {'requests':>9} {'time':>10}" print(header) print("-" * len(header)) for label, selection in selections.items(): results = {} for mode, config in (("off", NO_COALESCING), ("default", DEFAULT_COALESCING)): array = zarr.open_array(store=store, mode="r").with_config(config) requests, elapsed = measure_read(array, selection) results[mode] = (requests, elapsed) print(f"{label:<24} {mode:<12} {requests:>9} {elapsed * 1000:>9.1f}ms") off_requests, off_time = results["off"] on_requests, on_time = results["default"] if on_requests < off_requests: print( f"{'':<24} {'->':<12} " f"{off_requests // on_requests:>8}x fewer {off_time / on_time:>9.1f}x faster" ) else: print(f"{'':<24} {'->':<12} {'no change (ranges already adjacent)':>30}") print() # Correctness is the point: coalescing must not change what you read back. for mode, config in (("off", NO_COALESCING), ("default", DEFAULT_COALESCING)): array = zarr.open_array(store=store, mode="r").with_config(config) assert np.array_equal(array[::128], source[::128]), mode print("Both configurations return identical data; coalescing only changes how it is fetched.") if __name__ == "__main__": main() zarr-developers-zarr-python-a2f2415/lychee.toml000066400000000000000000000021611527014137600216330ustar00rootroot00000000000000# Configuration for the lychee link checker (https://lychee.cli.rs/). # Auto-discovered as ./lychee.toml by the lychee GitHub Action. # Treat redirect status codes as success rather than failures. accept = ["200..=299"] # Files lychee should not scan for links. exclude_path = [ # mkdocs-material theme overrides: hrefs are Jinja expressions like # `{{ '../' ~ base_url }}`, not real URLs, so lychee cannot resolve them. "docs/overrides", # Design notes: working records that point at transient artifacts (commits, # fork branches, compare URLs) which are expected to disappear over time. "design", ] # URL patterns to ignore (regex, matched against the full URL). exclude = [ # Local docs preview server shown in the contributing guide ("just docs-serve"), # documentation of a command rather than a reachable link. '^https?://0\.0\.0\.0', '^https?://(localhost|127\.0\.0\.1)(:\d+)?', # SPEC 0 page times out but is valid. '^https://scientific-python\.org/specs/spec-0000', # Stack Overflow answers link-checker requests with 403; the tag page is valid. '^https://stackoverflow\.com/questions/tagged/', ] zarr-developers-zarr-python-a2f2415/mkdocs.yml000066400000000000000000000375311527014137600215010ustar00rootroot00000000000000# Based on https://github.com/developmentseed/obspec/blob/main/mkdocs.yml site_name: zarr-python repo_name: zarr-developers/zarr-python repo_url: https://github.com/zarr-developers/zarr-python site_description: An implementation of chunked, compressed, N-dimensional arrays for Python. site_author: Alistair Miles site_url: !ENV [READTHEDOCS_CANONICAL_URL, 'https://zarr.readthedocs.io/'] docs_dir: docs use_directory_urls: true nav: - "index.md" - "quick-start.md" - User Guide: - user-guide/index.md - user-guide/installation.md - user-guide/arrays.md - user-guide/groups.md - user-guide/attributes.md - user-guide/data_types.md - user-guide/storage.md - user-guide/config.md - user-guide/performance.md - user-guide/cli.md - user-guide/extending.md - user-guide/gpu.md - user-guide/consolidated_metadata.md - user-guide/experimental.md - user-guide/v3_migration.md - user-guide/glossary.md - Examples: - user-guide/examples/custom_dtype.md - user-guide/examples/rectilinear_chunks.md - user-guide/examples/codec_pipeline_performance.md - user-guide/examples/sharding_coalescing.md - subprojects.md - API Reference: - api/zarr/index.md - ' zarr.abc': - api/zarr/abc/index.md - ' zarr.abc.buffer': api/zarr/abc/buffer.md - ' zarr.abc.codec': api/zarr/abc/codec.md - ' zarr.abc.metadata': api/zarr/abc/metadata.md - ' zarr.abc.numcodec': api/zarr/abc/numcodec.md - ' zarr.abc.store': api/zarr/abc/store.md - ' zarr.api': - api/zarr/api/index.md - ' zarr.api.asynchronous': api/zarr/api/asynchronous.md - ' zarr.api.synchronous': api/zarr/api/synchronous.md - ' zarr.Array': api/zarr/array.md - ' zarr.array': api/zarr/functions/array.md - ' zarr.buffer': - api/zarr/buffer/index.md - ' zarr.buffer.cpu': api/zarr/buffer/cpu.md - ' zarr.buffer.gpu': api/zarr/buffer/gpu.md - ' zarr.codecs': api/zarr/codecs.md - ' zarr.codecs.numcodecs': api/zarr/codecs/numcodecs.md - ' zarr.config': api/zarr/config.md - ' zarr.consolidate_metadata': api/zarr/functions/consolidate_metadata.md - ' zarr.create': api/zarr/functions/create.md - ' zarr.create_array': api/zarr/functions/create_array.md - ' zarr.create_group': api/zarr/functions/create_group.md - ' zarr.create_hierarchy': api/zarr/functions/create_hierarchy.md - ' zarr.dtype': api/zarr/dtype.md - ' zarr.empty': api/zarr/functions/empty.md - ' zarr.empty_like': api/zarr/functions/empty_like.md - ' zarr.errors': api/zarr/errors.md - ' zarr.experimental': api/zarr/experimental.md - ' zarr.from_array': api/zarr/functions/from_array.md - ' zarr.full': api/zarr/functions/full.md - ' zarr.full_like': api/zarr/functions/full_like.md - ' zarr.Group': api/zarr/group.md - ' zarr.group': api/zarr/functions/group.md - ' zarr.load': api/zarr/functions/load.md - ' zarr.metadata': api/zarr/metadata.md - ' zarr.ones': api/zarr/functions/ones.md - ' zarr.ones_like': api/zarr/functions/ones_like.md - ' zarr.open': api/zarr/functions/open.md - ' zarr.open_array': api/zarr/functions/open_array.md - ' zarr.open_consolidated': api/zarr/functions/open_consolidated.md - ' zarr.open_group': api/zarr/functions/open_group.md - ' zarr.open_like': api/zarr/functions/open_like.md - ' zarr.print_debug_info': api/zarr/functions/print_debug_info.md - ' zarr.registry': api/zarr/registry.md - ' zarr.save': api/zarr/functions/save.md - ' zarr.save_array': api/zarr/functions/save_array.md - ' zarr.save_group': api/zarr/functions/save_group.md - ' zarr.storage': api/zarr/storage.md - ' zarr.testing': - api/zarr/testing/index.md - ' zarr.testing.buffer': api/zarr/testing/buffer.md - ' zarr.testing.stateful': api/zarr/testing/stateful.md - ' zarr.testing.store': api/zarr/testing/store.md - ' zarr.testing.strategies': api/zarr/testing/strategies.md - ' zarr.testing.utils': api/zarr/testing/utils.md - ' zarr.zeros': api/zarr/functions/zeros.md - ' zarr.zeros_like': api/zarr/functions/zeros_like.md # The companion packages are Read the Docs subprojects of this one; link # to the /projects/ paths Read the Docs advertises as canonical rather # than to their standalone *.readthedocs.io domains, so following one # keeps the reader on this site's domain. - 'zarr-metadata ↪': https://zarr.readthedocs.io/projects/zarr-metadata/ - 'zarr-indexing ↪': https://zarr.readthedocs.io/projects/zarr-indexing/ - release-notes.md - roadmap.md - contributing.md - Blog: - blog/index.md hooks: - mkdocs_hooks.py watch: - src/zarr - docs theme: language: en name: material custom_dir: docs/overrides logo: _static/logo_bw.png favicon: _static/favicon-96x96.png palette: # Light mode - media: "(prefers-color-scheme: light)" scheme: default primary: custom accent: custom toggle: icon: material/brightness-7 name: Switch to dark mode # Dark mode - media: "(prefers-color-scheme: dark)" scheme: slate primary: custom accent: custom toggle: icon: material/brightness-4 name: Switch to light mode font: text: Roboto code: Roboto Mono features: - content.code.annotate - content.code.copy - navigation.indexes - navigation.instant - navigation.tracking - search.suggest - search.share extra: social: - icon: fontawesome/brands/mastodon link: https://fosstodon.org/@zarr - icon: fontawesome/brands/bluesky link: https://bsky.app/profile/zarr.dev extra_css: - overrides/stylesheets/extra.css plugins: - autorefs - blog: blog_dir: blog post_dir: "{blog}/posts" post_url_format: "{slug}" # The blog is a simple reverse-chronological list of posts; the archive # and category indexes add navigation we don't have the volume to justify. archive: false categories: false - search - markdown-exec - mkdocstrings: enable_inventory: true handlers: python: paths: [src/zarr] options: allow_inspection: true docstring_section_style: list docstring_style: numpy inherited_members: true line_length: 60 separate_signature: true show_root_heading: true show_signature_annotations: true show_source: true show_symbol_type_toc: true signature_crossrefs: true show_if_no_docstring: true extensions: - griffe_inherited_docstrings inventories: - https://docs.python.org/3/objects.inv - https://docs.xarray.dev/en/stable/objects.inv - https://numpy.org/doc/stable/objects.inv - https://numcodecs.readthedocs.io/en/stable/objects.inv - https://msgspec.dev/objects.inv - https://developmentseed.org/obstore/latest/objects.inv - https://filesystem-spec.readthedocs.io/en/latest/objects.inv - https://requests.readthedocs.io/en/latest/objects.inv - https://docs.aiohttp.org/en/stable/objects.inv - https://s3fs.readthedocs.io/en/latest/objects.inv - https://docs.h5py.org/en/stable/objects.inv - https://icechunk.io/en/stable/objects.inv - https://lithops-cloud.github.io/docs/objects.inv - https://docs.dask.org/en/stable/objects.inv - redirects: redirect_maps: 'spec/index.md': 'https://zarr-specs.readthedocs.io' 'spec/v1.md': 'https://zarr-specs.readthedocs.io/en/latest/v1/v1.0.html' 'spec/v2.md': 'https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html' 'spec/v3.md': 'https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html' 'license.md': 'https://github.com/zarr-developers/zarr-python/blob/main/LICENSE.txt' 'genindex.html.md': 'index.md' 'py-modindex.html.md': 'index.md' 'search.html.md': 'index.md' 'tutorial.md': 'user-guide/installation.md' 'getting-started.md': 'quick-start.md' 'installation.md': 'user-guide/installation.md' 'release.md': 'release-notes.md' 'about.html.md': 'index.md' 'arrays.html.md': 'user-guide/arrays.md' 'attributes.html.md': 'user-guide/attributes.md' 'cli.html.md': 'user-guide/cli.md' 'config.html.md': 'user-guide/config.md' 'consolidated_metadata.html.md': 'user-guide/consolidated_metadata.md' 'data_types.html.md': 'user-guide/data_types.md' 'extending.html.md': 'user-guide/extending.md' 'gpu.html.md': 'user-guide/gpu.md' 'groups.html.md': 'user-guide/groups.md' 'installation.html.md': 'user-guide/installation.md' 'performance.html.md': 'user-guide/performance.md' 'quickstart.html.md': 'quick-start.md' 'release-notes.html.md': 'release-notes.md' 'storage.html.md': 'user-guide/storage.md' 'v3_migration.html.md': 'user-guide/v3_migration.md' 'user-guide/arrays.html.md': 'user-guide/arrays.md' 'user-guide/attributes.html.md': 'user-guide/attributes.md' 'user-guide/cli.html.md': 'user-guide/cli.md' 'user-guide/config.html.md': 'user-guide/config.md' 'user-guide/consolidated_metadata.html.md': 'user-guide/consolidated_metadata.md' 'user-guide/data_types.html.md': 'user-guide/data_types.md' 'user-guide/extending.html.md': 'user-guide/extending.md' 'user-guide/gpu.html.md': 'user-guide/gpu.md' 'user-guide/groups.html.md': 'user-guide/groups.md' 'user-guide/installation.html.md': 'user-guide/installation.md' 'user-guide/performance.html.md': 'user-guide/performance.md' 'user-guide/storage.html.md': 'user-guide/storage.md' 'user-guide/v3_migration.html.md': 'user-guide/v3_migration.md' 'developers/contributing.html.md': 'contributing.md' 'developers/index.html.md': 'contributing.md' 'developers/roadmap.html.md': 'https://zarr.readthedocs.io/en/v3.0.8/developers/roadmap.html' 'api.md': 'api/zarr/index.md' 'api/zarr/create.md': 'api/zarr/functions/create.md' 'api/zarr/open.md': 'api/zarr/functions/open.md' 'api/zarr/save.md': 'api/zarr/functions/save.md' 'api/zarr/load.md': 'api/zarr/functions/load.md' 'api/zarr/metadata/migrate_v3.md': 'api/zarr/metadata.md' # Based on https://github.com/developmentseed/titiler/blob/50934c929cca2fa8d3c408d239015f8da429c6a8/docs/mkdocs.yml#L115-L140 markdown_extensions: - admonition - attr_list - codehilite: guess_lang: false - def_list - footnotes - md_in_html - pymdownx.arithmatex - pymdownx.betterem - pymdownx.caret: insert: false - pymdownx.details - pymdownx.escapeall: hardbreak: true nbsp: true - pymdownx.magiclink: hide_protocol: true repo_url_shortener: true - pymdownx.smartsymbols - pymdownx.superfences - pymdownx.tasklist: custom_checkbox: true - pymdownx.tilde - pymdownx.emoji: emoji_index: !!python/name:material.extensions.emoji.twemoji emoji_generator: !!python/name:material.extensions.emoji.to_svg - toc: permalink: true - pymdownx.highlight: anchor_linenums: true line_spans: __span pygments_lang_class: true - pymdownx.inlinehilite - pymdownx.snippets zarr-developers-zarr-python-a2f2415/mkdocs_hooks.py000066400000000000000000000052741527014137600225320ustar00rootroot00000000000000"""MkDocs hook that renders validation-marked code fences as ordinary code blocks. The docs validation convention (see ``tests/test_docs.py`` and the contributing guide) requires every python fence to carry ``exec="true"``, ``test="true"``, or ``exec="false" reason="..."``. Markdown Exec's superfences fence only claims ``exec="true"`` blocks; without this hook the remaining marked fences fail superfences validation and their contents spill into the page as raw markdown (e.g. the PEP 723 header of the custom dtype example rendered as headings). This hook registers a second ``python`` fence, tried when Markdown Exec's declines, that strips the validation attributes and delegates to the standard superfences highlighter so the block renders exactly like a plain code fence. """ from __future__ import annotations from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from markdown import Markdown from mkdocs.config.defaults import MkDocsConfig # Mirrors markdown_exec's _to_bool: everything but these means "true". _FALSY = {"", "no", "off", "false", "0"} def _validator( language: str, inputs: dict[str, str], options: dict[str, Any], attrs: dict[str, Any], md: Markdown, ) -> bool: """Claim fences marked test="true" or exec="false"; leave the rest alone.""" if "exec" not in inputs and "test" not in inputs: # Plain fence: let the default superfences pathway highlight it. return False if str(inputs.get("exec", "false")).lower() not in _FALSY: # Executable fence: Markdown Exec's own custom fence handles it. return False # Consume the validation attributes so they don't leak into the output. inputs.clear() return True def _formatter( source: str, language: str, css_class: str, options: dict[str, Any], md: Markdown, classes: list[str] | None = None, id_value: str = "", attrs: dict[str, Any] | None = None, **kwargs: Any, ) -> str: """Render with the same highlighter superfences uses for plain fences.""" fenced = md.preprocessors["fenced_code_block"] fenced.get_hl_settings() return fenced.highlight( src=source, language=language, options={}, md=md, classes=classes, id_value=id_value, attrs=attrs or {}, ) def on_config(config: MkDocsConfig) -> MkDocsConfig: superfences = config.setdefault("mdx_configs", {}).setdefault("pymdownx.superfences", {}) custom_fences = superfences.setdefault("custom_fences", []) custom_fences.append( { "name": "python", "class": "python", "validator": _validator, "format": _formatter, } ) return config zarr-developers-zarr-python-a2f2415/packages/000077500000000000000000000000001527014137600212435ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/mkdocs-base.yml000066400000000000000000000065601527014137600241650ustar00rootroot00000000000000# Shared Material for MkDocs configuration for the sub-package docs sites. # Each package's mkdocs.yml pulls this in via INHERIT and defines only its # identity (site_name, repo_*, site_description, site_url) and nav. # # INHERIT deep-merges mappings but *replaces* lists: a package that sets # theme.features or markdown_extensions drops every entry below. So the lists # here are the superset used by all packages (features, extensions, snippet # base paths, and inventories a package never exercises are harmless); add to # them here rather than in a package's mkdocs.yml. site_author: Davis Bennett use_directory_urls: true watch: - src theme: language: en name: material logo: _static/logo_bw.png favicon: _static/favicon-96x96.png palette: # Light mode - media: "(prefers-color-scheme: light)" scheme: default toggle: icon: material/brightness-7 name: Switch to dark mode # Dark mode - media: "(prefers-color-scheme: dark)" scheme: slate toggle: icon: material/brightness-4 name: Switch to light mode font: text: Roboto code: Roboto Mono features: - content.code.annotate - content.code.copy - content.tabs.link - navigation.indexes - navigation.instant - navigation.tracking - search.suggest - search.share plugins: - autorefs - search - mkdocstrings: enable_inventory: true handlers: python: paths: [src] options: allow_inspection: true docstring_section_style: list docstring_style: numpy inherited_members: true line_length: 60 separate_signature: true show_root_heading: true show_signature_annotations: true show_source: true show_symbol_type_toc: true signature_crossrefs: true show_if_no_docstring: true extensions: - griffe_inherited_docstrings inventories: - https://docs.python.org/3/objects.inv - https://numpy.org/doc/stable/objects.inv - https://zarr.readthedocs.io/en/stable/objects.inv # Override the header source widget's version fact (the monorepo's latest # release) with the latest tag matching the package's prefix. extra_javascript: - _static/source-version.js markdown_extensions: - admonition - attr_list - def_list - footnotes - md_in_html - pymdownx.details - pymdownx.superfences - toc: permalink: true - pymdownx.highlight: anchor_linenums: true line_spans: __span pygments_lang_class: true - pymdownx.inlinehilite # Content tabs; content.tabs.link in the theme features keeps tab sets # switched together. - pymdownx.tabbed: alternate_style: true - pymdownx.snippets: # The package root is last so docs/release-notes.md can include the # towncrier-managed CHANGELOG.md without moving it out of the package # root, where it is distributed and linked from the package metadata. # examples/ holds runnable example scripts (zarr-indexing); a missing # base path is skipped. base_path: [docs, examples, .] # Fail the build on an unresolvable include or missing region instead # of silently rendering nothing. zarr-indexing's # tests/test_doc_examples.py mirrors base_path. check_paths: true zarr-developers-zarr-python-a2f2415/packages/source-version.js000066400000000000000000000060101527014137600245610ustar00rootroot00000000000000/* Material's header source widget shows the monorepo's latest release (e.g. v3.3.0). Replace it with the latest tag for this package, and patch the theme's sessionStorage cache so instant navigation keeps it. The GitHub repo and the package's tag prefix (e.g. zarr_metadata-) are derived from repo_url, whose final path segment is the package directory, so this file is identical across all sub-package docs sites. */ (() => { const source = document.querySelector('[data-md-component="source"]') if (!source || !source.href) return const segments = new URL(source.href).pathname.split("/").filter(Boolean) const REPO = segments.slice(0, 2).join("/") const PREFIX = segments[segments.length - 1].replace(/-/g, "_") + "-" /* Final releases only (v1.2.3), matching the theme's own releases/latest lookup; pre-release parts like 0rc1 would otherwise parse as 0 and tie with (or outrank) the final release. */ const RELEASE = /^v?\d+(\.\d+)*$/ const parse = version => version .replace(/^v/, "") .split(".") .map(part => parseInt(part, 10)) const newestFirst = (a, b) => { const va = parse(a), vb = parse(b) for (let i = 0; i < Math.max(va.length, vb.length); i++) if ((vb[i] || 0) !== (va[i] || 0)) return (vb[i] || 0) - (va[i] || 0) return 0 } /* Cache the tag for the tab's lifetime (like the theme's own __source cache) so navigation doesn't burn GitHub's unauthenticated API rate limit. Keyed by prefix in case sub-package sites share an origin. */ const CACHE_KEY = `__package_tag/${PREFIX}` async function latestPackageTag() { const cached = sessionStorage.getItem(CACHE_KEY) if (cached) return cached const response = await fetch( `https://api.github.com/repos/${REPO}/tags?per_page=100` ) if (!response.ok) return undefined const tags = await response.json() const versions = tags .map(tag => tag.name) .filter(name => name.startsWith(PREFIX)) .map(name => name.slice(PREFIX.length)) .filter(version => RELEASE.test(version)) .sort(newestFirst) if (versions[0]) sessionStorage.setItem(CACHE_KEY, versions[0]) return versions[0] } function patchDom(version) { for (const el of document.querySelectorAll(".md-source__fact--version")) el.textContent = version } function patchCache(version) { const facts = __md_get("__source", sessionStorage) if (!facts) return false facts.version = version __md_set("__source", facts, sessionStorage) return true } latestPackageTag().then(version => { if (!version) return patchDom(version) if (patchCache(version)) return /* The theme's own API request hasn't resolved yet — patch both the cache and the DOM once it renders the facts list. */ new MutationObserver((_, observer) => { if (patchCache(version)) { patchDom(version) observer.disconnect() } }).observe(source, { childList: true, subtree: true }) }) })() zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/000077500000000000000000000000001527014137600245025ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/.readthedocs.yaml000066400000000000000000000052771527014137600277440ustar00rootroot00000000000000# Read the Docs configuration for the zarr-http-server docs site, separate from # the zarr-python site configured by the repo-root .readthedocs.yaml. The RTD # project for zarr-http-server must set its configuration-file path to # packages/zarr-http-server/.readthedocs.yaml. version: 2 build: os: ubuntu-22.04 tools: python: "3.12" jobs: post_checkout: # Cancel pull request builds that do not touch this package. Exit code # 183 cancels the build and reports success to the Git provider. Scoped # to PR builds ("external" versions) because origin/main is only a # meaningful diff base there. The shared docs config and header script # live outside the package directory but change this site too. # The diff is taken against the merge base (origin/main...HEAD), not # against the tip of main: a two-dot diff also reports everything that # landed on main after the PR branched, which made every PR that was # behind main build every site. The checkout is shallow, so fetch the # full history first; without a merge base the diff fails and the build # runs, which is the safe default. - | if [ "$READTHEDOCS_VERSION_TYPE" = "external" ]; then git fetch --quiet --unshallow origin main pull/$READTHEDOCS_VERSION_NAME/head || true; if git diff --quiet origin/main...HEAD -- packages/zarr-http-server packages/mkdocs-base.yml packages/source-version.js; then exit 183; fi fi install: - pip install --upgrade pip - pip install ./packages/zarr-http-server --group packages/zarr-http-server/pyproject.toml:docs build: html: # Build from inside the package rather than pointing `-f` at its config # from the repo root. mkdocs resolves some settings relative to the # current working directory rather than to the config file, so building # from elsewhere looks for them in the wrong place -- and silently, since # the paths are valid, just wrong. zarr-indexing hit this: with # `pymdownx.snippets` and a relative `base_path`, its snippets were # searched for under the repo-root docs/ and the build failed with # SnippetMissingError, while `just docs-check` passed because it runs # from here. Building from the package directory makes this identical to # the local and CI invocations, so a green build there means a green # build here. # # $READTHEDOCS_OUTPUT is absolute, so the cd does not affect it. - cd packages/zarr-http-server && mkdocs build --strict --site-dir $READTHEDOCS_OUTPUT/html mkdocs: configuration: packages/zarr-http-server/mkdocs.yml zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/CHANGELOG.md000066400000000000000000000000701527014137600263100ustar00rootroot00000000000000# Release notes zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/LICENSE.txt000066400000000000000000000021441527014137600263260ustar00rootroot00000000000000The MIT License (MIT) Copyright (c) 2015-2025 Zarr Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/README.md000066400000000000000000000044461527014137600257710ustar00rootroot00000000000000# zarr-http-server HTTP server for Zarr stores, arrays, and groups. Documentation: `zarr-http-server` exposes a Zarr `Store`, `Array`, or `Group` over HTTP via an ASGI app, so any HTTP-capable client — including zarr-python itself, via `FsspecStore` or `ObjectStore` — can read the data. The app is built on [Starlette](https://www.starlette.io/) and can be run with any ASGI server; the `serve` / `serve_background` helpers run it with [Uvicorn](https://uvicorn.dev/). > [!WARNING] > This package is experimental. Its API may change or be removed at any point. ## Installation ```bash pip install zarr-http-server ``` ## Quick start ```python import zarr from zarr_http_server import node_app, serve_background store = zarr.storage.MemoryStore() array = zarr.create_array(store, shape=(100,), chunks=(10,), dtype="float64") with serve_background(node_app(array)) as server: print(server.url) # e.g. http://127.0.0.1:8000 ``` Building an app and running it are separate steps, and either app works with either runner: - **Build** with `store_app` to serve every key in a store, or `node_app` to serve only the keys belonging to one `Array` or `Group` — requests for keys outside that node return 404 even when those keys exist in the underlying store. - **Run** with `serve`, which blocks, or `serve_background`, which returns a handle you can shut down later. Reads are all that is enabled by default: `GET` and `HEAD` are served, and `PUT`, `POST`, `DELETE` and `PATCH` are answered with 405. > [!CAUTION] > `store_app` applies no per-key filtering. Only point it at a store whose full > contents are safe to serve, and note that enabling `PUT` grants write access > to everything the store contains. The > [user guide](https://zarr-http-server.readthedocs.io/en/latest/guide/#read-only-serving) > covers the read-only guarantees available. The [user guide](https://zarr-http-server.readthedocs.io/en/latest/guide/) covers byte ranges, CORS, writes, serving several nodes, running from a notebook, and Uvicorn configuration. ## Examples [`examples/`](examples/) holds a runnable script and a notebook, both executed by the test suite so neither can drift from the code: ```bash uv run examples/serve.py ``` ## License MIT — see [LICENSE.txt](LICENSE.txt). zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/changes/000077500000000000000000000000001527014137600261125ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/changes/3732.feature.md000066400000000000000000000002701527014137600304630ustar00rootroot00000000000000Initial release of `zarr-http-server`: an HTTP server exposing Zarr stores, arrays, and groups over an ASGI app, extracted from the `zarr.experimental.serve` prototype in zarr-python. zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/changes/4291.doc.md000066400000000000000000000001001527014137600275660ustar00rootroot00000000000000Updated the MDN CORS link in the user guide to its current URL. zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/changes/4311.doc.md000066400000000000000000000002161527014137600275670ustar00rootroot00000000000000The docs now render the release notes as a page instead of linking to `CHANGELOG.md` on GitHub, so each docs version shows its own changelog. zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/changes/4406.misc.md000066400000000000000000000003421527014137600277620ustar00rootroot00000000000000`pyproject.toml` no longer tells uv to resolve `zarr` from the zarr-python repository, so `just test` works from an unpacked sdist, against `zarr` from PyPI. Inside the repository, the justfile recipes add the in-repo `zarr`. zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/changes/README.md000066400000000000000000000017061527014137600273750ustar00rootroot00000000000000Writing a changelog entry for `zarr-http-server` --------------------------------------------- Fragments in **this** directory are released notes for the `zarr-http-server` package only — kept separate from the parent zarr-python `changes/` directory so a PR touching only `packages/zarr-http-server/` produces a release note for this package only. Please put a new file in this directory named `xxxx..md`, where - `xxxx` is the pull request number associated with this entry - `` is one of: - feature - bugfix - doc - removal - misc Inside the file, please write a short description of what you have changed, and how it impacts users of `zarr-http-server`. A `zarr-http-server` release runs `towncrier build` in `packages/zarr-http-server/`, which consumes the fragments here and updates `CHANGELOG.md`. Fragments that describe parent zarr-python changes (not the server package) belong in the top-level `changes/` directory, not here. zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/docs/000077500000000000000000000000001527014137600254325ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/docs/_static/000077500000000000000000000000001527014137600270605ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/docs/_static/favicon-96x96.png000066400000000000000000000306521527014137600320240ustar00rootroot00000000000000PNG  IHDR_`{` pHYsodtEXtSoftwarewww.inkscape.org< IDATx}y|Uι7&lU* Ep\@[kTlUYVZpׯ-l, [s}~$dAk|>瓛s3g33̼!_L;[wG4kڽ3>B~I Nj4!oPD1۔Ec.:Џ[O觎/B}J1 8]Q?~jkPp_zkg-~޽)1 Ժ\Q6e`HJܗ6s\3_|77=el0"d~z"oeOU05uम8?:ՑLs\E!A @*G`EG3nyv,Ƽy``JZ{SNJOV:sϏbgav LI++?j]-HL2 @m4Pe?  W=SRG*>s~RL+Wzh##yx(f'Ҥf3Uu(w"T<ʍ1qsR*ں|QQI]g̘Ƭxibhӭ 5ulɭOqJ[;;Ң*& g@TYU2yj,P>x{ xɩcֵʚ`?}6x )鷉Ӳ+\ [Ϙ:D$bU)+[0{qwE7pg#;HNCP@.' Y譠XKʜ/P/B8Am (9ɩ=a 8m/SJP@䌽i臦 ,ŰaN~RMI @9Tp&Ӧ4pĽPP6JP@: 5a Q2X qYsABj:X<.+cFK5*U(bjpWc?!}t'SNNnZOBL،!\*PxkO (,VhUTJAIDêKTh=]OO0j܂{Rn+f q/' ܝ:Vw *DX`ЗWǧ6i,}nS{{0* / UU (B 0ķ'VZר@AțT^ Zma ~-RYok{"|ܓr;STp8M1= N]xԩ^DJ4OǍkC~)ό,E6p nV+WT,b*5By D:Ú aJw05q1QO ;~q僸d51}b,VG0J??Dl7.2 ot@⪺ZyKlJ!U-zXBc}(A@8|FͣJKߕ:2Q [<6()2N8` sZ,)eq)缚;kȖJs|Zeו7*n2o7.7.E-lϩW%aŷ64 Hxq5 z8Фp>tQ$ d47&DLݿx4?)DZV "y6Xg( ~e<@ߙRYR "H FZwqoѲ*t|2W|Ci>F?0{vwzB7~/0*9h+0C ͡##)OhNT >w`h#% '`( DD!Pט+cP( 'xT*a_Z5e x'Zע;y6D4- ϪЮL|rOmiЂ;ҴᢶOM-/6?ߞziB9P*04S Eu[EG'T 6"ʩɇ4RBѾ@c |d!,T]CEd)W>Z/17=S.ʽl^pGڈcǛwFA `KڠȦd]?9 e< ($!.wΚy`9nP MAαl>Xt6)4(!BԪqwP0ɢʐ29R/0yPp|YhV9֬TV4)hZS FE,VGZ^ aè7rN6@S@h~"꧍JE&oP4!olʱFbTI`w礼a"bEzͅbN~ b(dTJZuN`IQ%~Im1aӞmܜ'x0[Ӯ0/9Pg:B՚MH*:٬c O|>TjoMPgO~NMy$5PGřlmo+w4gE:gPz*QxǧaQ((/pTLe!7ՎP&ݒr[ȴUWCT,#FNu8~3?EřTD<6:oTUNLFk>7W5`ѨuY[9OzaFaNC_B y<  "T=ϟ"F>2U`(+Ð!bJEWK@ ^PbaO Fv!haM1~sj]vuK[?X?2'h kB-nB4Ӓ⛩R0ECi,Fbj a: Tx9 ŹUv@nr1}N`JDOl']Ee+P@ c^\(^v*|V֌V[ ot8Qv-(1#(ڏtQTWVy^H{5,n9 _hcn"332 iDR +AѼTO+EDs?Ӳ-9EUcmeybǍV1< ࣇ]-b!D(1fsS GԀ } \!hy!^֎@?cRD!!1<:-$({“YK/]/ynq A5\c%a͑Yn44հxZ_ +jpt q`&irOF:w/~*\ 2WXz9M-^*pݣQȱy_?@zcf^N0hr86\)x[‚ugXZ_(e~ kS{Ѱ#do OCo4/{Mפ-=={>+n-I >6v̈́Nt|P : z>('9"*!  KoJHd)GE\S9fŽ YT: 9b~9;}Æ9y^N`EG']s6Wj%7匵Y;44Vϻ6/+ŏ Jc"JƊ3&K:pgm˽&9/lkTEIR\.}UCRA _5aW( RϘm]?lﶻl;ImVtd ux÷uO ax$r!jY2Hv5`DK(o—o.E_'dP!lTQ2kjw%{$6cg%W"WGp:aPdCؓsXSH a5*Q<Ab1mJ|E_0. r,Q$?P`(R_m DQbuTk0jḡř g &Lrx0P_!1jʹ&&&Ð(xD,C·X|*FQo AH>[N1Q ^-[FLO`̵ KcJAVN:q*_F~qu) [~?BcwxhI]q.B,(ixsyE9Z=[Rb|@C9(HX[l_&)]Ȁ7aGoi,:U\|dgH3Q =GV7{tP5 7~E֢WCAi ֬R;8A#Ty>MZ1s. |ǮFq}qYv qlr`с|EnRs!Q:Xf+xEx~u\1FH8(bǯ-iJIdR'D++prx8GI):.{ K__xXرaK"6 X9K^w߻ӰмE(AhXDT mOcbRQ|@8MD$ @ @b\$bt8״Pk& fOtBە~ո6-}W$_0&@1VeίJKSCŪhZJ.IJM5r3 U6F"ʭ\h`!?PDLZ_("\͸[gToP}KHKT嶫fm-zn;#6v@Ҫ̑'>r/NYrq1RUw|_Օ(P\t= H;Ϫ)Zճ1lS+Ix'ňK#]}6u6e;!du_C pX**kK! ⥊ CT.\/ f Ljoʏ+δ5u+z*H@L\xәG[gVgV铲5Y=>^1B!K|?MH^仔)iܾq 5B .O|?=}Kڛ蚤)\PĔ x8!ѫG@(3h ^!r}y@LB0 @O(e @  &nF0a]:^0bcdJ@e8u'^A+rW6볖$&R>ԛhfv^ \UZJU/۩Ԭ|%u A_cؔ r)aIiD4% ;J\[<_Ghx| aY1G@IBX,p։׋h*^A;aY̱'>䁹|7`3qmf6벖" & #2J %oj$|9Ǹq8m<}k3tBPgkάWi&~[ˡ ~`\[mȮm? 4ZE%RQyD-y+ ˡӫ19B0"==/#ȦUt4]G*e/XDZuygUP-&l=9cLwOy\y f'3I0y&;7_ ~ygϲI\T­q"T+`Z8*ēˡ;ib ]4ŠsP#ė;;ܳ'~!Z8B\2VnkCNipeE‡Y*9gJ>=8+쉷>kbg1/I,*-D\@=v;{mZ;*Uv/?g9,ļ'aFV\C :ʤL6Q1miZ dhaJ<,GYb5;_X ⨴X*VvM^(EPq|3b( @a/vgc"_fTšt= H_%m^ `uNKA?kXk7V T3_d|ܮ/hQ:6Uv[2Ƽ.)'M@FtMF?jqgАٱ <F@\<6$Y/,J ,!Ey%,σEZW><[Q"-l"X4wŚQ 5C{Q+v:g25dlX)DXv>9ּNą3/rD?P*th/9cmpVmV,PLcIb@ E),h Og2@b1w">*+iCz2h7lYxfb ,$]Nu$ V&^tGtvfΧE;%/NT^M{[pUcM=UxNW(QM(ׯM+@5>I 8Aܤ1՛UpQJ'C` g7 J#!JhL[(1 p !&7cs˹CL*NH9@nSwN,}hy92XptxP͵`PkRxl(( p˜/`uE=%bVLObBkZbU>AOg2@M>6SyQ-,z(-[Z|V8S_hKKP U`)[O򛤼&5f썟\U62 6qbU>"%kN [ZΟZ&7rZ{ZOnTR*]7N-OIO=c]sZyШ+IDATC/ez0jk&Q1Xl'7œpuPϱchx$i<#=LLbn5`F\*MdkU1ݱfHӊp6f= yֈ@?fTW=T8Y|V!6C }%mfX/]*QlX@oqB6]~~!"cU 9J6튲'9 E^''|0?x4I1K|7"'69wMy6)IۃY𼈳F;|o29 1EN y@rH845r@9MdQǻ ]y쨈l-%1nͤRxai}R R N1*qn1eNN;u<2wǤ& 3<#|#Ltb '#IK`Mvib"'q@>oS43J׭QEW $x-0 !v&LwERQ"tbα4_y[B^ IROF8eNns |3m ^~~G L}5' ؽ>|Jq\[¬`u7I0bgl&`˕>%1)n>s%?d.|hS]Q\z9|(no|a-9f+ʚK1,q՞;ɗU%*_Ew t]PV' C Sꞕp6-$ \-*;j! &8CWp=4ri6aM>Pó%c*h jΘ.|Q۵=9-YYs-o$ulC+e"¸=FĵPTx>l@a I^<I#K;2/uIjn,=&(XW)F<|xڢHkt8\wJ7w(̚-:#"먦l06"ozL]CKk!kO㼢YA7$|8^ jMLr.gԵ b8AN}pEYaszulI/;\U@JҫZf#\Xs$:f?4|`-Q,B޼Nز%6h#y}UU#ו7"hֹFQW8%"$h4 OA]㳏Յ}K@ Եp|;~=:Ʀ@5\-ƴH+Q-:N?@H:75z":Ŷش3}ug$n "5rК֯gj&.TDU\C]5o -άʲ56 @vDo8r8if4[KH*B:8`-6"\E]JH)(hSTAYu^ Z\$EꉾBy񱣱җ:ޢWu$Mgͨ1]2>V ^ojr#&K7 4II &]=}w5Zlp|C;(Dt й£a@m롚s(X `@zqf:8FL6*3[F8:KB+_$1\'|z90\}t_QK.qW* N)] JjTf, V䁮p&$h6G^/" o|yƷ3RR0oGt"sLrM| "za/xOD2IċJ0ӎ՘JFu}]4.< K57L#o B):N͖ `w](́J\aoZl} # M;9~{!}| (G+ Њ4mE00/Qrt zi3\ߤqYUfSlZ4D`rb ÜM1F V VpХh[HuG6ŤjU݋BPū$k(-(kec =\}a da"N*.2˻ɊP " "(pw?xe%Fw<:iPk.6.S0ad=•K闱G2ĹVZwK}@;7Lic<l;Q{wX%A_@ML ŀB]xc'FkTOAVSo M49ENUO1KC9f}t; /kf>wUo~MiK,"po $*ec&jTzhJ,Ö˻MIfa}6 }#qB,{F߂_&xUŴ<0@@Ǵu|{CɁ5V'Tn~/.K꿬롩l$BdgC@9@HK)>(oSG]m09_//l`ZAUC{i[3lAl`vEE3=~GĔ,":jƝ@@VO2.HLD\rNԢ#~'$Q8@|v56/Üc:'E;)籓Y='Y-O&9}yF~ $ho$J߳MZe10*b{p2NG /7G{?occ7V#Qğ퐠(3*D8GoI* G{C Ю_>V̥a@sϟs:+"Aa"U;~1AnkL~0Muwc$W}Chsw0ۧ}raCB A.@yWS?AdژJoRT>pMO'#ek"Hȫ :PRO)J!K޷Afm $xQRϢ{ߣ>Tom _WXڻhNN:Ax*)zʚ?ڷW!ekYн42E"T>?U>F~l/+ LS1:F+}KNKh:P XtQ| ?X;^A~(V/Wf%SVg7و_U y9D>wpIIk~FޟX;/[ˊ욹?[+*vZ_ %S^8P= 0+/-ӷ^Jч >i}r$S_Oi*2ފ"z%~qWbE>PP|EStEGIENDB`zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/docs/_static/logo_bw.png000066400000000000000000001302301527014137600312150ustar00rootroot00000000000000PNG  IHDR;' pHYsodtEXtSoftwarewww.inkscape.org< IDATxweſ'*z/JE$XAPDP,"EcB'$T@swݛ[vw&a3;3LfO;Mm| i`S} p x #:~O\I@G>"lO}0&!3@҂fk-mhA$_9|\VOg;$D/V}8bʬےfOFO!\F?pgRFDaڡfm-mLhA=%\ '`U%m/nJNnCkafFDd~XAZgI Mmt IHpq&َdugr1{Jr "aljTN :5†>`;*02j;Hon* e{퇳6Npm' a n "h|mVCl-=cI>&96@4@((؜ WCIH&@5l?j{o"Ev Q5N~~ĹDk!=^3*ʮ6ʇ6 %]O [mOg֖t%Q}Av2;W$%%V"%Q{y=*Kfj-mm"#ͪ$|emDEp'HZ]߁ZnU ؾ gˀ%)ٽL'b oo!ۻ"@[ʹҖ6ʅ?֐4wh{vw%$b½;U*ؾvD@e"PbV:f2 83Q0hq@{D>eHn+(Ǡz? m_ߑ_J?Hپ6DK*%sA;rj>&V3O2ydžO؟HZS?Aw( 1nD CI%͞a?Dr$/i1= 4z=牂zPjm"zpJ9QEڶi; `" lntl?b{wo?ptw"42+ߵճr۷؜eETM3$ S/ؾ$G(AO$*r? կ/}H׏/IkBfD?{%}Նھd_9&m"A#]2T]?ARC ukcny J "69)#m"!ipJ o0gٞ i>I?&V'ja*y,~Ib:v[;+$ݚ ]}uG`:%]@^˒Z"`yg`I)'j$yo=IPҠt̵D#]@Ds"p /k$),mm" izY/CϘWTϰ$ 'V+'hc~MJTz?.bI;p*bg &TؖxOP^M KT<t0տhNopgyG%#iDWph.o V++$N;~&#|rVvtLt2MaDLpe6;F؏StES϶oYVi/b;p$mJ[=DPRHZJ9 @~JO耤Y$} s% |pyu o 2:Hŝ$4I ?Ib[mD^B- Fm@JT\+b`:rֿKo }ۯf;$C+)Ⱦ}SmN̾ W1"sm+>̹S#B[moDc}"%]/iVƴ&@b$*fWZgyߵRcV%ϲ}4%N_y8C$R;X81<.\I-%+QDTUA+mi>DP0rյ2W}nyIRbFU"N?+N+UO=Is~=O'dAV3Ոt笪*IҖ6Mv΢ !f f߰ tمh\7hi Y]7t 04/iY\Dp`%}w{,Dq&I:"rޱ}*q?!fAVgcۗ+ɃehA!iy>&WOf;$KQy=B&nm?퐴BfOw8-N9H!NeH*[ 矑SCp:jپ4.:E4 PY|&Բֵ}IHP-Ⱦ*nAu3q5?afrE~Y"|BY*ꨡ~IIJ*oG*})ؖO Dd(hv*Pwkjl͒6bfYf6{%%_TTz !OA%imD<.o"S(< 54ۯ8<*n &qDѽFJZ@[>hA 3N Oy\g{wۏdJP("fQ[logCIJ xhΘ,NE' }๞aOT֕ڣHGnKR"XI^X|xc$]*ilTM Fr  2 CII1nS6}[%ܓ&Ni$A[?7ڎО Bc%Yۇ߉zBB)7D 췉>V%JYhD (:l~ SkMmoo{tB=_̸Z@G Wтm*%%i۴ k 7\PC |pYΫm#^E:jHBkc&~B) άJ@eFD)^%T" M !.!(t"eKJ5諔 5tY!oßTu/Wo&{]^:"[.HL"X{I[iˌ6Gi?Jv9we*$,/ 'S>TN%~PJ@'!I>w:A#Jӻ7!ZRMg+K=sKWeu7qDr6;=̃B`8i:`=W#`SSvrhAp.!(#ђsmܷ%P5`:۟%rnǁnJDo%-J[g`P&1%1+#:e9~$&*yW};gY)G#. f$"iṱ7'Խ>tߊ|!;hSNJ#D $-H(OYҢt/-%rć>Q{q"B(C])TJ^7"gsٴHC]fqؾM[ vj*}o-DPI`s'$zìl?(WzxH\9,nN)ˣOaT%z.e90%eFHZ" "Chs._}:9꩒pE~S Y{nM~Wbu71* _}4B8f{Jfhfx"HѾ$\e†i|OFPzMbt<#neZuM&+TvqH좟P?]U1^F-!HE[Typ K"ݭ,%|@K ZLGei{gCxa.xT?R8%."bGS$+i4>3qJRDgÈ(eJex 8P҅` 3?N$mlPQɹ;AP:28G_ee|uv7&k]s~o)7DJw9s&ί׼8?~؏-DBND3E\ߧ{%Dc[1r+܀H F L,IS`Ԓu쾩A\~k4|D aʄAD /%GwbaG;P"ȝw!6uoRA-k5r]X!  M:"yJ6L%s(V x A5g3޹sOlݍ&1+wMO={I:G/4r_9KYryTk4r? {O:>"3/SYU/֋7r[d[Mt#PZd-ˀmSI;/gaBd-~,!iI)δ^?ƶwtgq%U( ?ۇI $!,m<{r:WRpG1Ht>BDږm_'ދ&+OL\h#>!ҫ>l$ \IyVm_6` ؾ3PjFk;X< ih"L~%:"s%[5Y~&GDҟȉKHc}oJ]_''EދU 7F%(HFҹEQBkI[H h[f\Omqu$]In`;ۛپ%PJ.%{ucI:D[\m\8{?LRJiU؅8#W9l]VW&4A%j5?$7D"<{K*~FHRҭ(`âJ`{}(icI!^M gWt9ک0z/CED ɾ}fu pOCtZ, jwf+R},WsM OHru4yG,%X,Nؤ;DNe Dζ>BvJqY~%IQnmqDKZFA(qf!tn8{};fBZ!T\߈Ⱦa´+˄2fDCgNn#'%f%𴤣LkD!"ZÍ&}O$]AUu=h`9z iDcLR fu(Akؾ̎tIK@p3>K<!\JX{.Ljb1$$͞Kt(n/iaIlߢY 8ʹ}+s7)`AW+Ժzί*cVLdD{&E%]Md{l_-2 >#-JA,nx=O$RJL ?hZ,nNIJ*itL^Kθ"~vI)ؾǡE|lWK!ym'lo)j=Jk $B^JURv,†:C;usI-" Db\L8Z/?S .$ t*TcnF[nVIrt1/?$zť&Ky#b<[%m^uDZZ] }ǒ]㊲eIJ2ɝ~'eUA@#'왻FZLda"PV;#[LdW{Zr_N IYG[#iBd:h"xh-Хg `{h85pAlе㬈:c5uL w ծHj=1pHزi8T=磓}˥:fC(Lvԋk4r2&\BhkHʒ.#L9S*/q%I3)>[cb0Ii qm_DpE |䟳klGZ%mYdI7B{~/͟*2UB.gyNI!{|^]={ A)wWqL(x8h=. KWZ 8/$-JjWqI!<9ϓl:"go]yk%_ #ϑ=i$ T4K Ot=N1ˊwslCʶ~:ܟ>W%~ؙP޳K*]v#N퓈GkRtL̾7&ۑeq{ *y({+{$"b#Enۛ1C}]!jW4&JS"f%ճY%}hA0.+'V)t=?FgCl`{ tD+m !(#IG*T;O"N DrJDs~$iUΑ6ȗ/r!TV9%2V9;꟱}-NI#"4HBuf3K2[IXYk}G@$DR@*)>'>O6t RN lL\OΚuaDjLz8I~? Srg5өr~Ba:۟#3)t4zv-(:SOu?Um i&I_"rEClf0Nˈ[0 %m-p|aM17PQ:D,_}<1PJJG[#cG ߓ4WZpH[!2&K:9V9W}} DȆڙ00m7DxbWvShy'$>Db++>" H :f5[oZhT6= z晴hڞh7}',>? +&1XpLmI~|'Jsg5 <漶fq0"+y6IwJƮDOHE`ݹyieOo_~'}@WD]'JodKysߚ56Zg!8Թ˛`3zB&C3N`ihV"?&ɇP"ȝwM\תڿ Y`>^D;`"po Rq{NuN5103dh/"ȝgh6TM8iM W\N'\RELxֶuV&*Eonp31{9q6!Te}Cۏ;J+8L~3=:Thzn3$m,?RXOT6 b87cx*]ew+AdRyλH9y%*LyDϨ$( ˕yqys\Ūp:4J-+܊cV:a74Օ$*1CCv%^Jc SWzHPBQ Yn];"M4.YP^ }9R{pZ"2ȶ=I+4 Rs&n]Lص59X0& qZxX`7,DpE'`00pTt]Ev'n&B܊ n~9T q*D68b%Qpc( Rg̻(6L+ad-DH-s/KhI;lHrnݒ=LqʥaP @{^L/Yhd!wEm>I?&ϣ e"|PZG?i(bvZ/QU#(:v71`ygvG5bBL8lEbQķ}Ʊ"=+Dͫ%DйfHHMUԂl?nv3i= Nq y@7=,L;&v-!r_&dų+ON :jM=r.^",?1X ݥhx>O_爕ڜ=vB't|&1AP,Nm9{4p^/128QO"s,O#]> gMwVMlε7]L'^%bKk7s bD!{ N3`f[ N3E`:캿h6Nvď0krߝ*HN&~'RKQh԰gX7Ib;[zY~DPu?q]]++,Wڅ{mH:^}^È`<1뙩EFg[3f|105A}Mgs YDP1nԵFsGtAdX>%zve.@ -vcܱ?=ǵFUe:fӯQ)MB'8f#edb%17L!zg\ubQG[#ۆ\4h~k$NU͆\DDo4Ca p"ȝkA |؟!QǞ@ gc{Ufn TV9 'ܹf%T͒/t̿DPtnLO lߋܗ` Um_`#I+zМԇs׃ eGPM 6l!n)s*kW_Q_},`."qay2^ j>ù4VAl%V;9̒ftBcۗ_%R鉖y(s = ¾+"Gv_xr:ۛ^dA瞠7DQ9WRthZVO>q% 'f?iel!yQ  > C:^s" /0ۥ?kFS`I$_z$a,TI :d@dJ+JXRf&'[IK%B;^IT(/uTմ!tI&=9".olWwt,A'K߲-b2I߂qC /)H^xך=Ol/ ʊw኶ t.}$>]B\GdVђ" `NIG+B DI?V`{Qkq _km7UgIZQ@x)Qu.|PHIe{'Idժi(FLntvwz>W~*l_:kBY~Mmb\fI߭oBY~M Sg[DTM>T;T(3|L ٶoT(3 a Sz/Qʍp '#M69aBCTbf<jԴA:_]a{q4J>;[8+w7Mss2NdqQzu]Q.KdAS?U)5@?u9Y"d QP]Ki-M@ɛteљޢB1^G#yrˆ wשLa2B7A9k<%sQDl IOV$Ot!\{x"#D;ih"EBT҃kvJkfp?,P]}D.yt&7FhOqy#e%[BƎZ J@c]1 {FJ}tB wluMt& WFjYQ!oP ʔ^ FD1yp][{z"f^9rT/QJ"CZHxM" "}gc#jU1JB`UޔsTִ 2?Ϫ +?nk%Gq{:z&jCOg[Dpף~nzw[XUOde:_`bN`/Oo z"'>/rͷy1Ι۟/T=矦fz?ـê_|%[_VAe5ZI\@=Wo$ eMKwUm bUѝy@ Ltܶ͏ "ȝcUBѩp4D;Dzw̞?o{3T=_7]>ӥEfoV5:& zkᏤmEQĤHVV 8b `abV]-x<F w]Od0y9=~WY"ȝkyCGD5~oQ]Jo"ȝkq"e!uκ5jk|BH>,"YNg\5qC7:$T1.:} `Qn`~^'O"ȝs}B*SJ7w.1~ȕkιדI~pyA>\aD;T@~d&uM4nΏxE "ةqzm5'J5[?Z-,I 0:BBk8Ν|jy>\D;jD|u~gM!{xA܇Cz mdê*2@snc{ wϫNmE969T ө}IKX-[ݗ́ԖHJxgJߦ5jsw4 7d!J'>p5BO5G~,7,imI"~!=.=PNy[v>ˁ+3Ǹ!١,$i sّ _eNIKH:Xz,[e%w_.6t qw_"N$eR[tm&P"᭪G( =D5&,#Fc$!IqoTWY<$ T m2TfϷe*4\Bvܳ燉fP8Uz =O$3VWW9tf" 2ysE3 /$af0 "$":[!&bf}-0YDP tlQ ;pG>*i?Ie2L< ih"'m:(PIݖq91fDǿy qZxN(z6N !A\!y!8 IDAT `DEVt1ޗ?loݠ0"0 . o"c1+cms{آ쭅1ߙ(+rKnj=Qk<熢D+Ξt 1{>(R)+'f$?{87 $ϿI9:-_4S."fx \H}C4^@}@t %DBI1#[UnVO娵ؚHIoM ~l!cdBY_1F/[XU0 B{BY_s|{I,JFmD3S2E8 Y*_o5Gޑ#eF bJS sܫJ^5sݺ e}ݺO7zof֙a6B# aof;(?O/h6LJ )]+k؝DDO=#VKec#OKI*]FÄi좣UTTmA~BZ7TB`53b?w+y?o@5.Oa6rhdT?3=ښA)isfk|@F#_k{pIQi\Wk{H\({y?y"x&}}z FʈX]^ :DE8Sh$UY9YzRtQo"x`ߺM(^~Z[)iK9맵M~MNj&,K6üt/iր؃Ӧt6X(x_] /&Hy\hT%]gJ>&ӆM,N*Mf?gI! {zaзs2xw#xxʢ);#ʻW#x2gd=}A4l27f@ۓ>&:~nPY7ǁTCJHRM|d2m -5p+in>I4R$~pt@%$i~G|D|йh8һ lIK8RkB FKR_[>)R~Zov{@'y'"TxLܾ<j˰&XH/X i*Yq^:E;yߟ2(v :k*U6:STabB,%F;'ǷI)\"xG;qsz-zswc4`*) z"e" j_0@TKgD0 eDK Bx2#!bpz a~&0"4%I}rNFgEm`"s/o,t!>مXd(+ؚh"ڳӾi Dp `U"]`7 9,A)t#UILt'r$\,SeR"ڲϚ??p21~p,J | NԎRks>AaSB7(C Y8Dp7}{XwsLEsL7<;.L2s?/*v=,&N=Jϋӿ@$7L/꼫Sk*g #˃ҾB|$W9ྡFu $D+KD:5sm_j" آ=FD{+n_ϋ:+OGׄ,0g6^%ēzyaU.'ܗYm`bJ‰ Ust],JGɈs0ߤVpK$ZIԸu込'[+KӞ=Okk%lFR{g$Q4DW Ù{q/˧OyA#.IzLuƔ/T4D;ZTzFZ,IrGS!.>AsSSAyl"K]ql"حt[F5aIyjBwZ/o36"@op(U}ʶٞDYqԮ 5Ν2lWHnݏ{7ΕuYc6o"ȝ+Br{AşרYDpU[+}=ښE}˳,pgϽݚI 2&Y5m I+{q/u>F?ݪ:6zAg"]cGt. t~(Џ5r׼Z@%*}w^z #:y< $L`:7?{5R{]^h"k칯[3` UAt;uW4dySIcmѽETdךsJW?_7dY*j:% ?Շk5rޜ jW.M^"ȝ{з^CCYeJ+h-HW2aL]H'u*I Ve mI`(Z`U@'Ś/ΡT$q>l= $]At_ݝRwMzYl^%/9&lY6s/wƃv)-TzL 2ۤ?$&ƏJb{סia18c gSi$=PzlKYg>_y=sDG@Ҽk2yJ)a0{R^D*qAlV HXҵ|(1F?q?.C""Ë;HF|#%]5LstmK̾ Hڱ$"i3=Ah <"iIKaDB\s}RB~|;۟LQ#h&nmON򜓉xh"C"Ck 9%};+"h]q;j" M-$HvGٞ؏kOW]߯;Ɂ$Mjo9E!p;w ,pK"OůIڭUN _-n8Q"]VdXC칖Z=pTLevwT%ixLjk/#k߱A&n WFzف HY# 9Ջńnm$NloDI[fmmlAL>C~Q "")kt_0ř8MdxuWqc{, B^X1-Cd$qCO}uqtvt7QXXk@{ppr#N%G4Y+/<!/#Dђ)it@ڊhY1K6"x=dyÞHIY͞>"xL9ED̯:î^A `8 ,\1ll`$Jخ$jIt^ ؙhi]4lHmeb2&D;u56x>:x;RK5+B6ğԓ_FDd܍I@1[=*I!VOa*fu]L`FM[ݷc+󋘚f/R2._:70ϡk.l5=Ap" N U){6 P sx@58 o)0 ~l=#Y}d*dFR'*'J\>Ͽ cVagc4E<4 :3`0D5?c &`6< 1 < )y .;PyK œ,`s@0ػK ˍ;Z ؤ뀍n;œxX`t5ɘ | $a>󆑰t X$1 F=s,my[cj/ONtṖ޽Ԓ 6sA"`YnJ/"7vJ/92>[XKD~h94~;ƹAֶzY0UʁԯQNu}#nKkf}Մ,]Ǒp,_31Kle^=j:smD`הv}#:N}M@45ihVe/2?^=EkWyj] O`klJQv`8aJ-hb;gǩ:5!'y)&(C ߺotsA=,TOP< /y. :i @$fǥzkT4OW`-&u%vR ?vExڤ>y'mf}t;m\xAAY|3{pNF193asDuw Sɳne"ۙ<&%o;0۸hV:ҙs5 - g]C+gx:y}@(J>w6LM{ֹWT\G61zbS*9oM^]=gL/8B3:  O1&/#4#? ZNYscҙ|ys]֍F;0Vݻ)ST6?7-4`j0aXI?갎4 Ob!m3ԬS/^#=G]7:( sݎZr}o^1X6 trfeq.|=:@pSDѨHhU`zQ0ؤ>o ;+>^ 1 @:]Jjd1;Q"j,-Ldqձt~ ̍#R{QI7Kq&@ib!&mOk`+fe;Sa%*I[Ž3hBE[?DFaoKuQO6>JR#mܗǗ 8gu9a69 I1cA9T{G W`v>F`D=A٪>`˔{hܾ@uNب~/1}7fg-O~\zc]V!,k㫐LcowJ!f7|)y? _i;$R΢nE=gi@0:à"9{U0wKv^~T*QGs F`;bQ\`#2T5&_FD.G9Xؚ1hζM{Rnrvt[\¾Cqrgk`:PW2ԹfeOUy{*jrƃ-UD-"sBrэk+NTuOq"rl?a^{Ն>{3|‹Yѭe[D#KAۯs&Qh-=ϰҾXBxDd]}Z(|9܋l*_ZU/P眘A8/S+38@SS|3hHYrܩDn*%"wzMXUw8KU.U<.66?!"r)zn**ZU$P~rm"%1-RED6aδ$lt݉7cYdLwE1)\Tum/"W#`"rl6U@w[q܄ƞ#!ߦ(.XTVU=;X*`5Uc?ueEp5~sY;rgv" ni1/ߡ\ȳ?i=owwb~uebϻ`ܤFN%VDn t$nW̑n&8^Un?_D\DDUoP 1­˼xN]YUx\DB2S~rFg)쎥- xՐ|ADnż?6M9R/bdI-`goϩOs`7"2n>_cQ;aEDVMSUט ,0Yȝ/̠5E&^D6(uv9`}DD>)"sPa}|wM%(*G!zݬ, "ce70eU4M\e@#܅%ZrۘzNcc.L $MKw*g`n^&'߉ )]Tu *PDd-8p,{f(c?J0cm\)HWs AD1[}6]FUG"2 S̕ee#pu,՘671ak~EGUC}ܔI_]T u~  evl?^DVO%"lmP< "r|"RDlVYA2yLD~#"PkUj"" @S_d[Dʧ> =L׶{3l"f8K\63JOv"2DD+6=װ~|XW1<9/~o\K)qvEhWU0я1{S"r,EO-`!"GF睱\[IQ~`m0PZ >z~3'beHeT("+O4e)X׃JYEd{؄˘}foL[lE-sgU}3b襃v&Y3!pt3TVeb_YeǕU|9H\-V|cq$b*Tɍdmjwϐ}^#矩\C>vI*bgEd]U)e"" w.qҹ)\*qy}XҒMV%ލx P;NúvMly4WI1gȍT`nآy'$aoߐˠy4r0xύ \qQlBI@-9FݱI:Wi1sN uR?Qad{L}r!v^VN^QUwW`XIQE= dłnR]"6KAv"2W`':tH n6뤝@NTջ6U[OSTS2Tf|TTuoˆW]DdV]}jrv^E)j#10+[z`p3EUvrz}ʹy`ێ}|$n;:X,=D9_Ի2 m ^.Ŏ آ2 p"o\i00[]9}p.R똼UC̰HNa+4ͨR'̻ظ`/lXjձNVP,yMj\s;E}iדL#yAFHXEΙSx9sxV`;˝u+C۫{؂Pl15$o܀3O`zsh#,0I|!IE^W@p-ٔ{X [=y@mn༅&܁}\+ar}Nd)kD̹ KFuٯp;k 4Բt- MTu];C+>,y&$oPU{> 9Ey. BF.%o XA?у+MF 0Y=7*uX Mx0ZتzNKDm|Té7OQO9vC:uWot#MQUb;ϰ8rswCU͘3N͂ǿ~8a݌:"Ra~2 6<xπ hV`? , r?c`m(Ẹ(Z(6^t5XTu[U}b~G_&h>wEdu+F?WakO~JTm(p<:FUl wajDdj9L|CDDU_R=1OIb-ȡEdg .vD(⤥'N 8Kڬ펠9E^I'N5d'oU^YUN1P``gJ-x+Q^:ק^MtN?vioo!ro4 MO;Q׻ uѹESzr1A ɓ1!hsGp=nEd-ڽ2N{iz~8UTې'w93|SUM7vqW/Wv [zW:7qK\P\0j(eiK64;9{K}}|@UǪ6,3"qM1 Q# q$)Fv&?[h/Fn T0z^O."b.rSnvgU3m "Ww0X-f;ޫSռs`;c/lFap(ˋHsyj0[곆'@UP/Ӏ47XGx6sǟR`1/'Ed)Xr%zX=9lHDf @p'nEd%6nC 9iyd|U V8]jUU&ȩy#TXt7{<^y ˅dn軬s jIrԵ1p$"u[0oTjx4-"2^FI}('@USo`;'.X)4 s~skw|&7`jرW 0o Tz_y>y `ҟJVkD珹X \=)~%UDvEDl3>&/aY䗘q<9DdI5 [ٕFx.ƀOl+sF ە螳{5<z9T:ф~Kla1IU6[?hX:)Zg|NyROƂ[^{ =Lu1Ot0p{6:^5F\07УgI:}~ |,8l6{ kȍ1GD +߂5vQl]L0Kñzt&S khAws܉cvu/K5fo+h {Sa4KNPLB6?\?l\ Surl)ob.g<+#'[@1[INԄi՗$FQ=/nK-~1u[QףwFZf>]ds),eXm^(II{ Q3˯BoM {~FybapM\6h'@?.t] |6z0u#>-> e,z 3FzTsyUGU_la6>-LR0{8\Px6Deչ$"si-i11]Tml?D$YHGQqXD|VEI*FN,kGLm7;9}+)G"UT]*"ct?V 0X%q6A{K>--duy|LVS3T#j5':?P|؊Y_DZX9]D84DX_lr/a RG2 SȢjRrIDV.P x8#U-."u.T}71/ae^EUwVXqrYr>I9CUn.+"{0U}GUD"" n^r5f=#DdUg`p8.LY׮=]Gz!s>*"8HߤH nd˫mEԧ4. {Kl% ],( 9:K/~#LQlS^#rN;܆IԅᣱLyDs fG#Ryck ωߝTuo#Hujap4?ʢ1x[US!9/v9`ivOȟEl@`YU=Z={ß\Vnc\DV.m֪QD61~M\ӛ&"},/d?Ezy9oZnݱ03"X7Tl2= MTLPvϊNhvAgbk8=slLLe~[UDd# PPڕU˪zQD>-/qY6U1CmY,$7q=bǼ\t9 S IVՋQՇׯ9^\馷st˥4J!~$aڙ<n}fC8zQWTul¿|VDK1j@oL?"r'N7>dk6:MD>+`zמCYV~0ض9O(716w6똧G9$uIT>`x&"ϩ.)Ɋ:U}BU=H0RDfqc;;T^xex+"?tиmr.3MdѠ51pR^JPD=svR¤sQ>aYLwbmq SsCuPNC_3SUbovtX;׀2WgfJjɧ'_׾Lm U%c !; mӷnem)cWx+܄ֈ!"TY[S;U}-眡RUB0]jn""C05f^4INS6j-tMIU?ݕcDj,] 'I_d8Xs, 9cމr v9ޙC,-7鐜er(0F51yWj4̟<<2feU㨝ȸΉ|SYpۤ 3֟V{=<1,dHm/? 4`]w`A;P}:u~$uӁ"  pNH9gL߶i㯍r0qNJir. ލǗvS4ߩ :ԟI5Mu[zo,lg|Pl UBH15)=g^UO)* nƶƹ V\UO:B?TZ5_ñ08*lz[t/sB+~4йn7")!/}AYdXqwMBé^ $cgqr/ȭ"RLwIZ_Hܷz r1Ꝫ%$d :YD@/GWxڀ^D vo^,:'uCOE%\l4>>4էM?|!A Ys<: e_U}.ώ-hבX,w:8F;@ΡC )p\."hy*ޯK&`1"7u">[rH,.'0yZON+,A͌&ǰTuFgYʭuj)瓱ޔ5ZrE`lpɄ1X`ap^9uZ?EUPʩzx( F(ux#1qsbtʐ\mOlGX~TzUY.`y[U g鿈ccٶDp,DYr5Xn!͈8fHBΠxHc SSLI^X<r>XY{,S0='ՠ«U=cmU'ۖ*P19oay޾'4i7ƿ77XkJLy`q 7zj,vu$q&yu~c 0IJu8Z/EA; kV A\=̉yDq) >|OEm}MlK\>SfsbiI?ckn6np$`; +^Ԓzr 'Q|R0Db'l0 s_%:$^H>W)gs5)@,F1kt"X NWUr Qm( x3Xvv SiFQ }04l낼>1sh:SǶz0{a/:gg6s"S!vf.@?ph4&n~w ͎Ss C7Lt/.54i=c%o Ixɩ_)6ű'0% IDAT#ð߷9KDFtoo<*m2N\QC0/'iJ}bf2"Iw}n@1lE=x8 :>'5wΙSo\ 7BHZϩՕ z~~} &aa 1ف`{ÛCvcx$n zFEmu.Y'OֹSN^vaʶv4ZM+1>w $fd{X`nI/1a_ѹ6z\@!]"_ ႣR8j2Ap:pF/-dHiܤGyyfǢ:̡~7mȦF<rG@$rǜXv\ 5 5 c-ژ` oC4,J)|sMQ/CY;@Jd=$U P{Y^_%\S:'!;z3ECbF= C15 9fR@1\sc7HMQ$@V֘,<@0ծ З75c5k>Π%I|L.FotYww5s}c1Liiw^?B-m"+(`a [?A,n|65L~ҹ/ >2Z)>$횢]8t3Z DɬVIiCRELHzy3x 7gӤ_+`oܗrcc 8cI$,]qZ`ie1q .'1:LWY {ݢmD|b>SkحY$b`6IrxqB .)=4X5vPQ>8rфB_\tB^aWz&$j+qG$Ĥ6{~puh|)F#Ra .V`I,tfkyXXӀM3ֹ%mQY$]{ S;7*Apl]ȭÝ2UJP j;"b~0H. ^baEmL&۸@5Z$DL8vyqژ sZ:>b65y Koy\Ӄ?P  B?@=9rXG$;X@\YƦcIj % ?6Hi|-L=MIYaܾo@d,%m"e;Sj<U5 9$zOw:9H7|AV}M6uj`Tu)ouHlG! ,/px@jn>iT u4Lw1f^wm8:>L.-,~8l܃vVvJN:=hɓm=cN'"{ xG=dԍ- RWMQ?ek{'M`.|a5Lډ]ʩS--!y@P }KMJJw\A$;Sˆ19HP5-Ap|V,CxX%^w-8m>8υ:otð#(v`y,l[خlF`WUB:b❽@ϳo ys`>-{*(?!a+lhuX1/i)5$wn׻ygLp/&M}raicrmh6ha,[Е^gWy[`m:סqAC,Emw_ G_m9^:ݍ:w  rP IR/,V (ՂI7Y8Y1?nM^M̨=_/? 5ޝ]ڣvA:5KD ߐѣ@6~l!ݮVGiIxpDM-9} z}"8>.T|0lgZ,+@zp7},Km &c9Buq\Fo`hK s Em#NX[ F) `.yjMٝ׷)yR}G;O`ޯr/ ׇc|,&^# nG6}- <|Z z3{%9v?IIkaPj>>R@пL@3f OM;? 1 <oos<\D~Zs!!q`nH1^,ݜl( hVƬ~BzSו/zN県̓y:3:h&Lr}g_H/6vBR[+C<isb>!ߙdd#g2j126y AҞ_f Y_ǼVe-BJs XYg~dm@e!5?;QU!$S>`*Ȯ>U%@azFP4x> n$6>4NmJC>rަ_Z=J8bǀp-s@01c19O]JNF] Jfyh a!Do>{H_:s!jc4S9(,Z ز< 4I=(|!gGa`~rg&0u,#V6`TZ/6&$/zޙUO$[a_Ƒn̍vRAb>B do8KӔ>G6>SMcr"`zL*azLͲ,ALAbAEa;es u,.MkŔTf\ A!HVvCj3!!OD,@pNJ/ 7a{|nCo2^V=a.|jAp4̮/OsChH2w yv1,khHNS1oq Kz4+azC?``M,㢟 _P}<^TY fd ?̉ޚ'R,Z5|<灎KSf !# q=H>΋r]Ӏ;jwoApFM27va`);%xka^eJ֠aA|\zţk8|Fie6*Qv>T[ Ga\tt}p;nL-92@;6DEXyTxhStY99jxtX.Z_CByBi˧LQbI`B"p0{%mPql}F?AÈ}% --,|6vgs`ѫ{NMoppZ@i ~>t ^< qtިu09xNT^d2ps6Ki'F@qz/3yM'ȟC;sQ-vu2o >8i"ӱE:E3=z..qYf@ Gb}xՑFN?kM>ϲkbrLnp*nhev`>1=qoQ;gE\Q7b0}z:D<`ǐFFB & eJ;S$ ZtNHN ]f%' xK6A#e 4gt}O_B}@`.҄:F?97~dVxT*RH8c'28=1w@lW+fk(ރ@")cev`]}e*KVb =PSDlQ;!Xj a@N, ac@1 WƖ@9gǂT[:~2 hLbhL=jҨ27;< \rkdBw2,12Ņg:0ƂI&%ȨQyѹ濍@@&!& PeX,Ѧ/{1Cw6jΞ#c">lr ziF*[r֜|g*6~,UtkjIb-^vϔl2Sk._ |AmO4_tlԇp|lkq >hp?fy5`v0%-t6i!@O (cpR=@:25c+Iks7_@sWB8JI2c0OxB!Ā:-݇Es  N>WjaG[߁ 8/$X880nA }pw_3Z<C~@ijjwyo$jv>N@1vEu~tWƂEOƈ3E`W/{9@?NM#rOGG%@pf<UQ09m)s\:q^-y1;X8]@p׬mv؏PSK6[78>;>wtaMLv,6rtk$ح*3۷g?뻡0+M>-aғw'ΦN [q*G,ƍgC;+'=Rv`tpͳ5@>NmꁏβԦ3V*2NR,Vt?g@ [ITs <BV Y3} iP(yD7l׌v(LG5yK,nX3p^2]F:'{hU-Bmd76= /|)&$ ?vAOꝌsd]"Va}7!}.{OTgKm +;^K4! IM 6vyAP$Aot\@#7V@KF3c}"ى@~#N,U>R)91a 3 \}ׇ(l}1JP{d1,˽-ޙ//:P( `/I7f 6 ˛Xg?U;^ ޻+5C؝Z`h6z/j =yn ~_ s+ *wk[!d Pl 3v6aX^wϩATOa:a0AKqS,yxUji@$p6qF?$벯0Vh&`.|Ϫt ,mz$Ir=GCXm0/,A=Ob*@+x޷q1_Un H*`T! 4eHLI!Vtd˫!z|ј`L;NU3NVꙪ111K;eTOٺY~ EjleyUN{+)LTUK2/𜈜'"KDOQ1ϜK#hY/\fT%JI|szVRQg),a/39QU uIh<!;}rKOva"܉YVSߧ]SI$T|U; C8]D5S^@0DtY "܅0|X^?TPtjS2\gCy3/gw#!Q pS},)hG!ZDҗ@/>!he(RzVq|+ y Vt; *i[T5U=R3b@p;`po@Iq*(?/ (zJ!)8$`vB@)Kt bK2Cy-þwA+ ,ͷ8@X<: NN=$cTPvvxȻ6^+)KΨ6B]@ N3T0@0l"2|::۵*/f1) 4 niPy_fm{.ʮUIi$ J @/ fxm&"bju#҃:K#YEd$XP([WI+a솒NC@_GD2&IDAT~R X5X[U.L%v"ҿ3ƅ6RտVc2>{b w; 1*X#C| u^x*K hPv"2qY\mCKf5+h/; u~pB.!zT obpv?Fp^U},B@DȞl =<:.`KU]WUt%YT9Uй8 /"NjȂSU"`a ~UX"p vc1PJ*UFc*aO3mG]2vn1PU{+lb6oC8@X;?=8A-F "sXʇ0Pn6QT2I`gsc0NDYP0k܉`B%=#p/f9F Tu3U_̅9Dl~R,╪Q͊PUoWխ1k01&+@t2 @Dv0Gr;u*-NuTuku<UZxsœJz. Dd9GAznU?*S 7Cpۋo2 l ,r=nw~% 9;/mMu+DdVU鸾<DdX{XD(U}+sL`?3ډ+[,%w"ӱ-Ѫd+d0 RJ ,?cT3^LDvf%S|FKDdj=ŗ\7uLmM4TvJ fߋaY!x@\)$7KDdML=}'h0ފKUu|vK?:qlT QI%^RK<`F; !"ñ~wL>p!pڳ+EDd>,oQ,ӁX;EERHWP3|ʇW(U}]#.}nc * _DrQYs6aLtǩޡJ:@m/LwSU})UR`pIT޾,Ӂ׈7V6 HW.J*) ORXU}ȾUҾ 1o1ATVI%e~h,2NRY%K534UPp*PIl>lhR* "" cʷJ*pR@,/".MTէ[%TRI#i1SnKSJJ(ITYZ2 ̃%G;M_%$J ^x g<C%TRI^2ˀ/m,Z-#_%TRI4#"]/ epl/sTRI%386 hY/\$:ZTR ^xiuPJ*?""CP c6 X2(K/ TT*rKGaia40܎!._TRIdf/3x |K>ZUo(cTRI2 /H۱ WֱJ*T Xe/ 8X\B% rVfj UX|'p){%T26`/qJ燀+@)4 R$c'PB% i` 4[aQT2U^T*R@gRA""|.\3w:URI%TНT@Ёpq*@¤`mH]l?~zET7iUhRAFqpN_EJz%K* (Xpׁ_;EJ )LDd5 ,Yv}nrK%Ys;.pTPɀfG* (*۩0(U}ȾURI#iW@I"",CNQ7[%x``JLDdla"pZd*y- P0}B̆Jq=dfppbG* ""KPT"V T@0HDDvf>.VЧ p`,w'?aǿA̴y!MN lH1/;knc bah ;*14mj166hC?{S:~哪[om7s {OK+`58$k}u 5pLw.ҤFJ>A @3( WM>D'.}smR] @(-af7w l".}smR @;)-cf7 Zw,$Bƀ3Xh r-A.E`o˹633b$3ES0\OI> vSZF!lz}FAx8Cؒsm; LG!I AX,w7d Lc@@J~5 Qw'd @fdfo<=nkr ̅B Rr4Vl IJx7 RdNl/*`8:ǁ!w+Z`I+2+ ̋ \ Ǥqk6 B R*4 Cx5׺N^R'R.'K .l k 3XG7Rd^)38C8>'ӴqF B }QpBN쩬 AIv]&H_]o6l5@E!(| ա.m(2h T!ci'^?wgG=(g dQV>Zv!ϸ&HV%w.4@HU(R ӽ6 UHLsxɶRt\ F_AxNAH9i  HR9+  uHm FQwߙk]S)R7 N (pI}ls?3KZRRwl3 L]'ĥ+@jA!3 =;AxUi 3;xbـ3z@F!1D.crtK\F 4B ef+G`6ϙ O( 4F Owuߛ Hk٩2J`wg!`RT6Ruqx0=`p~mD6 @L!2SN;gxH(zfv @?v[()[ IENDB`zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/docs/_static/source-version.js000077700000000000000000000000001527014137600365562../../../source-version.jsustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/docs/api/000077500000000000000000000000001527014137600262035ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/docs/api/index.md000066400000000000000000000014041527014137600276330ustar00rootroot00000000000000--- title: API reference --- # API reference Everything public is re-exported from the top-level `zarr_http_server` namespace; the private `_serve` and `_keys` modules are implementation detail and carry no compatibility guarantee. ## Building an app ::: zarr_http_server.store_app ::: zarr_http_server.node_app ## Running a server ::: zarr_http_server.serve ::: zarr_http_server.serve_background ::: zarr_http_server.BackgroundServer ## Configuration ::: zarr_http_server.CorsOptions ::: zarr_http_server.HTTPMethod ::: zarr_http_server.ReadOnlyHTTPMethod ::: zarr_http_server.READ_ONLY_HTTP_METHODS ::: zarr_http_server.READ_WRITE_HTTP_METHODS ::: zarr_http_server.AUTO_PORT ::: zarr_http_server.DEFAULT_PORT ::: zarr_http_server.DEFAULT_MAX_BODY_SIZE zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/docs/guide.md000066400000000000000000000350371527014137600270610ustar00rootroot00000000000000--- title: User guide --- # User guide ## Building an ASGI app [`store_app`][zarr_http_server.store_app] creates an ASGI app that exposes every key in a store. Only point it at a store whose full contents are safe to serve publicly — it grants read (and, if `PUT` is enabled, write) access to everything the store contains, with no per-key filtering: ```python import zarr from zarr_http_server import store_app store = zarr.storage.MemoryStore() zarr.create_array(store, shape=(100, 100), chunks=(10, 10), dtype="float64") app = store_app(store) # Run with any ASGI server, e.g. Uvicorn: # uvicorn my_module:app --host 0.0.0.0 --port 8000 ``` [`node_app`][zarr_http_server.node_app] creates an ASGI app that only serves keys belonging to a specific `Array` or `Group`. Requests for keys outside the node receive a 404, even if those keys exist in the underlying store: ```python import zarr from zarr_http_server import node_app store = zarr.storage.MemoryStore() root = zarr.open_group(store) root.create_array("a", shape=(10,), dtype="int32") root.create_array("b", shape=(20,), dtype="float64") # Only serve the array at "a" — requests for "b" will return 404. app = node_app(root["a"]) ``` ## Running the server Build an app, then run it. Either app works with either runner. [`serve`][zarr_http_server.serve] blocks until the server is stopped, which is the shape for a script or a container entrypoint: ```python from zarr_http_server import serve, store_app serve(store_app(store), host="127.0.0.1", port=8000) ``` [`serve_background`][zarr_http_server.serve_background] instead starts the server in a daemon thread and returns a [`BackgroundServer`][zarr_http_server.BackgroundServer] as soon as the socket is listening, so the caller can carry on. These are two functions rather than one with a flag, because they differ in the only thing that matters at a call site: whether control comes back. ### Choosing a port Both default to `port="auto"`, which prefers port 8000 but falls back to any free port if it is taken, reporting the result through `server.url` and Uvicorn's startup line. An **explicit** port means the opposite — bind exactly that or fail — because a caller who names one usually has a proxy or a container port mapping expecting the server there, and silently moving would break it while looking healthy. `port=0` keeps its usual meaning of "any free port, no preference". ### Reading back with a zarr client `BackgroundServer` is a context manager, so the server stops when the block exits. The example below also *reads back* over HTTP, which is a client-side concern: `zarr.open_array(server.url)` goes through `FsspecStore`, which needs an HTTP-capable fsspec that `zarr-http-server` does not pull in (`pip install "fsspec[http]"`). ```python import numpy as np import zarr from zarr.storage import MemoryStore from zarr_http_server import node_app, serve_background store = MemoryStore() arr = zarr.create_array(store, shape=(100,), chunks=(10,), dtype="float64") arr[:] = np.arange(100, dtype="float64") with serve_background(node_app(arr), host="127.0.0.1") as server: remote = zarr.open_array(server.url, mode="r") np.testing.assert_array_equal(remote[:], arr[:]) # Server is shut down automatically when the block exits. ``` ### Shutting down `BackgroundServer.shutdown()` — and leaving the `with` block — waits up to `shutdown_timeout` seconds, 5 by default, for in-flight requests to finish before forcing the server closed. It raises if the thread will not stop, so a silent failure cannot leave you believing the port is free when it is not. ```python with serve_background(node_app(arr), shutdown_timeout=30) as server: ... ``` ## Serving several nodes Serving two arrays does not mean running two servers. Which approach fits depends on where the arrays live. If they share a parent group, serve the parent — `node_app` recurses through its members, so both are reachable under one port and node scoping still applies to everything outside it: ```python server = serve_background(node_app(root)) # -> /a/zarr.json, /a/c/0, /b/zarr.json, ... ``` If everything in the store is safe to expose, `store_app(store)` does the same for the whole key space. Otherwise — arrays in *different* stores, or nodes that are not siblings — `store_app` and `node_app` return plain Starlette apps, so mount them and run the result: ```python from starlette.applications import Starlette from starlette.routing import Mount from zarr_http_server import node_app, serve_background app = Starlette(routes=[ Mount("/first", app=node_app(one)), Mount("/second", app=node_app(other)), ]) server = serve_background(app) ``` Each mount keeps its own validation, so a request under one cannot reach another's data — `/first/../second/zarr.json` and its percent-encoded spellings all return 404. Both runners take any ASGI app, so the split is clean: what an app *serves* (`methods`, `cors_options`, `max_body_size`) is settled when the app is built, while `serve` / `serve_background` only decide how it runs (`host`, `port`, `shutdown_timeout`, `uvicorn_options`). ## Serving from a notebook A notebook needs a server that outlives the cell that started it, so the `with` form above is the wrong shape — it shuts the server down as soon as the block ends. Start it, keep the handle, and stop it later: ```python # cell 1 — start server = serve_background(node_app(array), host="127.0.0.1") print(server.url) # e.g. http://127.0.0.1:54635 # cell 2..n — use it, across as many cells as you like httpx.get(f"{server.url}/zarr.json") # last cell — stop server.shutdown() ``` Two things make this comfortable in a kernel you re-run. `serve_background` runs Uvicorn in a daemon thread with its own event loop, so it never touches the kernel's loop and cannot block it. And its default `port="auto"` falls back to a free port when 8000 is taken — re-running a start cell without stopping the previous server is the classic notebook mistake, and a fixed port fails there with *address already in use*. `server.url` reports the port actually bound. If you forget to stop one, the thread is a daemon, so restarting the kernel always clears it — and since each start takes a fresh port, a forgotten server does not block the next one. [`examples/serve_notebook.ipynb`](https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-http-server/examples/serve_notebook.ipynb) is a runnable version of this. It is executed by the test suite, so it cannot drift from the code. ## Uvicorn configuration `serve` and `serve_background` name the options most callers need — `host`, `port`, `shutdown_timeout` — and forward anything else to `uvicorn.Config` through `uvicorn_options`, so nothing Uvicorn can do is out of reach: ```python server = serve_background( store_app(store), host="0.0.0.0", port=8443, uvicorn_options={ "ssl_keyfile": "key.pem", "ssl_certfile": "cert.pem", "proxy_headers": True, "forwarded_allow_ips": "10.0.0.0/8", "log_level": "warning", }, ) ``` Keys you pass are merged over the ones set for you, so they win. `server.url` reflects the scheme actually in use (`https` when TLS is configured) and is `None` when the server is not bound to a TCP host and port — a `uds` or `fd` bind has no URL to report. ## CORS Both app builders accept a [`CorsOptions`][zarr_http_server.CorsOptions] parameter to enable [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/CORS) middleware for browser-based clients: ```python from zarr_http_server import CorsOptions, store_app app = store_app( store, cors_options=CorsOptions( allow_origins=["*"], allow_methods=["GET"], ), ) ``` `CorsOptions` carries every parameter Starlette's `CORSMiddleware` accepts — `allow_headers`, `allow_credentials`, `allow_origin_regex`, `allow_private_network`, `expose_headers` and `max_age` as well as the two above — so configuring CORS never means reaching around this package. All keys are optional. Two defaults differ from Starlette's, because the server knows things the caller should not have to. It emits `Content-Range` on every ranged response, which is *not* a CORS-safelisted response header, so `expose_headers` defaults to `["Content-Range"]` — otherwise a browser client can read the bytes but not learn which bytes it got. And it accepts a `Range` request header, so `allow_headers` defaults to `["Range"]` — otherwise a preflight naming `Range` is rejected. A key you supply replaces the default outright, so `expose_headers=[]` means "expose nothing". `allow_methods` is checked against what the app actually serves: advertising a method the route rejects raises `ValueError`, and `"*"` expands to what is served rather than to every verb Starlette knows. ## HTTP range requests The server supports the standard `Range` header for partial reads. The three forms defined by [RFC 9110](https://httpwg.org/specs/rfc9110.html#field.range) are supported: | Header | Meaning | | ------------ | ------------------------ | | `bytes=0-99` | First 100 bytes | | `bytes=100-` | Everything from byte 100 | | `bytes=-50` | Last 50 bytes | A successful range request returns HTTP 206 (Partial Content) with a `Content-Range` header, including for suffix ranges — the server resolves `bytes=-50` against the object's size so the response says which bytes it carries. A range that is well-formed but names nothing readable — one lying wholly beyond the end of the object, an inverted one such as `bytes=5-2`, or `bytes=-0` — returns 416 (Range Not Satisfiable). A last-byte-position past the end of the object is *not* in that category: per RFC 9110 §14.1.2 it is clamped, so `bytes=0-999999` on a short object returns the whole thing. A `Range` header the server cannot use is **ignored** rather than refused, per RFC 9110 §14.2: an unrecognized unit (`chars=0-7`), a multi-range request (`bytes=0-7, 10-20`, which this server does not build multipart responses for), or malformed syntax all return 200 with the full representation. ## Read-only serving Read-only is the default. `store_app(store)` and `node_app(node)` accept `GET` and `HEAD` and answer **405** to `PUT`, `POST`, `DELETE` and `PATCH` — no argument is needed to get there. `HEAD` is served wherever `GET` is, as RFC 9110 §9.3.2 asks of every origin server, and is answered from the value's size without transferring it. `POST` is not merely unrouted, it is unconfigurable: the accepted methods are `GET`, `HEAD` and `PUT`, and asking for anything else raises `ValueError` when the app is built. Two named sets let a call site state which it is, instead of leaving it to the presence or absence of an argument: ```python from zarr_http_server import READ_ONLY_HTTP_METHODS, READ_WRITE_HTTP_METHODS, store_app app = store_app(store, methods=READ_ONLY_HTTP_METHODS) # GET, HEAD app = store_app(store, methods=READ_WRITE_HTTP_METHODS) # GET, HEAD, PUT ``` The distinction also exists in the type domain. [`ReadOnlyHTTPMethod`][zarr_http_server.ReadOnlyHTTPMethod] is a `Literal["GET", "HEAD"]`, so a read-only set can be *declared* rather than merely configured — a `frozenset[ReadOnlyHTTPMethod]` containing `"PUT"` is a type error, not a runtime surprise: ```python from zarr_http_server import ReadOnlyHTTPMethod reads: frozenset[ReadOnlyHTTPMethod] = frozenset({"GET", "HEAD"}) # ok reads = frozenset({"GET", "PUT"}) # type error ``` Both constants are derived from those Literals, so the runtime sets and the static types cannot disagree about what the server serves. `READ_ONLY_HTTP_METHODS` is exactly the default, so passing it changes nothing except that the intent is written down. The practical value is the other direction: a writable app *must* name a method set, so `grep -r 'methods='` finds every place that opts into writes. For a guarantee that does not depend on getting `methods` right, make the *store* read-only. The store refuses writes itself, so no routing mistake — now or in a later edit — can produce one: ```python app = store_app(store.with_read_only(True)) # For a node, open it read-only and node_app inherits that store: app = node_app(zarr.open_array(store, mode="r")) ``` These two layers are independent, and the store is the stronger one: it holds even if the HTTP layer is misconfigured. Asking for both at once — `READ_WRITE_HTTP_METHODS` on a read-only store — is a contradiction that can never succeed, so it raises `ValueError` at construction rather than turning into a 403 for whichever client tries to write first. ## Writes To enable writes, name a method set that includes `PUT`: ```python from zarr_http_server import READ_WRITE_HTTP_METHODS, store_app app = store_app(store, methods=READ_WRITE_HTTP_METHODS) ``` A `PUT` stores the request body at the given path and returns 204 (No Content). Bodies are capped at [`DEFAULT_MAX_BODY_SIZE`][zarr_http_server.DEFAULT_MAX_BODY_SIZE] (256 MiB) and a larger one returns 413 (Content Too Large) — `Store.set` takes a whole buffer, so an accepted body is held in memory in full. Raise or remove the cap with `max_body_size`: ```python app = store_app(store, methods=READ_WRITE_HTTP_METHODS, max_body_size=None) ``` !!! danger "Writes through `store_app` are unvalidated" `store_app` exposes every key in the store, so `PUT` grants unrestricted write access to all of it. It also does not *validate* keys, because it proxies the raw key space and has no array semantics to check against: a client that misspells a chunk key — `c/00/00` where zarr writes `c/0/0` — gets a successful write to a key no reader will ever consult, so the data is stored but invisible to anyone opening the array. `node_app` rejects such a key with 404, since it knows which node it is serving and therefore which keys are real. If you are serving a zarr hierarchy to clients you do not control and writes are enabled, prefer `node_app`. Note also that a client that can write a node's metadata can change what that node contains, and so what it will serve. ## Examples Both live in [`examples/`](https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-http-server/examples) and are executed by the test suite, so neither can drift from the code. `serve.py` creates an in-memory Zarr array, serves it, and fetches the `zarr.json` metadata document and a raw chunk with `httpx`. It declares its own dependencies inline, so uv installs them for you: ```bash uv run examples/serve.py ``` `serve_notebook.ipynb` is the notebook equivalent, showing how to start a server in one cell and stop it in another. zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/docs/index.md000066400000000000000000000051421527014137600270650ustar00rootroot00000000000000# zarr-http-server HTTP server for Zarr stores, arrays, and groups. `zarr-http-server` is developed in the [zarr-python repository](https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-http-server) and released independently of `zarr` itself. Install it with: ``` pip install zarr-http-server ``` !!! warning "Experimental" This package is experimental. Its API may change or be removed at any point. ## What this is `zarr-http-server` exposes a Zarr `Store`, `Array`, or `Group` over HTTP via an ASGI app, so any HTTP-capable client — including zarr-python itself, via `FsspecStore` or `ObjectStore` — can read the data. The app is built on [Starlette](https://www.starlette.io/) and can be run with any ASGI server; the `serve` / `serve_background` helpers run it with [Uvicorn](https://uvicorn.dev/). Building an app and running it are separate steps, and either app works with either runner: - **Build** with [`store_app`][zarr_http_server.store_app] to serve every key in a store, or [`node_app`][zarr_http_server.node_app] to serve only the keys belonging to one `Array` or `Group` — requests for keys outside that node return 404 even when those keys exist in the underlying store. - **Run** with [`serve`][zarr_http_server.serve], which blocks, or [`serve_background`][zarr_http_server.serve_background], which returns a handle you can shut down later. Byte-range reads, configurable CORS headers, and a configurable set of allowed HTTP methods are handled by the app. ## Quick start ```python import zarr from zarr_http_server import node_app, serve_background store = zarr.storage.MemoryStore() array = zarr.create_array(store, shape=(100,), chunks=(10,), dtype="float64") with serve_background(node_app(array)) as server: print(server.url) # e.g. http://127.0.0.1:8000 ``` Reads are all that is enabled by default: `GET` and `HEAD` are served, and `PUT`, `POST`, `DELETE` and `PATCH` are answered with 405. !!! danger "Serving a whole store grants access to all of it" `store_app` applies no per-key filtering. Only point it at a store whose full contents are safe to serve, and note that enabling `PUT` grants write access to everything the store contains. See [read-only serving](guide.md#read-only-serving) for the guarantees available. ## Next steps - [User guide](guide.md) — building apps, running them, byte ranges, CORS, writes, notebooks, and Uvicorn configuration - [API reference](api/index.md) - [Release notes](release-notes.md) - [License (MIT)](https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-http-server/LICENSE.txt) zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/docs/release-notes.md000066400000000000000000000006661527014137600305320ustar00rootroot00000000000000 --8<-- "CHANGELOG.md" zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/examples/000077500000000000000000000000001527014137600263205ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/examples/serve.py000066400000000000000000000031371527014137600300220ustar00rootroot00000000000000# /// script # requires-python = ">=3.12" # dependencies = [ # "zarr-http-server @ git+https://github.com/zarr-developers/zarr-python.git@main#subdirectory=packages/zarr-http-server", # "httpx", # ] # /// """ Serve a Zarr array over HTTP and fetch its metadata and chunks. This example creates an in-memory array, serves it in a background thread, then uses ``httpx`` to request the ``zarr.json`` metadata document and a raw chunk. """ import json import httpx import numpy as np import zarr from zarr.storage import MemoryStore from zarr_http_server import node_app, serve_background # -- create an array -------------------------------------------------------- store = MemoryStore() data = np.arange(1000, dtype="uint8").reshape(10, 10, 10) # no compression arr = zarr.create_array(store, data=data, chunks=(5, 5, 5), write_data=True, compressors=None) # -- serve it in the background --------------------------------------------- # port=0 asks the OS for a free port, so running this twice -- or running it # while something else holds 8000 -- works. `server.url` reports what it bound. with serve_background(node_app(arr), host="127.0.0.1") as server: # -- fetch metadata ------------------------------------------------------ resp = httpx.get(f"{server.url}/zarr.json") assert resp.status_code == 200 meta = resp.json() print("zarr.json:") print(json.dumps(meta, indent=2)) # -- fetch a raw chunk --------------------------------------------------- resp = httpx.get(f"{server.url}/c/0/0/0") assert resp.status_code == 200 print(f"\nchunk c/0/0/0: {len(resp.content)} bytes") zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/examples/serve_notebook.ipynb000066400000000000000000000142121527014137600324070ustar00rootroot00000000000000{ "cells": [ { "cell_type": "markdown", "id": "6cb56289", "metadata": {}, "source": [ "# Serving a Zarr array from a notebook\n", "\n", "A notebook needs a server that outlives the cell that started it, which is the\n", "one thing the `with serve_background(...)` form in the README cannot give\n", "that shuts the server down as soon as the block ends.\n", "\n", "The notebook pattern is instead:\n", "\n", "1. start with `serve_background` and keep the handle,\n", "2. use the server across as many cells as you like,\n", "3. `shutdown()` when you are done.\n", "\n", "Two details make this comfortable in a kernel you re-run:\n", "\n", "- **`serve_background`** runs uvicorn in a daemon thread with its own event\n", " loop, so it never touches the kernel's loop and cannot block it.\n", "- **`port=\"auto\"`**, its default, prefers port 8000 but takes a free one if\n", " it is busy. Re-running a start cell without stopping the previous server is\n", " the classic notebook mistake, and a fixed port fails there with *address\n", " already in use*. `server.url` reports the port actually bound." ] }, { "cell_type": "code", "execution_count": null, "id": "e2e217fa", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import zarr\n", "from zarr.storage import MemoryStore\n", "\n", "from zarr_http_server import node_app, serve_background\n", "\n", "store = MemoryStore()\n", "array = zarr.create_array(\n", " store,\n", " data=np.arange(1000, dtype=\"uint8\").reshape(10, 10, 10),\n", " chunks=(5, 5, 5),\n", " compressors=None,\n", " write_data=True,\n", ")\n", "array.info" ] }, { "cell_type": "markdown", "id": "35752272", "metadata": {}, "source": [ "## Start\n", "\n", "`serve_background` returns a `BackgroundServer` as soon as the socket is\n", "so the next cell can use it immediately." ] }, { "cell_type": "code", "execution_count": null, "id": "b5010cdb", "metadata": {}, "outputs": [], "source": [ "server = serve_background(node_app(array), host=\"127.0.0.1\")\n", "\n", "print(f\"serving at {server.url}\")\n", "assert server.url is not None" ] }, { "cell_type": "markdown", "id": "d4e1f8d6", "metadata": {}, "source": [ "## Use it\n", "\n", "The server is alive across cells now. Anything that speaks HTTP can read from\n", "it -- here `httpx`, but a browser or another zarr client works the same way.\n", "\n", "(With `fsspec[http]` installed you can also do\n", "`zarr.open_array(server.url, mode=\"r\")` to read the array back through zarr\n", "itself.)" ] }, { "cell_type": "code", "execution_count": null, "id": "5ac501fa", "metadata": {}, "outputs": [], "source": [ "import httpx\n", "\n", "metadata = httpx.get(f\"{server.url}/zarr.json\", timeout=30)\n", "print(metadata.status_code, metadata.headers[\"content-type\"])\n", "assert metadata.status_code == 200\n", "assert metadata.json()[\"shape\"] == [10, 10, 10]" ] }, { "cell_type": "code", "execution_count": null, "id": "c7a94eff", "metadata": {}, "outputs": [], "source": [ "chunk = httpx.get(f\"{server.url}/c/0/0/0\", timeout=30)\n", "print(f\"chunk c/0/0/0: {len(chunk.content)} bytes\")\n", "assert chunk.status_code == 200\n", "\n", "# Byte ranges work too, and say which bytes came back.\n", "part = httpx.get(f\"{server.url}/c/0/0/0\", headers={\"Range\": \"bytes=0-9\"}, timeout=30)\n", "print(part.status_code, part.headers[\"content-range\"], part.content)\n", "assert part.status_code == 206\n", "assert part.content == chunk.content[:10]" ] }, { "cell_type": "markdown", "id": "c8fd3ea2", "metadata": {}, "source": [ "## Writes are off unless you ask\n", "\n", "The default is read-only, so a stray `PUT` from a notebook cell -- or from\n", "anyone else who can reach the port -- is refused." ] }, { "cell_type": "code", "execution_count": null, "id": "2f6bee45", "metadata": {}, "outputs": [], "source": [ "refused = httpx.put(f\"{server.url}/c/0/0/0\", content=b\"nope\", timeout=30)\n", "print(\"PUT ->\", refused.status_code)\n", "assert refused.status_code == 405" ] }, { "cell_type": "markdown", "id": "74f4bebc", "metadata": {}, "source": [ "## Stop\n", "\n", "`shutdown()` waits for in-flight requests, then forces the server closed. It\n", "raises if the thread will not stop, so a silent failure cannot leave you\n", "believing the port is free when it is not." ] }, { "cell_type": "code", "execution_count": null, "id": "53a3e8d4", "metadata": {}, "outputs": [], "source": [ "server.shutdown()\n", "\n", "# The port really is closed now.\n", "try:\n", " httpx.get(f\"{server.url}/zarr.json\", timeout=5)\n", "except httpx.HTTPError as exc:\n", " print(f\"as expected, no longer serving: {type(exc).__name__}\")\n", "else:\n", " raise AssertionError(\"server still responding after shutdown\")" ] }, { "cell_type": "markdown", "id": "8a7742e0", "metadata": {}, "source": [ "## If you forget to stop one\n", "\n", "The server thread is a daemon, so it dies with the kernel -- restarting the\n", "kernel always clears it. Because `port=0` picks a fresh port each time, a\n", "forgotten server does not block the next one either; it just holds a port\n", "until the kernel exits.\n", "\n", "If you want the shutdown tied to a block rather than a cell, the context\n", "manager form still works inside a single cell:\n", "\n", "```python\n", "with serve_background(node_app(array)) as server:\n", " ... # everything must happen in this cell\n", "```" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "name": "python" } }, "nbformat": 4, "nbformat_minor": 5 } zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/justfile000066400000000000000000000056101527014137600262540ustar00rootroot00000000000000# Development verbs for the zarr-http-server package. Recipes run with this # directory as the working directory regardless of where `just` is invoked. # # CI calls these recipes rather than repeating their commands, so a green run # in .github/workflows/zarr-http-server.yml means the same thing as a green # `just check` here. # Pinned to the ruff that .pre-commit-config.yaml uses, so this and the # pre-commit gate enforce one standard. An unpinned `uvx ruff` floats to the # newest release: when ruff 0.16 began selecting BLE001 under the root # config's `B` prefix, this job failed on rules the pinned ruff never enforced, # with no code change to blame. Bump alongside the pre-commit rev. # Quoted arguments must survive delegation from the root Justfile. set positional-arguments ruff_version := "0.16.0" # In the repository, recipes overlay the in-repo zarr (editable) so they run # against the code beside them. pyproject.toml stays standalone, so from an # unpacked sdist, where ../.. is not the repository, zarr resolves from PyPI. repo_zarr := if path_exists("../../src/zarr/__init__.py") == "true" { "--with-editable ../.." } else { "" } # List available recipes default: @just --list # The `examples` group carries the deps the README examples need, so the test # that reads a served array back with a zarr client runs here instead of # silently skipping. # Run the test suite; extra args are passed to pytest test *args: uv run {{ repo_zarr }} --group test --group examples pytest tests "$@" # Lint the package sources and tests lint: uvx ruff@{{ ruff_version }} check . # This package type-checks with mypy (see [tool.mypy] in pyproject.toml) # rather than the pyright used by the other packages under packages/. # Type-check the package sources typecheck: uv run {{ repo_zarr }} --group test --with mypy mypy src # Run everything CI runs for this package check: lint typecheck test docs-check # Preview the changelog that the next release would generate changelog-draft: uvx towncrier build --draft --version Unreleased # Build this package's documentation site, warnings as errors docs-check: env DISABLE_MKDOCS_2_WARNING=true uv run {{ repo_zarr }} --group docs mkdocs build --strict # With no argument, uses port 8000 if free, otherwise an ephemeral free port; # an explicitly requested port is used as-is so a conflict fails loudly. # Serve this package's documentation site docs-serve port="": #!/usr/bin/env bash set -euo pipefail port="{{ port }}" if [ -z "$port" ]; then port=$(uv run --group docs python -c ' import socket s = socket.socket() try: s.bind(("127.0.0.1", 8000)) except OSError: s.close() s = socket.socket() s.bind(("127.0.0.1", 0)) print(s.getsockname()[1]) s.close() ') fi exec env DISABLE_MKDOCS_2_WARNING=true uv run {{ repo_zarr }} --group docs mkdocs serve -a "localhost:$port" zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/mkdocs-base.yml000077700000000000000000000000001527014137600325362../mkdocs-base.ymlustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/mkdocs.yml000066400000000000000000000014061527014137600265060ustar00rootroot00000000000000INHERIT: mkdocs-base.yml site_name: zarr-http-server # The package lives in the zarr-python monorepo; point the header source # widget at the package directory rather than the repository root. repo_name: zarr-python/packages/zarr-http-server repo_url: https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-http-server # Absolute because mkdocs would otherwise append this to repo_url's subpath. edit_uri: https://github.com/zarr-developers/zarr-python/edit/main/packages/zarr-http-server/docs/ site_description: HTTP server for Zarr stores, arrays, and groups. site_url: !ENV [READTHEDOCS_CANONICAL_URL, 'https://zarr-http-server.readthedocs.io/'] nav: - index.md - guide.md - API Reference: - api/index.md - Release notes: release-notes.md zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/pyproject.toml000066400000000000000000000144251527014137600274240ustar00rootroot00000000000000[build-system] requires = ["hatchling>=1.29.0", "hatch-vcs", "setuptools-scm>=9"] build-backend = "hatchling.build" [project] name = "zarr-http-server" dynamic = ["version"] description = "HTTP server for Zarr stores, arrays, and groups." readme = "README.md" requires-python = ">=3.12" license = "MIT" license-files = ["LICENSE.txt"] authors = [ { name = "Davis Bennett", email = "davis.v.bennett@gmail.com" }, ] classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Intended Audience :: Information Technology", "Intended Audience :: Science/Research", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Topic :: Scientific/Engineering", "Topic :: Internet :: WWW/HTTP :: HTTP Servers", "Typing :: Typed", ] keywords = ["zarr", "http", "server", "asgi"] dependencies = [ "zarr>=3.1", "starlette>=1.0", "uvicorn>=0.29", ] [project.urls] Homepage = "https://github.com/zarr-developers/zarr-python" Source = "https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-http-server" Issues = "https://github.com/zarr-developers/zarr-python/issues" Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-http-server/CHANGELOG.md" Documentation = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-http-server/README.md" [dependency-groups] test = [ "pytest", "httpx", "httpx2", "hypothesis", # Executing examples/serve_notebook.ipynb in the suite: nbformat reads it, # nbclient runs it, ipykernel is the kernel it runs in. "nbformat", "nbclient", "ipykernel", ] examples = [ # Optional dependencies to run the README examples. Reading a served # array back with `zarr.open_array(url)` goes through FsspecStore, which # needs an HTTP-capable fsspec; `examples/serve.py` uses httpx. "fsspec[http]", "httpx", ] docs = [ # Pins match the zarr-python docs environment in the repo-root # pyproject.toml so every site renders with the same toolchain, and the # shared packages/mkdocs-base.yml sees one set of plugin versions. # Bump them together with the root. "mkdocs-material==9.7.7", "mkdocs==1.6.1", "mkdocstrings==1.0.6", # pymdown-extensions 12.3 changed the Highlight extension API; mkdocstrings 1.0.6 # cannot drive it (Highlight.__init__() missing argument "md"). Match the # version in the repo-root uv.lock until mkdocstrings is bumped everywhere. "pymdown-extensions==11.0.1", "mkdocstrings-python==2.0.8", "griffe-inherited-docstrings==1.1.3", # mkdocstrings uses ruff to format rendered signatures "ruff==0.16.8", ] [tool.hatch.version] source = "vcs" tag-pattern = '^zarr_http_server-v(?P.+)$' # `git_describe_command` ensures we get the zarr_http_server tags instead of latest. # `local_scheme` strips the git commit info so the appending info is just a counter from latest tag. # test-pypi doesn't accept git commit info in tags, and the count should be enough to distinguish unique runs. raw-options = { root = "../..", scm = { git = { describe_command = "git describe --dirty --tags --long --match zarr_http_server-v*" } }, local_scheme = "no-local-version" } [tool.hatch.build.targets.wheel] packages = ["src/zarr_http_server"] # An allowlist, so nothing that merely happens to sit in the package directory # — a scratch script, a stray notebook — can ride along in a release. With no # sdist section at all hatchling defaults to "everything not gitignored", which # is a blocklist by another name. The list keeps an sdist self-testing: # `tests/test_examples.py` runs `examples/serve.py` and executes every cell of # `examples/serve_notebook.ipynb`, so those are part of the suite rather than # decoration, and `/docs` plus `/mkdocs.yml` are a self-contained site. # `changes/` and `.readthedocs.yaml` are deliberately absent: towncrier # fragments are consumed into `CHANGELOG.md` at release time, and the RTD # config only means anything in the repository. `pyproject.toml`, `README.md` # and `LICENSE.txt` are added by hatchling itself. [tool.hatch.build.targets.sdist] include = [ "/src", "/tests", "/docs", "/examples", "/mkdocs.yml", "/justfile", "/CHANGELOG.md", ] # The shared docs config and header script are symlinks into packages/ in the # repository; ship the files they point at, since a link out of the sdist # dangles and pip refuses to unpack it. exclude = ["/mkdocs-base.yml", "/docs/_static/source-version.js"] [tool.hatch.build.targets.sdist.force-include] "../mkdocs-base.yml" = "mkdocs-base.yml" "../source-version.js" = "docs/_static/source-version.js" [tool.ruff] extend = "../../pyproject.toml" target-version = "py312" [tool.pytest.ini_options] minversion = "7" testpaths = ["tests"] xfail_strict = true addopts = ["-ra", "--strict-config", "--strict-markers"] filterwarnings = [ "error", ] [tool.mypy] files = ["src"] python_version = "3.12" ignore_missing_imports = true namespace_packages = false pretty = true show_error_code_links = true show_error_context = true strict = true warn_unreachable = true enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool", "truthy-iterable"] [tool.numpydoc_validation] # Mirrors the root zarr-python config so moved docstrings (written for that # config) don't trip stricter defaults just because this package has its own # pyproject.toml. See https://numpydoc.readthedocs.io/en/latest/validation.html#built-in-validation-checks checks = [ "GL10", "SS04", "PR02", "PR03", "PR05", "PR06", ] [tool.towncrier] # Fragments for this package live alongside the package source, separate # from the parent zarr-python `changes/` directory, so a PR touching only # `packages/zarr-http-server/` produces a release note for this package only. directory = "changes" filename = "CHANGELOG.md" package = "zarr_http_server" underlines = ["", "", ""] title_format = "## {version} ({project_date})" issue_format = "[#{issue}](https://github.com/zarr-developers/zarr-python/issues/{issue})" start_string = "\n" zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/src/000077500000000000000000000000001527014137600252715ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/src/zarr_http_server/000077500000000000000000000000001527014137600306745ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/src/zarr_http_server/__init__.py000066400000000000000000000014051527014137600330050ustar00rootroot00000000000000"""Zarr-http-server: HTTP server for Zarr stores, arrays, and groups.""" from importlib.metadata import version from zarr_http_server._serve import ( AUTO_PORT, DEFAULT_MAX_BODY_SIZE, DEFAULT_PORT, READ_ONLY_HTTP_METHODS, READ_WRITE_HTTP_METHODS, BackgroundServer, CorsOptions, HTTPMethod, ReadOnlyHTTPMethod, node_app, serve, serve_background, store_app, ) __version__ = version("zarr-http-server") __all__ = [ "AUTO_PORT", "DEFAULT_MAX_BODY_SIZE", "DEFAULT_PORT", "READ_ONLY_HTTP_METHODS", "READ_WRITE_HTTP_METHODS", "BackgroundServer", "CorsOptions", "HTTPMethod", "ReadOnlyHTTPMethod", "__version__", "node_app", "serve", "serve_background", "store_app", ] zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/src/zarr_http_server/_keys.py000066400000000000000000000160771527014137600323730ustar00rootroot00000000000000"""Utilities for determining the set of valid store keys for zarr nodes. A zarr node (array or group) implicitly defines a subset of keys in the underlying store. For an **array** the valid keys are: * metadata documents (``zarr.json`` for v3, ``.zarray`` / ``.zattrs`` for v2) * chunk (or shard) keys whose decoded coordinates fall within the storage grid For a **group** the valid keys are: * its own metadata documents * any path ``/`` where ```` is a direct member and ```` is recursively valid for that child """ from __future__ import annotations from typing import TYPE_CHECKING, Any ZARR_JSON = "zarr.json" ZARRAY_JSON = ".zarray" ZGROUP_JSON = ".zgroup" ZATTRS_JSON = ".zattrs" ZMETADATA_V2_JSON = ".zmetadata" if TYPE_CHECKING: from zarr import Array, Group _ARRAY_METADATA_KEYS_V3 = frozenset({ZARR_JSON}) _ARRAY_METADATA_KEYS_V2 = frozenset({ZARRAY_JSON, ZATTRS_JSON}) _GROUP_METADATA_KEYS_V3 = frozenset({ZARR_JSON}) _GROUP_METADATA_KEYS_V2 = frozenset({ZGROUP_JSON, ZATTRS_JSON, ZMETADATA_V2_JSON}) def array_metadata_keys(zarr_format: int) -> frozenset[str]: """Return the metadata key basenames an array owns, for a zarr format. Parameters ---------- zarr_format : int The zarr format version (2 or 3). Returns ------- frozenset of str """ if zarr_format == 3: return _ARRAY_METADATA_KEYS_V3 return _ARRAY_METADATA_KEYS_V2 def group_metadata_keys(zarr_format: int) -> frozenset[str]: """Return the metadata key basenames a group owns, for a zarr format. Parameters ---------- zarr_format : int The zarr format version (2 or 3). Returns ------- frozenset of str """ if zarr_format == 3: return _GROUP_METADATA_KEYS_V3 return _GROUP_METADATA_KEYS_V2 def decode_chunk_key(array: Array[Any], key: str) -> tuple[int, ...] | None: """Try to decode *key* into chunk coordinates for *array*. Parameters ---------- array : Array The array whose chunk key encoding should be used. key : str The candidate chunk key string. Returns ------- tuple of int, or None The decoded coordinates, or ``None`` if *key* is not a valid chunk key. """ try: if array.metadata.zarr_format == 2: coords = tuple(int(p) for p in key.split(array.metadata.dimension_separator)) # A 0-d v2 array holds its single chunk under "0", which decodes # to a 1-tuple that no 0-d grid could match. if len(array.shape) == 0: return () if coords == (0,) else None return coords # Ask zarr rather than predicting it: the encoding owns its own # grammar, so a new or third-party chunk key encoding decodes here # without this package knowing anything about it. return array.metadata.chunk_key_encoding.decode_chunk_key(key) except (ValueError, TypeError, NotImplementedError): return None def _shard_grid_shape(array: Array[Any]) -> tuple[int, ...]: """Shape of the shard grid, falling back to the chunk grid when unsharded.""" shard_shape = array.shards if array.shards is not None else array.chunks return tuple(-(-s // c) for s, c in zip(array.shape, shard_shape, strict=True)) def is_valid_chunk_key(array: Array[Any], key: str) -> bool: """Check whether *key* is a valid chunk key for *array*. Decodes the key, checks that the resulting coordinates fall within the storage grid (shard grid if sharding is used, chunk grid otherwise), and requires the key to be spelled exactly as zarr itself would spell it. That last check is what makes the accepted key set equal to the set of keys zarr can actually read. Decoding alone is lenient -- `int` accepts leading zeros, a leading `+`/`-`, surrounding whitespace, underscore separators, and non-ASCII decimal digits -- so `c/00/00` and `c/0/0` decode to the same coordinates while naming *different* store keys. A write to the non-canonical spelling would be stored under a key no reader ever looks up: the client sees success and the data is invisible. Parameters ---------- array : Array The array to validate against. key : str The candidate chunk key string. Returns ------- bool """ coords = decode_chunk_key(array, key) if coords is None: return False grid = _shard_grid_shape(array) if len(coords) != len(grid): return False if not all(0 <= c < g for c, g in zip(coords, grid, strict=True)): return False return array.metadata.encode_chunk_key(coords) == key def is_valid_array_key(array: Array[Any], key: str) -> bool: """Check whether *key* is a valid store key for *array*. Valid keys are metadata documents and chunk keys. Parameters ---------- array : Array The array to validate against. key : str The candidate key, relative to the array's root. Returns ------- bool """ if key in array_metadata_keys(array.metadata.zarr_format): return True return is_valid_chunk_key(array, key) def is_valid_node_key(node: Array[Any] | Group, key: str) -> bool: """Check whether *key* is a valid store key relative to *node*. For an ``Array``, valid keys are metadata documents and chunk keys. For a ``Group``, valid keys are the group's own metadata documents, or a path of the form ``/`` where ```` is a direct member and ```` is recursively valid for that child. Parameters ---------- node : Array or Group The zarr node to validate against. key : str The candidate key, relative to the node's root. Returns ------- bool """ from zarr import Array if isinstance(node, Array): return is_valid_array_key(node, key) # Group if key in group_metadata_keys(node.metadata.zarr_format): return True # Try to match the first path component against a child member. if "/" in key: child_name, remainder = key.split("/", 1) else: # A bare name with no slash can't be a valid group-level key — # groups contain children (which have subkeys), not bare keys. return False try: child = node[child_name] except KeyError: # There is no such member, so no key beneath it can be valid. # # Only a missing name is caught here. Anything else -- an I/O error # reading the child's metadata, unparsable JSON, a codec from a # plugin this process lacks -- means the key could not be *judged*, # which is not the same as judging it absent. Reporting those as 404 # would be a lie with teeth: under the v3 spec an absent chunk is an # uninitialized one, so a correct reader answers a 404 by silently # substituting the array's fill value over data that exists. Letting # them propagate surfaces a 500, which is the honest answer and the # one a client cannot mistake for data. return False return is_valid_node_key(child, remainder) zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/src/zarr_http_server/_serve.py000066400000000000000000001332511527014137600325360ustar00rootroot00000000000000from __future__ import annotations import asyncio import errno import logging import ntpath import socket import sys import threading import time from enum import Enum, auto from functools import partial from typing import TYPE_CHECKING, Any, Literal, Self, TypedDict, cast, get_args from zarr.abc.store import OffsetByteRequest, RangeByteRequest, SuffixByteRequest from zarr.buffer import cpu from zarr_http_server._keys import array_metadata_keys, group_metadata_keys, is_valid_node_key if TYPE_CHECKING: from collections.abc import Mapping from collections.abc import Set as AbstractSet import uvicorn from starlette.applications import Starlette from starlette.requests import Request from starlette.responses import Response from zarr import Array, Group from zarr.abc.store import ByteRequest, Store __all__ = [ "AUTO_PORT", "DEFAULT_MAX_BODY_SIZE", "DEFAULT_PORT", "READ_ONLY_HTTP_METHODS", "READ_WRITE_HTTP_METHODS", "BackgroundServer", "CorsOptions", "HTTPMethod", "ReadOnlyHTTPMethod", "node_app", "serve", "serve_background", "store_app", ] class CorsOptions(TypedDict, total=False): """Options forwarded to Starlette's `CORSMiddleware`. Every parameter the middleware accepts appears here, so configuring CORS never requires reaching around this package. Keys left out fall back to the defaults described below; a key that is present is used verbatim, including an empty list. Two defaults differ from Starlette's own, because this server knows something its caller should not have to. It emits `Content-Range` on every ranged response, which is *not* a CORS-safelisted response header -- with Starlette's empty `expose_headers` a browser client can read the bytes but not learn which bytes it got. And it accepts a `Range` request header, which Starlette's empty `allow_headers` would reject at preflight. * `expose_headers` defaults to `["Content-Range"]` * `allow_headers` defaults to `["Range"]` Everything else defaults to the middleware's own value: no origins, `GET` only, no credentials, no origin regex, no private-network access, and a 600-second preflight cache. """ allow_origins: list[str] allow_methods: list[str] allow_headers: list[str] allow_credentials: bool allow_origin_regex: str | None allow_private_network: bool expose_headers: list[str] max_age: int _CORS_DEFAULTS: CorsOptions = { "expose_headers": ["Content-Range"], "allow_headers": ["Range"], } ReadOnlyHTTPMethod = Literal["GET", "HEAD"] """An HTTP method that cannot modify the store. Distinguished from `HTTPMethod` in the type domain, not only at runtime, so a read-only interface can be *declared* rather than merely configured: a parameter annotated `AbstractSet[ReadOnlyHTTPMethod]` cannot be handed `"PUT"` without a type error, whatever the value turns out to be at runtime. `HEAD` belongs here because Starlette routes it wherever `GET` goes, which is what RFC 9110 §9.3.2 asks of an origin server. It is answered from the value's size rather than by building and discarding a body. """ _WriteHTTPMethod = Literal["PUT"] """An HTTP method that modifies the store. Private because nothing needs to name "the write methods" on its own -- it exists so `HTTPMethod` can be defined as the union rather than as a third hand-written list of the same strings. """ HTTPMethod = ReadOnlyHTTPMethod | _WriteHTTPMethod """An HTTP method this server implements. `GET` and `HEAD` read a key; `PUT` writes one. Other verbs are not accepted: the handler has no behavior for them, so serving them would silently answer as if they were `GET`. """ # Derived from the types above rather than restated, so the runtime sets and # the static types cannot disagree about what this server serves. Adding a # method to a Literal is then the only edit needed. READ_ONLY_HTTP_METHODS: frozenset[ReadOnlyHTTPMethod] = frozenset(get_args(ReadOnlyHTTPMethod)) """Methods that only read. The default for every app in this package. Naming the set makes a read-only deployment say so at the call site, rather than being the absence of an argument: store_app(store, methods=READ_ONLY_HTTP_METHODS) Typed as a set of `ReadOnlyHTTPMethod`, so a caller building on it keeps the static guarantee: adding `"PUT"` to a `frozenset[ReadOnlyHTTPMethod]` is a type error, not a runtime surprise. The stronger guarantee is a read-only store, which holds however `methods` is configured -- see `store.with_read_only(True)`. """ READ_WRITE_HTTP_METHODS: frozenset[HTTPMethod] = frozenset( get_args(ReadOnlyHTTPMethod) + get_args(_WriteHTTPMethod) ) """Methods that read and write. Serving these grants clients write access. Every writable app must name a method set, so this constant is also what makes writable deployments findable: grepping for `READ_WRITE_HTTP_METHODS` (or for `methods=` generally) turns up every place that opts in. """ _SUPPORTED_METHODS: frozenset[str] = READ_WRITE_HTTP_METHODS _LOGGER = logging.getLogger("uvicorn.error") """uvicorn's own logger, so a port fallback appears alongside its startup lines.""" DEFAULT_PORT = 8000 """Port tried first when `port="auto"`.""" AUTO_PORT: Literal["auto"] = "auto" """Sentinel for `port`: prefer `DEFAULT_PORT`, but settle for any free port. An explicit port is a requirement -- it binds that port or fails -- because a caller who names one usually has something else expecting the server there. `"auto"` says the opposite: no particular port is needed, so a collision should not stop the server from starting. `port=0` keeps its usual meaning of "any free port", with no preference. """ _STARTUP_TIMEOUT = 5.0 """Seconds to wait for a background server to report that it is listening.""" _STARTUP_ABANDON_TIMEOUT = 5.0 """Seconds to wait for a server that failed to start to stop again.""" _SHUTDOWN_JOIN_MARGIN = 1.0 """Seconds to wait beyond uvicorn's graceful bound before forcing shutdown. uvicorn spends a fixed ~0.2s tearing down (a 0.1s loop tick plus a 0.1s sleep) before its own `timeout_graceful_shutdown` wait begins, so a join that merely equals that bound is guaranteed to expire first and escalate to `force_exit` -- which makes uvicorn skip ASGI lifespan shutdown. """ DEFAULT_MAX_BODY_SIZE = 256 * 1024 * 1024 """Default cap on a `PUT` body, in bytes. `Store.set` takes a whole `Buffer`, so an accepted body is held in memory in full; the body is read incrementally and abandoned once it passes this cap, so one request cannot size the server's memory use. Pass `max_body_size=None` to lift the cap and read the body whole. """ class BackgroundServer: """A running background HTTP server that can be used as a context manager. Wraps a ``uvicorn.Server`` running in a daemon thread. When used as a context manager the server is shut down automatically on exit. Parameters ---------- server : uvicorn.Server The running uvicorn server instance. thread : threading.Thread The daemon thread running the server. host : str or None The host the server was asked to bind, or ``None`` when it is not listening on a TCP socket. port : int or None The port actually bound, or ``None`` when the server is not listening on a TCP socket. scheme : str, optional URL scheme the server is reachable over. Defaults to ``"http"``. shutdown_timeout : int, optional Seconds to wait for in-flight requests to finish gracefully during :meth:`shutdown` before forcing the server closed. Defaults to ``5``. Examples -------- >>> with serve_background(node_app(arr)) as server: # doctest: +SKIP ... print(f"Listening on {server.host}:{server.port}") ... # server is shut down when the block exits """ def __init__( self, server: uvicorn.Server, thread: threading.Thread, *, host: str | None, port: int | None, scheme: str = "http", shutdown_timeout: int = 5, ) -> None: self._server = server self._thread = thread self.host = host self.port = port self.scheme = scheme self._shutdown_timeout = shutdown_timeout @property def url(self) -> str | None: """The base URL of the running server. ``None`` when the server is not listening on a TCP socket -- a unix socket or an inherited file descriptor has no host and port, and inventing one would be a URL that connects to nothing. """ if self.host is None or self.port is None: return None return f"{self.scheme}://{self.host}:{self.port}" def shutdown(self) -> None: """Signal the server to shut down and wait for it to stop. Waits for the server thread to exit on its own, then escalates to ``force_exit`` if it has not, so a request wedged outside uvicorn's loop cannot block here forever. Raises ------ RuntimeError If the thread is still running after both waits. Returning normally would report success for a server that is still bound to its port and still serving, which the caller cannot detect any other way. """ self._server.should_exit = True # Outlast uvicorn's own graceful wait rather than matching it. uvicorn # spends roughly 0.2s on teardown (a 0.1s loop tick plus a 0.1s sleep) # *before* its `timeout_graceful_shutdown` wait even begins, so an # equal bound here always expires first -- escalating to force_exit on # the path that is supposed to be the orderly one, which makes uvicorn # skip ASGI lifespan shutdown entirely. self._thread.join(timeout=self._graceful_timeout + _SHUTDOWN_JOIN_MARGIN) if self._thread.is_alive(): self._server.force_exit = True self._thread.join(timeout=self._shutdown_timeout) if self._thread.is_alive(): raise RuntimeError( "Server thread did not stop within " f"{self._graceful_timeout + _SHUTDOWN_JOIN_MARGIN + self._shutdown_timeout:.1f}s, " "even after force_exit. The server may still be serving and " "holding its port; a request blocked in a store call cannot be " "cancelled from here." ) @property def _graceful_timeout(self) -> float: """uvicorn's own graceful-shutdown bound, whoever configured it.""" configured = self._server.config.timeout_graceful_shutdown return float(configured) if configured is not None else float(self._shutdown_timeout) def __enter__(self) -> Self: return self def __exit__(self, *args: object) -> None: self.shutdown() class _RangeVerdict(Enum): """The outcome of a Range header that does not name a readable range.""" IGNORE = auto() """Serve the full representation with 200, as if no Range had been sent.""" UNSATISFIABLE = auto() """Answer 416: the range is well-formed but names nothing readable.""" _MAX_BYTE_POS = sys.maxsize """Largest byte position this server will pass to a store. Range bounds arrive as arbitrary-precision Python integers, but a store ultimately turns them into an index-sized `seek`/`read`. Feeding an oversized value through raises `OverflowError`/`ValueError` from deep inside the store rather than producing a response, so bounds are clamped or rejected here. """ def _parse_int(text: str) -> int | None: """Parse a byte position, accepting only the canonical spelling of one. `int` is lenient in ways an HTTP byte position is not -- it accepts surrounding whitespace, a leading `+`/`-`, underscore separators, and non-ASCII decimal digits -- so `bytes=+0-1` and `bytes=0_0-1` would parse. RFC 9110 defines a byte position as 1*DIGIT. """ if not text.isascii() or not text.isdigit(): return None return int(text) def _parse_range_header(range_header: str) -> ByteRequest | _RangeVerdict: """Parse an HTTP Range header into a ByteRequest. A header this server cannot turn into a single read is *ignored* rather than rejected. RFC 9110 §14.2 requires a server to ignore a Range whose unit it does not recognize, and permits ignoring one it cannot parse; in both cases the correct answer is the full representation, not 416. Answering 416 would tell a client the object is unreadable when it is merely the request that was unsupported -- and a client coalescing two chunk reads into one multi-range request would take that at face value. 416 is reserved for a well-formed range that genuinely names nothing. Parameters ---------- range_header : str The value of the Range header, e.g. ``"bytes=0-99"`` or ``"bytes=-100"``. Returns ------- ByteRequest or _RangeVerdict A ``RangeByteRequest``, ``OffsetByteRequest``, or ``SuffixByteRequest`` for a readable range, otherwise the verdict to apply. """ if not range_header.startswith("bytes="): # An unrecognized range unit; RFC 9110 §14.2 says MUST ignore. return _RangeVerdict.IGNORE range_spec = range_header[len("bytes=") :] if "," in range_spec: # A multipart range. Legal to send, and legal to answer with the whole # representation; this server does not build multipart/byteranges. return _RangeVerdict.IGNORE if range_spec.startswith("-"): # suffix request: bytes=-N suffix = _parse_int(range_spec[1:]) if suffix is None: return _RangeVerdict.IGNORE if suffix == 0: # "the last zero bytes" names nothing. return _RangeVerdict.UNSATISFIABLE return SuffixByteRequest(suffix=min(suffix, _MAX_BYTE_POS)) parts = range_spec.split("-", 1) if len(parts) != 2: return _RangeVerdict.IGNORE start_str, end_str = parts start = _parse_int(start_str) if start is None: return _RangeVerdict.IGNORE if start > _MAX_BYTE_POS: # No object can be this long, so the range starts past every end. return _RangeVerdict.UNSATISFIABLE if end_str == "": # offset request: bytes=N- return OffsetByteRequest(offset=start) end_pos = _parse_int(end_str) if end_pos is None: return _RangeVerdict.IGNORE # HTTP end is inclusive, ByteRequest end is exclusive. A last-byte-pos at # or past the end of the object is satisfiable -- RFC 9110 §14.1.2 says to # clamp it -- so an oversized bound is capped rather than refused. end = min(end_pos, _MAX_BYTE_POS - 1) + 1 if start >= end: # An inverted range like "bytes=5-2" is unsatisfiable, not a read # of negative length. return _RangeVerdict.UNSATISFIABLE return RangeByteRequest(start=start, end=end) def _is_drive_qualified(path: str) -> bool: """Whether *path* carries a drive or UNC prefix that would discard a store root. A drive-qualified key such as `C:/Windows`, or the drive-relative `a:b`, replaces the root it is joined to rather than extending it. This is rejected on every platform, not just Windows: the check is a string-level gate in front of an arbitrary `Store`, and this package cannot know how a given implementation resolves keys. The cost is that a node whose *first* path segment looks like `:...` is unreachable -- `ntpath` treats any single character before a colon as a drive, so there is no safe subset to admit. Such names are legal but rare, and later segments are unaffected (`sub/a:b` is served normally). Parameters ---------- path : str The candidate key, with separators already folded to `/`. Returns ------- bool """ return ntpath.splitdrive(path)[0] != "" def _names_nothing(exc: OSError) -> bool: """Whether an `OSError` answers about the *name*, rather than reporting failure. `ENAMETOOLONG` is the store saying no such name is expressible here. That is an answer about the key -- nothing can be stored under it, so a miss is honest -- and it is unreachable for real data, because `encode_chunk_key` never produces a segment near a filesystem's length limit. Answering 404 therefore cannot make a reader substitute fill values over a chunk that exists, and it keeps a client from turning a freely chosen key into a 5xx. Every other `errno` describes a failure to complete the operation and must surface as one. `EINVAL` was accepted here and was the dangerous case: it is POSIX's catch-all, reachable on a perfectly ordinary short key through a bad seek or an unsupported filesystem feature. Under the v3 spec an absent chunk is an uninitialized one, so reporting such a failure as 404 has a correct reader write fill values over data that is merely unreadable. When in doubt the store's own signal is the one to trust: `None` means absent, a raised error means the request could not be answered. Parameters ---------- exc : OSError The error raised by the store. Returns ------- bool """ return exc.errno == errno.ENAMETOOLONG def _content_range(byte_range: RangeByteRequest | OffsetByteRequest, length: int) -> str: """Build a `Content-Range` value for a 206 response. Parameters ---------- byte_range : RangeByteRequest or OffsetByteRequest The range that was served. A suffix request is resolved to an absolute range before reaching here, because RFC 9110 §15.3.7 requires every single-part 206 to carry a `Content-Range` and a suffix's first-byte position is not knowable without the object's size. length : int The number of bytes actually returned. Returns ------- str A `bytes -/*` value. """ start = byte_range.start if isinstance(byte_range, RangeByteRequest) else byte_range.offset # The total length is unknown here; RFC 9110 permits "*" in its place. return f"bytes {start}-{start + length - 1}/*" async def _resolve_suffix( store: Store, path: str, byte_range: SuffixByteRequest ) -> RangeByteRequest | _RangeVerdict: """Turn a suffix request into an absolute range using the object's size. A suffix range names its bytes relative to an end this server does not otherwise need to know. Resolving it here is what lets the 206 carry a `Content-Range`, which RFC 9110 requires and which a caller reading a shard index needs in order to locate what it was given. """ try: size = await store.getsize(path) except FileNotFoundError: return _RangeVerdict.UNSATISFIABLE if size == 0: return _RangeVerdict.UNSATISFIABLE # A suffix longer than the object is satisfiable and yields the whole # object, per RFC 9110 §14.1.2. return RangeByteRequest(start=max(0, size - byte_range.suffix), end=size) _JSON_BASENAMES = ( array_metadata_keys(2) | array_metadata_keys(3) | group_metadata_keys(2) | group_metadata_keys(3) ) """Metadata documents that are JSON, for every zarr format. Derived from the same tables that decide which keys a node owns, so a v2 array's `.zarray` is typed as JSON rather than as opaque bytes -- and adding a document in one place cannot leave the media type behind in the other. """ def content_type_for(path: str) -> str: """Media type for a store key, chosen by its basename.""" if path.rsplit("/", 1)[-1] in _JSON_BASENAMES: return "application/json" return "application/octet-stream" async def _head_response(store: Store, path: str, content_type: str) -> Response: """Answer a HEAD without transferring the value. A HEAD body is discarded at the wire, so routing HEAD through the GET handler reads the whole object -- megabytes of chunk or shard -- to report a length. `Store.getsize` is a `stat` on a filesystem store and an info/HEAD call on a remote one. """ from starlette.responses import Response try: size = await store.getsize(path) except FileNotFoundError: return Response(status_code=404) except OSError as exc: if not _names_nothing(exc): raise return Response(status_code=404) return Response(status_code=200, media_type=content_type, headers={"Content-Length": str(size)}) async def _get_response( store: Store, path: str, byte_range: RangeByteRequest | OffsetByteRequest | None = None, ) -> Response: """Fetch a key from the store and return an HTTP response.""" from starlette.responses import Response proto = cpu.buffer_prototype content_type = content_type_for(path) try: buf = await store.get(path, proto, byte_range=byte_range) except (MemoryError, OverflowError): # The client's last-byte-pos is wider than the store can materialize # -- it sizes its read from the range, not from the object. RFC 9110 # §14.1.2 makes a last-byte-pos at or past the end of the object # satisfiable and clamps it to the end, so re-read from the same start # to EOF, which is what the clamped range denotes. Refusing with 416 # would deny a request that is merely over-wide, and a client asking # for "from here to well past the end" is asking a normal question. if not isinstance(byte_range, RangeByteRequest): raise byte_range = OffsetByteRequest(offset=byte_range.start) buf = await store.get(path, proto, byte_range=byte_range) except OSError as exc: if not _names_nothing(exc): # A real I/O failure, which must not be reported as a miss. Under # the v3 spec an absent chunk is an uninitialized one, and a # reader is right to substitute the array's fill value for it -- # so 404 asserts something about the store's contents. An # unreadable chunk is not an uninitialized chunk, and answering # 404 would have a correct client silently materialize fill # values over data that exists. raise return Response(status_code=404) if buf is None: return Response(status_code=404) if byte_range is None: return Response(content=buf.to_bytes(), status_code=200, media_type=content_type) body = buf.to_bytes() if len(body) == 0: # The range lies wholly beyond the end of the object. return Response(status_code=416) headers = {"Content-Range": _content_range(byte_range, len(body))} return Response(content=body, status_code=206, media_type=content_type, headers=headers) async def _handle_request(request: Request) -> Response: """Handle a request, optionally filtering by node validity.""" from starlette.responses import Response store: Store = request.app.state.store node: Array[Any] | Group | None = request.app.state.node prefix: str = request.app.state.prefix path = request.path_params.get("path", "") # Reject non-canonical / traversal-prone keys before touching the store. # Starlette percent-decodes path params, so "..%2f" arrives as a literal # ".." segment and "%2f"-encoded leading slashes arrive as an empty leading # segment (making the key absolute, which escapes a filesystem store root). # Backslashes are separators on Windows and drive-qualified or # root-relative keys discard a filesystem store's root entirely, so fold # separators before checking segments -- mirroring zarr's normalize_path # (src/zarr/storage/_utils.py) plus a drive check it doesn't need. Legitimate # zarr keys never contain empty, ".", or ".." segments, or a drive letter. segments = path.replace("\\", "/").split("/") if any(segment in ("", ".", "..") for segment in segments) or _is_drive_qualified(path): return Response(status_code=404) # A NUL can never appear in a store key, and reaches the filesystem layer # as a raised error rather than a miss. Rejecting it here keeps that out # of the store, so the store's own errors always mean real I/O trouble. if "\x00" in path: return Response(status_code=404) # If serving a node, validate the key before touching the store. Group # validation opens children through zarr's synchronous API, which drives # the store to completion and would otherwise block the event loop for # the duration -- serializing every concurrent request behind it, and # outlasting the shutdown timeout. if node is not None and not await asyncio.to_thread(is_valid_node_key, node, path): return Response(status_code=404) # Resolve the full store key by prepending the node's prefix. store_key = f"{prefix}/{path}" if prefix else path if request.method == "PUT": if store.read_only: # The store will refuse this with a ValueError from deep inside # `set`, which is a 500 -- a server fault. Refusing to write to a # read-only store is not a fault, it is the answer. return Response(status_code=403) max_body_size: int | None = request.app.state.max_body_size if max_body_size is None: body = await request.body() else: declared = request.headers.get("content-length") if declared is not None and declared.isdigit() and int(declared) > max_body_size: return Response(status_code=413) # Read incrementally and stop at the cap. `request.body()` would # buffer the whole body first, which a chunked request can use to # exceed the cap by any amount before it is ever checked. chunks: list[bytes] = [] received = 0 async for chunk in request.stream(): received += len(chunk) if received > max_body_size: return Response(status_code=413) chunks.append(chunk) body = b"".join(chunks) buf = cpu.buffer_prototype.buffer.from_bytes(body) try: await store.set(store_key, buf) except OSError as exc: if not _names_nothing(exc): # A real write failure -- a full disk, a read-only mount, a # permissions problem. Reporting it as 404 would tell the # client the write is pointless rather than failed. raise return Response(status_code=404) return Response(status_code=204) if request.method == "HEAD": # A HEAD body is discarded at the wire, so reading the value to build # one transfers the whole object to answer a question about its size. # `getsize` is a stat on a filesystem store and a HEAD/info call on a # remote one. return await _head_response(store, store_key, content_type_for(path)) range_header = request.headers.get("range") byte_range: RangeByteRequest | OffsetByteRequest | None = None if range_header is not None: parsed = _parse_range_header(range_header) if parsed is _RangeVerdict.UNSATISFIABLE: return Response(status_code=416) if isinstance(parsed, SuffixByteRequest): parsed = await _resolve_suffix(store, store_key, parsed) if parsed is _RangeVerdict.UNSATISFIABLE: return Response(status_code=416) # _RangeVerdict.IGNORE falls through with byte_range still None, which # serves the full representation with 200. if not isinstance(parsed, _RangeVerdict): byte_range = parsed return await _get_response(store, store_key, byte_range) def _make_starlette_app( *, methods: AbstractSet[HTTPMethod] | None = None, cors_options: CorsOptions | None = None, ) -> Starlette: """Create a Starlette app with the request handler. Raises ------ ValueError If `methods` contains anything outside `GET`, `PUT`, and `HEAD`. """ from starlette.applications import Starlette from starlette.middleware.cors import CORSMiddleware from starlette.routing import Route if methods is None: methods = READ_ONLY_HTTP_METHODS # An empty set must not reach Starlette: `Route` treats a falsy `methods` # as "match every method", so asking for no methods would serve them all. if not methods: raise ValueError( "methods must name at least one HTTP method; " f"accepted methods are {', '.join(sorted(_SUPPORTED_METHODS))}." ) unsupported = sorted(set(methods) - _SUPPORTED_METHODS) if unsupported: raise ValueError( f"Unsupported HTTP method(s): {', '.join(unsupported)}. " f"Accepted methods are {', '.join(sorted(_SUPPORTED_METHODS))}." ) app = Starlette( routes=[Route("/{path:path}", _handle_request, methods=list(methods))], ) if cors_options is not None: # Typed loosely on purpose: unpacking a merged TypedDict loses the # per-key types, so the looseness is contained to this block rather # than spread across casts at each use. merged: dict[str, Any] = {**_CORS_DEFAULTS, **cors_options} if "allow_methods" in merged: # Only when the caller said something. Absent, Starlette's own # `GET`-only default stands: widening it to everything served # would newly advertise `PUT` cross-origin on a write-enabled app # that never asked for it. merged["allow_methods"] = _reconcile_allow_methods( merged["allow_methods"], served=_served_methods(methods) ) app.add_middleware( CORSMiddleware, # Our defaults first, so a key the caller supplied wins outright # rather than being merged into -- an explicit `expose_headers: []` # means "expose nothing", not "expose our default". **merged, ) return app def _reject_writes_to_a_read_only_store( store: Store, methods: AbstractSet[HTTPMethod] | None ) -> None: """Refuse a configuration whose writes can never succeed. A store's `read_only` is fixed when it is built, so asking to serve `PUT` from one is a contradiction that would only reveal itself as a 403 on the first write a client attempts -- possibly long after deployment, and to the client rather than to whoever misconfigured it. Saying so at construction matches how unsupported `methods` and contradictory `cors_options` are already handled. Raises ------ ValueError If `methods` asks for `PUT` on a read-only store. """ if methods is not None and "PUT" in methods and store.read_only: raise ValueError( "methods asks for PUT, but the store is read-only, so no write " "could ever succeed. Drop PUT to serve reads, or pass a writable " "store (`store.with_read_only(False)`)." ) def _served_methods(methods: AbstractSet[HTTPMethod]) -> frozenset[str]: """The methods the route will actually answer. Starlette adds `HEAD` to any route that serves `GET`, which RFC 9110 §9.3.2 asks of every origin server, so `HEAD` is served whenever `GET` is whether or not it was named. """ served = set(methods) if "GET" in served: served.add("HEAD") return frozenset(served) def _reconcile_allow_methods(allow_methods: list[str], *, served: frozenset[str]) -> list[str]: """Check `cors_options["allow_methods"]` against what the route serves. An advertised method the route rejects is a promise the server cannot keep: a browser caches the preflight and every later cross-origin call fails with 405 after a successful handshake. The reverse is worse -- the same silence lets `allow_methods=["*"]` on a write-enabled app hand every origin on the internet write access, which is exactly the footgun the `methods` validation above exists to prevent. `"*"` expands to what is actually served rather than being rejected: it is the idiomatic spelling of "everything this app does", and the app cannot do more than it serves. """ if "*" in allow_methods: return sorted(served) unserved = sorted(set(allow_methods) - served) if unserved: raise ValueError( f"cors_options['allow_methods'] advertises {', '.join(unserved)}, " f"which this app does not serve (it serves {', '.join(sorted(served))}). " "A browser would cache that preflight and every such request would " "then fail with 405." ) return list(allow_methods) def _build_server( app: Starlette, *, host: str, port: int | Literal["auto"], shutdown_timeout: int, uvicorn_options: Mapping[str, object] | None, ) -> tuple[uvicorn.Server, dict[str, object], socket.socket | None]: """Configure a `uvicorn.Server` for *app*, and report how it will bind. Returns the options actually used -- a key from `uvicorn_options` may have replaced one passed here -- and, for `port="auto"`, the socket already bound on the caller's behalf, which must be handed to `Server.run`. """ import uvicorn options: dict[str, object] = { "host": host, "port": port, "timeout_graceful_shutdown": shutdown_timeout, } if uvicorn_options is not None: options.update(uvicorn_options) sock: socket.socket | None = None if options.get("port") == AUTO_PORT: if _binds_without_a_port(options): # A uds or fd bind ignores host and port; leave a valid int in # place of the sentinel so `Config` still type-checks. options["port"] = DEFAULT_PORT else: # Bind here rather than probing and handing uvicorn a port number: # probing would release the port before uvicorn claimed it, which # is the bind-then-close race that makes "find a free port" helpers # flaky. Holding the socket means nothing can take it in between. sock = _bind_preferred_or_free(str(options["host"]), DEFAULT_PORT) # Keep Config agreeing with reality, so uvicorn's own "running on # ..." line names the port it is really serving. options["port"] = sock.getsockname()[1] # uvicorn.Config's parameters are individually typed and there are ~50 of # them; `Mapping[str, object]` is the honest type for the public argument, # so the cast is confined to the call itself. return uvicorn.Server(uvicorn.Config(app, **cast("dict[str, Any]", options))), options, sock def _binds_without_a_port(options: Mapping[str, object]) -> bool: """Whether these options bind something other than a TCP host and port.""" return options.get("uds") is not None or options.get("fd") is not None def _bind_preferred_or_free(host: str, preferred: int) -> socket.socket: """Bind *preferred* on *host* if it is free, otherwise any free port. The address family comes from `getaddrinfo` rather than being assumed: hard-coding `AF_INET` would fail for an IPv6 host such as ``"::1"``. """ for candidate in (preferred, 0): family, socktype, proto, _, sockaddr = socket.getaddrinfo( host, candidate, type=socket.SOCK_STREAM )[0] sock = socket.socket(family, socktype, proto) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: sock.bind(sockaddr) except OSError: sock.close() if candidate == 0: # Nothing is free, which is a real failure rather than a # reason to keep looking. raise _LOGGER.info( "port %d is in use; binding a free port instead. Pass an " "explicit port to require a particular one.", preferred, ) continue return sock raise AssertionError("unreachable") # pragma: no cover def serve( app: Starlette, *, host: str = "127.0.0.1", port: int | Literal["auto"] = AUTO_PORT, shutdown_timeout: int = 5, uvicorn_options: Mapping[str, object] | None = None, ) -> None: """Run an ASGI app under Uvicorn, blocking until it is stopped. Returns only when the server stops, so this is the shape for a process whose job is to serve -- a script, a container entrypoint. Use :func:`serve_background` when the caller has more to do. Build the app first with :func:`store_app` or :func:`node_app`, or compose several of them; what an app serves is settled when it is built, and this only decides how it runs. .. code-block:: python serve(store_app(store), host="0.0.0.0", port=8000) Parameters ---------- app : Starlette The ASGI app to run. host : str, optional The host to bind to. Defaults to ``"127.0.0.1"``. port : int, optional The port to bind to. Defaults to ``8000``. shutdown_timeout : int, optional Seconds to wait for in-flight requests to finish gracefully when the server is shut down, via uvicorn's ``timeout_graceful_shutdown``. Defaults to ``5``. uvicorn_options : Mapping[str, object], optional Extra options passed straight to `uvicorn.Config`, merged over the ones set here (`host`, `port`, `timeout_graceful_shutdown`), so a caller key wins. This is the escape hatch for anything uvicorn can do that this signature does not name -- TLS via `ssl_keyfile` / `ssl_certfile`, `proxy_headers` and `forwarded_allow_ips` behind a reverse proxy, `root_path` when mounted under a prefix, `log_level`, `limit_concurrency`, or a `uds` / `fd` bind. """ server, _, sock = _build_server( app, host=host, port=port, shutdown_timeout=shutdown_timeout, uvicorn_options=uvicorn_options, ) server.run(sockets=[sock] if sock is not None else None) def serve_background( app: Starlette, *, host: str = "127.0.0.1", port: int | Literal["auto"] = AUTO_PORT, shutdown_timeout: int = 5, uvicorn_options: Mapping[str, object] | None = None, ) -> BackgroundServer: """Start an ASGI app under Uvicorn in a daemon thread and return at once. Returns once the socket is listening, so the next statement can use the server. The returned handle is also a context manager: .. code-block:: python with serve_background(node_app(array)) as server: httpx.get(f"{server.url}/zarr.json") In a notebook, where the server must outlive the cell that started it, keep the handle instead and call :meth:`BackgroundServer.shutdown` later. Parameters ---------- app : Starlette The ASGI app to run. host : str, optional The host to bind to. Defaults to ``"127.0.0.1"``. port : int, optional The port to bind to. Defaults to ``0``, which asks the OS for a free one -- unlike :func:`serve`, whose caller usually needs a port others already know. A background server is normally reached through :attr:`BackgroundServer.url`, and a fixed default would make starting a second one, or re-running a notebook cell, fail on a port collision. shutdown_timeout : int, optional Seconds to wait for in-flight requests to finish gracefully, both for uvicorn's ``timeout_graceful_shutdown`` and for the wait :meth:`BackgroundServer.shutdown` uses before forcing the thread closed. Defaults to ``5``. uvicorn_options : Mapping[str, object], optional Extra options passed straight to `uvicorn.Config`, merged over the ones set here so a caller key wins. See :func:`serve`. Binding somewhere other than a TCP host and port leaves :attr:`BackgroundServer.url` as ``None``. Returns ------- BackgroundServer A handle for the running server. Raises ------ RuntimeError If the server does not start -- most often because the port is already in use. """ server, options, sock = _build_server( app, host=host, port=port, shutdown_timeout=shutdown_timeout, uvicorn_options=uvicorn_options, ) # uvicorn skips signal-handler installation off the main thread # (Server.capture_signals), so no workaround is needed here. thread = threading.Thread( target=partial(server.run, sockets=[sock] if sock is not None else None), daemon=True ) thread.start() deadline = time.monotonic() + _STARTUP_TIMEOUT while not server.started: if not thread.is_alive(): # uvicorn logs the underlying error and calls sys.exit, which in a # thread ends it without surfacing anything to the caller. The # overwhelmingly common cause is a port already in use. raise RuntimeError( f"Server thread exited before startup completed; {host}:{port} " "may already be in use. See the server log for the cause." ) if time.monotonic() > deadline: # The thread is still alive here, unlike the branch above, and it # is about to finish starting. Raising without stopping it would # leave a server bound to the port with no handle to shut it down # -- a daemon thread serving for the rest of the process, and a # retry on the same port failing with the other error above. server.should_exit = True server.force_exit = True thread.join(timeout=_STARTUP_ABANDON_TIMEOUT) raise RuntimeError( f"Server failed to start within {_STARTUP_TIMEOUT:g} seconds; " "it has been signalled to stop." ) time.sleep(0.01) # Report the port the socket actually bound rather than the one asked for, # so `port=0` ("pick a free port") yields a usable `url`. A unix-socket or # file-descriptor bind has no host and port at all, so both stay None and # `url` reports None rather than naming an address nothing is listening on. bound_port: int | None = None for bound in server.servers: for sock in bound.sockets: sockname = sock.getsockname() if isinstance(sockname, tuple) and len(sockname) >= 2: bound_port = int(sockname[1]) break break # The host is taken from the request rather than the socket: a wildcard # bind reports "0.0.0.0", which is not an address a client can connect to. requested_host = options.get("host") bound_host = ( str(requested_host) if bound_port is not None and requested_host is not None else None ) return BackgroundServer( server, thread, host=bound_host, port=bound_port, scheme="https" if server.config.is_ssl else "http", shutdown_timeout=shutdown_timeout, ) def store_app( store: Store, *, methods: AbstractSet[HTTPMethod] | None = None, cors_options: CorsOptions | None = None, max_body_size: int | None = DEFAULT_MAX_BODY_SIZE, ) -> Starlette: """Create a Starlette ASGI app that serves every key in a zarr ``Store``. Parameters ---------- store : Store The zarr store to serve. methods : set of HTTPMethod, optional The HTTP methods to accept: any of `"GET"`, `"HEAD"`, and `"PUT"`. Defaults to `{"GET"}`, which also serves `HEAD`. Passing any other method raises `ValueError`. cors_options : CorsOptions, optional If provided, CORS middleware will be added with the given options. max_body_size : int or None, optional Largest `PUT` body to accept, in bytes; larger requests get a 413. `Store.set` takes a whole `Buffer`, so bodies cannot be streamed and are held in memory in full. Defaults to `DEFAULT_MAX_BODY_SIZE`; pass `None` to lift the cap. Returns ------- Starlette An ASGI application. """ _reject_writes_to_a_read_only_store(store, methods) app = _make_starlette_app(methods=methods, cors_options=cors_options) app.state.store = store app.state.node = None app.state.prefix = "" app.state.max_body_size = max_body_size return app def node_app( node: Array[Any] | Group, *, methods: AbstractSet[HTTPMethod] | None = None, cors_options: CorsOptions | None = None, max_body_size: int | None = DEFAULT_MAX_BODY_SIZE, ) -> Starlette: """Create a Starlette ASGI app that serves only the keys belonging to a zarr ``Array`` or ``Group``. For an ``Array``, the served keys are the metadata document(s) and all chunk (or shard) keys whose coordinates fall within the array's grid. For a ``Group``, the served keys are the group's own metadata plus any path that resolves through the group's members to a valid array metadata document or chunk key. Requests for keys outside this set receive a 404 response, even if the underlying store contains data at that path. Parameters ---------- node : Array or Group The zarr array or group to serve. methods : set of HTTPMethod, optional The HTTP methods to accept: any of `"GET"`, `"HEAD"`, and `"PUT"`. Defaults to `{"GET"}`, which also serves `HEAD`. Passing any other method raises `ValueError`. cors_options : CorsOptions, optional If provided, CORS middleware will be added with the given options. max_body_size : int or None, optional Largest `PUT` body to accept, in bytes; larger requests get a 413. `Store.set` takes a whole `Buffer`, so bodies cannot be streamed and are held in memory in full. Defaults to `DEFAULT_MAX_BODY_SIZE`; pass `None` to lift the cap. Returns ------- Starlette An ASGI application. """ _reject_writes_to_a_read_only_store(node.store_path.store, methods) app = _make_starlette_app(methods=methods, cors_options=cors_options) app.state.store = node.store_path.store app.state.node = node app.state.prefix = node.store_path.path app.state.max_body_size = max_body_size return app zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/src/zarr_http_server/py.typed000066400000000000000000000000001527014137600323610ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/tests/000077500000000000000000000000001527014137600256445ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/tests/conftest.py000066400000000000000000000015471527014137600300520ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING, Literal import pytest from zarr.storage import MemoryStore if TYPE_CHECKING: from zarr.abc.store import Store ZarrFormat = Literal[2, 3] @pytest.fixture def store(request: pytest.FixtureRequest) -> Store: """Store fixture resolved via indirect parametrization.""" if request.param != "memory": raise ValueError(f"unsupported store param: {request.param!r}") return MemoryStore() @pytest.fixture(params=(2, 3), ids=["zarr2", "zarr3"]) def zarr_format(request: pytest.FixtureRequest) -> ZarrFormat: """Zarr format version fixture, parametrized over v2 and v3.""" if request.param == 2: return 2 elif request.param == 3: return 3 msg = f"Invalid zarr format requested. Got {request.param}, expected one of (2, 3)." raise ValueError(msg) zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/tests/test_examples.py000066400000000000000000000042601527014137600310750ustar00rootroot00000000000000"""The shipped examples must keep working. An example that has quietly rotted is worse than no example: it is the first thing a new user copies. These run the real files rather than a paraphrase of them, so a signature change or a behavior change fails here rather than in someone's notebook. """ from __future__ import annotations import pathlib import runpy import pytest EXAMPLES = pathlib.Path(__file__).resolve().parent.parent / "examples" NOTEBOOK = EXAMPLES / "serve_notebook.ipynb" SCRIPT = EXAMPLES / "serve.py" @pytest.mark.parametrize("path", [NOTEBOOK, SCRIPT], ids=["notebook", "script"]) def test_example_exists(path: pathlib.Path) -> None: """Guards the paths above: a renamed or moved example would otherwise turn its execution test into a skip, or a no-op, that nobody notices.""" assert path.is_file(), f"missing example at {path}" def test_serve_script_runs() -> None: """Run examples/serve.py top to bottom. In-process rather than as a subprocess: the script's inline uv metadata resolves `zarr-http-server` from git, so `uv run` on it would test whatever is on main instead of the working tree. """ runpy.run_path(str(SCRIPT), run_name="__main__") def test_serve_notebook_executes() -> None: """Run every cell in a real kernel. The notebook asserts its own expectations -- status codes, byte ranges, that a PUT is refused, that the port is closed after `shutdown()` -- so this is not merely a check that nothing raised. `NotebookClient.execute` defaults to ``allow_errors=False``, so any failed cell raises `CellExecutionError` and fails this test with that cell's traceback. """ nbformat = pytest.importorskip("nbformat") nbclient = pytest.importorskip("nbclient") pytest.importorskip("ipykernel", reason="a kernel is needed to execute the notebook") notebook = nbformat.read(NOTEBOOK, as_version=4) # Run with the notebook's own directory as cwd, so any relative path it # uses means the same thing as when a reader opens it. nbclient.NotebookClient( notebook, timeout=300, kernel_name="python3", resources={"metadata": {"path": str(EXAMPLES)}}, ).execute() zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/tests/test_properties.py000066400000000000000000000541741527014137600314640ustar00rootroot00000000000000"""Property-based tests driven against a real HTTP endpoint. Every test here talks to an actual uvicorn server over a socket, not to an in-process ASGI client, so the properties cover the parts of the stack that only exist on the wire: header parsing, method dispatch, and status codes. Each property checks **two** things after a request -- the response, and the state of the backing store. That pairing is the point. A server can answer correctly and corrupt the store, or refuse a request and write anyway, and a response-only assertion sees neither. The bug that motivated this module did exactly that: a `PUT` to a non-canonically spelled chunk key ("c/00/00" instead of "c/0/0") answered `204 No Content` and stored the body under a key no reader ever looks up, so the client saw success and the data was invisible. The vocabulary below splits keys into two families: * **in-band** -- keys the served node genuinely owns: its metadata documents and the canonical spelling of each chunk key in its grid. These must be served, and writes to them must be visible to a zarr reader. * **out-of-band** -- everything else: non-canonical spellings of a valid chunk key, coordinates outside the grid, traversal attempts, and keys belonging to a sibling node. These must be refused *and* must leave the store byte-for-byte unchanged. """ from __future__ import annotations import re from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal from urllib.parse import quote import httpx import numpy as np import pytest import zarr from hypothesis import assume, given, settings from hypothesis import strategies as st from zarr.buffer import cpu from zarr.core.sync import sync from zarr_http_server import node_app, serve_background, store_app from zarr_http_server._keys import _shard_grid_shape if TYPE_CHECKING: from collections.abc import Iterator from zarr import Array from zarr.abc.store import Store # Real HTTP round trips are slower and jumpier than hypothesis' default # deadline allows, and a CI runner under load makes that worse. _HTTP = settings(deadline=None, max_examples=50) # An uncompressed array so a chunk's bytes are exactly its raw values: a PUT # body of the right length is a legitimate chunk, which lets these tests # assert that a write is readable through a zarr client rather than merely # present in the store. _SHAPE = (6, 4) _CHUNKS = (2, 2) _DTYPE = "int32" @dataclass(frozen=True) class Served: """A running server plus the handles needed to reason about its keys.""" url: str store: Store array: Array[Any] kind: Literal["store", "node"] http_prefix: str """Prepended to an array-relative key to form the request path.""" store_prefix: str """Prepended to an array-relative key to form the backing store key.""" def path(self, array_relative_key: str) -> str: return f"{self.http_prefix}{array_relative_key}" def store_key(self, array_relative_key: str) -> str: return f"{self.store_prefix}{array_relative_key}" def _snapshot(store: Store) -> dict[str, bytes]: """Every key in *store* mapped to its bytes. Comparing two snapshots is how these tests assert that a rejected request changed nothing -- not just that it added no key, but that it modified none either. """ async def _read() -> dict[str, bytes]: out: dict[str, bytes] = {} async for key in store.list(): buf = await store.get(key, cpu.buffer_prototype) if buf is not None: out[key] = buf.to_bytes() return out return sync(_read()) @pytest.fixture( scope="module", params=[ ("store", 2, "memory"), ("store", 3, "memory"), ("node", 2, "memory"), ("node", 3, "memory"), # LocalStore reads through the filesystem, which sizes its read from # the range rather than from the object and so raises on an over-wide # one. MemoryStore just slices a `bytes` and is happy with any bound, # so a memory-only matrix cannot tell a clamped range from a refused # one -- the store backend is part of what these properties test. ("store", 3, "local"), ("node", 3, "local"), ], ids=["store-v2", "store-v3", "node-v2", "node-v3", "store-v3-local", "node-v3-local"], ) def served( request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory ) -> Iterator[Served]: """One real server per (app kind, zarr format, store backend). Module-scoped because hypothesis drives hundreds of requests per test and a server per example would dominate the runtime. `port=0` lets the OS choose the port and `server.url` reports what it actually bound, which avoids the bind-then-close race of picking a port up front. """ kind, zarr_format, backend = request.param store: Store if backend == "local": root_dir = tmp_path_factory.mktemp(f"{kind}-v{zarr_format}") store = zarr.storage.LocalStore(root_dir) else: store = zarr.storage.MemoryStore() root = zarr.open_group(store, mode="w", zarr_format=zarr_format) array = root.create_array( "inside", shape=_SHAPE, chunks=_CHUNKS, dtype=_DTYPE, compressors=None ) array[:] = np.arange(int(np.prod(_SHAPE)), dtype=_DTYPE).reshape(_SHAPE) # A sibling the node_app must never serve, and whose keys therefore make # good out-of-band probes. sibling = root.create_array( "outside", shape=_CHUNKS, chunks=_CHUNKS, dtype=_DTYPE, compressors=None ) sibling[:] = 1 if kind == "node": server = serve_background(node_app(array, methods={"GET", "PUT"}), host="127.0.0.1", port=0) http_prefix = "" else: server = serve_background( store_app(store, methods={"GET", "PUT"}), host="127.0.0.1", port=0 ) http_prefix = "inside/" assert server is not None try: yield Served( url=server.url, store=store, array=array, kind=kind, http_prefix=http_prefix, store_prefix="inside/", ) finally: server.shutdown() def _grid(array: Array[Any]) -> tuple[int, ...]: return _shard_grid_shape(array) @st.composite def chunk_coords(draw: st.DrawFn, array: Array[Any]) -> tuple[int, ...]: """Coordinates of a chunk that exists in *array*'s storage grid.""" return tuple(draw(st.integers(min_value=0, max_value=g - 1)) for g in _grid(array)) @st.composite def out_of_grid_coords(draw: st.DrawFn, array: Array[Any]) -> tuple[int, ...]: """Coordinates outside the grid, so the key is well-formed but names nothing.""" grid = _grid(array) axis = draw(st.integers(min_value=0, max_value=len(grid) - 1)) coords = list(draw(chunk_coords(array))) coords[axis] = draw(st.integers(min_value=grid[axis], max_value=grid[axis] + 50)) return tuple(coords) # The confusability with ASCII digits is the point: `int()` accepts these as # decimal digits, so they name a different store key while decoding to the # same coordinate. That is the defect these strategies probe for. _ARABIC_INDIC = "٠١٢٣٤٥٦٧٨٩" _FULLWIDTH = "０１２３４５６７８９" # noqa: RUF001 def _respellings(digits: str) -> list[str]: """Strings `int()` maps to the same value as *digits*, spelled differently. These are exactly the spellings that make decode-only validation unsafe: each names a *different* store key while decoding to the same coordinate. """ value = int(digits) return [ f"0{digits}", f"00{digits}", f"+{digits}", f" {digits}", f"{digits} ", f"\t{digits}", "".join(_ARABIC_INDIC[int(d)] for d in digits), "".join(_FULLWIDTH[int(d)] for d in digits), *([f"-{digits}"] if value == 0 else []), ] @st.composite def non_canonical_chunk_keys(draw: st.DrawFn, array: Array[Any]) -> str: """A chunk key that decodes into the grid but is not zarr's own spelling. Built by taking the canonical key and re-spelling one of its digit runs, which keeps this strategy independent of the chunk key encoding's grammar. """ coords = draw(chunk_coords(array)) canonical = array.metadata.encode_chunk_key(coords) runs = list(re.finditer(r"\d+", canonical)) assume(runs) run = draw(st.sampled_from(runs)) replacement = draw(st.sampled_from(_respellings(run.group()))) key = canonical[: run.start()] + replacement + canonical[run.end() :] assume(key != canonical) return key TRAVERSAL_KEYS = [ # Percent-encoded on purpose. An HTTP client resolves dot-segments before # it sends -- httpx turns "../secret" into "/secret" and ".." into "/" per # RFC 3986 §5.2.4 -- so a literal "../" probe never reaches the server as # traversal and asserts nothing. Encoded, it survives the client intact # and Starlette decodes it back into a real ".." segment on arrival, which # is the form the server's own guard has to catch. "..%2Fsecret", "%2e%2e%2Fsecret", "%2e%2e%2F%2e%2e%2Fetc%2Fpasswd", "%2e%2e", "%2e", "%2e%2Fzarr.json", "a%2F..%2F..%2Fb", "%2Fabsolute", "sub%2F%2Fempty", "C%3A%2Fwindows", "..%5Cwindows", "a%5C..%5C..%5Cb", "%00nul", ] def _chunk_payload(array: Array[Any]) -> bytes: """Bytes of a full, uncompressed chunk for *array*.""" return np.zeros(_CHUNKS, dtype=array.dtype).tobytes() def _url(served: Served, array_relative_key: str) -> str: """Request URL for a key, percent-encoded so it survives the wire verbatim. Generated keys contain characters a URL cannot carry literally -- a tab makes httpx raise `InvalidURL`, and a space or a non-ASCII digit would be re-encoded on the way out anyway. Encoding here means Starlette decodes the path param back to exactly the key the strategy produced, so the property really is "for any key K, a request for K is refused" rather than "for any key the URL parser happened to leave alone". """ return f"{served.url}/{quote(served.path(array_relative_key), safe='/')}" def _require_node_scope(served: Served) -> None: """Skip a property that only a node-scoped app can satisfy. `store_app` proxies the store's raw key space and has no array semantics to validate against, so every syntactically acceptable key is in-band for it by contract -- including a non-canonical chunk spelling. Only `node_app` claims to serve exactly one node's keys, so only `node_app` can be held to what that set contains. """ if served.kind != "node": pytest.skip("store_app serves the raw key space; node scoping does not apply") class TestInBandRequests: """Keys the node owns are served, and writes to them are visible.""" @given(data=st.data()) @_HTTP def test_get_of_a_canonical_key_returns_the_stored_bytes( self, served: Served, data: st.DataObject ) -> None: """A GET of an in-band key returns exactly what the store holds, and reading never changes the store.""" coords = data.draw(chunk_coords(served.array)) relative = served.array.metadata.encode_chunk_key(coords) before = _snapshot(served.store) response = httpx.get(_url(served, relative), timeout=30) expected = before.get(served.store_key(relative)) if expected is None: assert response.status_code == 404 else: assert response.status_code == 200 assert response.content == expected assert _snapshot(served.store) == before @given(data=st.data()) @_HTTP def test_put_of_a_canonical_key_is_stored_and_readable( self, served: Served, data: st.DataObject ) -> None: """The headline property: a PUT that reports success must be visible. Success means three things at once -- a 2xx, the bytes landing under the key the client named, and a zarr client subsequently reading back the values that were written. The original defect satisfied the first and failed the other two. """ coords = data.draw(chunk_coords(served.array)) fill = data.draw(st.integers(min_value=-(2**31), max_value=2**31 - 1)) relative = served.array.metadata.encode_chunk_key(coords) payload = np.full(_CHUNKS, fill, dtype=served.array.dtype).tobytes() before = _snapshot(served.store) response = httpx.put(_url(served, relative), content=payload, timeout=30) assert response.status_code == 204 after = _snapshot(served.store) key = served.store_key(relative) assert after[key] == payload, "the body did not land under the key the client named" assert set(after) - set(before) <= {key}, "the write touched a key the client did not name" # The write is not merely present, it is legible: reopen the array and # read the chunk the coordinates address. reread = zarr.open_array(served.store, path="inside") block = tuple(slice(c * s, (c + 1) * s) for c, s in zip(coords, _CHUNKS, strict=True)) assert np.array_equal(reread[block], np.full(_CHUNKS, fill, dtype=served.array.dtype)) @given(data=st.data()) @_HTTP def test_range_of_a_stored_key_returns_the_matching_slice( self, served: Served, data: st.DataObject ) -> None: """A satisfiable range returns exactly the bytes it names, and says so in Content-Range.""" relative = served.array.metadata.encode_chunk_key(tuple(0 for _ in _grid(served.array))) key = served.store_key(relative) before = _snapshot(served.store) assume(key in before) body = before[key] start = data.draw(st.integers(min_value=0, max_value=len(body) - 1)) end = data.draw(st.integers(min_value=start, max_value=len(body) - 1)) response = httpx.get( _url(served, relative), headers={"Range": f"bytes={start}-{end}"}, timeout=30 ) assert response.status_code == 206 assert response.content == body[start : end + 1] # RFC 9110 §15.3.7: a single-part 206 must carry Content-Range, and it # must describe the bytes actually returned. content_range = response.headers["content-range"] assert content_range.startswith(f"bytes {start}-{start + len(response.content) - 1}/") assert _snapshot(served.store) == before @given(suffix=st.integers(min_value=1, max_value=200)) @_HTTP def test_suffix_range_returns_the_tail_and_locates_it( self, served: Served, suffix: int ) -> None: """A suffix range must report where in the object its bytes came from. Zarr's sharding codec reads a shard index this way, so a 206 without Content-Range leaves the reader unable to tell a clamped whole-object read from the tail it asked for. """ relative = served.array.metadata.encode_chunk_key(tuple(0 for _ in _grid(served.array))) key = served.store_key(relative) before = _snapshot(served.store) assume(key in before) body = before[key] response = httpx.get( _url(served, relative), headers={"Range": f"bytes=-{suffix}"}, timeout=30 ) assert response.status_code == 206 expected = body[-suffix:] if suffix <= len(body) else body assert response.content == expected first = len(body) - len(expected) assert response.headers["content-range"].startswith(f"bytes {first}-{len(body) - 1}/") assert _snapshot(served.store) == before class TestOutOfBandRequests: """Keys the node does not own are refused, and change nothing.""" @given(data=st.data()) @_HTTP def test_non_canonical_chunk_key_is_refused_and_writes_nothing( self, served: Served, data: st.DataObject ) -> None: """The regression that motivated this module. A key that decodes into the grid but is spelled differently from zarr's own rendering names a store key no reader consults. Accepting a write to it reports success and loses the data, so it must be refused and the store must be untouched. """ _require_node_scope(served) relative = data.draw(non_canonical_chunk_keys(served.array)) payload = _chunk_payload(served.array) before = _snapshot(served.store) put = httpx.put(_url(served, relative), content=payload, timeout=30) assert put.status_code == 404 assert _snapshot(served.store) == before, "a refused write still modified the store" get = httpx.get(_url(served, relative), timeout=30) assert get.status_code == 404 assert _snapshot(served.store) == before @given(data=st.data()) @_HTTP def test_out_of_grid_chunk_key_is_refused_and_writes_nothing( self, served: Served, data: st.DataObject ) -> None: """Coordinates past the end of the grid address no chunk of this array.""" _require_node_scope(served) coords = data.draw(out_of_grid_coords(served.array)) relative = served.array.metadata.encode_chunk_key(coords) before = _snapshot(served.store) put = httpx.put(_url(served, relative), content=_chunk_payload(served.array), timeout=30) get = httpx.get(_url(served, relative), timeout=30) assert put.status_code == 404 assert get.status_code == 404 assert _snapshot(served.store) == before @given(key=st.sampled_from(TRAVERSAL_KEYS)) @_HTTP def test_traversal_key_is_refused_and_writes_nothing(self, served: Served, key: str) -> None: """Nothing that tries to leave the served scope may be served or written.""" before = _snapshot(served.store) put = httpx.put(f"{served.url}/{key}", content=b"payload", timeout=30) get = httpx.get(f"{served.url}/{key}", timeout=30) assert put.status_code in (403, 404, 405), f"{key!r} was accepted for writing" assert get.status_code in (403, 404, 405), f"{key!r} was served" assert _snapshot(served.store) == before @given(data=st.data()) @_HTTP def test_node_app_never_serves_a_sibling(self, served: Served, data: st.DataObject) -> None: """A node_app is scoped to one node, so a sibling's keys are invisible even though they exist in the same store.""" if served.kind != "node": pytest.skip("store_app deliberately serves the whole store") coords = data.draw(chunk_coords(served.array)) chunk = served.array.metadata.encode_chunk_key(coords) relative = data.draw( st.sampled_from( [ f"outside/{chunk}", "outside/zarr.json", "outside/.zarray", f"../outside/{chunk}", ] ) ) before = _snapshot(served.store) get = httpx.get(f"{served.url}/{relative}", timeout=30) put = httpx.put(f"{served.url}/{relative}", content=b"payload", timeout=30) assert get.status_code == 404 assert put.status_code == 404 assert _snapshot(served.store) == before class TestMethodsAndRanges: """Transport-level invariants that hold for every key.""" @given( method=st.sampled_from(["DELETE", "POST", "PATCH", "OPTIONS"]), data=st.data(), ) @_HTTP def test_unconfigured_method_is_refused_and_writes_nothing( self, served: Served, method: str, data: st.DataObject ) -> None: """Only the methods the app was configured with may reach the store.""" coords = data.draw(chunk_coords(served.array)) relative = served.array.metadata.encode_chunk_key(coords) before = _snapshot(served.store) response = httpx.request(method, _url(served, relative), content=b"payload", timeout=30) assert response.status_code == 405 assert _snapshot(served.store) == before @given( header=st.sampled_from( [ "bytes=abc-def", "bytes=0-1,4-5", "chars=0-7", "bytes=", "nonsense", "bytes=+0-1", ] ) ) @_HTTP def test_unusable_range_serves_the_whole_object(self, served: Served, header: str) -> None: """RFC 9110 §14.2: a Range the server cannot use is ignored, not refused.""" relative = served.array.metadata.encode_chunk_key(tuple(0 for _ in _grid(served.array))) before = _snapshot(served.store) key = served.store_key(relative) assume(key in before) response = httpx.get(_url(served, relative), headers={"Range": header}, timeout=30) assert response.status_code == 200 assert response.content == before[key] assert "content-range" not in response.headers @given( header=st.sampled_from( [ "bytes=5-2", "bytes=-0", "bytes=99999999999999999999-", "bytes=100000-100001", ] ) ) @_HTTP def test_unsatisfiable_range_is_refused(self, served: Served, header: str) -> None: """A well-formed range that names nothing readable is a 416.""" relative = served.array.metadata.encode_chunk_key(tuple(0 for _ in _grid(served.array))) before = _snapshot(served.store) assume(served.store_key(relative) in before) response = httpx.get(_url(served, relative), headers={"Range": header}, timeout=30) assert response.status_code == 416 assert _snapshot(served.store) == before @given(end=st.integers(min_value=10**19, max_value=10**30)) @_HTTP def test_absurdly_wide_range_is_clamped_not_refused(self, served: Served, end: int) -> None: """RFC 9110 §14.1.2 clamps a last-byte-pos past the end of the object, so an over-wide range reads to EOF rather than erroring.""" relative = served.array.metadata.encode_chunk_key(tuple(0 for _ in _grid(served.array))) key = served.store_key(relative) before = _snapshot(served.store) assume(key in before) response = httpx.get( _url(served, relative), headers={"Range": f"bytes=0-{end}"}, timeout=30 ) assert response.status_code == 206 assert response.content == before[key] assert _snapshot(served.store) == before zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/tests/test_serve.py000066400000000000000000002271161527014137600304120ustar00rootroot00000000000000from __future__ import annotations import asyncio import errno import os import socket from typing import TYPE_CHECKING, Any, Literal, get_args import numpy as np import pytest import zarr from starlette.applications import Starlette from starlette.routing import Mount from starlette.testclient import TestClient from zarr.buffer import cpu from zarr.storage import LocalStore, MemoryStore from zarr_http_server._serve import ( _SHUTDOWN_JOIN_MARGIN, READ_ONLY_HTTP_METHODS, READ_WRITE_HTTP_METHODS, CorsOptions, ReadOnlyHTTPMethod, _bind_preferred_or_free, _parse_range_header, _RangeVerdict, node_app, serve_background, store_app, ) if TYPE_CHECKING: import pathlib from collections.abc import Coroutine, Iterator from zarr.abc.store import Store ZarrFormat = Literal[2, 3] SHUTDOWN_TIMEOUT = 1 """shutdown_timeout used by the bounded-shutdown test.""" SLOW_HANDLER_SECONDS = 5 """How long that test's handler sleeps -- far longer than the shutdown bound, so an unbounded join would be obvious.""" def sync[T](coro: Coroutine[Any, Any, T]) -> T: """Run a store coroutine to completion (tests use MemoryStore only).""" return asyncio.run(coro) @pytest.fixture def group_with_arrays(store: Store) -> zarr.Group: """Create a group containing a regular array and a sharded array.""" root = zarr.open_group(store, mode="w") zarr.create_array(root.store_path / "regular", shape=(4, 4), chunks=(2, 2), dtype="f8") zarr.create_array( root.store_path / "sharded", shape=(8, 8), chunks=(2, 2), shards=(4, 4), dtype="i4", ) return root @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestNodeAppDoesNotExposeNonZarrKeys: """node_app must never expose keys that are not part of the zarr hierarchy.""" def test_non_zarr_key_returns_404(self, store: Store, group_with_arrays: zarr.Group) -> None: """A key that is not valid zarr metadata or a valid chunk key should return 404, even if the underlying store contains data at that path.""" non_zarr_buf = cpu.buffer_prototype.buffer.from_bytes(b"secret data") sync(store.set("secret.txt", non_zarr_buf)) app = node_app(group_with_arrays) client = TestClient(app) # The non-zarr key must not be accessible. response = client.get("/secret.txt") assert response.status_code == 404 def test_non_zarr_key_nested_returns_404( self, store: Store, group_with_arrays: zarr.Group ) -> None: """A non-zarr key nested under a real array's path should return 404, even though the path prefix matches a valid zarr node.""" non_zarr_buf = cpu.buffer_prototype.buffer.from_bytes(b"not a chunk") sync(store.set("regular/notes.txt", non_zarr_buf)) app = node_app(group_with_arrays) client = TestClient(app) response = client.get("/regular/notes.txt") assert response.status_code == 404 def test_valid_metadata_is_accessible(self, group_with_arrays: zarr.Group) -> None: """Zarr metadata keys (zarr.json) for both the root group and child arrays should be served with a 200 status.""" app = node_app(group_with_arrays) client = TestClient(app) # Root group metadata response = client.get("/zarr.json") assert response.status_code == 200 # Array metadata response = client.get("/regular/zarr.json") assert response.status_code == 200 def test_valid_chunk_is_accessible(self, group_with_arrays: zarr.Group) -> None: """A valid, in-bounds chunk key for an array with written data should be served with a 200 status.""" arr = group_with_arrays["regular"] assert isinstance(arr, zarr.Array) arr[:] = np.ones((4, 4)) app = node_app(group_with_arrays) client = TestClient(app) # c/0/0 is a valid chunk key for a (4,4) array with (2,2) chunks. response = client.get("/regular/c/0/0") assert response.status_code == 200 def test_out_of_bounds_chunk_key_returns_404( self, store: Store, group_with_arrays: zarr.Group ) -> None: """A chunk key that is syntactically valid but references indices beyond the array's chunk grid should return 404.""" arr = group_with_arrays["regular"] assert isinstance(arr, zarr.Array) arr[:] = np.ones((4, 4)) # Put real data at the out-of-grid key, so that a 404 can only come # from the bounds check -- not from the key merely being absent. planted = cpu.buffer_prototype.buffer.from_bytes(b"out of grid") sync(store.set("regular/c/99/99", planted)) assert sync(store.get("regular/c/99/99", cpu.buffer_prototype)) is not None app = node_app(group_with_arrays) client = TestClient(app) # (4,4) array with (2,2) chunks has grid shape (2,2), so c/99/99 is # syntactically valid but out of bounds. response = client.get("/regular/c/99/99") assert response.status_code == 404 def test_empty_path_returns_404(self, group_with_arrays: zarr.Group) -> None: """A request to the root path '/' should return 404 because an empty string is not a valid zarr key.""" app = node_app(group_with_arrays) client = TestClient(app) response = client.get("/") assert response.status_code == 404 @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestShardedArrayByteRangeReads: """Byte-range reads against a sharded array served via node_app.""" def test_range_read_returns_206(self, group_with_arrays: zarr.Group) -> None: """A Range header requesting a specific byte range (e.g. bytes=0-7) should return 206 Partial Content with exactly those bytes.""" arr = group_with_arrays["sharded"] assert isinstance(arr, zarr.Array) arr[:] = np.arange(64, dtype="i4").reshape((8, 8)) app = node_app(group_with_arrays) client = TestClient(app) # c/0/0 is the first shard key for an (8,8) array with (4,4) shards. full_response = client.get("/sharded/c/0/0") assert full_response.status_code == 200 full_body = full_response.content # Request the first 8 bytes. range_response = client.get("/sharded/c/0/0", headers={"Range": "bytes=0-7"}) assert range_response.status_code == 206 assert range_response.content == full_body[:8] def test_suffix_range_read(self, group_with_arrays: zarr.Group) -> None: """A suffix byte range (e.g. bytes=-4) should return the last N bytes of the resource with a 206 status.""" arr = group_with_arrays["sharded"] assert isinstance(arr, zarr.Array) arr[:] = np.arange(64, dtype="i4").reshape((8, 8)) app = node_app(group_with_arrays) client = TestClient(app) full_response = client.get("/sharded/c/0/0") full_body = full_response.content # Request the last 4 bytes. range_response = client.get("/sharded/c/0/0", headers={"Range": "bytes=-4"}) assert range_response.status_code == 206 assert range_response.content == full_body[-4:] def test_offset_range_read(self, group_with_arrays: zarr.Group) -> None: """An offset byte range (e.g. bytes=4-) should return all bytes from the given offset to the end, with a 206 status.""" arr = group_with_arrays["sharded"] assert isinstance(arr, zarr.Array) arr[:] = np.arange(64, dtype="i4").reshape((8, 8)) app = node_app(group_with_arrays) client = TestClient(app) full_response = client.get("/sharded/c/0/0") full_body = full_response.content # Request everything from byte 4 onward. range_response = client.get("/sharded/c/0/0", headers={"Range": "bytes=4-"}) assert range_response.status_code == 206 assert range_response.content == full_body[4:] @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestUnusableRangeHeadersAreIgnored: """A Range this server cannot turn into a read is ignored, not refused. RFC 9110 §14.2 requires ignoring a Range whose unit is unrecognized and permits ignoring one that will not parse; either way the answer is 200 with the full representation. Refusing with 416 would tell a client the object is unreadable when only the request shape was unsupported -- and a multi-range request, which is legal to send and which proxies and download accelerators do send, would take that at face value. """ @pytest.mark.parametrize( "header", [ "bytes=abc-def", "bytes=-abc", "bytes=abc-", "bytes=0-7,10-20", "chars=0-7", "bytes=", "bytes=+0-1", ], ) def test_unusable_range_serves_full_representation(self, store: Store, header: str) -> None: """Non-numeric bounds, multi-range, a non-'bytes' unit, an empty spec and a non-canonical byte position all fall back to a plain 200.""" body = b"some data here" sync(store.set("key", cpu.buffer_prototype.buffer.from_bytes(body))) client = TestClient(store_app(store), raise_server_exceptions=False) response = client.get("/key", headers={"Range": header}) assert response.status_code == 200 assert response.content == body assert "content-range" not in response.headers class TestParseRangeHeader: """Unit tests for _parse_range_header.""" def test_parser_rejects_an_inverted_range(self) -> None: """Pin the parser itself: on a MemoryStore an unguarded inverted range happens to return b"" and still yields 416, so the status code alone cannot tell whether the guard is present.""" assert _parse_range_header("bytes=5-2") is _RangeVerdict.UNSATISFIABLE assert _parse_range_header("bytes=0-0") is not _RangeVerdict.UNSATISFIABLE def test_valid_range(self) -> None: """'bytes=0-99' should parse into a RangeByteRequest with start=0 and end=100 (end is exclusive, so the inclusive HTTP end is incremented).""" from zarr.abc.store import RangeByteRequest result = _parse_range_header("bytes=0-99") assert result == RangeByteRequest(start=0, end=100) def test_valid_suffix(self) -> None: """'bytes=-50' should parse into a SuffixByteRequest requesting the last 50 bytes of the resource.""" from zarr.abc.store import SuffixByteRequest result = _parse_range_header("bytes=-50") assert result == SuffixByteRequest(suffix=50) def test_valid_offset(self) -> None: """'bytes=10-' should parse into an OffsetByteRequest starting at byte 10 and reading to the end of the resource.""" from zarr.abc.store import OffsetByteRequest result = _parse_range_header("bytes=10-") assert result == OffsetByteRequest(offset=10) def test_non_bytes_unit(self) -> None: """An unrecognized range unit must be ignored, per RFC 9110 §14.2.""" assert _parse_range_header("chars=0-7") is _RangeVerdict.IGNORE def test_garbage_values(self) -> None: """Non-numeric bounds are ignored rather than raising a ValueError.""" assert _parse_range_header("bytes=abc-def") is _RangeVerdict.IGNORE def test_multi_range(self) -> None: """Multi-range requests (e.g. bytes=0-7,10-20) are legal to send; this server does not build multipart/byteranges, so it serves the whole representation instead of refusing.""" assert _parse_range_header("bytes=0-7,10-20") is _RangeVerdict.IGNORE def test_empty_spec(self) -> None: """A Range header with no range specifier after 'bytes=' is ignored.""" assert _parse_range_header("bytes=") is _RangeVerdict.IGNORE def test_non_canonical_byte_position(self) -> None: """`int` would accept these; RFC 9110 defines a byte position as 1*DIGIT, so they are not ranges and the header is ignored.""" assert _parse_range_header("bytes=+0-1") is _RangeVerdict.IGNORE assert _parse_range_header("bytes= 0-1") is _RangeVerdict.IGNORE assert _parse_range_header("bytes=0_0-1") is _RangeVerdict.IGNORE def test_oversized_start_is_unsatisfiable(self) -> None: """A first-byte-pos past any possible object names nothing.""" assert _parse_range_header("bytes=99999999999999999999-") is _RangeVerdict.UNSATISFIABLE def test_zero_length_suffix_is_unsatisfiable(self) -> None: """ "The last zero bytes" names nothing.""" assert _parse_range_header("bytes=-0") is _RangeVerdict.UNSATISFIABLE @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestWriteViaPut: """store_app and node_app can be configured to accept PUT writes.""" def test_put_writes_to_store(self, store: Store) -> None: """A PUT request to store_app with PUT enabled should write the request body into the store at the given key.""" app = store_app(store, methods={"GET", "PUT"}) client = TestClient(app) payload = b"hello zarr" response = client.put("/some/key", content=payload) assert response.status_code == 204 # Verify the data landed in the store. buf = sync(store.get("some/key", cpu.buffer_prototype)) assert buf is not None assert buf.to_bytes() == payload def test_put_then_get_roundtrip(self, store: Store) -> None: """Data written via PUT should be retrievable via a subsequent GET at the same key.""" app = store_app(store, methods={"GET", "PUT"}) client = TestClient(app) payload = b"\x00\x01\x02\x03" client.put("/data/blob", content=payload) response = client.get("/data/blob") assert response.status_code == 200 assert response.content == payload def test_put_rejected_when_not_configured(self, store: Store) -> None: """PUT requests should return 405 Method Not Allowed when the server is created with the default methods (GET only).""" app = store_app(store) client = TestClient(app) response = client.put("/some/key", content=b"data") assert response.status_code == 405 def test_put_on_node_validates_key(self, store: Store, group_with_arrays: zarr.Group) -> None: """PUT requests via node_app should be rejected with 404 when the target key is not a valid zarr key (metadata or chunk).""" app = node_app(group_with_arrays, methods={"GET", "PUT"}) client = TestClient(app) response = client.put("/not_a_zarr_key.bin", content=b"data") assert response.status_code == 404 def test_put_to_valid_chunk_key_succeeds(self, group_with_arrays: zarr.Group) -> None: """PUT requests via node_app to a valid chunk key should succeed with 204, and the written data should be retrievable via GET.""" app = node_app(group_with_arrays, methods={"GET", "PUT"}) client = TestClient(app) payload = b"\x00" * 32 response = client.put("/regular/c/0/0", content=payload) assert response.status_code == 204 # Confirm it round-trips. get_response = client.get("/regular/c/0/0") assert get_response.status_code == 200 assert get_response.content == payload @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestMethodValidation: """Only the methods the handler implements may be served.""" def test_supported_methods_are_served(self, store: Store) -> None: """GET, PUT and HEAD each behave as their verb implies.""" app = store_app(store, methods={"GET", "PUT", "HEAD"}) client = TestClient(app) assert client.put("/zarr.json", content=b'{"a":1}').status_code == 204 assert client.get("/zarr.json").content == b'{"a":1}' # HEAD reports the same status as GET but carries no body. head = client.head("/zarr.json") assert head.status_code == 200 assert head.content == b"" def test_empty_method_set_raises(self, store: Store) -> None: """Asking for no methods must be rejected rather than producing a server that answers every verb, including writes: Starlette treats a falsy `methods` on a Route as "match anything".""" for build in ( lambda: store_app(store, methods=set()), lambda: node_app(zarr.open_group(store, mode="a"), methods=set()), ): with pytest.raises(ValueError, match="at least one"): build() @pytest.mark.parametrize("method", ["DELETE", "POST", "PATCH", "OPTIONS", "TRACE"]) def test_unsupported_method_raises(self, store: Store, method: str) -> None: """A verb the handler cannot implement is rejected when the app is built, rather than silently answering as if it were a GET.""" for build in ( lambda: store_app(store, methods={"GET", method}), # type: ignore[arg-type] lambda: node_app(zarr.open_group(store, mode="a"), methods={"GET", method}), # type: ignore[arg-type] ): with pytest.raises(ValueError, match=method): build() @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestStoreAppEdgeCases: """Edge cases for store_app.""" def test_get_nonexistent_key_returns_404(self, store: Store) -> None: """GET for a key that does not exist in the store should return 404.""" app = store_app(store) client = TestClient(app) response = client.get("/no/such/key") assert response.status_code == 404 def test_empty_path_returns_404(self, store: Store) -> None: """GET to the root path '/' (empty key) should return 404 because an empty string is not a valid store key.""" app = store_app(store) client = TestClient(app) response = client.get("/") assert response.status_code == 404 @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestNodeAppDirectArray: """Serve a single array directly (not through a group).""" def test_serve_nested_array_directly(self, store: Store) -> None: """When node_app is given a nested array (not a group), requests should use keys relative to that array's path. Metadata and in-bounds chunks should return 200, and out-of-bounds chunks should return 404.""" root = zarr.open_group(store, mode="w") arr = zarr.create_array( root.store_path / "sub/nested", shape=(4,), chunks=(2,), dtype="f8", ) arr[:] = np.arange(4, dtype="f8") # Serve the array directly — its prefix is "sub/nested". app = node_app(arr) client = TestClient(app) # Metadata should be accessible at the array root. response = client.get("/zarr.json") assert response.status_code == 200 # Chunk keys are relative to the array. response = client.get("/c/0") assert response.status_code == 200 response = client.get("/c/1") assert response.status_code == 200 # Out of bounds. response = client.get("/c/99") assert response.status_code == 404 def test_serve_root_array(self, store: Store) -> None: """When node_app is given an array stored at the root of a store (empty prefix), metadata and chunk keys should be accessible at their natural paths.""" arr = zarr.create_array( store, shape=(6,), chunks=(3,), dtype="i4", ) arr[:] = np.arange(6, dtype="i4") # Root-level array has prefix = "". app = node_app(arr) client = TestClient(app) response = client.get("/zarr.json") assert response.status_code == 200 response = client.get("/c/0") assert response.status_code == 200 response = client.get("/c/1") assert response.status_code == 200 response = client.get("/c/2") assert response.status_code == 404 @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestContentType: """Responses should have the correct Content-Type.""" def test_metadata_has_json_content_type(self, group_with_arrays: zarr.Group) -> None: """Zarr metadata files (zarr.json) should be served with Content-Type: application/json.""" app = node_app(group_with_arrays) client = TestClient(app) response = client.get("/zarr.json") assert response.status_code == 200 assert response.headers["content-type"] == "application/json" def test_chunk_has_octet_stream_content_type(self, group_with_arrays: zarr.Group) -> None: """Chunk data should be served with Content-Type: application/octet-stream since it is binary data.""" arr = group_with_arrays["regular"] assert isinstance(arr, zarr.Array) arr[:] = np.ones((4, 4)) app = node_app(group_with_arrays) client = TestClient(app) response = client.get("/regular/c/0/0") assert response.status_code == 200 assert response.headers["content-type"] == "application/octet-stream" @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestCorsMiddleware: """CORS middleware should add the expected headers.""" def test_cors_headers_present(self, store: Store) -> None: """When cors_options are provided, responses should include the Access-Control-Allow-Origin header matching the request origin.""" buf = cpu.buffer_prototype.buffer.from_bytes(b"data") sync(store.set("key", buf)) cors = CorsOptions(allow_origins=["https://example.com"], allow_methods=["GET"]) app = store_app(store, cors_options=cors) client = TestClient(app) response = client.get("/key", headers={"Origin": "https://example.com"}) assert response.status_code == 200 assert response.headers["access-control-allow-origin"] == "https://example.com" def test_cors_preflight(self, store: Store) -> None: """CORS preflight OPTIONS requests should return 200 with the Access-Control-Allow-Origin header when CORS is configured.""" cors = CorsOptions(allow_origins=["*"], allow_methods=["GET", "PUT"]) app = store_app(store, methods={"GET", "PUT"}, cors_options=cors) client = TestClient(app) response = client.options( "/any/path", headers={ "Origin": "https://example.com", "Access-Control-Request-Method": "PUT", }, ) assert response.status_code == 200 assert "access-control-allow-origin" in response.headers def test_no_cors_headers_without_option(self, store: Store) -> None: """When no cors_options are provided, responses should not include any CORS headers, even if the request includes an Origin header.""" buf = cpu.buffer_prototype.buffer.from_bytes(b"data") sync(store.set("key", buf)) app = store_app(store) client = TestClient(app) response = client.get("/key", headers={"Origin": "https://example.com"}) assert response.status_code == 200 assert "access-control-allow-origin" not in response.headers def _metadata_key(zarr_format: ZarrFormat) -> str: """Return the metadata key for the given zarr format.""" return "zarr.json" if zarr_format == 3 else ".zarray" def _chunk_key(zarr_format: ZarrFormat, coords: str) -> str: """Return a chunk key for the given format. *coords* is a dot-separated string like ``"0.0"``. For v3 this becomes ``"c/0/0"``; for v2 it is returned unchanged. """ if zarr_format == 3: return "c/" + coords.replace(".", "/") return coords @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestNodeAppV2AndV3: """Test node_app with both v2 and v3 arrays side by side.""" def test_metadata_accessible(self, store: Store, zarr_format: ZarrFormat) -> None: """The format-appropriate metadata key should be served with 200.""" arr = zarr.create_array(store, shape=(4,), chunks=(2,), dtype="f8", zarr_format=zarr_format) app = node_app(arr) client = TestClient(app) response = client.get(f"/{_metadata_key(zarr_format)}") assert response.status_code == 200 def test_chunk_accessible(self, store: Store, zarr_format: ZarrFormat) -> None: """An in-bounds chunk key should be served with 200 for both formats.""" arr = zarr.create_array(store, shape=(4,), chunks=(2,), dtype="f8", zarr_format=zarr_format) arr[:] = np.ones(4) app = node_app(arr) client = TestClient(app) response = client.get(f"/{_chunk_key(zarr_format, '0')}") assert response.status_code == 200 def test_out_of_bounds_chunk_returns_404(self, store: Store, zarr_format: ZarrFormat) -> None: """An out-of-bounds chunk key should return 404 for both formats.""" arr = zarr.create_array(store, shape=(4,), chunks=(2,), dtype="f8", zarr_format=zarr_format) arr[:] = np.ones(4) # Plant data at the out-of-grid key so the 404 must come from the # bounds check rather than from the key being absent. key = _chunk_key(zarr_format, "99") sync(store.set(key, cpu.buffer_prototype.buffer.from_bytes(b"out of grid"))) assert sync(store.get(key, cpu.buffer_prototype)) is not None app = node_app(arr) client = TestClient(app) response = client.get(f"/{key}") assert response.status_code == 404 def test_non_zarr_key_returns_404(self, store: Store, zarr_format: ZarrFormat) -> None: """A non-zarr key should return 404 regardless of format.""" arr = zarr.create_array(store, shape=(4,), chunks=(2,), dtype="f8", zarr_format=zarr_format) non_zarr_buf = cpu.buffer_prototype.buffer.from_bytes(b"secret") sync(store.set("secret.txt", non_zarr_buf)) app = node_app(arr) client = TestClient(app) response = client.get("/secret.txt") assert response.status_code == 404 def test_data_roundtrip(self, store: Store, zarr_format: ZarrFormat) -> None: """Data written to an array should be readable via store_app for both formats.""" arr = zarr.create_array(store, shape=(4,), chunks=(2,), dtype="f8", zarr_format=zarr_format) arr[:] = np.arange(4, dtype="f8") app = store_app(store) client = TestClient(app) # Metadata should be accessible. response = client.get(f"/{_metadata_key(zarr_format)}") assert response.status_code == 200 # First chunk should be accessible. response = client.get(f"/{_chunk_key(zarr_format, '0')}") assert response.status_code == 200 assert len(response.content) > 0 class TestPathTraversalProtection: """store_app and node_app must reject path-traversal attempts before touching the store, regardless of URL-encoding tricks.""" def test_get_traversal_outside_store_root_returns_404(self, tmp_path: Any) -> None: """A GET for a percent-encoded '../secret.txt' must not escape the store root and read a file outside it.""" from zarr.storage import LocalStore root = tmp_path / "store_root" root.mkdir() secret = tmp_path / "secret.txt" secret.write_text("top secret contents") store = LocalStore(root) app = store_app(store, methods={"GET", "PUT"}) client = TestClient(app) response = client.get("/..%2fsecret.txt") assert response.status_code == 404 assert b"top secret" not in response.content def test_put_traversal_outside_store_root_returns_404(self, tmp_path: Any) -> None: """A PUT to a percent-encoded '../pwned.txt' must not escape the store root and write a file outside it.""" from zarr.storage import LocalStore root = tmp_path / "store_root" root.mkdir() pwned = tmp_path / "pwned.txt" store = LocalStore(root) app = store_app(store, methods={"GET", "PUT"}) client = TestClient(app) response = client.put("/..%2fpwned.txt", content=b"pwned") assert response.status_code == 404 assert not pwned.exists() @pytest.mark.parametrize( ("encoded_path", "climb_depth"), [ ("/..%2fsecret.txt", 1), ("/%2e%2e/secret.txt", 1), ("/..%2f..%2fsecret.txt", 2), ], ) def test_encoded_traversal_variants_return_404( self, tmp_path: Any, encoded_path: str, climb_depth: int ) -> None: """Various percent-encoded traversal spellings must all be rejected. The store root is nested exactly ``climb_depth`` directories below ``tmp_path`` and the secret lives at ``tmp_path/secret.txt`` -- the exact location each traversal's ".." segments resolve to -- so a case with a missing or deleted guard would actually reach the secret instead of just returning 404 for an unrelated reason (e.g. a two-level climb landing on a directory that happens to be empty). """ from zarr.storage import LocalStore parts = [f"level{i}" for i in range(climb_depth - 1)] + ["store_root"] root = tmp_path.joinpath(*parts) root.mkdir(parents=True) secret = tmp_path / "secret.txt" secret.write_text("top secret contents") store = LocalStore(root) app = store_app(store, methods={"GET", "PUT"}) client = TestClient(app) response = client.get(encoded_path) assert response.status_code == 404 assert b"top secret" not in response.content def test_get_absolute_key_bypass_returns_404(self, tmp_path: Any) -> None: """A percent-encoded leading slash decodes to an ABSOLUTE path param (e.g. request '/%2fetc%2fhostname' -> path param '/etc/hostname'). '/etc/hostname'.split('/') -> ['', 'etc', 'hostname'] has no '.' or '..' segment, so the two-element guard misses it, but LocalStore resolves an absolute key by discarding its root entirely -- an arbitrary-file read. The empty leading segment must be rejected.""" from zarr.storage import LocalStore root = tmp_path / "store_root" root.mkdir() secret = tmp_path / "secret_abs.txt" secret.write_text("top secret absolute contents") store = LocalStore(root) app = store_app(store, methods={"GET", "PUT"}) client = TestClient(app) # Mirror the exploit: percent-encode every "/" (including the # leading one) in the absolute secret path as "%2f". encoded_path = "/" + str(secret).replace("/", "%2f") response = client.get(encoded_path) assert response.status_code == 404 assert b"top secret absolute" not in response.content assert secret.read_text() == "top secret absolute contents" def test_put_absolute_key_bypass_returns_404(self, tmp_path: Any) -> None: """Same absolute-key vector as above, but for PUT: a percent-encoded leading slash must not allow writing a file outside the store root.""" from zarr.storage import LocalStore root = tmp_path / "store_root" root.mkdir() pwned = tmp_path / "pwned_abs.txt" store = LocalStore(root) app = store_app(store, methods={"GET", "PUT"}) client = TestClient(app) encoded_path = "/" + str(pwned).replace("/", "%2f") response = client.put(encoded_path, content=b"pwned") assert response.status_code == 404 assert not pwned.exists() @pytest.mark.parametrize( "encoded_path", [ "/..%5C..%5Cwin.ini", "/%5CWindows%5Cwin.ini", "/C:/Windows/win.ini", "/C:%5CWindows", "/%5C%5Chost%5Cshare%5Cx", ], ) def test_backslash_and_drive_traversal_variants_return_404(self, encoded_path: str) -> None: """Backslash is a path separator on Windows, and a drive-qualified or root-relative key discards a filesystem store's root entirely on Windows, even though POSIX only ever treats '/' as a separator. The guard must reject these purely from the string, before the store is ever touched -- verified here by making the store raise if called.""" from unittest.mock import AsyncMock from zarr.storage import MemoryStore store = MemoryStore() store.get = AsyncMock(side_effect=AssertionError("store.get should not be called")) # type: ignore[method-assign] store.set = AsyncMock(side_effect=AssertionError("store.set should not be called")) # type: ignore[method-assign] app = store_app(store, methods={"GET", "PUT"}) client = TestClient(app) response = client.get(encoded_path) assert response.status_code == 404 def test_put_backslash_traversal_returns_404(self) -> None: """A PUT to a backslash-encoded '..\\..\\pwned.txt' must be rejected before the store is touched, mirroring the GET case above.""" from unittest.mock import AsyncMock from zarr.storage import MemoryStore store = MemoryStore() store.set = AsyncMock(side_effect=AssertionError("store.set should not be called")) # type: ignore[method-assign] app = store_app(store, methods={"GET", "PUT"}) client = TestClient(app) response = client.put("/..%5C..%5Cpwned.txt", content=b"pwned") assert response.status_code == 404 def _get_free_port() -> int: """Return an unused TCP port on localhost.""" import socket with socket.socket() as s: s.bind(("127.0.0.1", 0)) port: int = s.getsockname()[1] return port @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestServeBackground: """Test serve_background with store- and node-scoped apps.""" def test_background_server_over_a_store_app(self, store: Store) -> None: """serve_background over a store app should return a BackgroundServer that responds to HTTP requests and can be used as a context manager.""" import httpx from zarr_http_server import serve_background buf = cpu.buffer_prototype.buffer.from_bytes(b"hello") sync(store.set("key", buf)) port = _get_free_port() with serve_background(store_app(store), host="127.0.0.1", port=port) as server: assert server.host == "127.0.0.1" assert server.port == port assert server.url == f"http://127.0.0.1:{port}" response = httpx.get(f"{server.url}/key") assert response.status_code == 200 assert response.content == b"hello" def test_background_server_over_a_node_app(self, store: Store) -> None: """serve_background over a node app should return a BackgroundServer that responds to HTTP requests and can be used as a context manager.""" import httpx from zarr_http_server import serve_background arr = zarr.create_array(store, shape=(4,), chunks=(2,), dtype="f8") arr[:] = np.arange(4, dtype="f8") port = _get_free_port() with serve_background(node_app(arr), host="127.0.0.1", port=port) as server: response = httpx.get(f"{server.url}/zarr.json") assert response.status_code == 200 class TestStoreFailuresAreNotReportedAsMisses: """Under the v3 spec an absent chunk is an uninitialized one, and a reader is right to substitute the array's fill value for it. A 404 therefore asserts something about the store's contents, and an I/O failure must not borrow it -- that would have a correct client materialize fill values over data that exists.""" @pytest.mark.skipif(os.geteuid() == 0, reason="root bypasses the permission bits under test") def test_unreadable_key_is_a_server_error(self, tmp_path: pathlib.Path) -> None: root = tmp_path / "root" root.mkdir() (root / "key").write_bytes(b"real data") os.chmod(root / "key", 0o000) client = TestClient(store_app(LocalStore(str(root))), raise_server_exceptions=False) try: assert client.get("/key").status_code >= 500 finally: os.chmod(root / "key", 0o600) @pytest.mark.skipif(os.geteuid() == 0, reason="root bypasses the permission bits under test") def test_unwritable_store_is_a_server_error(self, tmp_path: pathlib.Path) -> None: root = tmp_path / "ro" root.mkdir() os.chmod(root, 0o500) client = TestClient( store_app(LocalStore(str(root)), methods={"GET", "PUT"}), raise_server_exceptions=False, ) try: assert client.put("/key", content=b"data").status_code >= 500 finally: os.chmod(root, 0o700) @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestShardGridBounds: """A sharded array's storage grid is its shard grid, not its chunk grid.""" def test_out_of_shard_grid_key_returns_404(self, store: Store) -> None: arr = zarr.create_array(store, shape=(8, 8), chunks=(2, 2), shards=(4, 4), dtype="i4") arr[:] = np.arange(64, dtype="i4").reshape(8, 8) # (8,8) with (4,4) shards has a 2x2 shard grid, so c/3/3 is out of it. # Plant data there so the 404 must come from the bounds check. sync(store.set("c/3/3", cpu.buffer_prototype.buffer.from_bytes(b"PLANTED"))) assert sync(store.get("c/3/3", cpu.buffer_prototype)) is not None client = TestClient(node_app(arr)) assert client.get("/c/0/0").status_code == 200 assert client.get("/c/3/3").status_code == 404 @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestWrongArityChunkKeys: """A chunk key with the wrong number of coordinates is invalid, and must not reach the grid comparison -- zip(strict=True) would raise there.""" @pytest.mark.parametrize("key", ["c/0", "c/0/0/0", "c/0/0/0/0"]) def test_wrong_arity_returns_404(self, store: Store, key: str) -> None: arr = zarr.create_array(store, shape=(4, 4), chunks=(2, 2), dtype="f8") arr[:] = np.ones((4, 4)) sync(store.set(key, cpu.buffer_prototype.buffer.from_bytes(b"PLANTED"))) client = TestClient(node_app(arr), raise_server_exceptions=False) assert client.get(f"/{key}").status_code == 404 @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestNodeNamesContainingAColon: """A leading `:` is a drive reference to `ntpath`, so the guard rejects it on every platform to keep it a pure string gate in front of any store. The documented cost is that such a name is unreachable in the first segment -- but only there.""" def test_colon_named_node_in_a_later_segment_is_served(self, store: Store) -> None: root = zarr.open_group(store, mode="w") sub = root.create_group("sub") sub.create_array("a:b", shape=(2,), chunks=(2,), dtype="f8") client = TestClient(node_app(root)) assert client.get("/sub/a:b/zarr.json").status_code == 200 def test_colon_named_node_in_the_first_segment_is_rejected(self, store: Store) -> None: root = zarr.open_group(store, mode="w") root.create_array("a:b", shape=(2,), chunks=(2,), dtype="f8") client = TestClient(node_app(root)) assert client.get("/a:b/zarr.json").status_code == 404 class TestChunkedBodyIsCapped: """A chunked request carries no Content-Length, so the cap has to hold while the body is being read rather than after it is buffered.""" def test_chunked_body_over_cap_is_rejected(self, tmp_path: pathlib.Path) -> None: store = LocalStore(str(tmp_path / "root")) client = TestClient( store_app(store, methods={"GET", "PUT"}, max_body_size=64), raise_server_exceptions=False, ) def body() -> Iterator[bytes]: for _ in range(20): yield b"x" * 32 # httpx sends an iterator body with Transfer-Encoding: chunked. assert client.put("/key", content=body()).status_code == 413 assert not (tmp_path / "root" / "key").exists() class TestHostileKeysAreNotServerErrors: """A key the store cannot express is a miss, not a server fault: the server must never answer 5xx for input a client can choose freely. This needs a filesystem-backed store -- a `MemoryStore` accepts any key as a dict key, so only `LocalStore` surfaces the underlying errors (an embedded NUL, a name longer than the filesystem allows). """ @pytest.mark.parametrize( "path", ["/x%00y", "/ok.txt%00", "/" + "a" * 3000, "/" + "b" * 3000 + "/zarr.json"] ) def test_unexpressable_key_returns_404_not_500(self, tmp_path: pathlib.Path, path: str) -> None: store = LocalStore(str(tmp_path / "root")) client = TestClient(store_app(store, methods={"GET", "PUT"})) assert client.get(path).status_code == 404 assert client.put(path, content=b"x").status_code == 404 class TestGenericStoreFailuresAreNotMisses: """Only an error that answers about the *name* may become a 404. `ENAMETOOLONG` says no such name is expressible, which is an answer about the key. `EINVAL` is POSIX's catch-all and is reachable on a perfectly ordinary short key -- a bad seek, an unsupported filesystem feature -- so reporting it as absence would have a v3 reader write fill values over a chunk that exists but could not be read. """ @staticmethod def _store_failing_with(code: int) -> Store: class Failing(MemoryStore): async def get(self, key: str, prototype: Any, byte_range: Any = None) -> Any: raise OSError(code, os.strerror(code)) async def set(self, key: str, value: Any) -> None: raise OSError(code, os.strerror(code)) return Failing() def test_einval_is_not_reported_as_absent(self) -> None: """The regression this class exists for.""" client = TestClient( store_app(self._store_failing_with(errno.EINVAL), methods={"GET", "PUT"}), raise_server_exceptions=False, ) assert client.get("/c/0/0").status_code >= 500 assert client.put("/c/0/0", content=b"data").status_code >= 500 def test_enametoolong_is_still_a_miss(self) -> None: """A name the store cannot express holds nothing, so 404 is honest.""" client = TestClient( store_app(self._store_failing_with(errno.ENAMETOOLONG), methods={"GET", "PUT"}), raise_server_exceptions=False, ) assert client.get("/c/0/0").status_code == 404 assert client.put("/c/0/0", content=b"data").status_code == 404 @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestPutBodyLimit: """`Store.set` takes a whole buffer, so an unbounded body would let one request size the server's memory use.""" def test_body_over_the_limit_is_rejected(self, store: Store) -> None: client = TestClient(store_app(store, methods={"GET", "PUT"}, max_body_size=64)) assert client.put("/key", content=b"x" * 65).status_code == 413 # Nothing was written. assert sync(store.get("key", cpu.buffer_prototype)) is None # A body within the limit still succeeds. assert client.put("/key", content=b"x" * 64).status_code == 204 def test_limit_can_be_lifted(self, store: Store) -> None: client = TestClient(store_app(store, methods={"GET", "PUT"}, max_body_size=None)) assert client.put("/key", content=b"x" * 5000).status_code == 204 @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestRangeResponseCorrectness: """A 206 must describe which bytes it carries, and a range that cannot be satisfied must say so rather than returning an empty 206.""" def test_206_carries_content_range(self, store: Store) -> None: sync(store.set("key", cpu.buffer_prototype.buffer.from_bytes(b"0123456789"))) client = TestClient(store_app(store)) response = client.get("/key", headers={"Range": "bytes=2-5"}) assert response.status_code == 206 assert response.content == b"2345" assert response.headers["Content-Range"] == "bytes 2-5/*" def test_range_beyond_end_is_416(self, store: Store) -> None: sync(store.set("key", cpu.buffer_prototype.buffer.from_bytes(b"0123456789"))) client = TestClient(store_app(store)) assert client.get("/key", headers={"Range": "bytes=1000-2000"}).status_code == 416 def test_inverted_range_is_416(self, store: Store) -> None: sync(store.set("key", cpu.buffer_prototype.buffer.from_bytes(b"0123456789"))) client = TestClient(store_app(store)) assert client.get("/key", headers={"Range": "bytes=5-2"}).status_code == 416 @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestZeroDimensionalArray: """A 0-d array has exactly one chunk, spelled `0` in v2 and `c` in v3.""" @pytest.mark.parametrize("zarr_format", [2, 3]) def test_sole_chunk_is_served(self, store: Store, zarr_format: ZarrFormat) -> None: arr = zarr.create_array(store, shape=(), dtype="i4", zarr_format=zarr_format) arr[...] = 7 client = TestClient(node_app(arr)) chunk_key = "0" if zarr_format == 2 else "c" assert client.get(f"/{chunk_key}").status_code == 200 # A 1-d coordinate is not valid for a 0-d grid. assert client.get("/0/0").status_code == 404 @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestGroupAndArrayMetadataKeysAreDistinct: """A v2 group owns `.zgroup`; `.zarray` belongs to arrays, and vice versa.""" def test_node_does_not_claim_the_other_kind_of_metadata(self, store: Store) -> None: root = zarr.open_group(store, mode="w", zarr_format=2) arr = root.create_array("a", shape=(2,), chunks=(2,), dtype="f8") # Plant both documents so a 404 reflects the key set, not absence. sync(store.set(".zarray", cpu.buffer_prototype.buffer.from_bytes(b"{}"))) sync(store.set("a/.zgroup", cpu.buffer_prototype.buffer.from_bytes(b"{}"))) assert TestClient(node_app(root)).get("/.zarray").status_code == 404 assert TestClient(node_app(arr)).get("/.zgroup").status_code == 404 @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestBackgroundServerReportsBoundPort: """`port=0` asks the OS for a free port, so the server must report the port it actually bound rather than the zero it was asked for.""" def test_port_zero_reports_the_bound_port(self, store: Store) -> None: import httpx from zarr_http_server import serve_background sync(store.set("key", cpu.buffer_prototype.buffer.from_bytes(b"hello"))) with serve_background(store_app(store), host="127.0.0.1", port=0) as server: assert server.port != 0 assert server.url == f"http://127.0.0.1:{server.port}" # The reported URL is the one that actually serves the data. assert httpx.get(f"{server.url}/key").content == b"hello" @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestUnopenableChildIsAnError: """A child that cannot be *judged* must not be reported as absent. 404 is a claim about the store's contents, and under the v3 spec an absent chunk is an uninitialized one -- so a correct reader answers 404 by silently substituting the array's fill value. Returning it for a child whose metadata could not be read would materialize zeros over data that exists. Only a genuinely missing member is a 404; corrupt metadata, an I/O error, or a codec this process lacks all surface as 5xx. """ def test_child_with_unparseable_metadata_is_not_reported_as_missing(self, store: Store) -> None: """A corrupt child metadata document must not yield 404.""" root = zarr.open_group(store, mode="w") root.create_array("good", shape=(2,), chunks=(2,), dtype="f8") sync(store.set("junk/zarr.json", cpu.buffer_prototype.buffer.from_bytes(b"not json"))) client = TestClient(node_app(root), raise_server_exceptions=False) assert client.get("/good/zarr.json").status_code == 200 assert client.get("/junk/zarr.json").status_code >= 500 assert client.get("/junk/c/0").status_code >= 500 def test_absent_child_is_reported_as_missing(self, store: Store) -> None: """A member that simply is not there is still a plain 404.""" root = zarr.open_group(store, mode="w") root.create_array("good", shape=(2,), chunks=(2,), dtype="f8") client = TestClient(node_app(root), raise_server_exceptions=False) assert client.get("/nope/zarr.json").status_code == 404 assert client.get("/junk/c/0").status_code == 404 @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestReadBackWithZarrClient: """The round-trip the README leads with: serve an array, then open it with a zarr client over HTTP. This needs an HTTP-capable fsspec, which is a client-side concern that `zarr-http-server` deliberately does not depend on -- it lives in the `docs` dependency group, alongside the other deps the README examples need. """ def test_served_array_reads_back_identically(self, store: Store) -> None: """`zarr.open_array(server.url)` should return the same data that was served, so the README's headline example stays true.""" pytest.importorskip("fsspec") pytest.importorskip("aiohttp") from zarr_http_server import serve_background expected = np.arange(100, dtype="uint8").reshape(10, 10) arr = zarr.create_array(store, data=expected, chunks=(5, 5), write_data=True) port = _get_free_port() with serve_background(node_app(arr), host="127.0.0.1", port=port) as server: remote = zarr.open_array(server.url, mode="r") np.testing.assert_array_equal(remote[:], expected) class TestBackgroundServerBoundedShutdown: """BackgroundServer.shutdown() must not hang forever on a slow or stuck in-flight request.""" def test_shutdown_returns_promptly_with_slow_inflight_request(self) -> None: """A request that takes far longer than shutdown_timeout must not prevent shutdown() from returning within roughly shutdown_timeout, via uvicorn's force_exit rather than an unbounded thread join.""" import asyncio import threading import time import httpx from starlette.applications import Starlette from starlette.responses import Response from starlette.routing import Route async def slow(request: Any) -> Response: # Sleeps far longer than shutdown_timeout below, so a correct # implementation must force the connection closed rather than # wait for this to finish. await asyncio.sleep(SLOW_HANDLER_SECONDS) return Response(status_code=204) app = Starlette(routes=[Route("/slow", slow, methods=["GET"])]) port = _get_free_port() server = serve_background( app, host="127.0.0.1", port=port, shutdown_timeout=SHUTDOWN_TIMEOUT ) assert server is not None request_errors: list[BaseException] = [] def make_slow_request() -> None: try: httpx.get(f"http://127.0.0.1:{port}/slow", timeout=10) except Exception as exc: # noqa: BLE001 -- connection drop when the server force-closes is expected request_errors.append(exc) request_thread = threading.Thread(target=make_slow_request, daemon=True) request_thread.start() time.sleep(0.2) # give the request time to actually start start = time.monotonic() server.shutdown() elapsed = time.monotonic() - start # The property is that shutdown is *bounded*, not that it hits a # particular wall-clock number. Derive the bound from the timeouts # that produce it rather than hard-coding one: shutdown() waits # `shutdown_timeout + _SHUTDOWN_JOIN_MARGIN` for a graceful stop, then # `shutdown_timeout` more after force_exit. A literal here silently # loses its headroom whenever one of those constants changes -- which # is what happened when the margin was introduced. bound = (SHUTDOWN_TIMEOUT + _SHUTDOWN_JOIN_MARGIN) + SHUTDOWN_TIMEOUT + 1.0 assert elapsed < bound, f"shutdown() took {elapsed:.2f}s, expected under {bound:.1f}s" # ...and the point of it all: far less than the handler's own sleep, # which an unbounded join would have waited out in full. assert elapsed < SLOW_HANDLER_SECONDS request_thread.join(timeout=10) @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestCorsOptionsCoverTheMiddleware: """`CorsOptions` exposes every `CORSMiddleware` parameter, with our own defaults only for the two the server knows about.""" _ORIGIN = "https://viewer.example" def _client(self, store: Store, cors: CorsOptions) -> TestClient: return TestClient(store_app(store, methods={"GET"}, cors_options=cors)) def test_ranged_response_is_readable_cross_origin(self, store: Store) -> None: """`Content-Range` is not CORS-safelisted, so without `expose_headers` a browser reads the bytes but cannot learn which bytes it got.""" sync(store.set("k", cpu.buffer_prototype.buffer.from_bytes(b"0123456789"))) client = self._client(store, {"allow_origins": [self._ORIGIN], "allow_methods": ["GET"]}) response = client.get("/k", headers={"Origin": self._ORIGIN, "Range": "bytes=0-3"}) assert response.status_code == 206 assert response.headers["access-control-expose-headers"] == "Content-Range" def test_range_survives_a_preflight(self, store: Store) -> None: """A preflight naming `Range` must be allowed, not answered 400.""" client = self._client(store, {"allow_origins": [self._ORIGIN], "allow_methods": ["GET"]}) preflight = client.options( "/k", headers={ "Origin": self._ORIGIN, "Access-Control-Request-Method": "GET", "Access-Control-Request-Headers": "range", }, ) assert preflight.status_code == 200 assert "Range" in preflight.headers["access-control-allow-headers"] def test_caller_value_replaces_the_default(self, store: Store) -> None: """Our defaults apply only to absent keys; a supplied key wins outright so `expose_headers: []` means "expose nothing", not "expose ours".""" sync(store.set("k", cpu.buffer_prototype.buffer.from_bytes(b"data"))) base: CorsOptions = {"allow_origins": [self._ORIGIN], "allow_methods": ["GET"]} empty = self._client(store, {**base, "expose_headers": []}) assert ( "access-control-expose-headers" not in empty.get("/k", headers={"Origin": self._ORIGIN}).headers ) custom = self._client(store, {**base, "expose_headers": ["X-Custom"]}) assert ( custom.get("/k", headers={"Origin": self._ORIGIN}).headers[ "access-control-expose-headers" ] == "X-Custom" ) def test_parameters_beyond_the_original_two_are_reachable(self, store: Store) -> None: """The regression this class exists for: `CorsOptions` used to carry only `allow_origins` and `allow_methods`, sealing the rest away.""" sync(store.set("k", cpu.buffer_prototype.buffer.from_bytes(b"data"))) client = self._client( store, { "allow_origin_regex": r"https://.*\.example", "allow_credentials": True, "max_age": 30, }, ) origin = "https://sub.example" response = client.get("/k", headers={"Origin": origin}) assert response.headers["access-control-allow-origin"] == origin assert response.headers["access-control-allow-credentials"] == "true" preflight = client.options( "/k", headers={"Origin": origin, "Access-Control-Request-Method": "GET"} ) assert preflight.headers["access-control-max-age"] == "30" @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestUvicornOptionsAreNotSealedOff: """`uvicorn.Config` takes ~50 parameters; naming four of them and dropping the rest would put TLS, proxy headers, `root_path` and log level out of reach entirely.""" def test_options_reach_uvicorn_config(self, store: Store) -> None: """A key this signature does not name still lands on the Config.""" from zarr_http_server import serve_background server = serve_background( store_app(store), host="127.0.0.1", port=0, uvicorn_options={"root_path": "/api", "log_level": "warning"}, ) assert server is not None try: config = server._server.config assert config.root_path == "/api" assert config.log_level == "warning" # Ours still apply where the caller did not override them. assert config.timeout_graceful_shutdown == 5 finally: server.shutdown() def test_caller_options_win_over_ours(self, store: Store) -> None: """The merge order is ours-then-theirs, so a caller can override even an option this signature sets itself.""" from zarr_http_server import serve_background server = serve_background( store_app(store), host="127.0.0.1", port=0, shutdown_timeout=5, uvicorn_options={"timeout_graceful_shutdown": 11}, ) assert server is not None try: assert server._server.config.timeout_graceful_shutdown == 11 finally: server.shutdown() @pytest.mark.skipif(not hasattr(socket, "AF_UNIX"), reason="needs unix domain sockets") def test_non_tcp_bind_reports_no_url(self, store: Store, tmp_path: pathlib.Path) -> None: """A unix-socket bind has no host and port, so `url` must say so rather than naming an address nothing is listening on.""" import httpx from zarr_http_server import serve_background sock = str(tmp_path / "s.sock") server = serve_background(store_app(store), uvicorn_options={"uds": sock}) assert server is not None try: assert server.url is None assert server.host is None assert server.port is None # ...and it really is serving, just not over TCP. with httpx.Client(transport=httpx.HTTPTransport(uds=sock)) as client: response = client.get("http://localhost/zarr.json", timeout=10) assert response.status_code in (200, 404) finally: server.shutdown() class TestHeadDoesNotTransferTheBody: """A HEAD body is discarded at the wire, so building one is pure waste.""" def test_head_does_not_read_the_value(self, tmp_path: pathlib.Path) -> None: """The regression this class exists for: HEAD used to fall through to the GET handler and pull the whole object to report its length.""" read = {"bytes": 0} class CountingLocal(LocalStore): async def get(self, key: str, prototype: Any, byte_range: Any = None) -> Any: buf = await super().get(key, prototype, byte_range) if buf is not None: read["bytes"] += len(buf) return buf store = CountingLocal(str(tmp_path / "root")) payload = b"x" * 100_000 sync(store.set("big", cpu.buffer_prototype.buffer.from_bytes(payload))) client = TestClient(store_app(store)) read["bytes"] = 0 response = client.head("/big") assert response.status_code == 200 assert response.headers["content-length"] == str(len(payload)) assert read["bytes"] == 0, "HEAD read the value to report its length" @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_head_of_a_missing_key_is_404(self, store: Store) -> None: client = TestClient(store_app(store)) assert client.head("/nope").status_code == 404 @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestMetadataContentType: """Metadata documents are JSON in every zarr format, not just v3.""" @pytest.mark.parametrize( ("zarr_format", "key"), [(3, "zarr.json"), (2, ".zarray"), (2, ".zattrs")] ) def test_metadata_is_served_as_json( self, store: Store, zarr_format: ZarrFormat, key: str ) -> None: zarr.create_array( store, name="a", shape=(4,), chunks=(2,), dtype="i4", zarr_format=zarr_format ) sync(store.set(f"a/{key}", cpu.buffer_prototype.buffer.from_bytes(b"{}"))) response = TestClient(store_app(store)).get(f"/a/{key}") assert response.status_code == 200 assert response.headers["content-type"].startswith("application/json") @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestCorsAllowMethodsMatchTheRoute: """Advertising a method the route rejects is a promise the server cannot keep: the browser caches the preflight and every later call 405s.""" def test_wildcard_expands_to_what_is_served(self, store: Store) -> None: """`"*"` is the idiomatic "everything this app does", so it expands to exactly that rather than to every verb Starlette knows.""" app = store_app( store, methods={"GET"}, cors_options={"allow_origins": ["*"], "allow_methods": ["*"]} ) preflight = TestClient(app).options( "/k", headers={"Origin": "https://e.test", "Access-Control-Request-Method": "GET"} ) advertised = preflight.headers["access-control-allow-methods"] assert set(advertised.replace(" ", "").split(",")) == {"GET", "HEAD"} def test_advertising_an_unserved_method_is_rejected(self, store: Store) -> None: with pytest.raises(ValueError, match="does not serve"): store_app( store, methods={"GET"}, cors_options={"allow_origins": ["*"], "allow_methods": ["GET", "DELETE"]}, ) def test_head_counts_as_served_when_get_is(self, store: Store) -> None: """Starlette routes HEAD wherever GET goes, so naming it is not an error.""" store_app( store, methods={"GET"}, cors_options={"allow_origins": ["*"], "allow_methods": ["GET", "HEAD"]}, ) def test_absent_allow_methods_is_left_alone(self, store: Store) -> None: """Starlette's GET-only default stands; widening it to everything served would newly advertise PUT on a write-enabled app.""" app = store_app(store, methods={"GET", "PUT"}, cors_options={"allow_origins": ["*"]}) preflight = TestClient(app).options( "/k", headers={"Origin": "https://e.test", "Access-Control-Request-Method": "GET"} ) assert "PUT" not in preflight.headers["access-control-allow-methods"] class TestReadOnlyServing: """The guarantees a read-only deployment rests on. Two independent layers: `methods` decides what the route answers, and the store decides whether a write could succeed at all. The second is the one that survives a misconfiguration of the first, so both are pinned here. """ @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize("method", ["PUT", "POST", "DELETE", "PATCH"]) def test_default_app_refuses_every_mutating_method(self, store: Store, method: str) -> None: """The default is read-only: no argument is needed to get there, and nothing a client sends can write.""" sync(store.set("k", cpu.buffer_prototype.buffer.from_bytes(b"data"))) before = sync(store.get("k", cpu.buffer_prototype)).to_bytes() client = TestClient(store_app(store), raise_server_exceptions=False) response = client.request(method, "/k", content=b"overwritten") assert response.status_code == 405 assert sync(store.get("k", cpu.buffer_prototype)).to_bytes() == before @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_post_can_never_be_enabled(self, store: Store) -> None: """POST is not merely unrouted, it is unconfigurable: there is no handler behavior for it, so asking is an error rather than a no-op.""" with pytest.raises(ValueError, match="Unsupported HTTP method"): store_app(store, methods={"GET", "POST"}) # type: ignore[arg-type] def test_read_only_store_refuses_writes_independently_of_methods( self, tmp_path: pathlib.Path ) -> None: """The layer that survives getting `methods` wrong. Constructed through the private builder because the public entry points now reject this combination outright; the handler check stays as the backstop for a store whose `read_only` is not fixed. """ from zarr_http_server._serve import _make_starlette_app writable = LocalStore(str(tmp_path / "root")) sync(writable.set("k", cpu.buffer_prototype.buffer.from_bytes(b"data"))) app = _make_starlette_app(methods={"GET", "PUT"}) app.state.store = writable.with_read_only(True) app.state.node = None app.state.prefix = "" app.state.max_body_size = None response = TestClient(app, raise_server_exceptions=False).put("/k", content=b"x") assert response.status_code == 403 assert sync(writable.get("k", cpu.buffer_prototype)).to_bytes() == b"data" def test_put_on_a_read_only_store_is_rejected_at_construction( self, tmp_path: pathlib.Path ) -> None: """A write that could never succeed is a configuration error, not a runtime 403 delivered to whoever happens to try first.""" store = LocalStore(str(tmp_path / "root")).with_read_only(True) with pytest.raises(ValueError, match="store is read-only"): store_app(store, methods={"GET", "PUT"}) def test_read_only_node_is_rejected_at_construction(self, tmp_path: pathlib.Path) -> None: """The same check applies to a node, whose store it inherits.""" store = LocalStore(str(tmp_path / "root")) zarr.create_array(store, shape=(4,), chunks=(2,), dtype="i4", compressors=None) read_only_array = zarr.open_array(store, mode="r") with pytest.raises(ValueError, match="store is read-only"): node_app(read_only_array, methods={"GET", "PUT"}) class TestMethodSetConstants: """Named method sets let a call site state its intent, and make writable deployments findable: every writable app must name one.""" @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_read_only_constant_matches_the_default(self, store: Store) -> None: """Passing it explicitly and omitting `methods` are the same server, so saying so out loud costs nothing.""" sync(store.set("k", cpu.buffer_prototype.buffer.from_bytes(b"data"))) default = TestClient(store_app(store), raise_server_exceptions=False) named = TestClient( store_app(store, methods=READ_ONLY_HTTP_METHODS), raise_server_exceptions=False ) for method in ["GET", "HEAD", "PUT", "POST", "DELETE", "PATCH"]: assert default.request(method, "/k").status_code == ( named.request(method, "/k").status_code ) @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize("method", ["PUT", "POST", "DELETE", "PATCH"]) def test_read_only_constant_refuses_writes(self, store: Store, method: str) -> None: sync(store.set("k", cpu.buffer_prototype.buffer.from_bytes(b"data"))) client = TestClient( store_app(store, methods=READ_ONLY_HTTP_METHODS), raise_server_exceptions=False ) assert client.request(method, "/k", content=b"x").status_code == 405 assert sync(store.get("k", cpu.buffer_prototype)).to_bytes() == b"data" @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_read_write_constant_permits_exactly_put(self, store: Store) -> None: """It grants writes -- and still not POST, which has no handler.""" client = TestClient( store_app(store, methods=READ_WRITE_HTTP_METHODS), raise_server_exceptions=False ) assert client.put("/k", content=b"data").status_code == 204 assert client.post("/k", content=b"data").status_code == 405 assert sync(store.get("k", cpu.buffer_prototype)).to_bytes() == b"data" def test_constants_cannot_be_mutated_by_a_caller(self) -> None: """Frozen, so one caller cannot widen the default for every other.""" assert isinstance(READ_ONLY_HTTP_METHODS, frozenset) assert isinstance(READ_WRITE_HTTP_METHODS, frozenset) assert "PUT" not in READ_ONLY_HTTP_METHODS def test_constants_match_the_types_they_model(self) -> None: """The sets are derived from the Literals, so they cannot disagree about what this server serves. Pinning the contents here makes widening either type a deliberate, visible edit.""" assert frozenset(get_args(ReadOnlyHTTPMethod)) == READ_ONLY_HTTP_METHODS assert set(READ_ONLY_HTTP_METHODS) == {"GET", "HEAD"} assert set(READ_WRITE_HTTP_METHODS) == {"GET", "HEAD", "PUT"} # Read-only is a strict subset: the only difference is the write verb. assert READ_ONLY_HTTP_METHODS < READ_WRITE_HTTP_METHODS assert {"PUT"} == READ_WRITE_HTTP_METHODS - READ_ONLY_HTTP_METHODS class TestServeAnyApp: """`serve` runs whatever ASGI app it is handed, which is what makes several nodes on one port possible.""" @staticmethod def _two_mounted_arrays() -> tuple[Starlette, bytes, bytes]: """Two arrays in *separate* stores, so no common parent exists and mounting is the only way to serve both from one server.""" first, second = MemoryStore(), MemoryStore() one = zarr.create_array(first, shape=(4,), chunks=(2,), dtype="i4", compressors=None) other = zarr.create_array(second, shape=(4,), chunks=(2,), dtype="i4", compressors=None) one[:] = 7 other[:] = 9 app = Starlette( routes=[Mount("/first", app=node_app(one)), Mount("/second", app=node_app(other))] ) return app, np.full(2, 7, dtype="i4").tobytes(), np.full(2, 9, dtype="i4").tobytes() def test_mounted_apps_each_serve_their_own_node(self) -> None: app, first_chunk, second_chunk = self._two_mounted_arrays() client = TestClient(app, raise_server_exceptions=False) assert client.get("/first/zarr.json").status_code == 200 assert client.get("/second/zarr.json").status_code == 200 assert client.get("/first/c/0").content == first_chunk assert client.get("/second/c/0").content == second_chunk @pytest.mark.parametrize( "path", [ "/first/%2e%2e/second/zarr.json", "/first/..%2f..%2fsecond/zarr.json", "/first/%2e%2e%2fsecond/c/0", "/first/%2fsecond/zarr.json", ], ) def test_one_mount_cannot_reach_another(self, path: str) -> None: """Per-node validation runs inside each mount, so composing apps does not widen what any of them serves.""" app, _, _ = self._two_mounted_arrays() assert TestClient(app, raise_server_exceptions=False).get(path).status_code == 404 def test_serve_runs_a_composed_app_in_the_background(self) -> None: """The gap `serve` closes: a composed app previously had no way to use the background-server ergonomics, only a blocking `uvicorn.run`.""" import httpx app, first_chunk, second_chunk = self._two_mounted_arrays() server = serve_background(app, host="127.0.0.1", port=0) try: assert server.url is not None assert httpx.get(f"{server.url}/first/c/0", timeout=30).content == first_chunk assert httpx.get(f"{server.url}/second/c/0", timeout=30).content == second_chunk finally: server.shutdown() class TestPortSelection: """`port="auto"` prefers a predictable port but never fails over one. An explicit port means the opposite -- bind exactly that or fail -- because a caller who names one usually has something else expecting the server there, and silently moving would break it while looking healthy. """ @staticmethod def _free_port() -> int: """A port that was free a moment ago. Only ever used as the *preferred* port, never bound afterwards, so the usual bind-then-close race does not apply: if something takes it, that is the case under test.""" with socket.socket() as sock: sock.bind(("127.0.0.1", 0)) return int(sock.getsockname()[1]) def test_preferred_port_is_used_when_free(self) -> None: preferred = self._free_port() sock = _bind_preferred_or_free("127.0.0.1", preferred) try: assert sock.getsockname()[1] == preferred finally: sock.close() def test_falls_back_when_the_preferred_port_is_taken(self) -> None: with socket.socket() as squatter: squatter.bind(("127.0.0.1", 0)) squatter.listen() taken = int(squatter.getsockname()[1]) sock = _bind_preferred_or_free("127.0.0.1", taken) try: assert sock.getsockname()[1] != taken finally: sock.close() def test_address_family_follows_the_host(self) -> None: """Hard-coding AF_INET would bind the wrong family for an IPv6 host.""" try: sock = _bind_preferred_or_free("::1", 0) except OSError: # pragma: no cover - depends on the host's networking pytest.skip("no IPv6 loopback available") try: assert sock.family == socket.AF_INET6 finally: sock.close() @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_auto_produces_a_working_server(self, store: Store) -> None: """Whichever port it lands on, `url` names it and the server answers.""" import httpx server = serve_background(store_app(store)) try: assert server.url is not None assert server.port is not None assert httpx.get(f"{server.url}/nope", timeout=30).status_code == 404 finally: server.shutdown() # uvicorn answers a failed bind with `sys.exit` on its own thread, which # pytest reports as an unhandled thread exception -- and this package turns # warnings into errors. That exit is exactly what the RuntimeError below # reports to the caller, so it is expected here rather than a defect. @pytest.mark.filterwarnings("ignore::pytest.PytestUnhandledThreadExceptionWarning") @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_an_explicit_port_that_is_taken_fails(self, store: Store) -> None: """The regression this class exists for: `auto` must not leak into the explicit case, where a collision has to be loud.""" with socket.socket() as squatter: squatter.bind(("127.0.0.1", 0)) squatter.listen() taken = int(squatter.getsockname()[1]) with pytest.raises(RuntimeError, match="may already be in use"): serve_background(store_app(store), port=taken) @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_port_zero_still_means_any_free_port(self, store: Store) -> None: """`0` keeps its OS meaning rather than being folded into `auto`.""" server = serve_background(store_app(store), port=0) try: assert server.port not in (0, None) finally: server.shutdown() zarr-developers-zarr-python-a2f2415/packages/zarr-http-server/uv.lock000066400000000000000000014745221527014137600260250ustar00rootroot00000000000000version = 1 revision = 3 requires-python = ">=3.12" [[package]] name = "aiohappyeyeballs" version = "2.7.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ce/f4/eec0465c2f67b2664688d0240b3212d5196fd89e741df67ddb81f8d35658/aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d", size = 24757, upload-time = "2026-07-01T17:11:55.501Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472", size = 15038, upload-time = "2026-07-01T17:11:54.055Z" }, ] [[package]] name = "aiohttp" version = "3.14.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohappyeyeballs" }, { name = "aiosignal" }, { name = "attrs" }, { name = "frozenlist" }, { name = "multidict" }, { name = "propcache" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, { name = "yarl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/93/2f/6a91adaa2dc26877d6ed2f54c0370c8910f019db7d77c5c6a194611e93ea/aiohttp-3.14.4.tar.gz", hash = "sha256:831fc5bd39ec2517851e348f613ddb5447a47cf4b71cb09845af7ad7ed45d8f9", size = 8077839, upload-time = "2026-10-05T17:43:48.309Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/22/02/db5935d45347fa8ede56b777930740fb0e72bbb4473ddb6ff8541f6f2afa/aiohttp-3.14.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:55b0b335982b0db7117dff7f6a8e3aad43c4105f9e392ca04f9fc8d958a95cce", size = 817689, upload-time = "2026-10-05T17:38:50.69Z" }, { url = "https://files.pythonhosted.org/packages/e6/56/f00aa46127e6342f019d567853eeeb932e04fb9b9424b5fb981ad68f4178/aiohttp-3.14.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d0d8e435582b0d53009dac99e26a3d2be932ed7c9dbfcfc70f3fce1cf943bd5", size = 549599, upload-time = "2026-10-05T17:38:52.525Z" }, { url = "https://files.pythonhosted.org/packages/3f/bc/825c09b09962253a2103e4a88c04374ee939c0c8ea31c270daa0308efb07/aiohttp-3.14.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bf734016932d68e1324cfb638cac7a0866a83933a33badf15e836a7e24efb8c0", size = 548132, upload-time = "2026-10-05T17:38:54.629Z" }, { url = "https://files.pythonhosted.org/packages/40/80/8a1bc8036540fe7cb38219cc5c0aa67d32e9068165725e8199ea8200b08b/aiohttp-3.14.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:322e3d741b3133bcbb981a41d4f9b8ac3545c18de49c662c2b1f4e1c770b98ad", size = 1898485, upload-time = "2026-10-05T17:38:56.439Z" }, { url = "https://files.pythonhosted.org/packages/3c/fe/65b464d8520ace0d65b4ca69bc9abe175f842c8ce8d1522d363d9669903f/aiohttp-3.14.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:15a3bb5f90a4e515071f3617a45dfe26560beb834a6b6e108cdf4bc9e9719ec3", size = 1875334, upload-time = "2026-10-05T17:38:58.536Z" }, { url = "https://files.pythonhosted.org/packages/be/51/9c583be502720e927560a8414f4dfb7c7de41945e1a30ccf7003b93c6f85/aiohttp-3.14.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6c044e52df1a466af82cad3518c92ea80cc4ddcfea86966ff083bb34bab29bf4", size = 1937772, upload-time = "2026-10-05T17:39:00.376Z" }, { url = "https://files.pythonhosted.org/packages/a2/3d/b56512df7ea9cbb5f59099a6b375088dd578356a5a3fb27198ab82e19832/aiohttp-3.14.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d613d7d51bf06fe9a5e3aab86ce06d835bb0ceb1a31ea9e8766df4a403ae62a6", size = 2066220, upload-time = "2026-10-05T17:39:02.54Z" }, { url = "https://files.pythonhosted.org/packages/11/6c/a7a42701d9ea319140a3d44a24ec72a00b0b46d1b899b26d8a705e65bdd8/aiohttp-3.14.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5e93ab6e6330dce40f5b43996f1fc5f2543296d263a5e4b59ff09e73870b7d8", size = 1922895, upload-time = "2026-10-05T17:39:04.638Z" }, { url = "https://files.pythonhosted.org/packages/77/04/f8337bd824ca3fe3968d34dc5037ace1c4872551c4621f8266239311d301/aiohttp-3.14.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:20134dc8e68c67e7478124d516449428684c0545b2d3faa575691c8bcbd56678", size = 1712589, upload-time = "2026-10-05T17:39:06.671Z" }, { url = "https://files.pythonhosted.org/packages/fe/a6/d20e2c940b592d7c4a21690889dbff64fe0ca3facd9190696368694e1488/aiohttp-3.14.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f0daef5ef012369dddc391c61f604111424aa9a96313a69fb899018b09d1b01e", size = 1862251, upload-time = "2026-10-05T17:39:08.523Z" }, { url = "https://files.pythonhosted.org/packages/f5/e4/d50edd5ae5d024b380a8e892e333840bb22967a9eb9c8492108a403f678a/aiohttp-3.14.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:19290a108de0718b73bc69b86e788975c6a59855c671ebe2a98a07a6f968328f", size = 1895155, upload-time = "2026-10-05T17:39:10.305Z" }, { url = "https://files.pythonhosted.org/packages/c3/d3/f36f2931f807ccf8437ec15e8f8bfaa4748d4792639401cc767a866abaa0/aiohttp-3.14.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1494877625baeffab669f4a25a09ec4eea721b49e35363081a2a8e4231fe141f", size = 1906623, upload-time = "2026-10-05T17:39:12.411Z" }, { url = "https://files.pythonhosted.org/packages/5d/0a/94821b711493142bf5e11c27b0d7502776778f7792c77ecc6647357d20c1/aiohttp-3.14.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:a779ce4bf099ce4ad2cdfed62f1ca536d435f53237c54ff40172fddada24d1b5", size = 1698932, upload-time = "2026-10-05T17:39:14.446Z" }, { url = "https://files.pythonhosted.org/packages/d9/76/3d5a0d28362f0053c1c457024749911932481e8fd105fddf8305b1f93c81/aiohttp-3.14.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:0b4cec9fca876e2d4f4c6cb32b177390132d04618867689f43fefabf3f5c3980", size = 1934528, upload-time = "2026-10-05T17:39:16.354Z" }, { url = "https://files.pythonhosted.org/packages/a7/51/9784a21dc05d2e0f6acaf864f16db53dab0fbd7b0753cef168601eacb012/aiohttp-3.14.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:217707313e23aedde9bea67cf8365c7b138b549ae179121ef3f476cf67e3173a", size = 1901984, upload-time = "2026-10-05T17:39:18.299Z" }, { url = "https://files.pythonhosted.org/packages/44/4f/f98c88703e6d620243aa08473a946ca4e3756b6288690e38f0d9f9c1d007/aiohttp-3.14.4-cp312-cp312-win32.whl", hash = "sha256:18a9fb9a3f6e6e63f4d27f9adeef313a2d3c69a84d31170a844623824e722534", size = 480278, upload-time = "2026-10-05T17:39:20.206Z" }, { url = "https://files.pythonhosted.org/packages/b7/bc/8555013f84b53e4fdde423131b641af50e822c6d7d9118b24c52b31f6900/aiohttp-3.14.4-cp312-cp312-win_amd64.whl", hash = "sha256:69dcb02d33dfe415d5ee342cc63b7d320efcaf4b761701c7bd7bfb91c715481a", size = 506317, upload-time = "2026-10-05T17:39:22.706Z" }, { url = "https://files.pythonhosted.org/packages/21/3f/ca5bad7bfdaa7ddca053a044a1c7c0d4f3ad9d1f12f96dcab10bfb9c50e7/aiohttp-3.14.4-cp312-cp312-win_arm64.whl", hash = "sha256:d36b0263e7c2fbf1750b9f35d4e7e48b0442d8c45e4a89e3e74168821fb014bb", size = 487683, upload-time = "2026-10-05T17:39:24.482Z" }, { url = "https://files.pythonhosted.org/packages/23/ec/7fee140da700ec3f5aa5e0fd2319f67be5e86de197e2384d27e5e6209e0e/aiohttp-3.14.4-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:264b5a7568306590c44ae4a0237ad1715e35d447ea44ebf324e4c8fd9f78f67a", size = 539114, upload-time = "2026-10-05T17:39:26.322Z" }, { url = "https://files.pythonhosted.org/packages/4e/04/930c01ce9e186e787046433d7dc5b45b188885f8406c14b58435b4431974/aiohttp-3.14.4-cp313-cp313-android_24_x86_64.whl", hash = "sha256:17e5d7c7775d8dd8e894c7ef6ebfa26509de36868a9a6c7c4fcfaf6a1bf43d60", size = 553514, upload-time = "2026-10-05T17:39:28.104Z" }, { url = "https://files.pythonhosted.org/packages/2c/d7/983e3cea81c07e95f35d34da4d1e182b5c05489c49b7560b198a83f0da79/aiohttp-3.14.4-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:e91dc8fe9cbd052d16f7f1267e285cf8bbac4d2194bd5741f05ef201c77ed09e", size = 521381, upload-time = "2026-10-05T17:39:30.419Z" }, { url = "https://files.pythonhosted.org/packages/0e/2d/80baed11659093883f51245755105557d3253e45bd0b693d70df0059d68b/aiohttp-3.14.4-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8df6157d31703972aec9e3b93963f039ff06318301e30b007c3e941ff8a7ac42", size = 528347, upload-time = "2026-10-05T17:39:32.344Z" }, { url = "https://files.pythonhosted.org/packages/c8/18/bfda255c311096c08dab26dc65f3d219f090df37dc5f5f6345f8c3aefc1f/aiohttp-3.14.4-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:220912b351549dc736c59d104dc443fa615753102bb778a4e98f86e4d94bff6a", size = 539914, upload-time = "2026-10-05T17:39:34.265Z" }, { url = "https://files.pythonhosted.org/packages/51/fc/3b6ca6117c0c42481d719411fff7b5d3970ebd906543d3dc1ba40fe2462f/aiohttp-3.14.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2efa42b39bb3f524d5eca3760639b584afdd1143a6b047fda29910ffb75e4993", size = 813997, upload-time = "2026-10-05T17:39:36.061Z" }, { url = "https://files.pythonhosted.org/packages/b5/f6/86e3e02a9712aadc1ae0a1076bb2d1adf3d9addfe5542a28190c8f9b6d9b/aiohttp-3.14.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7e7356059c8262d2c1fd95242e2b7b70fbc6eee3135bf470049cbec3e968b5b0", size = 548058, upload-time = "2026-10-05T17:39:38.109Z" }, { url = "https://files.pythonhosted.org/packages/85/7e/6b9c2e271419d40ca76589613b3c7ab185fd3050a407bd2565b6aa46f695/aiohttp-3.14.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fac247d0cc956d732df1211ac2d8a9999ec8dc1a92a90401de9d52e4ead546f7", size = 545933, upload-time = "2026-10-05T17:39:40.193Z" }, { url = "https://files.pythonhosted.org/packages/14/fc/33d6e24a5f8ac32423230088c415e2a1c6efee23d902b62f20506631e40b/aiohttp-3.14.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e2eb0f8b03b4f2bd154ed8cf7c1a119a1ad04808c258bf58f90138c83b8c6d5", size = 1888265, upload-time = "2026-10-05T17:39:42.182Z" }, { url = "https://files.pythonhosted.org/packages/8d/e8/2eca1d468d83a514621d1280f9992d08649191db2028c9236b4edfa7c723/aiohttp-3.14.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7fbe827d27e0e369fd7bb88ceeaa5b46ea006e73e015b9dd77e3723a53162293", size = 1872300, upload-time = "2026-10-05T17:39:44.195Z" }, { url = "https://files.pythonhosted.org/packages/d6/f6/5872983de71b0214acdfb86822f5cca295b39be4832ec589fa224aee0f6c/aiohttp-3.14.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a70387146d03af9f047c629b71aa27d9594b3b31ec4c9c91e1a73d0d8bf38e0e", size = 1926213, upload-time = "2026-10-05T17:39:46.199Z" }, { url = "https://files.pythonhosted.org/packages/ee/65/ebfefa11a546c33c717af63c67868408ecf0b33b23a9d75669af95485407/aiohttp-3.14.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0d0934926fc65744e2fdd44ce68b2d79d5ee608a1e23f0596b35dd696519bb7c", size = 2056609, upload-time = "2026-10-05T17:39:48.221Z" }, { url = "https://files.pythonhosted.org/packages/69/c0/70667e82ec041f4122a17afce711bbcf94a71bec5888b69a49aed8817be2/aiohttp-3.14.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff4b366510c733974adb7a4102c094e4a6308e6f369315a4cf14d2d0a619cf82", size = 1914740, upload-time = "2026-10-05T17:39:50.304Z" }, { url = "https://files.pythonhosted.org/packages/bf/86/559e9dfea676206b20737d7ccb5b3716bff5e58d2a8d53682e5301724cd1/aiohttp-3.14.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:85fac7c99e0ac3dcbd6ae1c33774e0603bfc732ddb967a6fce4a731103c1888e", size = 1703390, upload-time = "2026-10-05T17:39:52.385Z" }, { url = "https://files.pythonhosted.org/packages/fc/91/c98897d4cf23e120b2f3980042835cb34c86bbe3c653f3eaacf4bf986349/aiohttp-3.14.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef1e9c3b4a2300023a5dc865889ce532cfaf6798ba4ef78fbbefe738a9d23efa", size = 1852642, upload-time = "2026-10-05T17:39:54.391Z" }, { url = "https://files.pythonhosted.org/packages/14/13/0258f352cb2c236f358a5b57d138e15fa2566e17accc0758d968dbf2ceb6/aiohttp-3.14.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:97e65916588be0f952b3307aaa8460c0771fa98dbea2dd27dabc9f5d84d53c3b", size = 1890616, upload-time = "2026-10-05T17:39:56.466Z" }, { url = "https://files.pythonhosted.org/packages/64/3c/f43e294ce1a06d1234a97bd862c1aa31acfa1a0eea3786c22fd0dc1ea731/aiohttp-3.14.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:840775cb39a9424f9edf142133a1192263ac5bd79117ad2a16602cef6a0c1e9d", size = 1896638, upload-time = "2026-10-05T17:39:58.585Z" }, { url = "https://files.pythonhosted.org/packages/9f/f7/ce1e3a85b73d05037d729d3d32da7d6959b9e433435e9abd46c2fb38410f/aiohttp-3.14.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:54f9256430ca040d58affbd551513de13a5cff7d065204fc52b9a1b0af09a049", size = 1693003, upload-time = "2026-10-05T17:40:00.698Z" }, { url = "https://files.pythonhosted.org/packages/1d/72/45eb900d6a9ee046728f5589f3d07f5d78f049b5fb84f48881fa4174d5df/aiohttp-3.14.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:6112d5931bf12c776188cab81b82bbb17d86e2e1381f2ade6edc43e6b240f0cb", size = 1927730, upload-time = "2026-10-05T17:40:02.943Z" }, { url = "https://files.pythonhosted.org/packages/b2/43/a83ff79dc8375953a1f00b808a21c82590f4e68304fecccdcc42a25fb7d8/aiohttp-3.14.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:facc6df39047c4732cbd885bb62e40df20885fcd709fc399e11f81b45ce0fff9", size = 1891177, upload-time = "2026-10-05T17:40:04.989Z" }, { url = "https://files.pythonhosted.org/packages/5c/ba/fbdc41608d4e2fb6084d960a41b392fa82beb5b126c2466de549b5c67fc8/aiohttp-3.14.4-cp313-cp313-win32.whl", hash = "sha256:7968634fa3a967a2bc0b9aeba1d5a1955501dcd7d0de44b96f7ebfb662797735", size = 479943, upload-time = "2026-10-05T17:40:07.402Z" }, { url = "https://files.pythonhosted.org/packages/35/51/d75629750e705832fd5cd52516a82943ed58b6e09c30e14830f5d87452fa/aiohttp-3.14.4-cp313-cp313-win_amd64.whl", hash = "sha256:c0a894b0265d139cd4c2c8bd4643822cc289a4a5331e21e810881bd000ffb2dc", size = 506209, upload-time = "2026-10-05T17:40:09.49Z" }, { url = "https://files.pythonhosted.org/packages/02/64/66452d2ffcc13e5e5ee4ba4a968656fa4a98c29ed5ff0205d4fde02b2943/aiohttp-3.14.4-cp313-cp313-win_arm64.whl", hash = "sha256:da16c3037178e72589d91de59536d271dffc92c2be2b47c38a45538cba54fd29", size = 487163, upload-time = "2026-10-05T17:40:11.526Z" }, { url = "https://files.pythonhosted.org/packages/d2/fd/b653b1f969e2e949fe4726af7a8043e8ef117f202bed7757ac8564e077a1/aiohttp-3.14.4-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:749bfc43bb1df71bbe6ed215044d5412d786698fc5771d2037436ca49622f3c9", size = 540738, upload-time = "2026-10-05T17:40:14.109Z" }, { url = "https://files.pythonhosted.org/packages/90/b5/eea49a0bb697deffac5165d707ec53a24748d6720b3af77a544107b49573/aiohttp-3.14.4-cp314-cp314-android_24_x86_64.whl", hash = "sha256:c113e4b489d711f606ca278cd8dc89179746244ea841aa0638b1b134600afad0", size = 554494, upload-time = "2026-10-05T17:40:16.231Z" }, { url = "https://files.pythonhosted.org/packages/11/ea/cf8f4efbbfa8572d9965cd43b3c6c84c581aa006772e5ce3ab4020027e9b/aiohttp-3.14.4-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:50969522bc7022026a2c7968e67da5c789f6b716b988edac55dc630390efd5df", size = 522653, upload-time = "2026-10-05T17:40:18.879Z" }, { url = "https://files.pythonhosted.org/packages/a3/24/9f6721f0b52323789cb1041ffc3afa729462d9925607e733a065b6fc5e17/aiohttp-3.14.4-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:cbf92cd7a1c800cee244558dd5630abb45c1c1093ef7b0d546a99f526ff7a9eb", size = 529697, upload-time = "2026-10-05T17:40:21.296Z" }, { url = "https://files.pythonhosted.org/packages/9c/a4/399debc3c2ca0ab88a08e3082b4c4f62d0f66978af60243c63f2345ef337/aiohttp-3.14.4-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:fc76be08ff407fba717a769aff04ca1b4bf9ecfdcb5caadef84a563fdbbd146d", size = 540526, upload-time = "2026-10-05T17:40:23.548Z" }, { url = "https://files.pythonhosted.org/packages/a2/97/90379813b18b795bccdeb3a5a85e9b93ac2c69f9c2a5d0faba59da3ad6d1/aiohttp-3.14.4-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:9b416f995b68a921069e6ec61fa07f2ac3ff033d92e97a69f673af89f5bb954d", size = 818866, upload-time = "2026-10-05T17:40:25.842Z" }, { url = "https://files.pythonhosted.org/packages/78/70/824fd0dac819652f1df895b3f621a75bf0a3050c30e38d8d2316e34ab821/aiohttp-3.14.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2c9585101cfca10e74a07cc0ef2bdc6d90616e6750926b20456f429d0608f740", size = 549666, upload-time = "2026-10-05T17:40:28.523Z" }, { url = "https://files.pythonhosted.org/packages/8c/45/eafb2dd200dd2b770ed25a2dc5731c4ba5d58ab3fa8e2e82494bb946808e/aiohttp-3.14.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3c4dcf611b095e7983421f49d01ffbed7ed574891533d606ea115fc23e7c4e90", size = 548961, upload-time = "2026-10-05T17:40:30.613Z" }, { url = "https://files.pythonhosted.org/packages/a7/bc/6c733a1c71300293471ac7ef1dffd12b5b4dc9a43c53461224b2b8a7f667/aiohttp-3.14.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f8f61a9b5fa56c58e3441f9dc64137f786232d40fb89127190d0e0f84e1e5865", size = 1883165, upload-time = "2026-10-05T17:40:33.062Z" }, { url = "https://files.pythonhosted.org/packages/37/34/885ddbd351679a31141b25e0d1a5e4f67738806e564c858b399709b61239/aiohttp-3.14.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d43e538dfc85b705fd832620d050970ef5613b3869a7169b44a554b9dab9eee9", size = 1844119, upload-time = "2026-10-05T17:40:35.266Z" }, { url = "https://files.pythonhosted.org/packages/cc/ef/0a64dac53c11d569851efb3e7eddb43551df72b2c172ebb34cccc93bcd3f/aiohttp-3.14.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4135c4bb4b7682a7633fadc7170ea969d843f83aa7224e70111a20675105c985", size = 1930222, upload-time = "2026-10-05T17:40:37.527Z" }, { url = "https://files.pythonhosted.org/packages/69/5f/774792ed8f973e0d234fcb06b0f0bc9f29e9a0a0dd847f6418c2904be7d9/aiohttp-3.14.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9055324f156a94fdf6dc9a825ae60f9a2947b88f591ea81ee80826633de62ae7", size = 2038842, upload-time = "2026-10-05T17:40:39.815Z" }, { url = "https://files.pythonhosted.org/packages/cb/05/e00f35581c97656d9e864edaa03292382c620dd1d9c836a4f73a2279acc4/aiohttp-3.14.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fb22c8e4b60385a9b414c70c6e8b2d0f4b23ff7fc4bb572ceb58297b228169f1", size = 1887845, upload-time = "2026-10-05T17:40:42.359Z" }, { url = "https://files.pythonhosted.org/packages/ca/2e/07e3eff5b4c1b082a2eaf0ffb508b409b21ea1ddc9fdc57f3eabb51469a1/aiohttp-3.14.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3b6945559a260112742ed58cde2165488d2fc51aeea34afbaa99d9371b43e1b5", size = 1702533, upload-time = "2026-10-05T17:40:44.569Z" }, { url = "https://files.pythonhosted.org/packages/75/44/816844b7f876e6f8fee4ecb088269fcd92241da712e4814a4d624eda13a9/aiohttp-3.14.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8f4078a90b174f15efb2c8725e8720c40dbd3b2cf6350dbc085c964a938d9de7", size = 1849058, upload-time = "2026-10-05T17:40:47.818Z" }, { url = "https://files.pythonhosted.org/packages/ae/bd/163242e12464a1f3e58a77e28beca949652ddce84abdd5500a91c70a1e0d/aiohttp-3.14.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:f433c5654e32f72de3ef074d715eb5c7604eb6d28a5c2f3c9e5056f20fe6b306", size = 1854600, upload-time = "2026-10-05T17:40:50.401Z" }, { url = "https://files.pythonhosted.org/packages/f2/59/5c1ae708a89524eddfd9c727b87cae26b5c50c3c76683b910ac06151ead0/aiohttp-3.14.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:871035fe79c636b6bc126ed78df37d72b830dfc751612cb4b1040360edf2719f", size = 1896565, upload-time = "2026-10-05T17:40:52.625Z" }, { url = "https://files.pythonhosted.org/packages/37/45/95300a7de69241fe47d3f838f3fb582c712782e172d447ff650a013ea673/aiohttp-3.14.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2617132524f66338f6c1d218cd16081576aefc40df4f2bcba338a99fbf67fec1", size = 1693092, upload-time = "2026-10-05T17:40:54.862Z" }, { url = "https://files.pythonhosted.org/packages/ff/33/01a8e61d48d0094121092c74125df14c0f8f10ee3149bfe68b6f869409a9/aiohttp-3.14.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d8bc326133272deee8ef337581550450e4bb735c0520be893b950a1f5baaf41a", size = 1917073, upload-time = "2026-10-05T17:40:57.656Z" }, { url = "https://files.pythonhosted.org/packages/9e/d6/1a4f24624cda6686d5b927e26d5d1ae3ebb692faf88e93017fab4b391fd0/aiohttp-3.14.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3ec57223e261b7fa9979c12dbce009ca53439be76cac3af27f860d3e4b1d6f7a", size = 1872656, upload-time = "2026-10-05T17:41:00.123Z" }, { url = "https://files.pythonhosted.org/packages/be/be/ed286fcfa2d22061ae021b568c790123505cd482df35e23023339ecaed96/aiohttp-3.14.4-cp314-cp314-win32.whl", hash = "sha256:7242043e71fc449a19a47d92f4e9fd1e0ae8d3f88488982381f422bf6d651218", size = 485461, upload-time = "2026-10-05T17:41:02.686Z" }, { url = "https://files.pythonhosted.org/packages/c0/74/e040e6f65f4da305777f1adb3023d47401693489e4a337c983b52d2a0a6b/aiohttp-3.14.4-cp314-cp314-win_amd64.whl", hash = "sha256:019587c7b3a44917dd61bcb312fd5c034f341dc9851b83e7b41b65e08b68ca0f", size = 512147, upload-time = "2026-10-05T17:41:04.98Z" }, { url = "https://files.pythonhosted.org/packages/2e/5e/bddb33ae15fdd5a68a10a986587aa9916c418a712fdd110d3b3ae9b51407/aiohttp-3.14.4-cp314-cp314-win_arm64.whl", hash = "sha256:60dc71d38db0988f37e78f6ad4a010a2f6355cfac3fff5f130cf5ef2231f3f0d", size = 494685, upload-time = "2026-10-05T17:41:07.366Z" }, { url = "https://files.pythonhosted.org/packages/22/2e/7b884014b40573f919fc3a528e5028bc37b249e31a93928ba61bdd25c960/aiohttp-3.14.4-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:674f53ab9d2794e2dd767619a3384bf1006f8d075e43c19d449a645f8ff641c2", size = 855696, upload-time = "2026-10-05T17:41:09.514Z" }, { url = "https://files.pythonhosted.org/packages/5f/e6/b6e2b4f580ab6ae39241bb9182aca60e6e1915cdad0148faaa04db3f366b/aiohttp-3.14.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:6d243221f73e5ac546c2ed9255100423e4bc092b08b944f23498e8cf3096c26c", size = 567931, upload-time = "2026-10-05T17:41:11.685Z" }, { url = "https://files.pythonhosted.org/packages/63/99/ae8e3a1a18adcbf636839d18b3cccc51007836ccdae5209381c5cf82d1ae/aiohttp-3.14.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0daa016eb74e7888ed5f9ceec6eea3158999c1c4a115ef5cbe88baa8d924f94c", size = 567421, upload-time = "2026-10-05T17:41:13.836Z" }, { url = "https://files.pythonhosted.org/packages/63/38/c5d4bb58be72b967a4090d2deeb38571f8c992852936c51dbbbeae49db1c/aiohttp-3.14.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70c031a622aa20add05ebd234b90adaf6bf51c1c88cc305ec760721af83308c3", size = 2054173, upload-time = "2026-10-05T17:41:16.098Z" }, { url = "https://files.pythonhosted.org/packages/eb/87/3c8a9c7670f56d478a75c5bb84b336fb61a67b908e36c30a4a96ed8c8e86/aiohttp-3.14.4-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7df363fa7c38945a896e12c69af4a1a56cbd4152791d9e41fd0bac396b72de38", size = 1918296, upload-time = "2026-10-05T17:41:18.478Z" }, { url = "https://files.pythonhosted.org/packages/49/97/bd00e499c0a1505b44f5e23b9739034ea9be792ad42537f2894211ba1c07/aiohttp-3.14.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:02a508ba5ad91c4d730523584c0adc65882274bba5e37c33bc2f475e37577256", size = 2027513, upload-time = "2026-10-05T17:41:20.874Z" }, { url = "https://files.pythonhosted.org/packages/23/1d/a3666a7c224f86a881f18d673a8ffdc92fa4c35274fb4361822f8f0dc566/aiohttp-3.14.4-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0cb3edece6fe6a944eaf97d803b88f606da6bc1600b1894c2969e78ffdc1fc29", size = 2152686, upload-time = "2026-10-05T17:41:23.181Z" }, { url = "https://files.pythonhosted.org/packages/46/7e/f4204b8c4959c46f6367c182afbbacef08a1b53874281912c52fbd80151c/aiohttp-3.14.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21ce1223b461bcc5fb389a7204001578f1c0b85c2da60a84d915ebb021294245", size = 2006431, upload-time = "2026-10-05T17:41:25.776Z" }, { url = "https://files.pythonhosted.org/packages/33/43/091e69d42967cbea833ceb355c5457f8f70748fe08b88c9d91491b3fafb8/aiohttp-3.14.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:35d5432f0dc1c8b71307e9112461b52f336da225e344283eec4779222c0a1dff", size = 1789101, upload-time = "2026-10-05T17:41:28.095Z" }, { url = "https://files.pythonhosted.org/packages/47/7b/b57cbe533c01098c5d30471f057e7c3c19f281122f756acc9c716b636e2c/aiohttp-3.14.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d6ec364493057c118d08292f7cd60865965c209167d8657b45643a26aef77495", size = 1974472, upload-time = "2026-10-05T17:41:30.499Z" }, { url = "https://files.pythonhosted.org/packages/9f/44/79acce03611450e4ed8594f96f1cea40d70bc84fb4e918f94d7d9454a4af/aiohttp-3.14.4-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:70492983b7aed9e61bda82fe1ce80bcba9199610886443133783a288638923c0", size = 1956340, upload-time = "2026-10-05T17:41:33.009Z" }, { url = "https://files.pythonhosted.org/packages/da/bb/36e6cbd6720e3377a02f8709aec6fe80de729ef41e78ba01f13d50363c73/aiohttp-3.14.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:a77776673331577147e9d25cd7833a72e8891b47556336cff06ba1569e488b99", size = 1965771, upload-time = "2026-10-05T17:41:35.506Z" }, { url = "https://files.pythonhosted.org/packages/69/0f/879735ac70ce5e6852916eed51c706a517b205150a8d384f9fa60933a3fc/aiohttp-3.14.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:73799000dd6fd5247094fd8e6d3e1eb871e41aab5cc35dff9d9f654cd230bc24", size = 1780297, upload-time = "2026-10-05T17:41:38.284Z" }, { url = "https://files.pythonhosted.org/packages/01/5d/7c030ca5b6baf3b02eccd11afc3d58f0b35e407031d5cc78cae9c24921c0/aiohttp-3.14.4-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:27b370fb532e707e8c96d04fa0c96301fb7dba2d50aaca57f6c456cdfffec5bd", size = 2001374, upload-time = "2026-10-05T17:41:40.888Z" }, { url = "https://files.pythonhosted.org/packages/80/1e/ddacbb8141d727034c51dd192d17749521d5b30b03f84aa034325396089e/aiohttp-3.14.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:884e0d1c707b6217fee3080b9b4029eee7551af788ee761010df19be158414b7", size = 1968531, upload-time = "2026-10-05T17:41:43.62Z" }, { url = "https://files.pythonhosted.org/packages/3a/08/2214f823701eebc3e57db141faeae76df7bfd415f254cc7c1692258f228d/aiohttp-3.14.4-cp314-cp314t-win32.whl", hash = "sha256:f2e3d34edb151d3e27f93a8338e2da33e2c7261cd4c879ce5f5fe280cc190ea5", size = 504126, upload-time = "2026-10-05T17:41:46.321Z" }, { url = "https://files.pythonhosted.org/packages/91/a1/90e9b6fd23310b4e82d0c0a37b5ecc2944053350473c57533c1da9647235/aiohttp-3.14.4-cp314-cp314t-win_amd64.whl", hash = "sha256:88cf889e51092537fe44adfbff96c2a22d6479d6f92b97b3d413fbfd0a4decf4", size = 531494, upload-time = "2026-10-05T17:41:48.849Z" }, { url = "https://files.pythonhosted.org/packages/97/ab/5b501472c92b9b1dc53742591d6210dccb262962f4721299300d3a9f664b/aiohttp-3.14.4-cp314-cp314t-win_arm64.whl", hash = "sha256:ca450c6d42fda7c0bfaf0e200f32f77c5f2c960a33be1e14d197c84588b7f5f6", size = 510544, upload-time = "2026-10-05T17:41:51.358Z" }, { url = "https://files.pythonhosted.org/packages/0f/77/ef165a197850573ea07c8d77bf5f4854a1db12fdb20e1cf0b218abf025fc/aiohttp-3.14.4-cp315-cp315-android_24_arm64_v8a.whl", hash = "sha256:3b10799df72f66c30935bc661cb89e5ac2918debfc575e350e226fbaa846cbb7", size = 540737, upload-time = "2026-10-05T17:41:53.761Z" }, { url = "https://files.pythonhosted.org/packages/e6/aa/1663f64d1316b3be9d3f6fafa02b8d888070171a6544cba515e82f3af595/aiohttp-3.14.4-cp315-cp315-android_24_x86_64.whl", hash = "sha256:7ffd7f9dd39caccc6b2f68ec0339435ecad2b20f6ea2e2b3de91bbe464ac7137", size = 554600, upload-time = "2026-10-05T17:41:56.183Z" }, { url = "https://files.pythonhosted.org/packages/98/f7/d4d535855123c1a774a798643e75fdfbac62c047b0c0729d6505d09df444/aiohttp-3.14.4-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:a0a7a227353f7693f73878ad14cd2623cafadd656ababcd20e2adfb7225172ea", size = 523184, upload-time = "2026-10-05T17:41:58.624Z" }, { url = "https://files.pythonhosted.org/packages/0a/5f/18a8d1cd2958a0b6164b047c18ccb192f2123bb70f1c0a145f6f01f858ad/aiohttp-3.14.4-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1c43b6af8e4708e8b3812a9ee4f79895207e12d555b7f6d70deb838e0011e12d", size = 530342, upload-time = "2026-10-05T17:42:00.96Z" }, { url = "https://files.pythonhosted.org/packages/7f/c4/d1dd2b1fcf8a3192a431375b287a8762ebfd22195ab5b201d3fac30e3a7a/aiohttp-3.14.4-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:97416b2b997e5e9b817e322b2f0a89b76c54cde22471f51ac17ac15e10901ba0", size = 540629, upload-time = "2026-10-05T17:42:03.324Z" }, { url = "https://files.pythonhosted.org/packages/37/51/d1868f3459f618f6b68a6ef549d8b3f76aa616fa083298911fd3ae65d6b9/aiohttp-3.14.4-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:ee2a5aa31e507c17841d592364f2148393b17f83bc2766e9211e1eb31ea72a70", size = 819347, upload-time = "2026-10-05T17:42:05.713Z" }, { url = "https://files.pythonhosted.org/packages/84/e6/53179b721b67e0bd20974d02eca97b5691559e8ce666ba23a65aecda6b7f/aiohttp-3.14.4-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:5d5d9843c34866e986dd24eef713452609e304550c471320ce7b54b3cfb1391e", size = 549825, upload-time = "2026-10-05T17:42:08.093Z" }, { url = "https://files.pythonhosted.org/packages/00/0f/d67ddf7a43d4899176954511b6c048328abec1a5df4901f811a3b408a4fc/aiohttp-3.14.4-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:efcbf93e31b52c21665b6ebc489eb6fdb0dfd86b592edc677b5cffdfa7b11677", size = 549315, upload-time = "2026-10-05T17:42:10.766Z" }, { url = "https://files.pythonhosted.org/packages/9d/db/42f372c7f8e259505f239cb49db75f23548deaec56d22c68b4866f1c3fe2/aiohttp-3.14.4-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f1a79977794592dcf6485ed964ec2448db527a1bc0c72ec89663658dad3a739", size = 1892181, upload-time = "2026-10-05T17:42:13.194Z" }, { url = "https://files.pythonhosted.org/packages/23/0d/8b1900216302a94f69b41bf0786bb992a6e67d703f4698e335c2442db615/aiohttp-3.14.4-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:515e7ac6d27509dc3ba2c35fb88970b00b9e7b07ee7cc947c8733dcd54ef5def", size = 1883513, upload-time = "2026-10-05T17:42:15.704Z" }, { url = "https://files.pythonhosted.org/packages/7b/5e/15c014313e567fff931ec4a5f81683cc40251257791fbd607ec37c0dad80/aiohttp-3.14.4-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:185154ffa3b54765b5de96f464f41cfa476f8815a03854ae5e0fe4d48fb06e9c", size = 1933283, upload-time = "2026-10-05T17:42:18.241Z" }, { url = "https://files.pythonhosted.org/packages/f5/b3/bf7dc39a450218efb1d5c692bac7dc85ef56da282e889bee594f3267211e/aiohttp-3.14.4-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6aece59f77959628a22cd2ffe48b102cda9611d680e090f204385b3998a1190f", size = 2041054, upload-time = "2026-10-05T17:42:20.728Z" }, { url = "https://files.pythonhosted.org/packages/b6/03/ffba18d7aaedb5eafac3f41f0c26621ebb0a740bb0115ea2fae1911d902b/aiohttp-3.14.4-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7205aa6589f1ad1306c6994ff2e1ab5efabacf938f9ffd7682fb225c197b06a", size = 1895494, upload-time = "2026-10-05T17:42:23.518Z" }, { url = "https://files.pythonhosted.org/packages/45/f4/68123365b86826eee82bdceb7fb4c429204cb3a104d6b7ae44bc6a543fcb/aiohttp-3.14.4-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f38c7b3ce60d55ac0f2a9fd2edc194789d043fefea15cfb7dcb40b7d5f561146", size = 1714098, upload-time = "2026-10-05T17:42:26.471Z" }, { url = "https://files.pythonhosted.org/packages/b1/db/b13c0b214afc571782e8667f3b9e0bbed6843397be90fa35acf2f3ecfbb0/aiohttp-3.14.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:b0a1274ee4ee6203c15a341cce87f9f3df8151d5496047b3608bb62bd969cffc", size = 1857111, upload-time = "2026-10-05T17:42:29.169Z" }, { url = "https://files.pythonhosted.org/packages/6b/7f/4f5d3cee3c23b57119aebba5174962fed0bc493f37f38edab2f5730411b6/aiohttp-3.14.4-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:69ce2407951516f413adf3132b88451eababd710ce631f83c81a9603a3640253", size = 1892123, upload-time = "2026-10-05T17:42:31.786Z" }, { url = "https://files.pythonhosted.org/packages/5e/2e/5a6a1f014c4e17d58d4e935d0dd64e612f4eaa2f33cca2475b48ca5e3fb5/aiohttp-3.14.4-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:b84d3d057c60eb8e0c2b457a1203e868d82709d3f0c272a4975dfd88c2414939", size = 1898809, upload-time = "2026-10-05T17:42:34.34Z" }, { url = "https://files.pythonhosted.org/packages/de/f3/d1cca5311a70fcc1c7c2082e02044578a69bcb10686db33a322df9fb849e/aiohttp-3.14.4-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:4ecc84f69c9e217984f6877e832e4786a74567432c0bf2b7512919fbdfe18ade", size = 1704814, upload-time = "2026-10-05T17:42:37.047Z" }, { url = "https://files.pythonhosted.org/packages/0a/cb/cd58eae366148a5b9b1baa233542151154f2b72cd32daa39fd9db97d0c0d/aiohttp-3.14.4-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:878f2d7950238b2b0a8cbb88054444a125f0cfbc77f54c811aca0483132a9be3", size = 1918390, upload-time = "2026-10-05T17:42:39.674Z" }, { url = "https://files.pythonhosted.org/packages/e3/d8/f5e6c4939432734033048ba631503edf4a1e693ee5e261773e169803bbe4/aiohttp-3.14.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:6ff9d3a0e6935f1cb0a6685f9f4b3f0d3e406456d7a434a03cf309b8cd5d7a2e", size = 1878321, upload-time = "2026-10-05T17:42:42.528Z" }, { url = "https://files.pythonhosted.org/packages/2b/71/462957d6b5d4eb208f78e67745c41480bad2e738aa7caf1d9b07536920c1/aiohttp-3.14.4-cp315-cp315-win32.whl", hash = "sha256:e07dfd7cd360f20be26dc2487ad4c7f21b0b39fdd593e9ebf37d2424ad52defb", size = 485536, upload-time = "2026-10-05T17:42:45.223Z" }, { url = "https://files.pythonhosted.org/packages/ba/6e/48f59266a8b8c6420894ff59b8e74175252da67df85967b96997e96b7092/aiohttp-3.14.4-cp315-cp315-win_amd64.whl", hash = "sha256:921fc4f1ad549091bfc39cb5e93925b04f307f83517756a3111843851130efce", size = 512087, upload-time = "2026-10-05T17:42:47.717Z" }, { url = "https://files.pythonhosted.org/packages/25/f5/46263b23cee4f312e28aaabbdec2e074c807700a12c9dfa18beff461f035/aiohttp-3.14.4-cp315-cp315-win_arm64.whl", hash = "sha256:5be8af106f96fd625f6c264fa9faaaa9e0104089684d9015d2fa5b6eba6407c7", size = 494709, upload-time = "2026-10-05T17:42:50.615Z" }, { url = "https://files.pythonhosted.org/packages/77/8d/e3b1d406c95a8fe436ae2e5f01ee9591196c6c22c44c645b24f20d73eaea/aiohttp-3.14.4-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:e69727ec04a4608200ff32ec29478299df7657e2eb78c416ae338d0ca6662596", size = 856014, upload-time = "2026-10-05T17:42:53.425Z" }, { url = "https://files.pythonhosted.org/packages/5d/fc/e9e78d1097830040bc0a4b8af073379f964064bfa7f0a852d774f96f3168/aiohttp-3.14.4-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:f8b658dd3ef2ebd7708b311ed96ff7cb5edaca001a30018a091504ef492bcb1c", size = 567724, upload-time = "2026-10-05T17:42:56.416Z" }, { url = "https://files.pythonhosted.org/packages/e6/ad/c813a705a1da1c111c30723b52e1efb67763e9f0b3b54a418cb419616e7a/aiohttp-3.14.4-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:3749e018123df1205161a73ba96d336e17e9cf6b947295e53a43571de420e7b1", size = 567888, upload-time = "2026-10-05T17:42:59.048Z" }, { url = "https://files.pythonhosted.org/packages/c2/1f/7676f46aa24554a45686853c61552d02d67ade2b4c51adfe8b2ec4fdecd2/aiohttp-3.14.4-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f28a3c4fca436b2e594090e7cd22558f5e93a30ca18516c436de982da6fc25a", size = 2054356, upload-time = "2026-10-05T17:43:01.781Z" }, { url = "https://files.pythonhosted.org/packages/44/79/b9ef7d7442061e0a3125990d6021230b30361a972f7b8ebfb011042877ce/aiohttp-3.14.4-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e099aae21246992a1d84303ff262262d054c474e46cace33b0d5a69810c71877", size = 1937678, upload-time = "2026-10-05T17:43:04.492Z" }, { url = "https://files.pythonhosted.org/packages/5c/98/5927a31540c8e6d0542effead5bd5c56e749a9d99d84e902fb80ae906ef8/aiohttp-3.14.4-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fafb521891e646d9cf5da89f01c75e7159d08be7254a024149630a698be28251", size = 2026705, upload-time = "2026-10-05T17:43:07.353Z" }, { url = "https://files.pythonhosted.org/packages/62/bd/1be001ed97dcfa57a292188c38979fb78be795b960b50f02660587bffa06/aiohttp-3.14.4-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1d385db2cd154bce0b3dd451cf2a52bec91ebb7be2c3a5d06b14b4391631d807", size = 2147075, upload-time = "2026-10-05T17:43:10.141Z" }, { url = "https://files.pythonhosted.org/packages/0b/f7/d59628ccedd7c86db4c1f904467d1c80fed6cc10a1831ed5d75ff12e5a71/aiohttp-3.14.4-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ae8a244889d8a53549851bf595b9f382f64acca57e8762da1ecb26ba786828f7", size = 2005016, upload-time = "2026-10-05T17:43:12.848Z" }, { url = "https://files.pythonhosted.org/packages/3c/dc/472db3ab1daafa12a61d7f83314f85f502b99c143c78d66554a5e24361fd/aiohttp-3.14.4-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:681dde68ff8d8d5e7d5abcad4feb45bfb457a218310c919825e21fc36cac3d0b", size = 1796363, upload-time = "2026-10-05T17:43:15.562Z" }, { url = "https://files.pythonhosted.org/packages/08/61/6f9834b131b9a42e30d68c469c196033d3cae2d8fe340d4668f362c4fd1f/aiohttp-3.14.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:b91a6441fcaed88ca7bca67726b189332df3ceb6ac0dfc98dcb9654bb723288c", size = 1974240, upload-time = "2026-10-05T17:43:18.884Z" }, { url = "https://files.pythonhosted.org/packages/b5/2f/b8eb2a276685ef1da7f7bc15348404b8b9ee12b8f32f2cd8895385cd1589/aiohttp-3.14.4-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:f15dbbac49bc15eaff95c0b8166525c4050996af578c7be141e145f40ee2cf24", size = 1973387, upload-time = "2026-10-05T17:43:21.661Z" }, { url = "https://files.pythonhosted.org/packages/5b/32/ad559b2d6817e85f7cb294d8641e975e64b526f4da98ffe941c9af67a047/aiohttp-3.14.4-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:7abae043e87692d1c2adda1fab501bb0612d28c4a23eb082005c1e684f792dac", size = 1968766, upload-time = "2026-10-05T17:43:24.544Z" }, { url = "https://files.pythonhosted.org/packages/60/67/4a16e398140bdabb363428b9f439af3f77706a6df8bea6e626bbde53c2ab/aiohttp-3.14.4-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:21f4624fb1051fc1aa10d57a636a80e829f7d0417462d6f1ce380b08fe1a904c", size = 1788193, upload-time = "2026-10-05T17:43:27.467Z" }, { url = "https://files.pythonhosted.org/packages/a9/38/17b326da7f52e530a675d789ade3e727a1b3db9959722f6049b9c392321a/aiohttp-3.14.4-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:2e8dc3a0fdaf9d12b954d3d06cc4c355d91b17fb6c671715b593624ca4c379f8", size = 1999645, upload-time = "2026-10-05T17:43:30.32Z" }, { url = "https://files.pythonhosted.org/packages/53/dc/91b87961250010e05eb3adabaf0d98e4229ee7543147c265e834922e1b0b/aiohttp-3.14.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:1410b6f6e3d52ea7c02c950496522ce6e85d5543bf154bc80a6141e505251cfa", size = 1970998, upload-time = "2026-10-05T17:43:33.364Z" }, { url = "https://files.pythonhosted.org/packages/72/66/9bcf2c2db7b75a3b51c11470ceb67941d028727d674276ff5d5120adc87e/aiohttp-3.14.4-cp315-cp315t-win32.whl", hash = "sha256:a65189a89f3e621f7c27a968b5131bbca29126e2b8517dc72b205747a39d4c96", size = 504036, upload-time = "2026-10-05T17:43:36.212Z" }, { url = "https://files.pythonhosted.org/packages/e2/20/349db4af12dbc16070d32028d30c2067123b4520f3dcb2d9013524efeb4f/aiohttp-3.14.4-cp315-cp315t-win_amd64.whl", hash = "sha256:2b2c95f5f769eec92b552969db1f79ea0282058ce5131ed0f2fa540b4af3f14e", size = 531284, upload-time = "2026-10-05T17:43:39.362Z" }, { url = "https://files.pythonhosted.org/packages/26/21/544c711bc5fad611278ebd153bdcab1bffce8b8581f8ec9096b275948a3c/aiohttp-3.14.4-cp315-cp315t-win_arm64.whl", hash = "sha256:54209fff79346ee1ef0d5cbf92fa80a5bb37396406bb4d932e5107f9c29b2d3f", size = 510623, upload-time = "2026-10-05T17:43:42.15Z" }, { url = "https://files.pythonhosted.org/packages/2b/27/5e1f8446545be98e9416af59c396b1ed1f9f3294afb0a40ee3a709889273/aiohttp-3.14.4-py3-none-any.whl", hash = "sha256:5c6758ba62aea282c537179cfc8474a90f2b5f7b089cc5ff66d8920d86a9bfdd", size = 279487, upload-time = "2026-10-05T17:43:44.905Z" }, ] [[package]] name = "aiosignal" version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "frozenlist" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, ] [[package]] name = "anyio" version = "4.14.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "idna" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, ] [[package]] name = "appnope" version = "0.1.4" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/35/5d/752690df9ef5b76e169e68d6a129fa6d08a7100ca7f754c89495db3c6019/appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee", size = 4170, upload-time = "2024-02-06T09:43:11.258Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/81/29/5ecc3a15d5a33e31b26c11426c45c501e439cb865d0bff96315d86443b78/appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c", size = 4321, upload-time = "2024-02-06T09:43:09.663Z" }, ] [[package]] name = "asttokens" version = "3.0.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/25/1e/faf0f247f6f881b98fc4d6d07e14085cb89d13665084e6d6ac1dc2c03d0b/asttokens-3.0.2.tar.gz", hash = "sha256:3ecdbd8f2cc195f53ccada3a613538bb5f9ef6f6869129f13e03c30a677b8fe2", size = 63136, upload-time = "2026-07-12T03:31:49.084Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d4/2b/04b8a15f3a1c77bc79ddf5c73875327f34b4fa75982df2b76e45e402d364/asttokens-3.0.2-py3-none-any.whl", hash = "sha256:9da13157f5b28becde0bd374fc677dcd3c290614264eff096f167c469cd9f933", size = 28702, upload-time = "2026-07-12T03:31:47.542Z" }, ] [[package]] name = "attrs" version = "26.1.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] [[package]] name = "babel" version = "2.18.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d", size = 9959554, upload-time = "2026-02-01T12:30:56.078Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, ] [[package]] name = "backrefs" version = "8.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ec/56/4744bcd0c82184e80c52b0ac4076c261a8ffa1f1b343ff2f6e89ce0e1cef/backrefs-8.0.tar.gz", hash = "sha256:b556cd7d36c3a3a2f256b89590b176b8eddfb73bcfaee3a3ddd84ea66d21ce50", size = 7013081, upload-time = "2026-07-26T19:54:24.638Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e3/fd/9bf53b6a6f6f519ffaac765df2f2a25e5c2fc6d32cfd2b2747099e72c911/backrefs-8.0-py310-none-any.whl", hash = "sha256:4a627b817fd2dce43b79ab48da63613340509381cd8ce0897078a0bce79a2ab8", size = 380377, upload-time = "2026-07-26T19:54:17.457Z" }, { url = "https://files.pythonhosted.org/packages/e1/29/4bd7ae72a2634da00379c2b3bcc5439e7c94620235c6afea8af15229a973/backrefs-8.0-py311-none-any.whl", hash = "sha256:f0c35cf0102ba6b6070c12a492be3c1c1d3f5839529784b9a9565d6d04569a01", size = 392169, upload-time = "2026-07-26T19:54:18.782Z" }, { url = "https://files.pythonhosted.org/packages/29/13/232505664e8e2a0c7a2eb0c505cfade9d715538f89a5d62bc4c272968f62/backrefs-8.0-py312-none-any.whl", hash = "sha256:87f0fae8c5f207fe9f4b2887efc71d42f4900ac78faa1af08d675ef303692dc5", size = 398084, upload-time = "2026-07-26T19:54:19.954Z" }, { url = "https://files.pythonhosted.org/packages/8a/69/47a3dc20abc4fa5486655fde681bd55e63211b46c886d8c02223d6468431/backrefs-8.0-py313-none-any.whl", hash = "sha256:601ce68ca12385dbda06ce264406b4c4210cf5b79fd0fd627592365c92f29a88", size = 400040, upload-time = "2026-07-26T19:54:21.194Z" }, { url = "https://files.pythonhosted.org/packages/1c/cf/e5f9b68a5b0e939a2fb933a66c20180d0c9241bf8927f7a47fa48c1675e9/backrefs-8.0-py314-none-any.whl", hash = "sha256:9ec96efa080938be92323e8e730e57718c9c88eb15ad70bbef4e1766df591408", size = 411903, upload-time = "2026-07-26T19:54:23.221Z" }, ] [[package]] name = "certifi" version = "2026.7.22" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, ] [[package]] name = "cffi" version = "2.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pycparser" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/10/69/43965eccfdead3b9220015fd1320e117be8c6ed01a62ffab76eeb752f5d5/cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0", size = 184821, upload-time = "2026-08-03T21:19:44.887Z" }, { url = "https://files.pythonhosted.org/packages/54/7d/16e5a096677b5e313ca80cd5e5170efa3ea44624a82bb111925522da64b1/cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf", size = 184719, upload-time = "2026-08-03T21:19:46.129Z" }, { url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" }, { url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" }, { url = "https://files.pythonhosted.org/packages/6a/5b/717f1526b9957b34456313c31645c5b82b8fb5c3fe9e4752999be7128bfc/cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50", size = 210249, upload-time = "2026-08-03T21:19:49.543Z" }, { url = "https://files.pythonhosted.org/packages/64/b3/f8aa4f3e34986c7e4ec45072d1b1b9dd295b6b18007b45518d79726dd725/cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e", size = 208775, upload-time = "2026-08-03T21:19:50.918Z" }, { url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" }, { url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" }, { url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" }, { url = "https://files.pythonhosted.org/packages/80/fb/0bb75b7039588c074b37ae99f40d9bfddf990ecb2fbc346ebccd2e56b9be/cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e", size = 175292, upload-time = "2026-08-03T21:19:55.566Z" }, { url = "https://files.pythonhosted.org/packages/d9/79/615cc094e2fb508cade7de88d3b4f6c4ec2bab695c97bce9153dc65aadf5/cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a", size = 185919, upload-time = "2026-08-03T21:19:56.89Z" }, { url = "https://files.pythonhosted.org/packages/70/c6/d0ea84713fe46b243a436a18fcd47d639732747e21635c8a27191b06dc30/cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80", size = 180093, upload-time = "2026-08-03T21:19:58.155Z" }, { url = "https://files.pythonhosted.org/packages/9d/f4/035513d4117049066b4779dc3b7c0c0fdad175fa13731c9f4003f1cd1478/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e", size = 194248, upload-time = "2026-08-03T21:19:59.399Z" }, { url = "https://files.pythonhosted.org/packages/76/af/2aeb4dbb5fc41a04161ae9ff1518de7cec08e164f44a8ce6a4cf7fd2cd1d/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c", size = 196908, upload-time = "2026-08-03T21:20:00.746Z" }, { url = "https://files.pythonhosted.org/packages/a7/46/2e5fdde8555706dd98139a910ca11be02809f3f605ce956f655d0214e100/cffi-2.1.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6", size = 184805, upload-time = "2026-08-03T21:20:02.02Z" }, { url = "https://files.pythonhosted.org/packages/55/41/4c7042f317b9217502988f0873af87e16ad606dc20f84e546e3e6ce9764c/cffi-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971", size = 184764, upload-time = "2026-08-03T21:20:03.141Z" }, { url = "https://files.pythonhosted.org/packages/43/1f/1c3d90d91811c8f86ced9ed637956c54bfe5b79ca98fe976d7f8c8979f6b/cffi-2.1.1-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c", size = 214722, upload-time = "2026-08-03T21:20:04.377Z" }, { url = "https://files.pythonhosted.org/packages/37/6f/3b5ce4c3b2192d250f04908f2bfd91ef34552ec8f7716a5d4abdb8d67bb2/cffi-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125", size = 222369, upload-time = "2026-08-03T21:20:05.544Z" }, { url = "https://files.pythonhosted.org/packages/02/10/4b3c75dde3d9663c9e02ba05c2668b954f671d4bbe346413ca8c696b295a/cffi-2.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264", size = 210175, upload-time = "2026-08-03T21:20:06.75Z" }, { url = "https://files.pythonhosted.org/packages/df/62/14f74b9543e605d17701dc797b815958b8bb70b7624ce1b832ddad48ed6c/cffi-2.1.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3", size = 208670, upload-time = "2026-08-03T21:20:08.04Z" }, { url = "https://files.pythonhosted.org/packages/95/95/86342356ff5953b3fb06f7ef7c5bee212d45e770abc7218d451b9148313c/cffi-2.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2", size = 221824, upload-time = "2026-08-03T21:20:09.274Z" }, { url = "https://files.pythonhosted.org/packages/eb/ff/7b3429ff53aafe931ed8a5fc69f481bbef7ba6de87ddcbb63d08f483f613/cffi-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b", size = 225148, upload-time = "2026-08-03T21:20:10.7Z" }, { url = "https://files.pythonhosted.org/packages/34/34/a95870b9221e09cf4f2ce3178b1a210abdfe63a1bd357da940418d7b8d15/cffi-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7", size = 223564, upload-time = "2026-08-03T21:20:12.165Z" }, { url = "https://files.pythonhosted.org/packages/70/ea/839b50531021a647fb5e929f72cf97bc1ff702b5472166164b5b6e76b851/cffi-2.1.1-cp313-cp313-win32.whl", hash = "sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac", size = 175263, upload-time = "2026-08-03T21:20:13.559Z" }, { url = "https://files.pythonhosted.org/packages/60/a6/8b149b2c3f2e11aaa1618ef64500b45f50f22c57a977a4dff1aff1f91042/cffi-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d", size = 185688, upload-time = "2026-08-03T21:20:14.69Z" }, { url = "https://files.pythonhosted.org/packages/01/9a/11f687cb39d6a3504060d5242f04f48c735afb4d3d533958a20594890cb2/cffi-2.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973", size = 180078, upload-time = "2026-08-03T21:20:15.917Z" }, { url = "https://files.pythonhosted.org/packages/d3/7b/d6bbf82b8b96e7391438898c42f5bd96dd02030fd5b64937d248220003e2/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c", size = 194064, upload-time = "2026-08-03T21:20:17.148Z" }, { url = "https://files.pythonhosted.org/packages/94/e6/bcc91b283be94735e268487a054004f0aa19947b6348fa367db53230abc8/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb", size = 196720, upload-time = "2026-08-03T21:20:18.268Z" }, { url = "https://files.pythonhosted.org/packages/d9/99/c4b0c17cacdc9c3b8f280026286a9826d6a208c0f047591a3c3ce99b91fd/cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54", size = 184964, upload-time = "2026-08-03T21:20:19.708Z" }, { url = "https://files.pythonhosted.org/packages/b3/a9/9db617d05d7367c1ad0ab00b3aa6e6f9281edd689b4ee9ea0e5a84e89c97/cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72", size = 184962, upload-time = "2026-08-03T21:20:20.833Z" }, { url = "https://files.pythonhosted.org/packages/67/b8/b42132ca113dc567d37684437b46ca1dafc885902b02a110a02d5b511857/cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1", size = 222328, upload-time = "2026-08-03T21:20:22.118Z" }, { url = "https://files.pythonhosted.org/packages/80/10/c5c0cbf0a657aecf59ef511409734230bf556f05a0d6c9eed7aa5c0a0166/cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062", size = 209985, upload-time = "2026-08-03T21:20:23.401Z" }, { url = "https://files.pythonhosted.org/packages/d5/6c/bfa0b87b03b9238148beca990292843c9396ba069b54496596594173de7b/cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03", size = 208530, upload-time = "2026-08-03T21:20:24.628Z" }, { url = "https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96", size = 221525, upload-time = "2026-08-03T21:20:25.758Z" }, { url = "https://files.pythonhosted.org/packages/82/1d/a4aaf9babd75acb4d5f223bff71533bee748dd770a382619a798960ee9ba/cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527", size = 225053, upload-time = "2026-08-03T21:20:26.985Z" }, { url = "https://files.pythonhosted.org/packages/81/10/5dc0e7bdd18e22107054288283380fc97a06ae3f1656a106908d666a3c88/cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13", size = 223213, upload-time = "2026-08-03T21:20:28.277Z" }, { url = "https://files.pythonhosted.org/packages/0b/e9/d0061c364cde06ee43168a0d076ac1da512cbc380d44767b844ba34fe2b6/cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c", size = 177682, upload-time = "2026-08-03T21:20:44.288Z" }, { url = "https://files.pythonhosted.org/packages/a7/06/1c3e01e3ba14c39f6d10bfbac52753b7e22259e38088e5cfe1d704918690/cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48", size = 187949, upload-time = "2026-08-03T21:20:45.623Z" }, { url = "https://files.pythonhosted.org/packages/87/5b/da4e39efe18eeb89cf580ea9cfc66b6a7c3eadb808fc0cc1d3a295cb5a5d/cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836", size = 182947, upload-time = "2026-08-03T21:20:46.955Z" }, { url = "https://files.pythonhosted.org/packages/23/59/40338bf421c5accea1d45158170c87006ef1cd371b05c077e76476949728/cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3", size = 188504, upload-time = "2026-08-03T21:20:29.495Z" }, { url = "https://files.pythonhosted.org/packages/7d/47/5ecf1023850036e674c77ec4de86182d309ae344e39e7cba984b7df5d647/cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2", size = 188259, upload-time = "2026-08-03T21:20:31.291Z" }, { url = "https://files.pythonhosted.org/packages/2a/9c/92934c3bea9f785b23eba304538c0b4d37a2a96d2431eb3a1bc87a11aa19/cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94", size = 223864, upload-time = "2026-08-03T21:20:32.571Z" }, { url = "https://files.pythonhosted.org/packages/4d/45/ba4c93527bc38616a8bd36488acb69a2212d60486794f0c1f318949bbb76/cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc", size = 211538, upload-time = "2026-08-03T21:20:33.808Z" }, { url = "https://files.pythonhosted.org/packages/80/e9/b6ef565e452acb932fb0cb5443f44a78efbd1233e566f02b5a83855e9115/cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29", size = 210688, upload-time = "2026-08-03T21:20:34.974Z" }, { url = "https://files.pythonhosted.org/packages/9a/95/eff5f0cee78d2eabc7eebffec40d3fc1876b5f3c95582e018bb4b99601f2/cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676", size = 223803, upload-time = "2026-08-03T21:20:36.564Z" }, { url = "https://files.pythonhosted.org/packages/fa/01/579d39fb8bef00a335a23d83757b44feb24cd6345a2c451b64cb67b9c362/cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e", size = 226763, upload-time = "2026-08-03T21:20:37.816Z" }, { url = "https://files.pythonhosted.org/packages/8d/b0/0b44f47c60b01b57b6e2bbd92343f13a85a1d93bc46ccf6e47e244acd99c/cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f", size = 225688, upload-time = "2026-08-03T21:20:38.959Z" }, { url = "https://files.pythonhosted.org/packages/eb/d2/3b7176cb570a1d3e27faf67b72f591af508036e0d8b2be2ef9af9e8c84bb/cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4", size = 182868, upload-time = "2026-08-03T21:20:40.388Z" }, { url = "https://files.pythonhosted.org/packages/56/78/31f00c1bcd97c9bbf55f1bfdf5bc809a5de8887473e90bb9960dca825e80/cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e", size = 194104, upload-time = "2026-08-03T21:20:41.725Z" }, { url = "https://files.pythonhosted.org/packages/7b/1b/58496f2ed0a35de575250c02a43ab3cc2c04d494a88fed31c1cabc0fd176/cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5", size = 186402, upload-time = "2026-08-03T21:20:43.042Z" }, { url = "https://files.pythonhosted.org/packages/c1/8f/9ebe220eab48a093d1a5a5e339ab0dc7316eef3bb04d63c42f0251b61f50/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d", size = 194043, upload-time = "2026-08-03T21:20:48.179Z" }, { url = "https://files.pythonhosted.org/packages/ff/69/844bad3ece306c4782c2ecb93597035b6690d48704b803914c199da1e8b3/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b", size = 196737, upload-time = "2026-08-03T21:20:49.457Z" }, { url = "https://files.pythonhosted.org/packages/1b/8a/af668013284634733f02d683458a0728739c7d6ddb5e14cb0c20832266fe/cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4", size = 184933, upload-time = "2026-08-03T21:20:50.639Z" }, { url = "https://files.pythonhosted.org/packages/0c/75/2f5207ff6d1a613133b23a5203cc0c2a628313b5eb3974d7956ae3c57950/cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8", size = 185002, upload-time = "2026-08-03T21:20:52.173Z" }, { url = "https://files.pythonhosted.org/packages/e2/31/9e1313b0a6e30e91b3b3d3fff51ae99c857c07738e3afcce1f7334e1b7ab/cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6", size = 222271, upload-time = "2026-08-03T21:20:53.462Z" }, { url = "https://files.pythonhosted.org/packages/50/e3/f6234a833e6e08c7007003074723c406559eecf9b48dfc97471e5a8eb7a0/cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80", size = 209919, upload-time = "2026-08-03T21:20:54.783Z" }, { url = "https://files.pythonhosted.org/packages/0d/fc/5f74e293fced6edb51af3a46c4ccf6c23c9943774ecb375ddbd522c76add/cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779", size = 208529, upload-time = "2026-08-03T21:20:56.066Z" }, { url = "https://files.pythonhosted.org/packages/44/16/29e6d01b388bef055ecd6ca8244b3f4d336bd09e92d5d892187b9601084e/cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399", size = 221630, upload-time = "2026-08-03T21:20:57.336Z" }, { url = "https://files.pythonhosted.org/packages/a4/18/fa7f1f6857d5eb88a4ca99ffcbfb7c387a287ccc154c64a73e86314745d7/cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688", size = 225134, upload-time = "2026-08-03T21:20:58.675Z" }, { url = "https://files.pythonhosted.org/packages/e0/9f/e8e3dfa04a1b4c241f8c91faacad872b4d4efd051d49764ad4e2fd4b9fea/cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7", size = 223197, upload-time = "2026-08-03T21:20:59.968Z" }, { url = "https://files.pythonhosted.org/packages/f8/7e/8debeb04f1ab9fe2a6963964cd6f1aaf7192627b83926586a6a4e089c9fa/cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac", size = 177683, upload-time = "2026-08-03T21:21:14.901Z" }, { url = "https://files.pythonhosted.org/packages/e0/31/5158704cc474ab65c1647932e88be78dc0873f47130e253be38bcaf13d01/cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960", size = 187897, upload-time = "2026-08-03T21:21:16.108Z" }, { url = "https://files.pythonhosted.org/packages/cc/4b/b3a2da8570c704ffc0f9762cdc3ec0f02c8573798e0b5cf7f11c82bbb70f/cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1", size = 182935, upload-time = "2026-08-03T21:21:17.271Z" }, { url = "https://files.pythonhosted.org/packages/d0/ef/5443574510a1207e6f6bc38ba6e1f1de36cb48fef07b2728bb896a21f430/cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc", size = 188464, upload-time = "2026-08-03T21:21:01.163Z" }, { url = "https://files.pythonhosted.org/packages/7e/ae/a56fa8c4686ad50e148fcbc8d3ae0d03915ff5c30d795058988c24118cef/cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab", size = 188262, upload-time = "2026-08-03T21:21:02.382Z" }, { url = "https://files.pythonhosted.org/packages/53/b2/6187f46f2912276a3ae284076109cc5c8680482f11f766ccf26db4a86427/cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e", size = 223779, upload-time = "2026-08-03T21:21:03.553Z" }, { url = "https://files.pythonhosted.org/packages/8a/f6/c3ad28bd19f77047a03084424fbd4cbe997303267c14423737324be0385d/cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358", size = 211520, upload-time = "2026-08-03T21:21:04.863Z" }, { url = "https://files.pythonhosted.org/packages/a0/cd/ccac9013a5bd9fd764de118674ab9c805b5ca10c19270d90ee273f8b2240/cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231", size = 210673, upload-time = "2026-08-03T21:21:06.223Z" }, { url = "https://files.pythonhosted.org/packages/52/86/2976131c639aead931c5bee5aba67e4b09fbeb8018b6f282f70803f923a7/cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6", size = 223835, upload-time = "2026-08-03T21:21:07.539Z" }, { url = "https://files.pythonhosted.org/packages/ac/0c/33a7aeab2f9c76918c52e084beb39c570db3588133412929e8ec06fab90b/cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94", size = 226705, upload-time = "2026-08-03T21:21:08.774Z" }, { url = "https://files.pythonhosted.org/packages/e3/26/2cde30fdde421130bfc18f70395731a6e6b2053c6a1978a5258ff04e72fa/cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5", size = 225539, upload-time = "2026-08-03T21:21:09.911Z" }, { url = "https://files.pythonhosted.org/packages/6d/cd/a361394c94b2129d604bb846f624a8e88255a3ee33129c434a00d715e64f/cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66", size = 182707, upload-time = "2026-08-03T21:21:11.226Z" }, { url = "https://files.pythonhosted.org/packages/9b/b5/ba2b299993c26577d529b6ae29841f9e15b9fcf004d65f423f4fcf94ade9/cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3", size = 193772, upload-time = "2026-08-03T21:21:12.39Z" }, { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, ] [[package]] name = "charset-normalizer" version = "3.4.9" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0", size = 319300, upload-time = "2026-07-07T14:33:15.666Z" }, { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" }, { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" }, { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" }, { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" }, { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" }, { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" }, { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" }, { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" }, { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" }, { url = "https://files.pythonhosted.org/packages/0e/42/6dbc00b8cd16011691203e33570fa42ed5746599a2e878112d16eab403a3/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642", size = 151185, upload-time = "2026-07-07T14:33:30.781Z" }, { url = "https://files.pythonhosted.org/packages/80/cc/f920afd1a23c58ccd53c1d36085a71893a4737ff5e66e0371efab6809850/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0", size = 162557, upload-time = "2026-07-07T14:33:32.176Z" }, { url = "https://files.pythonhosted.org/packages/f0/e6/0386d43a261ff4e4b30c5857af7df877254b46bec7b9d1b74b6bf969a90b/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2", size = 152665, upload-time = "2026-07-07T14:33:33.711Z" }, { url = "https://files.pythonhosted.org/packages/b2/06/97ec2aeae780b31d742b6352218b43841a6871e2564578ca522dce4a45c3/charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614", size = 317688, upload-time = "2026-07-07T14:33:35.408Z" }, { url = "https://files.pythonhosted.org/packages/d0/39/8ff066c672434225f8d25f8b739f992af250944392173dcc88362681c9bf/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698", size = 214982, upload-time = "2026-07-07T14:33:36.996Z" }, { url = "https://files.pythonhosted.org/packages/92/8f/3a47a3667c83c2df9483d91644c6c107de3bf8874aa1793da9d3012eb986/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b", size = 236460, upload-time = "2026-07-07T14:33:38.536Z" }, { url = "https://files.pythonhosted.org/packages/f1/60/b22cdbee7e4013dab8b0d7647fc6181120fbbbc8f7025c226d15bd5a47fc/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9", size = 232003, upload-time = "2026-07-07T14:33:40.059Z" }, { url = "https://files.pythonhosted.org/packages/ea/f8/72eb13dcabe7257035cea8aefd922caad2f110d252bf9f67c4c2ca763aee/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33", size = 223149, upload-time = "2026-07-07T14:33:41.631Z" }, { url = "https://files.pythonhosted.org/packages/b0/3e/faee8f9de92b14ee1198e9163252bb15efee7301b31256a3b6d9ebfdd0dd/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63", size = 207901, upload-time = "2026-07-07T14:33:43.209Z" }, { url = "https://files.pythonhosted.org/packages/3a/25/45f30093ae27dd7b92a793b61882a38685f993700113ca36e0c9c14965e1/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0", size = 219176, upload-time = "2026-07-07T14:33:44.725Z" }, { url = "https://files.pythonhosted.org/packages/48/18/c8f397329c35e32f6a837e488986f4ae03bd2abebc453b48714991630c2f/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe", size = 217356, upload-time = "2026-07-07T14:33:46.192Z" }, { url = "https://files.pythonhosted.org/packages/86/7e/5ce0bba863470fd1902d5e5843968951bddf38abe4742fc97116ef4598b3/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35", size = 209614, upload-time = "2026-07-07T14:33:47.705Z" }, { url = "https://files.pythonhosted.org/packages/6c/ef/2473d3c4d869155be4af1191111d59c4d5c4e0173026f7e85b176e23bf65/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8", size = 224991, upload-time = "2026-07-07T14:33:49.238Z" }, { url = "https://files.pythonhosted.org/packages/d0/a3/53ddae3db108a088156aa8ddfafd411ebbc1340f48c5573f697b27f69a39/charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9", size = 150622, upload-time = "2026-07-07T14:33:50.711Z" }, { url = "https://files.pythonhosted.org/packages/e8/ef/6953a77c7cf2c2ff9998e6f575ab3e380119f100223381565a4f94c1f836/charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115", size = 161947, upload-time = "2026-07-07T14:33:52.197Z" }, { url = "https://files.pythonhosted.org/packages/6e/fb/d560d1d1555debbfe7849d9cac6145c1b537709d79576bf22557ed803b82/charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012", size = 152594, upload-time = "2026-07-07T14:33:53.486Z" }, { url = "https://files.pythonhosted.org/packages/7e/8d/496817fa0944239ecae662dd57ea765cfeaec6a735f9f025d4b7b72e7143/charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380", size = 317253, upload-time = "2026-07-07T14:33:54.994Z" }, { url = "https://files.pythonhosted.org/packages/2b/f9/ef4a69ea338ad3c0deceea0f5f7d2380ae8b52132b06d652cb0d2cd86706/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9", size = 215898, upload-time = "2026-07-07T14:33:56.334Z" }, { url = "https://files.pythonhosted.org/packages/8c/e7/5ddfd76fc061eb52de219658a4aa431cbacadf0a0219c8854f00da50d289/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4", size = 236718, upload-time = "2026-07-07T14:33:57.9Z" }, { url = "https://files.pythonhosted.org/packages/49/ba/768fa3f36048d81c477a0ce61f813bc1454d80917ccfe550abd9f44f5e24/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a", size = 232519, upload-time = "2026-07-07T14:33:59.811Z" }, { url = "https://files.pythonhosted.org/packages/f4/c4/b3e049d2aa3766180c78507110543d9d50894cc97f57de543f1be521dcdc/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046", size = 223143, upload-time = "2026-07-07T14:34:01.517Z" }, { url = "https://files.pythonhosted.org/packages/19/79/55c32d06d76ae4feafe053f061f3e3ab70bcf19f4007797ce8c3efda7830/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81", size = 206742, upload-time = "2026-07-07T14:34:03.04Z" }, { url = "https://files.pythonhosted.org/packages/10/e0/47c079dd82d217c807479cd59ffd30af56307ea31c108b75758970459ad3/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917", size = 219191, upload-time = "2026-07-07T14:34:04.657Z" }, { url = "https://files.pythonhosted.org/packages/42/ab/b9bc2e77d6b44a7e46ef62ec5cac1c9a6ba7b9135a5d560f002696ec9995/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41", size = 218328, upload-time = "2026-07-07T14:34:06.115Z" }, { url = "https://files.pythonhosted.org/packages/f1/78/c9c71d599f5aa2d42bcdd35cbbd46d7f535351a57e40ff7d8e5a7e219401/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1", size = 207406, upload-time = "2026-07-07T14:34:07.554Z" }, { url = "https://files.pythonhosted.org/packages/f6/39/c914445c321a845097ce4f6ac7de9a18228a77b766272125a1ce00d851eb/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf", size = 225157, upload-time = "2026-07-07T14:34:09.061Z" }, { url = "https://files.pythonhosted.org/packages/9b/f2/c0d4b8508565a36bc5c624e88ed297f5b0b1095011034d7f5b83a69908b5/charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48", size = 151095, upload-time = "2026-07-07T14:34:10.901Z" }, { url = "https://files.pythonhosted.org/packages/49/fd/a1d26144398c67486422a72bf5812cda22cb4ccfcd95a290fb41ceb4b8e2/charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b", size = 162796, upload-time = "2026-07-07T14:34:12.47Z" }, { url = "https://files.pythonhosted.org/packages/20/95/d75e82f8ce9fd323ebf059c16c9aadefb22a1ecde13b7840b35835e4886c/charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519", size = 153334, upload-time = "2026-07-07T14:34:14.044Z" }, { url = "https://files.pythonhosted.org/packages/00/5e/17398df3a139985ba9d11ed072531986f408c8fca952835ef1ab1820c02b/charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198", size = 338848, upload-time = "2026-07-07T14:34:15.688Z" }, { url = "https://files.pythonhosted.org/packages/cd/91/7253a32e86b7e1d1239b1b36ba6dd0f021a21107ab33054b53119cc083b9/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32", size = 223022, upload-time = "2026-07-07T14:34:17.248Z" }, { url = "https://files.pythonhosted.org/packages/cb/32/2e64bd2be10e89c61e57ebe6a93fd98ae88eb7ebe414b5121f22c96c69eb/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632", size = 241590, upload-time = "2026-07-07T14:34:18.813Z" }, { url = "https://files.pythonhosted.org/packages/3d/ef/d96ec496cfea0c21db43b0ad03891308b02388d054cc902cf0e5a1ad6a88/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf", size = 239584, upload-time = "2026-07-07T14:34:20.52Z" }, { url = "https://files.pythonhosted.org/packages/d4/ce/9af95f7876194bd7a14e3dfe4a4de2e0bff02666a3910d72beafd06cc297/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990", size = 230224, upload-time = "2026-07-07T14:34:22.189Z" }, { url = "https://files.pythonhosted.org/packages/52/94/af74dde74a3996bd959c350709bfe50e297823d70a8c1cbd54b838880863/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d", size = 212667, upload-time = "2026-07-07T14:34:23.857Z" }, { url = "https://files.pythonhosted.org/packages/ee/f0/f1c4fe746c395922961b5916ed1d7d6e7d4c84851d19ed43cc89980ec953/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e", size = 227179, upload-time = "2026-07-07T14:34:25.586Z" }, { url = "https://files.pythonhosted.org/packages/e4/56/6c745619ac397e8871e2bcd3cea1eec86b877488f33888b3aef5c3ed506e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c", size = 225372, upload-time = "2026-07-07T14:34:27.212Z" }, { url = "https://files.pythonhosted.org/packages/78/ad/98aae8630ac71f16711968e38a5acfecce41b778bf2f0312851020f565a8/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2", size = 215222, upload-time = "2026-07-07T14:34:28.774Z" }, { url = "https://files.pythonhosted.org/packages/f7/40/9593d54209765207a7f11073c06494c1721e4ca4a0a426c597679bf7f91e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534", size = 231958, upload-time = "2026-07-07T14:34:30.345Z" }, { url = "https://files.pythonhosted.org/packages/b1/27/693ee5e8a18191eb38647360c51cd505013e2bd3b366aa43fd5344c21e3c/charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226", size = 155580, upload-time = "2026-07-07T14:34:31.884Z" }, { url = "https://files.pythonhosted.org/packages/80/3f/bd97d3d9c613013d07cb7733d299385b41df37f0471310f5a73dc359f0b8/charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177", size = 167620, upload-time = "2026-07-07T14:34:33.438Z" }, { url = "https://files.pythonhosted.org/packages/3d/c6/eee9dca4439b1061f76373f06ea855678cc4a64c1c3c90b50e479edbb8eb/charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501", size = 158037, upload-time = "2026-07-07T14:34:35.018Z" }, { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, ] [[package]] name = "click" version = "8.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] [[package]] name = "colorama" version = "0.4.6" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] [[package]] name = "comm" version = "0.2.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/4c/13/7d740c5849255756bc17888787313b61fd38a0a8304fc4f073dfc46122aa/comm-0.2.3.tar.gz", hash = "sha256:2dc8048c10962d55d7ad693be1e7045d891b7ce8d999c97963a5e3e99c055971", size = 6319, upload-time = "2025-07-25T14:02:04.452Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl", hash = "sha256:c615d91d75f7f04f095b30d1c1711babd43bdc6419c1be9886a85f2f4e489417", size = 7294, upload-time = "2025-07-25T14:02:02.896Z" }, ] [[package]] name = "debugpy" version = "1.8.21" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/f2/aa/12037145b7a56eaa5b29b41872f7a21b538e807e13f32c4d3c46e59be084/debugpy-1.8.21.tar.gz", hash = "sha256:a3c53278e84c94e11bd87c53970ec391d1a67396c8b22609fcac576520e611a6", size = 1697577, upload-time = "2026-06-01T19:30:35.156Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a2/df/bf625547431a9cadc9f4cbfeda38866e2b17f6aed147b625377e87834449/debugpy-1.8.21-cp312-cp312-macosx_15_0_universal2.whl", hash = "sha256:9f96713896f39c3dff0ee841f47320c3f2983d33c341e009361bb0ebc79adc4e", size = 2483609, upload-time = "2026-06-01T19:30:50.794Z" }, { url = "https://files.pythonhosted.org/packages/bf/09/59324b903599031ff9faaec1758292409f6561a0ec2492fe4b703327705a/debugpy-1.8.21-cp312-cp312-manylinux_2_34_x86_64.whl", hash = "sha256:c193d474f0a211191f2b4449d2d06157c689013035bd952f3b617e0ef422b176", size = 3968900, upload-time = "2026-06-01T19:30:52.341Z" }, { url = "https://files.pythonhosted.org/packages/14/cd/27f65b805d7fe005c44e1a36b9183ecdfbcdbf9d3e721a5115d461ecc7ee/debugpy-1.8.21-cp312-cp312-win32.whl", hash = "sha256:4743373c1cac7f9e74a1b9915bf1dbe0e900eca657ffb170ae07ac8363205ae9", size = 5336340, upload-time = "2026-06-01T19:30:54.047Z" }, { url = "https://files.pythonhosted.org/packages/77/1d/c84e30c0c674184948b66f076ab271c01d940618a2824c23cd035a27bc20/debugpy-1.8.21-cp312-cp312-win_amd64.whl", hash = "sha256:bd7ba9dd3daa7c2f942c6ca8d4695a16bf9ac16b63615261c7982bc74f7ed20c", size = 5374751, upload-time = "2026-06-01T19:30:55.891Z" }, { url = "https://files.pythonhosted.org/packages/77/6b/d817e1f8cc77aa055d37fba092e0febfdff40fe652d8d53d4cd7a86ad98d/debugpy-1.8.21-cp313-cp313-macosx_15_0_universal2.whl", hash = "sha256:13678151fc401e2d68c9880b91e28714f797d40422994572b24560ef80910a88", size = 2477398, upload-time = "2026-06-01T19:30:57.644Z" }, { url = "https://files.pythonhosted.org/packages/48/57/412421516afc3055fa577516f00beec3d663f9b0ab330639547ae6c57720/debugpy-1.8.21-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:ecbd158386c31ffe71d46f72d44d56e66331ab9b16cad649156d514368f23ab2", size = 3962096, upload-time = "2026-06-01T19:30:59.235Z" }, { url = "https://files.pythonhosted.org/packages/c1/62/2c616337cf6ba7b07ebbc97f02c6c945a8e2f76b365e33ee809c32ee36d1/debugpy-1.8.21-cp313-cp313-win32.whl", hash = "sha256:2c2ae706dec41d99a9ca1f7ebc987a83e65578363be6f6b3ac9067504917fae1", size = 5336288, upload-time = "2026-06-01T19:31:00.79Z" }, { url = "https://files.pythonhosted.org/packages/f8/99/9175103392f84c4b1bf7622888cdc68da07f0ff7d9e581266428f6776033/debugpy-1.8.21-cp313-cp313-win_amd64.whl", hash = "sha256:aa648733047443eb1d07682c4ef287d36a54507b643ffdf38b09a3ef002c72a0", size = 5376567, upload-time = "2026-06-01T19:31:02.56Z" }, { url = "https://files.pythonhosted.org/packages/ce/3d/f4bbb323a548bfab2af3d6b4ffd9bf22636e55956a1285d317a1de643aad/debugpy-1.8.21-cp314-cp314-macosx_15_0_universal2.whl", hash = "sha256:9bb2a685287a2ac9b181cde89edcec64845cb51de7faaa75badb9a698bc24782", size = 2477209, upload-time = "2026-06-01T19:31:04.157Z" }, { url = "https://files.pythonhosted.org/packages/8c/2d/6e7ec524984a1702777868de49a4c53202bddac2a432a76a093469587750/debugpy-1.8.21-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:3d6922439bf33fd38a3e2c447869ebc7b97da5cd3d329ff1ef9bc06c4903437e", size = 3927115, upload-time = "2026-06-01T19:31:05.863Z" }, { url = "https://files.pythonhosted.org/packages/97/47/d1aa6d64005a98a9144647d99306b419396f9ad7bf1d73c119e17a81fb4d/debugpy-1.8.21-cp314-cp314-win32.whl", hash = "sha256:15d4963bd5ffa48f0da0947fd06757fa7621945048a14ad7705431566d3c0e7c", size = 5336724, upload-time = "2026-06-01T19:31:07.711Z" }, { url = "https://files.pythonhosted.org/packages/5f/67/b905b90d163af11878c1af8abafa4a25206335e112e284e413454543a6da/debugpy-1.8.21-cp314-cp314-win_amd64.whl", hash = "sha256:fe0744a12353406de0ae8ccff0d0a4a666f00801a3db8fd04e7a5f761cd520e8", size = 5373803, upload-time = "2026-06-01T19:31:09.469Z" }, { url = "https://files.pythonhosted.org/packages/95/51/67e7cf11a53e40694f720457d5b3a1cdaaa3d5a9a633e482f225456b93ff/debugpy-1.8.21-py2.py3-none-any.whl", hash = "sha256:b1e37d333663c8851516a47364ef473da127f9caebe4417e6df6f5825a7e9a92", size = 5352888, upload-time = "2026-06-01T19:31:25.186Z" }, ] [[package]] name = "donfig" version = "0.8.1.post1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz", hash = "sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", size = 19506, upload-time = "2024-05-23T14:14:31.513Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl", hash = "sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", size = 21592, upload-time = "2024-05-23T14:13:55.283Z" }, ] [[package]] name = "executing" version = "2.2.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/cc/28/c14e053b6762b1044f34a13aab6859bbf40456d37d23aa286ac24cfd9a5d/executing-2.2.1.tar.gz", hash = "sha256:3632cc370565f6648cc328b32435bd120a1e4ebb20c77e3fdde9a13cd1e533c4", size = 1129488, upload-time = "2025-09-01T09:48:10.866Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, ] [[package]] name = "fastjsonschema" version = "2.22.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e4/98/474719c58eddaf77fa443b063693e76d49db32bbe851bcbaf58d2700119f/fastjsonschema-2.22.1.tar.gz", hash = "sha256:0b83d1ce8d7845b959dcb20e1a5c3c8883b6541d9c52ab02cce5166b75ec805f", size = 382291, upload-time = "2026-07-27T13:31:08.515Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/17/e1/62cc96341f01bdff2ba967441939178fcd1900d11ce7e6554d9954a5d7ec/fastjsonschema-2.22.1-py3-none-any.whl", hash = "sha256:cf377ff5c9a6f4f3125fb35f75a2c5767bd824ffbcf62c209a93cd48d1453999", size = 26239, upload-time = "2026-07-27T13:31:03.251Z" }, ] [[package]] name = "frozenlist" version = "1.8.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782, upload-time = "2025-10-06T05:36:06.649Z" }, { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594, upload-time = "2025-10-06T05:36:07.69Z" }, { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448, upload-time = "2025-10-06T05:36:08.78Z" }, { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014, upload-time = "2025-10-06T05:36:11.394Z" }, { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909, upload-time = "2025-10-06T05:36:12.598Z" }, { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049, upload-time = "2025-10-06T05:36:14.065Z" }, { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485, upload-time = "2025-10-06T05:36:15.39Z" }, { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619, upload-time = "2025-10-06T05:36:16.558Z" }, { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320, upload-time = "2025-10-06T05:36:17.821Z" }, { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820, upload-time = "2025-10-06T05:36:19.046Z" }, { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518, upload-time = "2025-10-06T05:36:20.763Z" }, { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985, upload-time = "2025-10-06T05:36:23.661Z" }, { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591, upload-time = "2025-10-06T05:36:24.958Z" }, { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102, upload-time = "2025-10-06T05:36:26.333Z" }, { url = "https://files.pythonhosted.org/packages/2d/40/0832c31a37d60f60ed79e9dfb5a92e1e2af4f40a16a29abcc7992af9edff/frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a", size = 85717, upload-time = "2025-10-06T05:36:27.341Z" }, { url = "https://files.pythonhosted.org/packages/30/ba/b0b3de23f40bc55a7057bd38434e25c34fa48e17f20ee273bbde5e0650f3/frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7", size = 49651, upload-time = "2025-10-06T05:36:28.855Z" }, { url = "https://files.pythonhosted.org/packages/0c/ab/6e5080ee374f875296c4243c381bbdef97a9ac39c6e3ce1d5f7d42cb78d6/frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40", size = 49417, upload-time = "2025-10-06T05:36:29.877Z" }, { url = "https://files.pythonhosted.org/packages/d5/4e/e4691508f9477ce67da2015d8c00acd751e6287739123113a9fca6f1604e/frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027", size = 234391, upload-time = "2025-10-06T05:36:31.301Z" }, { url = "https://files.pythonhosted.org/packages/40/76/c202df58e3acdf12969a7895fd6f3bc016c642e6726aa63bd3025e0fc71c/frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822", size = 233048, upload-time = "2025-10-06T05:36:32.531Z" }, { url = "https://files.pythonhosted.org/packages/f9/c0/8746afb90f17b73ca5979c7a3958116e105ff796e718575175319b5bb4ce/frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121", size = 226549, upload-time = "2025-10-06T05:36:33.706Z" }, { url = "https://files.pythonhosted.org/packages/7e/eb/4c7eefc718ff72f9b6c4893291abaae5fbc0c82226a32dcd8ef4f7a5dbef/frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5", size = 239833, upload-time = "2025-10-06T05:36:34.947Z" }, { url = "https://files.pythonhosted.org/packages/c2/4e/e5c02187cf704224f8b21bee886f3d713ca379535f16893233b9d672ea71/frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e", size = 245363, upload-time = "2025-10-06T05:36:36.534Z" }, { url = "https://files.pythonhosted.org/packages/1f/96/cb85ec608464472e82ad37a17f844889c36100eed57bea094518bf270692/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11", size = 229314, upload-time = "2025-10-06T05:36:38.582Z" }, { url = "https://files.pythonhosted.org/packages/5d/6f/4ae69c550e4cee66b57887daeebe006fe985917c01d0fff9caab9883f6d0/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1", size = 243365, upload-time = "2025-10-06T05:36:40.152Z" }, { url = "https://files.pythonhosted.org/packages/7a/58/afd56de246cf11780a40a2c28dc7cbabbf06337cc8ddb1c780a2d97e88d8/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1", size = 237763, upload-time = "2025-10-06T05:36:41.355Z" }, { url = "https://files.pythonhosted.org/packages/cb/36/cdfaf6ed42e2644740d4a10452d8e97fa1c062e2a8006e4b09f1b5fd7d63/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8", size = 240110, upload-time = "2025-10-06T05:36:42.716Z" }, { url = "https://files.pythonhosted.org/packages/03/a8/9ea226fbefad669f11b52e864c55f0bd57d3c8d7eb07e9f2e9a0b39502e1/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed", size = 233717, upload-time = "2025-10-06T05:36:44.251Z" }, { url = "https://files.pythonhosted.org/packages/1e/0b/1b5531611e83ba7d13ccc9988967ea1b51186af64c42b7a7af465dcc9568/frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496", size = 39628, upload-time = "2025-10-06T05:36:45.423Z" }, { url = "https://files.pythonhosted.org/packages/d8/cf/174c91dbc9cc49bc7b7aab74d8b734e974d1faa8f191c74af9b7e80848e6/frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231", size = 43882, upload-time = "2025-10-06T05:36:46.796Z" }, { url = "https://files.pythonhosted.org/packages/c1/17/502cd212cbfa96eb1388614fe39a3fc9ab87dbbe042b66f97acb57474834/frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62", size = 39676, upload-time = "2025-10-06T05:36:47.8Z" }, { url = "https://files.pythonhosted.org/packages/d2/5c/3bbfaa920dfab09e76946a5d2833a7cbdf7b9b4a91c714666ac4855b88b4/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94", size = 89235, upload-time = "2025-10-06T05:36:48.78Z" }, { url = "https://files.pythonhosted.org/packages/d2/d6/f03961ef72166cec1687e84e8925838442b615bd0b8854b54923ce5b7b8a/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c", size = 50742, upload-time = "2025-10-06T05:36:49.837Z" }, { url = "https://files.pythonhosted.org/packages/1e/bb/a6d12b7ba4c3337667d0e421f7181c82dda448ce4e7ad7ecd249a16fa806/frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52", size = 51725, upload-time = "2025-10-06T05:36:50.851Z" }, { url = "https://files.pythonhosted.org/packages/bc/71/d1fed0ffe2c2ccd70b43714c6cab0f4188f09f8a67a7914a6b46ee30f274/frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51", size = 284533, upload-time = "2025-10-06T05:36:51.898Z" }, { url = "https://files.pythonhosted.org/packages/c9/1f/fb1685a7b009d89f9bf78a42d94461bc06581f6e718c39344754a5d9bada/frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65", size = 292506, upload-time = "2025-10-06T05:36:53.101Z" }, { url = "https://files.pythonhosted.org/packages/e6/3b/b991fe1612703f7e0d05c0cf734c1b77aaf7c7d321df4572e8d36e7048c8/frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82", size = 274161, upload-time = "2025-10-06T05:36:54.309Z" }, { url = "https://files.pythonhosted.org/packages/ca/ec/c5c618767bcdf66e88945ec0157d7f6c4a1322f1473392319b7a2501ded7/frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714", size = 294676, upload-time = "2025-10-06T05:36:55.566Z" }, { url = "https://files.pythonhosted.org/packages/7c/ce/3934758637d8f8a88d11f0585d6495ef54b2044ed6ec84492a91fa3b27aa/frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d", size = 300638, upload-time = "2025-10-06T05:36:56.758Z" }, { url = "https://files.pythonhosted.org/packages/fc/4f/a7e4d0d467298f42de4b41cbc7ddaf19d3cfeabaf9ff97c20c6c7ee409f9/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506", size = 283067, upload-time = "2025-10-06T05:36:57.965Z" }, { url = "https://files.pythonhosted.org/packages/dc/48/c7b163063d55a83772b268e6d1affb960771b0e203b632cfe09522d67ea5/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51", size = 292101, upload-time = "2025-10-06T05:36:59.237Z" }, { url = "https://files.pythonhosted.org/packages/9f/d0/2366d3c4ecdc2fd391e0afa6e11500bfba0ea772764d631bbf82f0136c9d/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e", size = 289901, upload-time = "2025-10-06T05:37:00.811Z" }, { url = "https://files.pythonhosted.org/packages/b8/94/daff920e82c1b70e3618a2ac39fbc01ae3e2ff6124e80739ce5d71c9b920/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0", size = 289395, upload-time = "2025-10-06T05:37:02.115Z" }, { url = "https://files.pythonhosted.org/packages/e3/20/bba307ab4235a09fdcd3cc5508dbabd17c4634a1af4b96e0f69bfe551ebd/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41", size = 283659, upload-time = "2025-10-06T05:37:03.711Z" }, { url = "https://files.pythonhosted.org/packages/fd/00/04ca1c3a7a124b6de4f8a9a17cc2fcad138b4608e7a3fc5877804b8715d7/frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b", size = 43492, upload-time = "2025-10-06T05:37:04.915Z" }, { url = "https://files.pythonhosted.org/packages/59/5e/c69f733a86a94ab10f68e496dc6b7e8bc078ebb415281d5698313e3af3a1/frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888", size = 48034, upload-time = "2025-10-06T05:37:06.343Z" }, { url = "https://files.pythonhosted.org/packages/16/6c/be9d79775d8abe79b05fa6d23da99ad6e7763a1d080fbae7290b286093fd/frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042", size = 41749, upload-time = "2025-10-06T05:37:07.431Z" }, { url = "https://files.pythonhosted.org/packages/f1/c8/85da824b7e7b9b6e7f7705b2ecaf9591ba6f79c1177f324c2735e41d36a2/frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0", size = 86127, upload-time = "2025-10-06T05:37:08.438Z" }, { url = "https://files.pythonhosted.org/packages/8e/e8/a1185e236ec66c20afd72399522f142c3724c785789255202d27ae992818/frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f", size = 49698, upload-time = "2025-10-06T05:37:09.48Z" }, { url = "https://files.pythonhosted.org/packages/a1/93/72b1736d68f03fda5fdf0f2180fb6caaae3894f1b854d006ac61ecc727ee/frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c", size = 49749, upload-time = "2025-10-06T05:37:10.569Z" }, { url = "https://files.pythonhosted.org/packages/a7/b2/fabede9fafd976b991e9f1b9c8c873ed86f202889b864756f240ce6dd855/frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2", size = 231298, upload-time = "2025-10-06T05:37:11.993Z" }, { url = "https://files.pythonhosted.org/packages/3a/3b/d9b1e0b0eed36e70477ffb8360c49c85c8ca8ef9700a4e6711f39a6e8b45/frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8", size = 232015, upload-time = "2025-10-06T05:37:13.194Z" }, { url = "https://files.pythonhosted.org/packages/dc/94/be719d2766c1138148564a3960fc2c06eb688da592bdc25adcf856101be7/frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686", size = 225038, upload-time = "2025-10-06T05:37:14.577Z" }, { url = "https://files.pythonhosted.org/packages/e4/09/6712b6c5465f083f52f50cf74167b92d4ea2f50e46a9eea0523d658454ae/frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e", size = 240130, upload-time = "2025-10-06T05:37:15.781Z" }, { url = "https://files.pythonhosted.org/packages/f8/d4/cd065cdcf21550b54f3ce6a22e143ac9e4836ca42a0de1022da8498eac89/frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a", size = 242845, upload-time = "2025-10-06T05:37:17.037Z" }, { url = "https://files.pythonhosted.org/packages/62/c3/f57a5c8c70cd1ead3d5d5f776f89d33110b1addae0ab010ad774d9a44fb9/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128", size = 229131, upload-time = "2025-10-06T05:37:18.221Z" }, { url = "https://files.pythonhosted.org/packages/6c/52/232476fe9cb64f0742f3fde2b7d26c1dac18b6d62071c74d4ded55e0ef94/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f", size = 240542, upload-time = "2025-10-06T05:37:19.771Z" }, { url = "https://files.pythonhosted.org/packages/5f/85/07bf3f5d0fb5414aee5f47d33c6f5c77bfe49aac680bfece33d4fdf6a246/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7", size = 237308, upload-time = "2025-10-06T05:37:20.969Z" }, { url = "https://files.pythonhosted.org/packages/11/99/ae3a33d5befd41ac0ca2cc7fd3aa707c9c324de2e89db0e0f45db9a64c26/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30", size = 238210, upload-time = "2025-10-06T05:37:22.252Z" }, { url = "https://files.pythonhosted.org/packages/b2/60/b1d2da22f4970e7a155f0adde9b1435712ece01b3cd45ba63702aea33938/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7", size = 231972, upload-time = "2025-10-06T05:37:23.5Z" }, { url = "https://files.pythonhosted.org/packages/3f/ab/945b2f32de889993b9c9133216c068b7fcf257d8595a0ac420ac8677cab0/frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806", size = 40536, upload-time = "2025-10-06T05:37:25.581Z" }, { url = "https://files.pythonhosted.org/packages/59/ad/9caa9b9c836d9ad6f067157a531ac48b7d36499f5036d4141ce78c230b1b/frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0", size = 44330, upload-time = "2025-10-06T05:37:26.928Z" }, { url = "https://files.pythonhosted.org/packages/82/13/e6950121764f2676f43534c555249f57030150260aee9dcf7d64efda11dd/frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b", size = 40627, upload-time = "2025-10-06T05:37:28.075Z" }, { url = "https://files.pythonhosted.org/packages/c0/c7/43200656ecc4e02d3f8bc248df68256cd9572b3f0017f0a0c4e93440ae23/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d", size = 89238, upload-time = "2025-10-06T05:37:29.373Z" }, { url = "https://files.pythonhosted.org/packages/d1/29/55c5f0689b9c0fb765055629f472c0de484dcaf0acee2f7707266ae3583c/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed", size = 50738, upload-time = "2025-10-06T05:37:30.792Z" }, { url = "https://files.pythonhosted.org/packages/ba/7d/b7282a445956506fa11da8c2db7d276adcbf2b17d8bb8407a47685263f90/frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930", size = 51739, upload-time = "2025-10-06T05:37:32.127Z" }, { url = "https://files.pythonhosted.org/packages/62/1c/3d8622e60d0b767a5510d1d3cf21065b9db874696a51ea6d7a43180a259c/frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c", size = 284186, upload-time = "2025-10-06T05:37:33.21Z" }, { url = "https://files.pythonhosted.org/packages/2d/14/aa36d5f85a89679a85a1d44cd7a6657e0b1c75f61e7cad987b203d2daca8/frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24", size = 292196, upload-time = "2025-10-06T05:37:36.107Z" }, { url = "https://files.pythonhosted.org/packages/05/23/6bde59eb55abd407d34f77d39a5126fb7b4f109a3f611d3929f14b700c66/frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37", size = 273830, upload-time = "2025-10-06T05:37:37.663Z" }, { url = "https://files.pythonhosted.org/packages/d2/3f/22cff331bfad7a8afa616289000ba793347fcd7bc275f3b28ecea2a27909/frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a", size = 294289, upload-time = "2025-10-06T05:37:39.261Z" }, { url = "https://files.pythonhosted.org/packages/a4/89/5b057c799de4838b6c69aa82b79705f2027615e01be996d2486a69ca99c4/frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2", size = 300318, upload-time = "2025-10-06T05:37:43.213Z" }, { url = "https://files.pythonhosted.org/packages/30/de/2c22ab3eb2a8af6d69dc799e48455813bab3690c760de58e1bf43b36da3e/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef", size = 282814, upload-time = "2025-10-06T05:37:45.337Z" }, { url = "https://files.pythonhosted.org/packages/59/f7/970141a6a8dbd7f556d94977858cfb36fa9b66e0892c6dd780d2219d8cd8/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe", size = 291762, upload-time = "2025-10-06T05:37:46.657Z" }, { url = "https://files.pythonhosted.org/packages/c1/15/ca1adae83a719f82df9116d66f5bb28bb95557b3951903d39135620ef157/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8", size = 289470, upload-time = "2025-10-06T05:37:47.946Z" }, { url = "https://files.pythonhosted.org/packages/ac/83/dca6dc53bf657d371fbc88ddeb21b79891e747189c5de990b9dfff2ccba1/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a", size = 289042, upload-time = "2025-10-06T05:37:49.499Z" }, { url = "https://files.pythonhosted.org/packages/96/52/abddd34ca99be142f354398700536c5bd315880ed0a213812bc491cff5e4/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e", size = 283148, upload-time = "2025-10-06T05:37:50.745Z" }, { url = "https://files.pythonhosted.org/packages/af/d3/76bd4ed4317e7119c2b7f57c3f6934aba26d277acc6309f873341640e21f/frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df", size = 44676, upload-time = "2025-10-06T05:37:52.222Z" }, { url = "https://files.pythonhosted.org/packages/89/76/c615883b7b521ead2944bb3480398cbb07e12b7b4e4d073d3752eb721558/frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd", size = 49451, upload-time = "2025-10-06T05:37:53.425Z" }, { url = "https://files.pythonhosted.org/packages/e0/a3/5982da14e113d07b325230f95060e2169f5311b1017ea8af2a29b374c289/frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79", size = 42507, upload-time = "2025-10-06T05:37:54.513Z" }, { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, ] [[package]] name = "fsspec" version = "2026.7.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/00/78/f34251dadb8f3921264a1d9b8946f5e542014ee2614b285261b4e40e6775/fsspec-2026.7.0.tar.gz", hash = "sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88", size = 317040, upload-time = "2026-07-28T16:34:51.052Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", hash = "sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279", size = 206583, upload-time = "2026-07-28T16:34:49.538Z" }, ] [package.optional-dependencies] http = [ { name = "aiohttp" }, ] [[package]] name = "ghp-import" version = "2.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "python-dateutil" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d9/29/d40217cbe2f6b1359e00c6c307bb3fc876ba74068cbab3dde77f03ca0dc4/ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343", size = 10943, upload-time = "2022-05-02T15:47:16.11Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", size = 11034, upload-time = "2022-05-02T15:47:14.552Z" }, ] [[package]] name = "google-crc32c" version = "1.8.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e9/5f/7307325b1198b59324c0fa9807cafb551afb65e831699f2ce211ad5c8240/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113", size = 31300, upload-time = "2025-12-16T00:21:56.723Z" }, { url = "https://files.pythonhosted.org/packages/21/8e/58c0d5d86e2220e6a37befe7e6a94dd2f6006044b1a33edf1ff6d9f7e319/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb", size = 30867, upload-time = "2025-12-16T00:38:31.302Z" }, { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" }, { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" }, { url = "https://files.pythonhosted.org/packages/df/c0/87c2073e0c72515bb8733d4eef7b21548e8d189f094b5dad20b0ecaf64f6/google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962", size = 34437, upload-time = "2025-12-16T00:35:21.395Z" }, { url = "https://files.pythonhosted.org/packages/d1/db/000f15b41724589b0e7bc24bc7a8967898d8d3bc8caf64c513d91ef1f6c0/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b", size = 31297, upload-time = "2025-12-16T00:23:20.709Z" }, { url = "https://files.pythonhosted.org/packages/d7/0d/8ebed0c39c53a7e838e2a486da8abb0e52de135f1b376ae2f0b160eb4c1a/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27", size = 30867, upload-time = "2025-12-16T00:43:14.628Z" }, { url = "https://files.pythonhosted.org/packages/ce/42/b468aec74a0354b34c8cbf748db20d6e350a68a2b0912e128cabee49806c/google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa", size = 33344, upload-time = "2025-12-16T00:40:24.742Z" }, { url = "https://files.pythonhosted.org/packages/1c/e8/b33784d6fc77fb5062a8a7854e43e1e618b87d5ddf610a88025e4de6226e/google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8", size = 33694, upload-time = "2025-12-16T00:40:25.505Z" }, { url = "https://files.pythonhosted.org/packages/92/b1/d3cbd4d988afb3d8e4db94ca953df429ed6db7282ed0e700d25e6c7bfc8d/google_crc32c-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f", size = 34435, upload-time = "2025-12-16T00:35:22.107Z" }, { url = "https://files.pythonhosted.org/packages/21/88/8ecf3c2b864a490b9e7010c84fd203ec8cf3b280651106a3a74dd1b0ca72/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697", size = 31301, upload-time = "2025-12-16T00:24:48.527Z" }, { url = "https://files.pythonhosted.org/packages/36/c6/f7ff6c11f5ca215d9f43d3629163727a272eabc356e5c9b2853df2bfe965/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651", size = 30868, upload-time = "2025-12-16T00:48:12.163Z" }, { url = "https://files.pythonhosted.org/packages/56/15/c25671c7aad70f8179d858c55a6ae8404902abe0cdcf32a29d581792b491/google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2", size = 33381, upload-time = "2025-12-16T00:40:26.268Z" }, { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" }, { url = "https://files.pythonhosted.org/packages/08/a5/7b059810934a09fb3ccb657e0843813c1fee1183d3bc2c8041800374aa2c/google_crc32c-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2", size = 34878, upload-time = "2025-12-16T00:35:23.142Z" }, ] [[package]] name = "griffe-inherited-docstrings" version = "1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffelib" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cb/da/fd002dc5f215cd896bfccaebe8b4aa1cdeed8ea1d9d60633685bd61ff933/griffe_inherited_docstrings-1.1.3.tar.gz", hash = "sha256:cd1f937ec9336a790e5425e7f9b92f5a5ab17f292ba86917f1c681c0704cb64e", size = 26738, upload-time = "2026-02-21T09:38:44.312Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/16/20/4bc15f242181daad1c104e0a7d33be49e712461ea89e548152be0365b9ea/griffe_inherited_docstrings-1.1.3-py3-none-any.whl", hash = "sha256:aa7f6e624515c50d9325a5cfdf4b2acac547f1889aca89092d5da7278f739695", size = 6710, upload-time = "2026-02-20T11:06:38.75Z" }, ] [[package]] name = "griffelib" version = "2.1.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/33/e4/8d187ea29c2e30b3a09505c567513077d6117861bde1fbd997a167f262ec/griffelib-2.1.0.tar.gz", hash = "sha256:762a186d2c6fd6794d4ea20d428d597ffb857cb56b66421651cbba15bdd5e813", size = 216234, upload-time = "2026-06-19T12:05:42.278Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e4/d3/5268aeabf2ad82658c4e2ff3a060648d0f02f3926cb53247c0e4d0dab49e/griffelib-2.1.0-py3-none-any.whl", hash = "sha256:cc7b3d2d2865ad0b909fcc38086e3f554b5ea7acbaa7bbb7ecaa3f5dfb7d9f00", size = 142560, upload-time = "2026-06-19T12:05:38.742Z" }, ] [[package]] name = "h11" version = "0.16.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] [[package]] name = "httpcore" version = "1.0.9" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, { name = "h11" }, ] sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] [[package]] name = "httpcore2" version = "2.12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "h11" }, { name = "truststore" }, ] sdist = { url = "https://files.pythonhosted.org/packages/be/ad/f4f0e57345f1870f3e8cb624e058d7eca6e5a27d33bcc3311d9b618734cd/httpcore2-2.12.0.tar.gz", hash = "sha256:9293522bba0aa7c4c8e9e3f040c16575bd8868e155a77fa30c7a9085a5eae648", size = 67548, upload-time = "2026-08-18T13:22:08.211Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d2/74/d370e55600d9bcfa0d9794b0166126d49291a3d2b20c268fc98c453a4948/httpcore2-2.12.0-py3-none-any.whl", hash = "sha256:7e04258ce01013d7d615e5b910a3b27fac937d7a95038227e79652b4ba3b4ceb", size = 83074, upload-time = "2026-08-18T13:22:05.854Z" }, ] [[package]] name = "httpx" version = "0.28.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "certifi" }, { name = "httpcore" }, { name = "idna" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] [[package]] name = "httpx2" version = "2.12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio", marker = "sys_platform != 'emscripten'" }, { name = "httpcore2", marker = "sys_platform != 'emscripten'" }, { name = "httpx2-jsfetch", marker = "sys_platform == 'emscripten'" }, { name = "idna" }, { name = "truststore", marker = "sys_platform != 'emscripten'" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7f/f8/579a8b51e42e38ee32647df9f08aa25643ae788e275cc625b199829c4671/httpx2-2.12.0.tar.gz", hash = "sha256:7631fe9887a8a2275f4a2540e053aa670fcc50742864a9ae7c66e609fdcf12cf", size = 100040, upload-time = "2026-08-18T13:22:09.086Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c8/95/411ba65569158e862368917aaf56597f3e5fa3b91b0502919638465a08f3/httpx2-2.12.0-py3-none-any.whl", hash = "sha256:cc8b6eecb8661c146b8f89a60e97456ee086e91a784ed31ac450c3a9e613dd36", size = 95427, upload-time = "2026-08-18T13:22:06.834Z" }, ] [[package]] name = "httpx2-jsfetch" version = "1.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/0e5636363151a2a1795e0a77617168b9ca438e1748ec05fc9b5687f93d64/httpx2_jsfetch-1.0.tar.gz", hash = "sha256:70a0e3eabfef7cce5ad9c629f7d01ca05e418f586646f4ddf14782e4c1454c60", size = 6872, upload-time = "2026-08-07T00:13:07.492Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, ] [[package]] name = "hypothesis" version = "6.165.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "sortedcontainers" }, ] sdist = { url = "https://files.pythonhosted.org/packages/18/7a/7a277ac07776191be594f74f6425649d529e4876f7d3ff1ee96d393ffdbc/hypothesis-6.165.3.tar.gz", hash = "sha256:687c5abb1a9c11478577c2cf18685c0eb82150d278477d3e14da290a1ef2a098", size = 502263, upload-time = "2026-08-11T01:23:09.1Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f9/c7/18152acad5f85f91554b2030000319b952a54151509953651ec40f37d50d/hypothesis-6.165.3-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:56af539c811b11ab5475704c300b8f0b46cc6dd0edc267e02a16487e803c77f8", size = 781671, upload-time = "2026-08-11T01:22:09.176Z" }, { url = "https://files.pythonhosted.org/packages/d3/77/4293ea8a7fdb713956a8bf460b9070115df69f8216a900507633f9cdb225/hypothesis-6.165.3-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:f40c10cfdb1ea2cd75e5d4e6e0cfdcb6198ab8406e8922666480e6dc11eea341", size = 777291, upload-time = "2026-08-11T01:22:15.991Z" }, { url = "https://files.pythonhosted.org/packages/02/fa/fa2071a6afaefc082dc7a033f41ae61436caf442d5973ba8ca9c29a69460/hypothesis-6.165.3-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a0854b1de4577f7e1beb1d681360285b5d678b65a809787ff4eab5b8b25efca", size = 1106490, upload-time = "2026-08-11T01:22:07.858Z" }, { url = "https://files.pythonhosted.org/packages/ba/86/de724b7f9cd10e3be4efa21770457172e549d7576b1d8e29d6177eef5e47/hypothesis-6.165.3-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:360991cda8e488924905af48949033b90d4877ac97b9ad5d826d4d0f5a4b8cfb", size = 1135054, upload-time = "2026-08-11T01:22:29.499Z" }, { url = "https://files.pythonhosted.org/packages/12/6a/96721cf447bd3c64b5e6843dde4444b20f3ddd901ad366cc73d0e7314bf5/hypothesis-6.165.3-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf502000f4a8ef4c9ab9493ca3b4fe17ae3033c18a8e2a31cdd69515dc7d97be", size = 1155997, upload-time = "2026-08-11T01:21:48.496Z" }, { url = "https://files.pythonhosted.org/packages/0f/ae/a793cce6497f233b155f97684bf7d0e424c25613dd87b8af8a4e87820232/hypothesis-6.165.3-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:b9fcf47ad18f87f7c15bd36289bd45708bbfd250129d73bf554653e2f9afc931", size = 1111326, upload-time = "2026-08-11T01:22:21.9Z" }, { url = "https://files.pythonhosted.org/packages/28/8d/dc3cdfd55843d038effa2458a9c9bd73002218a8c0fd58c2c0ab7fa328db/hypothesis-6.165.3-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2fb05529cbcab5a317d03d7bb0e90d382f79ef1643e3568577916d0e24bfe70b", size = 1148079, upload-time = "2026-08-11T01:21:51.071Z" }, { url = "https://files.pythonhosted.org/packages/db/4b/2f62924ac41f3d3482b29ded4c213f27ff4a103e56e84eeb528d4900cac7/hypothesis-6.165.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:57eae10a64340cd621a78eae9cb0459bd68ea99fbaa933c4f00e34d5087b6376", size = 1281862, upload-time = "2026-08-11T01:21:39.274Z" }, { url = "https://files.pythonhosted.org/packages/7e/d4/01c78b7b7348b6e8cef9b999109dfb93b14c7e1e38bc22170129f8b17181/hypothesis-6.165.3-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:19df0f2239052e9a870634a1d9bcdff95e2a2ab508573e5dd5c3d1ca545f5b3c", size = 1408437, upload-time = "2026-08-11T01:22:13.243Z" }, { url = "https://files.pythonhosted.org/packages/35/76/e940b5a5aaf75bcd4784f1f3f9bf2b9a642a706bc0a9639077ca84f1325f/hypothesis-6.165.3-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:9781a8026adff4b4516404cf0e5f2cadcb471318c2882a264e1c57c4c092266f", size = 1281168, upload-time = "2026-08-11T01:21:58.964Z" }, { url = "https://files.pythonhosted.org/packages/fc/84/b153e81a614f45e0902e3b9e8a8b079e64214c50abb6fbe9acc62ccf686d/hypothesis-6.165.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1dd7e05f88e3e108a5e4f5f71a3eaf205559e8951e3c1f1ffd04cea82ed3b731", size = 1323263, upload-time = "2026-08-11T01:21:52.374Z" }, { url = "https://files.pythonhosted.org/packages/fb/5f/e5144d9e91ab7260650cb1ee032ca23208d49ebb1334845baf6407c1a9d9/hypothesis-6.165.3-cp310-abi3-win32.whl", hash = "sha256:d1389bda38cb222acc109aef5b31643ce799a39a76294a50ad8b84e32f92d76d", size = 667499, upload-time = "2026-08-11T01:21:47.36Z" }, { url = "https://files.pythonhosted.org/packages/a9/18/f008b6f1f1c293d51c2776f8815d95bccb777dcf87df2a0ab56b273b47dc/hypothesis-6.165.3-cp310-abi3-win_amd64.whl", hash = "sha256:10cda6988ca4b1da389548b6fdd71af236b588a601fc1757e56eb8988e4240d8", size = 673643, upload-time = "2026-08-11T01:22:46.975Z" }, { url = "https://files.pythonhosted.org/packages/e2/3d/e7eade134bd7f57d4071d82ae84691bc3017fe6945af0bb149578ba8b565/hypothesis-6.165.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f88fe4915f8dd4f8999a197f9e22c3a7177042aa994d35c0c7c7b22541d2885b", size = 783298, upload-time = "2026-08-11T01:22:14.706Z" }, { url = "https://files.pythonhosted.org/packages/d6/a5/668810f493feaf886a9240ad689792fb32450667c8e5770c3bcaa7fabeac/hypothesis-6.165.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c7d9f6c36b812f6069c7436492e12812cf541391954e21d5bd7fcafc9fb46700", size = 774856, upload-time = "2026-08-11T01:22:35.836Z" }, { url = "https://files.pythonhosted.org/packages/e4/a0/0af93a70f5128763079ab714277ccad4a7de42d442d67dd43e3029178162/hypothesis-6.165.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:423ca8e30087bb41db7e6f47dbf98690a2155241b9f1057e366dc138d7dcc4fe", size = 1105274, upload-time = "2026-08-11T01:23:03.591Z" }, { url = "https://files.pythonhosted.org/packages/7b/00/ddcdc99beee469573addf5cfcd817c9a20a71832b1ca88404b6d3f99c44c/hypothesis-6.165.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c58c66f3e1b8d4091bb52664d5af4b0c1715293c6822d5344b166494534cd498", size = 1155379, upload-time = "2026-08-11T01:22:23.371Z" }, { url = "https://files.pythonhosted.org/packages/de/10/d574b21e63f16a1cad9c0cf4592aa170285940f034d04bb33a1e08025397/hypothesis-6.165.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3774882c4685e5474b7940697da55963e591b71c6dce593d90ac4128766371ad", size = 1279259, upload-time = "2026-08-11T01:21:53.63Z" }, { url = "https://files.pythonhosted.org/packages/b9/3f/13f6b6c7d7d0b9bb570a18617eb659b015c312b1d8d1d3aaf9f2edea9628/hypothesis-6.165.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb3a397a5422c67387f4408989dbdfc2b1e0306f883f2aa79b9472c80958464", size = 1322605, upload-time = "2026-08-11T01:21:42.853Z" }, { url = "https://files.pythonhosted.org/packages/6c/dd/243317f5fb8497601dc65d3ded984834eb5f36b8ec59e7853ef753ec0ee1/hypothesis-6.165.3-cp312-cp312-win_amd64.whl", hash = "sha256:dbb74811d54b6317ba0d2047aad269c09afefaa25d1849f8f33f80a638b0c3af", size = 670812, upload-time = "2026-08-11T01:22:50.245Z" }, { url = "https://files.pythonhosted.org/packages/03/3e/95cba31dbe775b99a4548cdae192e1ad15cee7b64fbdfe6cd4c9d00031b4/hypothesis-6.165.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:447f139d6dd70a5d8b178ef507463fb0430ace9ce42e3b2351d2803a391fe774", size = 783183, upload-time = "2026-08-11T01:22:45.286Z" }, { url = "https://files.pythonhosted.org/packages/56/0e/51bf125cdf7855b69097b8f59c73ef3cf5f4e3d68a16e808d2d1f08a1ff1/hypothesis-6.165.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6152c718606f1705e673c6b30a6ebd3ff08d340da85291dd3c432c73b28a9b3a", size = 774820, upload-time = "2026-08-11T01:22:24.825Z" }, { url = "https://files.pythonhosted.org/packages/0a/69/b954f742b97441a5c49f8f8704826ee0637a6cce3a7d06ce85fbefc54ac5/hypothesis-6.165.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27fe7826ad83ccc2e8062f0fab43b137bab34cef1a149a926b34a7b8382ee22c", size = 1105186, upload-time = "2026-08-11T01:21:41.733Z" }, { url = "https://files.pythonhosted.org/packages/6e/8a/33e41d9cc1be7661e0b4129c225a93c3f12544714300aadb95ae7eedf894/hypothesis-6.165.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1ff92876a324f7b9cdb92cedf103e380b7a12aa7df55ebcb16dd0f495a879e8", size = 1155215, upload-time = "2026-08-11T01:22:06.604Z" }, { url = "https://files.pythonhosted.org/packages/ee/53/ba09526c9100ace5752908ac7251d2dc3960ce7e0e97a31152aaa26c33ee/hypothesis-6.165.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:53f1564c97d27fc109f212404d49cd71d7789777dbe0685628ffe9838df56240", size = 1279245, upload-time = "2026-08-11T01:21:56.182Z" }, { url = "https://files.pythonhosted.org/packages/c1/93/fc637d355791a65364a3409ff06ade7ba5d3fc6f1d07a729781dec315fa0/hypothesis-6.165.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:788a9b0a7aae719a2b71a1c2f07e51deb1d0fe990164a9090c686833ed4bfbad", size = 1322370, upload-time = "2026-08-11T01:22:48.492Z" }, { url = "https://files.pythonhosted.org/packages/b6/8d/826053ba0263143fed2b0e8af009dc868d8a932e7246e191deb8ca7ce8ff/hypothesis-6.165.3-cp313-cp313-win_amd64.whl", hash = "sha256:37830f0795abfdf738d2a5b6f829a73f3ab498de45a2e61b0bf3bd38d8c9ddb9", size = 670804, upload-time = "2026-08-11T01:22:00.103Z" }, { url = "https://files.pythonhosted.org/packages/e7/27/3230f8de3d853b2b547731916ae1d1026bd197cd3f2d35dafc0b445da46b/hypothesis-6.165.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:38826441dbf528cc156388d0a05526086a12da3e1348353d3fa14de03e57c4b2", size = 783286, upload-time = "2026-08-11T01:22:40.816Z" }, { url = "https://files.pythonhosted.org/packages/6c/28/9f9ca830d376c50babe55c616f6d99eea886c6ebcd8b512dcd5d56f9e40c/hypothesis-6.165.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:87490115edd34a246a4ba8b1144cbdf571438c46c406ece05caf65908667c9a9", size = 774963, upload-time = "2026-08-11T01:23:05.409Z" }, { url = "https://files.pythonhosted.org/packages/33/3c/3c81f08ec1edce160da509c5785d78c0e25a7913899c4b9ff724bfd01420/hypothesis-6.165.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f51f4346cfa26bca68c68f7bbbd2b1812208bc9f572187c95ecab080ed402153", size = 1105730, upload-time = "2026-08-11T01:22:42.311Z" }, { url = "https://files.pythonhosted.org/packages/bf/b5/f6f81b9aec9999ec63920d168617cab67a038be05487eff3410ccd072bfe/hypothesis-6.165.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4da89eb4b36b3260ff714d2ecc3274b9bd599fd96687d2d9ed53d5e1a801a7a7", size = 1155383, upload-time = "2026-08-11T01:21:57.58Z" }, { url = "https://files.pythonhosted.org/packages/dd/27/7f3a8c6101675bf95c80cd8c9173d65892ca0b7b640551156dc4537fab1f/hypothesis-6.165.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:eb6d31c14d7bdfe03e501d88ee296c149a74cc93e3d01c76ea335e64ee5f33ec", size = 1279606, upload-time = "2026-08-11T01:22:37.56Z" }, { url = "https://files.pythonhosted.org/packages/d9/16/0c23e06a24e421e532f62a95021fae34f685f3a194c081c6991b4ab202b3/hypothesis-6.165.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0863e1a9258bc103abe616fa9471cfa66a1535ea404dd8a0bf360e0a29502397", size = 1322697, upload-time = "2026-08-11T01:22:27.857Z" }, { url = "https://files.pythonhosted.org/packages/dc/56/8356dadf45e5c635b46aa2b57fa74f3210250a8e38b860b6b75f50ed0b42/hypothesis-6.165.3-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:53c56155f2cfbb45ec97fef9ea3b8453b4a34c48c3c5cacee16f97dd2a037994", size = 614859, upload-time = "2026-08-11T01:22:01.304Z" }, { url = "https://files.pythonhosted.org/packages/e3/79/124d4faf235219acd685c359760a5cb3995609bc50ce465e54c3249841ee/hypothesis-6.165.3-cp314-cp314-win_amd64.whl", hash = "sha256:c48f41e950b5e602e2fdf8f92dcc8ac7bf715a003bf822afb7c9d5cbc41bc344", size = 670600, upload-time = "2026-08-11T01:22:10.356Z" }, { url = "https://files.pythonhosted.org/packages/01/7a/41ac5e68d9ce079d1b76d4c54126354df61b948c3d519d1289aca877eedc/hypothesis-6.165.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:9563d3040178fb1f522665bcec6458cc0d21ab77d7c637058a8be4ea8c01d236", size = 781746, upload-time = "2026-08-11T01:22:34.472Z" }, { url = "https://files.pythonhosted.org/packages/5d/fb/7ecc21aae63a83dbc8036f9a0544c6b3d798db566b97b5202bdf8e770f80/hypothesis-6.165.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1fe1783543b43ba9808c016950e5e84b3804dc3365ba77c37c427b5896a558a1", size = 773382, upload-time = "2026-08-11T01:22:55.279Z" }, { url = "https://files.pythonhosted.org/packages/cb/f2/9cc2a4768f9a483b12e307ba585f5eb9c7f5500bd16ff82ddbf62a9a1b88/hypothesis-6.165.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ea34806a4df4e8305a096dcf8e53cdd903c96c1e0d2dd5b001d2283f639c3f1", size = 1103911, upload-time = "2026-08-11T01:23:00.286Z" }, { url = "https://files.pythonhosted.org/packages/54/9e/b551a494f84976ee5bb9374c197ccc126dea2ec6f22098d5f70705237473/hypothesis-6.165.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d188454b95ce46ba991e3c52161255d76af25170ad28591f6b30b045e501216e", size = 1154060, upload-time = "2026-08-11T01:22:20.413Z" }, { url = "https://files.pythonhosted.org/packages/69/37/8e22a236f1f1e599525549a34672fb0523109f571486fe209b12a84a942e/hypothesis-6.165.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a1c47b15ce97a9b1346bc7d7013c5f215380f78ae01c0f73a1638bd8b98bdd76", size = 1277631, upload-time = "2026-08-11T01:22:30.965Z" }, { url = "https://files.pythonhosted.org/packages/13/0f/feb33bfc23853b4ba6360ff5e34235cd8bea0d7dd1eb21e17491f581c4e2/hypothesis-6.165.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:996077ef7a3bb332b6638f698ddf7555c82784b58dde80eeba3f07c0a322b40f", size = 1321326, upload-time = "2026-08-11T01:21:45.089Z" }, { url = "https://files.pythonhosted.org/packages/9f/3b/ad56b56540a0719f493edec0dd442ebb21272147d2482ef505d19760a6d3/hypothesis-6.165.3-cp314-cp314t-win_amd64.whl", hash = "sha256:57a8273bdafe3f450afe66999fd130d4935d775eaf4ef63fcac0bee8015fc512", size = 670613, upload-time = "2026-08-11T01:22:05.294Z" }, ] [[package]] name = "idna" version = "3.18" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] [[package]] name = "iniconfig" version = "2.3.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] [[package]] name = "ipykernel" version = "7.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "appnope", marker = "sys_platform == 'darwin'" }, { name = "comm" }, { name = "debugpy" }, { name = "ipython" }, { name = "jupyter-client" }, { name = "jupyter-core" }, { name = "matplotlib-inline" }, { name = "nest-asyncio2" }, { name = "packaging" }, { name = "psutil" }, { name = "pyzmq" }, { name = "tornado" }, { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3d/c4/e4a38f579de4225a561305666f7541cdabb30075def2aa1ac17bd73c1fb5/ipykernel-7.3.0.tar.gz", hash = "sha256:9acaaaf97d16355166e4085afe9d225bfbdf2b7ef520f9df3be8f2b248275e09", size = 184899, upload-time = "2026-06-10T08:41:25.481Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/3d/02/77b271f5dc58bfbc0b577c877b2365d1ffea2afe66a80c13f2312820348c/ipykernel-7.3.0-py3-none-any.whl", hash = "sha256:897eb64da762549ef610698fca5e9675195ec6ac8ec7f19d81ce1ca20c876057", size = 120583, upload-time = "2026-06-10T08:41:23.648Z" }, ] [[package]] name = "ipython" version = "9.16.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "ipython-pygments-lexers" }, { name = "jedi" }, { name = "matplotlib-inline" }, { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, { name = "prompt-toolkit" }, { name = "psutil", marker = "sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, { name = "pygments" }, { name = "stack-data" }, { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/06/96/b150fe7e25a5a29ae9ac1374e71488639605d39a1ea4abb74c9ce33af235/ipython-9.16.1.tar.gz", hash = "sha256:5a3d1f9a47ff216d6cf9cf863124f6a2c1a198d1354c546a4d24a370a283b64c", size = 4515302, upload-time = "2026-08-03T08:36:15.571Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/bc/8e/1239df488393d61076653bfb29f759d0f60cab8e030abdf7c17c31539b51/ipython-9.16.1-py3-none-any.whl", hash = "sha256:4acae635506f6d352d94c4899a19d5f85f8bc4d230932342dca556fdab1c69b4", size = 625974, upload-time = "2026-08-03T08:36:13.654Z" }, ] [[package]] name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" }, ] [[package]] name = "jedi" version = "0.20.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "parso" }, ] sdist = { url = "https://files.pythonhosted.org/packages/46/b7/a3635f6a2d7cf5b5dd98064fc1d5fbbafcb25477bcea204a3a92145d158b/jedi-0.20.0.tar.gz", hash = "sha256:c3f4ccbd276696f4b19c54618d4fb18f9fc24b0aef02acf704b23f487daa1011", size = 3119416, upload-time = "2026-05-01T23:38:47.814Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl", hash = "sha256:7bdd9c2634f56713299976f4cbd59cb3fa92165cc5e05ea811fb253480728b67", size = 4884812, upload-time = "2026-05-01T23:38:43.919Z" }, ] [[package]] name = "jinja2" version = "3.1.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markupsafe" }, ] sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] [[package]] name = "jsonschema" version = "4.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, { name = "jsonschema-specifications" }, { name = "referencing" }, { name = "rpds-py" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, ] [[package]] name = "jsonschema-specifications" version = "2025.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "referencing" }, ] sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] [[package]] name = "jupyter-client" version = "8.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jupyter-core" }, { name = "python-dateutil" }, { name = "pyzmq" }, { name = "tornado" }, { name = "traitlets" }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7d/dc/5512503b088997c2250b8bf18258fba9d9ce5ead641183700960d3c9d342/jupyter_client-8.9.1.tar.gz", hash = "sha256:a58f730dd9e728ba16ba1d62ebccf7ffe1ebbdbce4e95cfae941b7321ae1f4fa", size = 359256, upload-time = "2026-06-09T13:15:01.033Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/3f/6f/56d39bf385c5c27988aebaf0c18a2a17e960575740100973511018bd904e/jupyter_client-8.9.1-py3-none-any.whl", hash = "sha256:0b7a295bc46e8751e9adae84781f726c851c1d911bd793edc4a3bde942e3da81", size = 109828, upload-time = "2026-06-09T13:14:58.835Z" }, ] [[package]] name = "jupyter-core" version = "5.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "platformdirs" }, { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/02/49/9d1284d0dc65e2c757b74c6687b6d319b02f822ad039e5c512df9194d9dd/jupyter_core-5.9.1.tar.gz", hash = "sha256:4d09aaff303b9566c3ce657f580bd089ff5c91f5f89cf7d8846c3cdf465b5508", size = 89814, upload-time = "2025-10-16T19:19:18.444Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl", hash = "sha256:ebf87fdc6073d142e114c72c9e29a9d7ca03fad818c5d300ce2adc1fb0743407", size = 29032, upload-time = "2025-10-16T19:19:16.783Z" }, ] [[package]] name = "markdown" version = "3.10.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/29/6f/da4c6aea59b3001f2e8c0ec7497475aadaf3b021c10cab5b2858f0f32b26/markdown-3.10.3.tar.gz", hash = "sha256:3589362618f743188b4d955b874402bc814f4f83f544dc207719f4baa7d9c45f", size = 372596, upload-time = "2026-07-30T19:05:29.005Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/64/69/4a5af2bc115a9a33fefe51709749de8262be3f9ba063d1753a837cdbc49c/markdown-3.10.3-py3-none-any.whl", hash = "sha256:fa6c92a00a4a3c98b22728c64a935ae1928250ae65058a6ded814d2cc29a4cea", size = 110757, upload-time = "2026-07-30T19:05:27.883Z" }, ] [[package]] name = "markupsafe" version = "3.0.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] [[package]] name = "matplotlib-inline" version = "0.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bd/c0/9f7c9a46090390368a4d7bcb76bb87a4a36c421e4c0792cdb53486ffac7a/matplotlib_inline-0.2.2.tar.gz", hash = "sha256:72f3fe8fce36b70d4a5b612f899090cd0401deddc4ea90e1572b9f4bfb058c79", size = 8150, upload-time = "2026-05-08T17:33:33.49Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" }, ] [[package]] name = "mergedeep" version = "1.3.4" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/3a/41/580bb4006e3ed0361b8151a01d324fb03f420815446c7def45d02f74c270/mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8", size = 4661, upload-time = "2021-02-05T18:55:30.623Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/2c/19/04f9b178c2d8a15b076c8b5140708fa6ffc5601fb6f1e975537072df5b2a/mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307", size = 6354, upload-time = "2021-02-05T18:55:29.583Z" }, ] [[package]] name = "mkdocs" version = "1.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "ghp-import" }, { name = "jinja2" }, { name = "markdown" }, { name = "markupsafe" }, { name = "mergedeep" }, { name = "mkdocs-get-deps" }, { name = "packaging" }, { name = "pathspec" }, { name = "pyyaml" }, { name = "pyyaml-env-tag" }, { name = "watchdog" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bc/c6/bbd4f061bd16b378247f12953ffcb04786a618ce5e904b8c5a01a0309061/mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2", size = 3889159, upload-time = "2024-08-30T12:24:06.899Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/22/5b/dbc6a8cddc9cfa9c4971d59fb12bb8d42e161b7e7f8cc89e49137c5b279c/mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e", size = 3864451, upload-time = "2024-08-30T12:24:05.054Z" }, ] [[package]] name = "mkdocs-autorefs" version = "1.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown" }, { name = "markupsafe" }, { name = "mkdocs" }, ] sdist = { url = "https://files.pythonhosted.org/packages/52/c0/f641843de3f612a6b48253f39244165acff36657a91cc903633d456ae1ac/mkdocs_autorefs-1.4.4.tar.gz", hash = "sha256:d54a284f27a7346b9c38f1f852177940c222da508e66edc816a0fa55fc6da197", size = 56588, upload-time = "2026-02-10T15:23:55.105Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/28/de/a3e710469772c6a89595fc52816da05c1e164b4c866a89e3cb82fb1b67c5/mkdocs_autorefs-1.4.4-py3-none-any.whl", hash = "sha256:834ef5408d827071ad1bc69e0f39704fa34c7fc05bc8e1c72b227dfdc5c76089", size = 25530, upload-time = "2026-02-10T15:23:53.817Z" }, ] [[package]] name = "mkdocs-get-deps" version = "0.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mergedeep" }, { name = "platformdirs" }, { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ce/25/b3cccb187655b9393572bde9b09261d267c3bf2f2cdabe347673be5976a6/mkdocs_get_deps-0.2.2.tar.gz", hash = "sha256:8ee8d5f316cdbbb2834bc1df6e69c08fe769a83e040060de26d3c19fad3599a1", size = 11047, upload-time = "2026-03-10T02:46:33.632Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/88/29/744136411e785c4b0b744d5413e56555265939ab3a104c6a4b719dad33fd/mkdocs_get_deps-0.2.2-py3-none-any.whl", hash = "sha256:e7878cbeac04860b8b5e0ca31d3abad3df9411a75a32cde82f8e44b6c16ff650", size = 9555, upload-time = "2026-03-10T02:46:32.256Z" }, ] [[package]] name = "mkdocs-material" version = "9.7.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "babel" }, { name = "backrefs" }, { name = "colorama" }, { name = "jinja2" }, { name = "markdown" }, { name = "mkdocs" }, { name = "mkdocs-material-extensions" }, { name = "paginate" }, { name = "pygments" }, { name = "pymdown-extensions" }, { name = "requests" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f1/cd/c05d3a530ba7934f144fb45f7203cd236adc25c7bdcc34673d202f4b0278/mkdocs_material-9.7.7.tar.gz", hash = "sha256:c0649c065b1b0512d60aad8c10f947f8e455284475239b364b610f2deb4d0855", size = 4097923, upload-time = "2026-07-17T16:21:33.156Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ad/21/17c1bc9e6f47c972ad66fb2ac2568f99f90f1207eeb6fc3b34d094dba7b5/mkdocs_material-9.7.7-py3-none-any.whl", hash = "sha256:8ea9bb1737a5b524a5f9dcf2e1b4ebda8274ae3008aa7845720a97083bef708f", size = 9305438, upload-time = "2026-07-17T16:21:30.017Z" }, ] [[package]] name = "mkdocs-material-extensions" version = "1.3.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/79/9b/9b4c96d6593b2a541e1cb8b34899a6d021d208bb357042823d4d2cabdbe7/mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443", size = 11847, upload-time = "2023-11-22T19:09:45.208Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/5b/54/662a4743aa81d9582ee9339d4ffa3c8fd40a4965e033d77b9da9774d3960/mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31", size = 8728, upload-time = "2023-11-22T19:09:43.465Z" }, ] [[package]] name = "mkdocstrings" version = "1.0.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jinja2" }, { name = "markdown" }, { name = "markupsafe" }, { name = "mkdocs" }, { name = "mkdocs-autorefs" }, { name = "pymdown-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/71/f85bdf13355073ae15a7375f09879375a830553552e58c1c4b7e0bbc5c8b/mkdocstrings-1.0.6.tar.gz", hash = "sha256:a0b8c2bdd29a6416c80d717aa369bbf7831946bd9f23c2a66db1b1dbe7693dbd", size = 100649, upload-time = "2026-07-11T19:38:05.732Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/5d/5b/4c1902e8bdd5c4db63284e9d101dece4038d4025d6d88850ffe0a1578980/mkdocstrings-1.0.6-py3-none-any.whl", hash = "sha256:2703708697487d1b6d6d7b412e176fa436edf120c1bf81dc9e126b12d00893c7", size = 35787, upload-time = "2026-07-11T19:38:04.417Z" }, ] [[package]] name = "mkdocstrings-python" version = "2.0.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffelib" }, { name = "mkdocs-autorefs" }, { name = "mkdocstrings" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c5/65/0db919b7c91bf033b42d04f15ff16a56f4edd6afb26e84d448359660ece3/mkdocstrings_python-2.0.8.tar.gz", hash = "sha256:34545b54681cf0b212aab95394d37b0f5326a93b102f6ad1df3d965c79384c9d", size = 201771, upload-time = "2026-08-31T20:16:06.698Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0b/b7/a5a854d4e44853f26decc0c5e0d68df052399f3da344a313da4178f00816/mkdocstrings_python-2.0.8-py3-none-any.whl", hash = "sha256:e555d9ef53b0febdeae4dd603144d4f86ec90fef672276645ea3bb61232c3d24", size = 105443, upload-time = "2026-08-31T20:16:05.25Z" }, ] [[package]] name = "msgspec" version = "0.21.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e3/60/f79b9b013a16fa3a58350c9295ddc6789f2e335f36ea61ed10a21b215364/msgspec-0.21.1.tar.gz", hash = "sha256:2313508e394b0d208f8f56892ca9b2799e2561329de9763b19619595a6c0f72c", size = 319193, upload-time = "2026-04-12T21:44:50.394Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/6e/cf/317224852c00248c620a9bcf4b26e2e4ab8afd752f18d2a6ef73ebd423b6/msgspec-0.21.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4248cf0b6129b7d230eacd493c17cc2d4f3989f3bb7f633a928a85b7dcfa251", size = 196188, upload-time = "2026-04-12T21:44:07.181Z" }, { url = "https://files.pythonhosted.org/packages/6d/81/074612945c0666078f7366f40000013de9f6ba687491d450df699bceebc9/msgspec-0.21.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5102c7e9b3acff82178449b85006d96310e690291bb1ea0142f1b24bcb8aabcb", size = 188473, upload-time = "2026-04-12T21:44:08.736Z" }, { url = "https://files.pythonhosted.org/packages/8a/37/655101799590bcc5fddb2bd3fe0e6194e816c2d1da7c361725f5eb89a910/msgspec-0.21.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:846758412e9518252b2ac9bffd6f0e54d9ff614f5f9488df7749f81ff5c80920", size = 218871, upload-time = "2026-04-12T21:44:09.917Z" }, { url = "https://files.pythonhosted.org/packages/b5/d1/d4cd9fe89c7d400d7a18f86ccc94daa3f0927f53558846fcb60791dce5d6/msgspec-0.21.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21995e74b5c598c2e004110ad66ec7f1b8c20bf2bcf3b2de8fd9a3094422d3ff", size = 225025, upload-time = "2026-04-12T21:44:11.191Z" }, { url = "https://files.pythonhosted.org/packages/24/bf/e20549e602b9edccadeeff98760345a416f9cce846a657e8b18e3396b212/msgspec-0.21.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6129f0cca52992e898fd5344187f7c8127b63d810b2fd73e36fca73b4c6475ee", size = 222672, upload-time = "2026-04-12T21:44:12.481Z" }, { url = "https://files.pythonhosted.org/packages/b4/68/04d7a8f0f786545cf9b8c280c57aa6befb5977af6e884b8b54191cbe44b3/msgspec-0.21.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ef3ec2296248d1f8b9231acb051b6d471dfde8f21819e86c9adaaa9f42918521", size = 227303, upload-time = "2026-04-12T21:44:13.709Z" }, { url = "https://files.pythonhosted.org/packages/cc/4d/619866af2840875be408047bf9e70ceafbae6ab50660de7134ed1b25eb86/msgspec-0.21.1-cp312-cp312-win_amd64.whl", hash = "sha256:d4ab834a054c6f0cbeef6df9e7e1b33d5f1bc7b86dea1d2fd7cad003873e783d", size = 190017, upload-time = "2026-04-12T21:44:14.977Z" }, { url = "https://files.pythonhosted.org/packages/5e/2e/a8f9eca8fd00e097d7a9e99ba8a4685db994494448e3d4f0b7f6e9a3c0f7/msgspec-0.21.1-cp312-cp312-win_arm64.whl", hash = "sha256:628aaa35c74950a8c59da330d7e98917e1c7188f983745782027748ee4ca573e", size = 175345, upload-time = "2026-04-12T21:44:16.431Z" }, { url = "https://files.pythonhosted.org/packages/7e/74/f11ede02839b19ff459f88e3145df5d711626ca84da4e23520cebf819367/msgspec-0.21.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:764173717a01743f007e9f74520ed281f24672c604514f7d76c1c3a10e8edb66", size = 196176, upload-time = "2026-04-12T21:44:17.613Z" }, { url = "https://files.pythonhosted.org/packages/bb/40/4476c1bd341418a046c4955aff632ec769315d1e3cb94e6acf86d461f9ed/msgspec-0.21.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:344c7cd0eaed1fb81d7959f99100ef71ec9b536881a376f11b9a6c4803365697", size = 188524, upload-time = "2026-04-12T21:44:18.815Z" }, { url = "https://files.pythonhosted.org/packages/ca/d9/9e9d7d7e5061b47540d03d640fab9b3965ba7ae49c1b2154861c8f007518/msgspec-0.21.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48943e278b3854c2f89f955ddc6f9f430d3f0784b16e47d10604ee0463cd21f5", size = 218880, upload-time = "2026-04-12T21:44:20.028Z" }, { url = "https://files.pythonhosted.org/packages/74/66/2bb344f34abb4b57e60c7c9c761994e0417b9718ec1460bf00c296f2a7ea/msgspec-0.21.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9aa659ebb0101b1cbc31461212b87e341d961f0ab0772aaf068a99e001ec4aa", size = 225050, upload-time = "2026-04-12T21:44:21.577Z" }, { url = "https://files.pythonhosted.org/packages/1a/84/7c1e412f76092277bf760cef12b7979d03314d259ab5b5cafde5d0c1722d/msgspec-0.21.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7b27d1a8ead2b6f5b0c4f2d07b8be1ccfcc041c8a0e704781edebe3ae13c484", size = 222713, upload-time = "2026-04-12T21:44:22.83Z" }, { url = "https://files.pythonhosted.org/packages/4e/27/0bba04b2b4ef05f3d068429410bc71d2cea925f1596a8f41152cccd5edb8/msgspec-0.21.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:38fe93e86b61328fe544cb7fd871fad5a27c8734bfda90f65e5dbe288ae50f61", size = 227259, upload-time = "2026-04-12T21:44:24.11Z" }, { url = "https://files.pythonhosted.org/packages/b0/2d/09574b0eea02fed2c2c1383dbaae2c7f79dc16dcd6487a886000afb5d7c4/msgspec-0.21.1-cp313-cp313-win_amd64.whl", hash = "sha256:8bc666331c35fcce05a7cd2d6221adbe0f6058f8e750711413d22793c080ac6a", size = 189857, upload-time = "2026-04-12T21:44:25.359Z" }, { url = "https://files.pythonhosted.org/packages/46/34/105b1576ad182879914f0c821f17ee1d13abb165cb060448f96fe2aff078/msgspec-0.21.1-cp313-cp313-win_arm64.whl", hash = "sha256:42bb1241e0750c1a4346f2aa84db26c5ffd99a4eb3a954927d9f149ff2f42898", size = 175403, upload-time = "2026-04-12T21:44:26.608Z" }, { url = "https://files.pythonhosted.org/packages/5a/ad/86954e987d1d6a5c579e2c2e7832b65e0fff194179fdac4f581536086024/msgspec-0.21.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fab48eb45fdbfbdb2c0edfec00ffc53b6b6085beefc6b50b61e01659f9f8757f", size = 196261, upload-time = "2026-04-12T21:44:27.807Z" }, { url = "https://files.pythonhosted.org/packages/d1/a1/c5e46c3e42b866199365e35d11dddfd1fbd8bba4fdb3c52f965b1607ce94/msgspec-0.21.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3cb779ea0c35bc807ff941d415875c1f69ca0be91a2e907ab99a171811d86a9a", size = 188729, upload-time = "2026-04-12T21:44:28.99Z" }, { url = "https://files.pythonhosted.org/packages/85/7d/1e29a319d678d6cb962ae5bdf32a6858ebdf38f73bc654c0e9c742a0c2c8/msgspec-0.21.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68604db36b3b4dd9bf160e436e12798a4738848144cea1aca1cb984011eb160f", size = 219866, upload-time = "2026-04-12T21:44:31.104Z" }, { url = "https://files.pythonhosted.org/packages/25/1f/cca084ca2572810fff12ea9dbdcbe39eac048f40daf4a9077b49fcbe8cee/msgspec-0.21.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d6b9dc50948eaf65df54d2fd0ff66e6d8c32f116037209ee861810eb9b676cb", size = 224993, upload-time = "2026-04-12T21:44:32.649Z" }, { url = "https://files.pythonhosted.org/packages/71/94/d2120fc9d419a89a3a7c13e5b7078798c4b392a96a02a6e2b3ce43a8766c/msgspec-0.21.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:52c5e21930942302394429c5a582ce7e6b62c7f983b3760834c2ce107e0dd6df", size = 223535, upload-time = "2026-04-12T21:44:33.839Z" }, { url = "https://files.pythonhosted.org/packages/75/17/42418b66a3ad972a89bab73dd78b79cc6282bb488a25e73c853cee7443b9/msgspec-0.21.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:abbb39d65681fa24ed394e01af3d59d869068324f900c61d06062b7fb9980f2f", size = 227222, upload-time = "2026-04-12T21:44:35.093Z" }, { url = "https://files.pythonhosted.org/packages/c4/33/265c894268cca88ff67b144ca2b4c522fc8b9a6f1966a3640c70516e78e1/msgspec-0.21.1-cp314-cp314-win_amd64.whl", hash = "sha256:5666b1b560b97b6ec2eb3fca8a502298ebac56e13bbca1f88523538ce83d01ea", size = 193810, upload-time = "2026-04-12T21:44:36.612Z" }, { url = "https://files.pythonhosted.org/packages/3b/8f/a6d35f25bf1fc63c492fdd88fdce01ba0875ead48c2b91f90f33653b4131/msgspec-0.21.1-cp314-cp314-win_arm64.whl", hash = "sha256:d8b8578e4c83b14ceea4cef0d0b747e31d9330fe4b03b2b2ad4063866a178f93", size = 179125, upload-time = "2026-04-12T21:44:38.198Z" }, { url = "https://files.pythonhosted.org/packages/c6/39/74839641e64b99d87da55af0fc472854d42b46e2183b9e2a67fe1bb2a512/msgspec-0.21.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:15f523d51c00ebad412213bfe9f06f0a50ec2b93e0c19e824a2d267cabb48ea2", size = 200171, upload-time = "2026-04-12T21:44:39.414Z" }, { url = "https://files.pythonhosted.org/packages/70/9b/ce0cca6d2d87fcd4b6ff97600790494e64f26a2c55d61507cd2755c16193/msgspec-0.21.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4e47390360583ba3d5c6cb44cf0a9f61b0a06a899d3c2c00627cedebb2e2884b", size = 192879, upload-time = "2026-04-12T21:44:40.882Z" }, { url = "https://files.pythonhosted.org/packages/a7/08/673a7bb05e5702dc787ddd3011195b509f9867927970da59052211929987/msgspec-0.21.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f60800e6299b798142dc40b0644da77ceac5ea0568be58228417eae14135c847", size = 226281, upload-time = "2026-04-12T21:44:42.181Z" }, { url = "https://files.pythonhosted.org/packages/7d/45/86508cf57283e9070b3c447e3ab25b792a7a0855a3ea4e0c6d111ac34c97/msgspec-0.21.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f8e9dfcd98419cf7568808470c4317a3fb30bef0e3715b568730a2b272a20d7", size = 229863, upload-time = "2026-04-12T21:44:43.442Z" }, { url = "https://files.pythonhosted.org/packages/2c/62/e7c9367cd08d590559faacd711edbae36840342843e669440363f33c7d36/msgspec-0.21.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:92d89dfad13bd1ea640dc3e37e724ed380da1030b272bdf5ecafb983c3ad7c75", size = 230445, upload-time = "2026-04-12T21:44:44.806Z" }, { url = "https://files.pythonhosted.org/packages/42/b4/c0f54632103846b658a10930025f4de41c8724b5e4805a5f3b395586cb7e/msgspec-0.21.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0d03867786e5d7ba25d666df4b11320c27170f4aeafcb8e3a8b0a50a4fb742ca", size = 231822, upload-time = "2026-04-12T21:44:46.343Z" }, { url = "https://files.pythonhosted.org/packages/ea/1d/0d85cc79d0ccf5508e9c846cc66552a6a16bf92abd1dbd8362617f7b35cd/msgspec-0.21.1-cp314-cp314t-win_amd64.whl", hash = "sha256:740fbf1c9d59992ca3537d6fbe9ebbf9eaf726a65fbf31448e0ecbc710697a63", size = 206650, upload-time = "2026-04-12T21:44:47.601Z" }, { url = "https://files.pythonhosted.org/packages/90/91/56c5d560f20e6c20e9e4f55bd0e458f7f162aa689ee350346c04c48eac0b/msgspec-0.21.1-cp314-cp314t-win_arm64.whl", hash = "sha256:0d2cc73df6058d811a126ac3a8ad63a4dfa210c82f9cf5a004802eaf4712de90", size = 183149, upload-time = "2026-04-12T21:44:48.833Z" }, ] [[package]] name = "multidict" version = "6.9.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/d6/99/1d4d69c3512d0ddbfa3a1b69cfd9a151012ab2eb4eabbb096201b1f0b7d8/multidict-6.9.1.tar.gz", hash = "sha256:0f06e60fa190aa7abd0914c2a766736fdc8e9f34878c4346338534b73d1b20e2", size = 182404, upload-time = "2026-09-21T17:59:05.362Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d9/0d/4b5afb6d3e545c9af0cdfe2db8f6f4c6664568c23d863d888674e447e6a4/multidict-6.9.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:29138fef49828542e859828107e42e50d0e587c513b7eb4b2d92bade2b0860fe", size = 98360, upload-time = "2026-09-21T17:55:11.508Z" }, { url = "https://files.pythonhosted.org/packages/89/ab/1b9ca66251899981b21138b87da9d5a9c2c81af12b1ea7d19466972f7fe2/multidict-6.9.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:19e31815d41cefc365489e591d105d2baceb2f65aa75d29471fbdbda8651e006", size = 59979, upload-time = "2026-09-21T17:55:12.866Z" }, { url = "https://files.pythonhosted.org/packages/36/eb/6ae44062466c26c8469ef43f2481a6a48d8cea0587b2d54514ec92e2adfd/multidict-6.9.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6ed30be8918e18c8bed0a2e8b70639ecf02feb61ed00ca2e41cfcb2a50fa3f42", size = 57736, upload-time = "2026-09-21T17:55:14.223Z" }, { url = "https://files.pythonhosted.org/packages/b9/5c/a67817593019257a4ac8b0d1b4c426030e637c047b0692ef405439ecea7a/multidict-6.9.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:637f4ae36264bd7b8d9a60193acddc1d735ad52e8ed53a19931ea6d921fea8e5", size = 333097, upload-time = "2026-09-21T17:55:15.591Z" }, { url = "https://files.pythonhosted.org/packages/90/bf/599ae2e6222822d88a247a8a7ae82fe6fd25d5700757b79603d5edafe6a0/multidict-6.9.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:35fc236507fb1b3138f0af5ecd5f94ed752d4d6d826248eae425f86204013eea", size = 334196, upload-time = "2026-09-21T17:55:17.015Z" }, { url = "https://files.pythonhosted.org/packages/15/10/d8aac5acacbe7f5c117866c776ec26d5f15a868759b6f37ad8e7ed3b5b02/multidict-6.9.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e58952f04772f59f11c6e007471449809a30165188669bca8fdb19dde40a8f24", size = 317465, upload-time = "2026-09-21T17:55:18.412Z" }, { url = "https://files.pythonhosted.org/packages/19/0a/714f796f7293a8b1c5c3f465a26996231d5450c54e85d64ef1258c091134/multidict-6.9.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d35a4f1c63f07fbb8c8f9946dea98b21eddf6c57421585f71d91864be3ba2a24", size = 345914, upload-time = "2026-09-21T17:55:20.306Z" }, { url = "https://files.pythonhosted.org/packages/da/b1/e37fbf769c567be277bcf32df6234035a4384677fcc1bd852752be3d6b93/multidict-6.9.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b5f8771aaaed7f80e84a4e471d2f29ab6721e4595075e54d03ae1ed951b2000a", size = 351163, upload-time = "2026-09-21T17:55:22.21Z" }, { url = "https://files.pythonhosted.org/packages/ed/5b/db08419c1e1f7c9d60cfd2787b2b517d7ae4ebbda8281b48a33eb5141467/multidict-6.9.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:976fd7689d69ec78d67d31d38d396d8adb562f7e8368279f76aed4aa451fa06d", size = 336881, upload-time = "2026-09-21T17:55:23.699Z" }, { url = "https://files.pythonhosted.org/packages/9b/07/cc9bc8a62651d2d53ab93ee4993b3a71b7cb78eb8ebfc9c757a5b6698617/multidict-6.9.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:95052e8777a86bae87c0bd0b5ab22d809e3d1d02bf69e3e66ddda5ba75a05805", size = 303405, upload-time = "2026-09-21T17:55:25.305Z" }, { url = "https://files.pythonhosted.org/packages/b6/0c/8e912afafa70e944dbb8bec4b66ca6e008511395278c0d3dd0e89567536a/multidict-6.9.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1a8adfcaf96f587ab138476eaddef95f29b8a2a8a9afbfea8d2fd62180995d02", size = 324265, upload-time = "2026-09-21T17:55:26.989Z" }, { url = "https://files.pythonhosted.org/packages/66/6a/62c2af80fb085e6234805017af857b8e913dfac7a55e9c1349c27768c58a/multidict-6.9.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1a53de2772cfb74559df2eb4456ec4eeb908435ec55a84b69370d9d745d62aa8", size = 322085, upload-time = "2026-09-21T17:55:28.732Z" }, { url = "https://files.pythonhosted.org/packages/f8/6b/35bf801b336fd960811207203ffdcdc24acc3558b3e7ff2e1c914b141e70/multidict-6.9.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f2ce963299d42fa3f22a90adc0fdf174792ffef5ff4c7ffb68260548fb05580", size = 338107, upload-time = "2026-09-21T17:55:30.303Z" }, { url = "https://files.pythonhosted.org/packages/80/41/495ef65bf5bba29d142d81b3fe1b8154b919bed70e96491b1e93c3c26f0f/multidict-6.9.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5b30ddf7234e611ca877575b62840e6af5977f92f1f9d532eedbb05a44ff8004", size = 339631, upload-time = "2026-09-21T17:55:32.012Z" }, { url = "https://files.pythonhosted.org/packages/19/bd/057fdff5f4e04dcd40a960e38f77d19d3c4b67dd243ffa5f43718edb29fc/multidict-6.9.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:63ada7ee2e9345695f9e9bc4c65d72222253f07b1ac94fd0e37555cc6f3c7f60", size = 302080, upload-time = "2026-09-21T17:55:33.817Z" }, { url = "https://files.pythonhosted.org/packages/3f/de/9ace933ee8dad808632523726f42255b09600087219e3d4ead7369820910/multidict-6.9.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3c95601ed98fad3f6e2f8fe809c3b526b0fab31ef525e00a155e227f3d17f58a", size = 341107, upload-time = "2026-09-21T17:55:35.471Z" }, { url = "https://files.pythonhosted.org/packages/d8/ac/7c1204406097bfc5c283d4a3287d61166807d189917567df4c1318484fc4/multidict-6.9.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c148e8b596000dd3e4bfe206e70f3e666be18d72032e0012555f2373c52e35d6", size = 333482, upload-time = "2026-09-21T17:55:37.002Z" }, { url = "https://files.pythonhosted.org/packages/ff/c0/a70c32ea3299ebe00f44533740cb46905717c51faf29bbd3ce8bb5886d9d/multidict-6.9.1-cp312-cp312-win32.whl", hash = "sha256:f9dad513626a33670f17cddc6078e30e311f444c957e8dbfc5b2b4603c8b4edb", size = 52548, upload-time = "2026-09-21T17:55:38.529Z" }, { url = "https://files.pythonhosted.org/packages/d7/2e/8c9c2591df01e5692ad1bf92febb082a17b1c6c3a19aa8b7ff49988fd989/multidict-6.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:a16a1dc8529f9e734a41c3b856f3eae7ebacdc061dde3f8a844e0c7889c97203", size = 59622, upload-time = "2026-09-21T17:55:39.86Z" }, { url = "https://files.pythonhosted.org/packages/d0/95/59f4472ec512bc180fd207899594c7803ece96262ff9aaa3a4b6d7da6940/multidict-6.9.1-cp312-cp312-win_arm64.whl", hash = "sha256:361f7206cf341ba94fb015688f5c8b480f8e63bd58a4c14a48aeca7851a241cc", size = 54930, upload-time = "2026-09-21T17:55:41.155Z" }, { url = "https://files.pythonhosted.org/packages/eb/45/ddf7c76860f5f553a23ad3ca38463ccf5247081618977742c8dc4625355e/multidict-6.9.1-cp313-cp313-android_24_x86_64.whl", hash = "sha256:d7bf9e43282d69561618e8a0ea33368d532ebef42f15c096f427090521dd74f3", size = 63266, upload-time = "2026-09-21T17:55:42.676Z" }, { url = "https://files.pythonhosted.org/packages/6d/d3/f4ae5945ea2de597eaeb4eca3a74c59783ace54482c9b49435442b63efa8/multidict-6.9.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:03d47df72f084f757c1cb771188d5f4e3a805e4abc4d67e32509272343ae9382", size = 55771, upload-time = "2026-09-21T17:55:44.016Z" }, { url = "https://files.pythonhosted.org/packages/93/7d/15468239920040d01c686e5ae669e6382f7bf31fc3e5e0a6b8f1c3b32d3b/multidict-6.9.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:6bc94fe17c3c56e5418f79515b786b101845f70609b0d19d0c1ba13448e5633a", size = 57116, upload-time = "2026-09-21T17:55:45.315Z" }, { url = "https://files.pythonhosted.org/packages/17/1b/b958f06aac2d8b1e485eb1c105b1b159cbf3868249e5142142451577dad2/multidict-6.9.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8f2973bbd2bebd9d2e0cd6394c1292a1a19ccd56bdcbe1e174059f1a39be5b40", size = 97692, upload-time = "2026-09-21T17:55:46.674Z" }, { url = "https://files.pythonhosted.org/packages/93/6c/d6cfe18e61010166d7237d7527c775eb9843e7078feadea45b7628751b60/multidict-6.9.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:de7738b8c0bb74c4cc16bbd7fb49fc2bcf6430dba11b3432cd52768ae40933e8", size = 59556, upload-time = "2026-09-21T17:55:48.22Z" }, { url = "https://files.pythonhosted.org/packages/46/46/9b4c1127cece0289fb207d04aae5116382ddfcb2a4dc8e0cb49a33f3c7b3/multidict-6.9.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e5ccad4b7bac125722f48d6f862bed3b514d8526deea06316bb72f152cd30a7c", size = 57357, upload-time = "2026-09-21T17:55:49.998Z" }, { url = "https://files.pythonhosted.org/packages/bc/fc/c18b07100a6064573e49e538d13d47eb2b5221f48080512df78aef54ab04/multidict-6.9.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a49ff5cdb33654cb7d6a3c377aa2a83ddefaa1db31eb10bcf3c180aa84f9af8a", size = 330433, upload-time = "2026-09-21T17:55:51.435Z" }, { url = "https://files.pythonhosted.org/packages/62/ff/52a0082adeb69656b634609d4fb0ae65456deaae4cca3d5a12656626dc50/multidict-6.9.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b22ff30006a2f28f8bff878fb93413cbe3a4d1fd517c28081d848c90e9cfd2c8", size = 332487, upload-time = "2026-09-21T17:55:52.937Z" }, { url = "https://files.pythonhosted.org/packages/39/3e/80bd4729635a7af371ff4dcc312594cf96e73f98854491be8740792de430/multidict-6.9.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3adf06c66041aa21eeb8a71e82379b74773298c8e6d3d839b151aae441a99b94", size = 316911, upload-time = "2026-09-21T17:55:54.751Z" }, { url = "https://files.pythonhosted.org/packages/15/f4/7ea4a907e053e924e2e1694d90560f79f72056504bfd5e8483695f9527d2/multidict-6.9.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:963a8d8f97057082679523d0fd4c53a38f86bc58cabe4556faef682ae53fa2fa", size = 344950, upload-time = "2026-09-21T17:55:56.5Z" }, { url = "https://files.pythonhosted.org/packages/0d/16/9642ae41fbfbdc546aa481c2b07ae1bc087a94ab5f5020b4d9ab77da401f/multidict-6.9.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b66ccc5c2cdd26e74fa5d4c29ffae424cc6148bf93ce574821783fb3b6d452c5", size = 350264, upload-time = "2026-09-21T17:55:58.073Z" }, { url = "https://files.pythonhosted.org/packages/93/f2/e06c8e42074d0a4b8419bfe92afbd1d262190b69549ecbcd2dffa4f103c2/multidict-6.9.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fd79c521f6290c69125fa2b85fa65d9e657e6a8ffaf722dc881b926bef4aa5c", size = 336213, upload-time = "2026-09-21T17:55:59.723Z" }, { url = "https://files.pythonhosted.org/packages/98/43/d7cf9ef4700e7c25244590d1b4f20cf12cbed86beba6c17be4d9e48c1099/multidict-6.9.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1d804e4caf5d5da37d6dac1325da5629ebef1e27a294c2b568b295814aa36c7b", size = 302555, upload-time = "2026-09-21T17:56:01.666Z" }, { url = "https://files.pythonhosted.org/packages/7d/3a/54209920324bc3928f5f2ba28b01a6dd957a9d48c3aaff77e38faaf5668d/multidict-6.9.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bb0d664505f4b112f384cffeee82e91e3f6448d8e574989479db4439b68cba05", size = 322247, upload-time = "2026-09-21T17:56:03.265Z" }, { url = "https://files.pythonhosted.org/packages/33/50/df96f961b178b621ab0adbf9221d4fd21534e1064fd9f5e85c1fa27fea55/multidict-6.9.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:9e14d17773b1b3c758ff153659a1824608a0cb562c45f484b5ed8a433428444a", size = 321787, upload-time = "2026-09-21T17:56:04.894Z" }, { url = "https://files.pythonhosted.org/packages/a7/9b/37f354562a8f82f9c1f63d3a94300fc82126d50320175f0702bbd544f87c/multidict-6.9.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:083735b7f395894e43adb278d5dae901448883a835ff8f1977e285fefdb10418", size = 335498, upload-time = "2026-09-21T17:56:06.783Z" }, { url = "https://files.pythonhosted.org/packages/1e/44/78e366efc6c004185295cc9eb9711c10f13eadf3b547603b9c5526329976/multidict-6.9.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b65121091567847a8cb520d364ab22ba90e00d3cc55fa9eb34bb439f0684bcd1", size = 338244, upload-time = "2026-09-21T17:56:09.162Z" }, { url = "https://files.pythonhosted.org/packages/ad/2b/ab5bd3964691abe4d14b43bdbdb8a621b77cea2ca352dc93159ec0a4575b/multidict-6.9.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:ea999ae6e80e66ad5eea287860951b033d0104ca34d6d87c7b5125ebe0e12721", size = 301641, upload-time = "2026-09-21T17:56:11.227Z" }, { url = "https://files.pythonhosted.org/packages/ea/a8/f6bc899f5aafaba755edc8e4bb934bc00b1e405f8d3fbc1dd10283738074/multidict-6.9.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:095d900c242e00fbe5f321ee072e7278b4153e78c5ce9c1efde167d62c1e4771", size = 340162, upload-time = "2026-09-21T17:56:13.076Z" }, { url = "https://files.pythonhosted.org/packages/eb/03/a8fc809ef364b8c231c065ea2ea2f97ae86d4f9a0e36b5721759fd606778/multidict-6.9.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6441cc837aea58be7d9baef1b2383eb8311ab9303f500f99ac90b584cd78bb14", size = 332826, upload-time = "2026-09-21T17:56:15.175Z" }, { url = "https://files.pythonhosted.org/packages/f2/27/32dde80245e2024e9bb7a6ca3c2fb7c695dc74014cba557cc35be6f91e24/multidict-6.9.1-cp313-cp313-win32.whl", hash = "sha256:9c4880d017555d70dea367dd49271830842d48e3891c2da97da7ce8c4abcee40", size = 52378, upload-time = "2026-09-21T17:56:16.959Z" }, { url = "https://files.pythonhosted.org/packages/be/78/1bac2987edac273a6b27fa50e9204bc3466c94a7b4f6a44828d730f8810f/multidict-6.9.1-cp313-cp313-win_amd64.whl", hash = "sha256:ac51cd64bae51c462ea58ad2492c9b8209667a4ef60c45c4a304518b67598d5d", size = 59449, upload-time = "2026-09-21T17:56:18.336Z" }, { url = "https://files.pythonhosted.org/packages/e1/32/2a77ce19eea48cb9b3521a202b513ba3349aafa702c8317be0b645ffb74b/multidict-6.9.1-cp313-cp313-win_arm64.whl", hash = "sha256:37a9ebe00c698279213d56e6c64e1962ab1e092918270649b397cac3dc196ca4", size = 54713, upload-time = "2026-09-21T17:56:19.774Z" }, { url = "https://files.pythonhosted.org/packages/ec/0f/c6041015aa2cdc11e1dff0cae58898ae5525d19e35efd65a4ecf3c6ea235/multidict-6.9.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:fc0dcb22fa9aeabfe3fa4e0430099acff985ec5d77a851382f76cc6146e780e5", size = 62336, upload-time = "2026-09-21T17:56:21.181Z" }, { url = "https://files.pythonhosted.org/packages/23/db/95bd0afb130a0149c955f1f066143528d60864e64f41ff5a1e1313609361/multidict-6.9.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:024123f0ab402ab33828e24eb80fa8f25167d0d3783ba5f287e39ed741e6abf9", size = 54892, upload-time = "2026-09-21T17:56:22.63Z" }, { url = "https://files.pythonhosted.org/packages/94/cc/8aa34ae8d09498e8da003f485c03371b55cf1bc6d56cff861b3a7de61251/multidict-6.9.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:af1b5a92315048c3e36bbebfa7d4760a9c3e910bc4166f11b20d77d20d6bcfca", size = 56251, upload-time = "2026-09-21T17:56:24.36Z" }, { url = "https://files.pythonhosted.org/packages/52/48/22baa3b95375096369b04c4348af14139570f302f951773d2c68bf0ccf2e/multidict-6.9.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b2483da477932ad1983d1d33c18bc3771c6fb00cfbaaed70a875fd547ef8e840", size = 96345, upload-time = "2026-09-21T17:56:25.777Z" }, { url = "https://files.pythonhosted.org/packages/2f/3e/b96779dcac28ec6d491fd821112a0156b519b6701bba186cdf6f6df737f4/multidict-6.9.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:db4d697b18b6ef5528b1f36bfa25072cd2a421869f5963bc0e92c8a34b9e2800", size = 58901, upload-time = "2026-09-21T17:56:27.364Z" }, { url = "https://files.pythonhosted.org/packages/f9/b4/8e6d950f02ca6ecc00ecf671f2ddb5dab7017671a8d197326f75621f5786/multidict-6.9.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:854fd2f1bc6e8a56b89910b5cd7261a8b40f13ebb31572da985ce59c7da0886d", size = 56491, upload-time = "2026-09-21T17:56:28.901Z" }, { url = "https://files.pythonhosted.org/packages/ed/8f/212ed7e03282c7bc271e3796bf985b53d5727c2827361e3dd9c5ee6b093f/multidict-6.9.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:03731f6fc036180700c9dc2308205a48e5ca6f3ff03087739ab746f294022201", size = 332297, upload-time = "2026-09-21T17:56:30.495Z" }, { url = "https://files.pythonhosted.org/packages/a0/97/555aab000e03ecba85c0a1837fd5674d8bf10949490df7800da3a40660ce/multidict-6.9.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:989261c5f1735a165f2e4e87cf6d5f17ab734fa18f9ad0383d5adfcdaefce701", size = 329197, upload-time = "2026-09-21T17:56:32.652Z" }, { url = "https://files.pythonhosted.org/packages/3d/25/f3539b5bdc147beba66015795dfe589eedcc580348fb1a8c69b1df4196b5/multidict-6.9.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b4c9e5d05b126b267ac048a89a0e2d9b48b1b648add62d4872906304a8590610", size = 308456, upload-time = "2026-09-21T17:56:34.657Z" }, { url = "https://files.pythonhosted.org/packages/1e/7d/4665cad8fc326787d218879be879d86dfe32f006c1f88ae47b04d3b52fd5/multidict-6.9.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9d0a21cf76153de8f2d96a877991d6bc59b9ab5180b949e50db73cc193b4a694", size = 343382, upload-time = "2026-09-21T17:56:36.606Z" }, { url = "https://files.pythonhosted.org/packages/41/01/e1e9abe27f492b22dede492bc423015597412b14ce074f7475f4ecd80185/multidict-6.9.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7386aa18d98d6b8af44b92173654ec469237fda35f8e8523e43b581a86f476a", size = 346848, upload-time = "2026-09-21T17:56:38.344Z" }, { url = "https://files.pythonhosted.org/packages/f8/e5/8d54118bc228e64e1087f1729647b93bbea225eda4a3f914663a6f410bca/multidict-6.9.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b69651732c64afb691e50cdc3387cae305e0eeff8804fe3e3ce203876494932a", size = 333612, upload-time = "2026-09-21T17:56:40.041Z" }, { url = "https://files.pythonhosted.org/packages/76/e1/9509dafc1fc68948e75842a74533ddfa64af57f2b17810e9cacfa68a829b/multidict-6.9.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1fee9a16d88a1c4865610de31ef5c666671020d7a81d1f510eaf3c97d00ebeba", size = 304246, upload-time = "2026-09-21T17:56:42.313Z" }, { url = "https://files.pythonhosted.org/packages/1d/b6/96b93808e1bec0b51d49d89859e49b82a94f7b2e58eca0fc44ffcafd3ae5/multidict-6.9.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7814bbee202acd3bd240204c17d8b87a4c48c81064fd8674dbe527d94d5a4290", size = 321983, upload-time = "2026-09-21T17:56:44.399Z" }, { url = "https://files.pythonhosted.org/packages/d9/1c/08e5184df026b9d8e05a6dce5da22628e46bc4a7c7f03b8bf3c6939a8c46/multidict-6.9.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:95f273bea318a194f656527ee2ed19494327bc500b8e87d9358e3222579aa28d", size = 315927, upload-time = "2026-09-21T17:56:46.208Z" }, { url = "https://files.pythonhosted.org/packages/d7/5d/9ff39e7387f6fcc9a81af5dbf055fde08149849a04f8df10ca5b086b0d39/multidict-6.9.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:73bcafa21a78d0776b3ee7cd2a63c66f968eb7db8e8d594e32d7329950f6e828", size = 336827, upload-time = "2026-09-21T17:56:48.021Z" }, { url = "https://files.pythonhosted.org/packages/5a/be/30fbb2cab22203128928066de94be247dc9a3230bf2bfc54437ac3b5f9f7/multidict-6.9.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:1a938761c77e0e6edb0c93d02f4e988d44a69e5195e5a3b893e5553311347132", size = 336317, upload-time = "2026-09-21T17:56:49.778Z" }, { url = "https://files.pythonhosted.org/packages/bf/f0/2480aef6b7d8e7ab89b064ae14f12ec069570dab3a5c13e0155912293247/multidict-6.9.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:37245ca4105386194dd1d292a6f2aae09bfe1bd7ac6f9ec25093e3cf8e9b143b", size = 303258, upload-time = "2026-09-21T17:56:51.829Z" }, { url = "https://files.pythonhosted.org/packages/4c/04/dbe59cf4ea3778600e8e70e30b31aa9118ad6da2844bc9f717804cbaa000/multidict-6.9.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f0700527dd5bfa8b7204b08330542f4f388899d3c14d885d8a368992e0eb562d", size = 337232, upload-time = "2026-09-21T17:56:53.537Z" }, { url = "https://files.pythonhosted.org/packages/f4/8c/abe55548f06f12878588f08eb803c5a391283f01243f784cb00adac79231/multidict-6.9.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6b7e54fd883671d1a8810704851c044b7173287c552b9f5c3d9e0eb9f00ae194", size = 330261, upload-time = "2026-09-21T17:56:55.335Z" }, { url = "https://files.pythonhosted.org/packages/d0/f1/ba67f668478f152bdfa18abe72e7588bd078bf8b42c172fcf542fcf45826/multidict-6.9.1-cp314-cp314-win32.whl", hash = "sha256:e81ae656b9935ac4528a71f96bb7a14d949778ed1897c573d3e7ebb9187f8841", size = 51715, upload-time = "2026-09-21T17:56:57.048Z" }, { url = "https://files.pythonhosted.org/packages/33/81/014bb2128aedc8157d2848f0ec8a0209c6b822fb88deea012834e1c4a4e8/multidict-6.9.1-cp314-cp314-win_amd64.whl", hash = "sha256:acddcac38adc8342ba48aba98896faa7928854bebb62542362138655b5367ee3", size = 58621, upload-time = "2026-09-21T17:56:58.531Z" }, { url = "https://files.pythonhosted.org/packages/66/34/8f43c03c2d8821a2320b8c560d8a49d0be2920627edfac3f546922f91969/multidict-6.9.1-cp314-cp314-win_arm64.whl", hash = "sha256:32217133dddc58c927805cf6c0731d8144584176b768042ee886d51e71860bc9", size = 53959, upload-time = "2026-09-21T17:57:00.15Z" }, { url = "https://files.pythonhosted.org/packages/08/64/4f2a5eeea10b6d42634ba168ec6efc732ef218772e57b8239cd485c202f6/multidict-6.9.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0b2fb8c349d1103863750b5d8cb5ace766917f4b35f3d883c8f778853eaa9f76", size = 107694, upload-time = "2026-09-21T17:57:01.785Z" }, { url = "https://files.pythonhosted.org/packages/32/ce/c68d08ac2096c1528ff20d77fd10f7e401db3e534ac5a1c7726e36a77c2f/multidict-6.9.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:56d834b74c993a7d7cb2b8ab33a0d55c3e0d4a3d2f2da2808a4ad3d79189711b", size = 64766, upload-time = "2026-09-21T17:57:03.52Z" }, { url = "https://files.pythonhosted.org/packages/3a/9a/9f5c270c88fe289b4876fe28f9932a70e015a7916aff1e5b0325f7597b11/multidict-6.9.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:783ba7d845d79ce976afd9c1e91a4e5714671defa198ee789e8b23316083a485", size = 62723, upload-time = "2026-09-21T17:57:05.104Z" }, { url = "https://files.pythonhosted.org/packages/7c/63/c9a0b131354dd949426166cef712d708f436a8cf261114f8d5d2aec440ca/multidict-6.9.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d718fa1b5f0d0dd75e86fbbc5b0c93ea3a5d65216c85c61cd5d7cdddfe08455", size = 290445, upload-time = "2026-09-21T17:57:06.687Z" }, { url = "https://files.pythonhosted.org/packages/9d/41/ed27d1d20ba91f4e13e8f7dcfaa592614ae0e1d5f45e03171d23c3b098ac/multidict-6.9.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a313bad717dde740959d50850c315b75fd4eb0c5e6b4dc8535db0f1c369be125", size = 304375, upload-time = "2026-09-21T17:57:08.717Z" }, { url = "https://files.pythonhosted.org/packages/66/6f/a7b26167173c3b73d6e5ebb0856c5154ad005156c3a2fa4062329921819c/multidict-6.9.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ef01d29fca550ab871fd99154f82c6472aacf8e7def272dfb07b460123850390", size = 281746, upload-time = "2026-09-21T17:57:10.713Z" }, { url = "https://files.pythonhosted.org/packages/34/1d/da27bdd49b0cb84263288f5f48dbd915f2d17fd228fee2078ca607e80277/multidict-6.9.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f3bd290711c6e9486173a6ee7cd4e7f00c3971c7908c1ec1b6e5437c5e4c6f9", size = 311499, upload-time = "2026-09-21T17:57:12.467Z" }, { url = "https://files.pythonhosted.org/packages/d0/36/ab0299a7abf53e38ff75d96c64480f1e2ff8edde985f93f7a1ad5e08bd94/multidict-6.9.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:81cdc537e0a42e3c0170752fcadbe450246d5c3b4b7231d6eb9672456605ac94", size = 313553, upload-time = "2026-09-21T17:57:14.548Z" }, { url = "https://files.pythonhosted.org/packages/5a/44/e21be702efc9c9ba097c8be874c68860b1cb6cf9c85611263039bf95b37e/multidict-6.9.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8b193bf7a443c97d81c47052f60c486071a4bdef4a573fa2514f920089414d45", size = 299898, upload-time = "2026-09-21T17:57:16.192Z" }, { url = "https://files.pythonhosted.org/packages/00/8d/e93bc0dc947c7f872e502cfaa7ad9d98f001e1c93cfa6f4940457a6e187a/multidict-6.9.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7a0c98f6a636adf0d7edd60c61589eaf52d149239af754d0bfeb0effedba53a8", size = 273054, upload-time = "2026-09-21T17:57:17.793Z" }, { url = "https://files.pythonhosted.org/packages/5e/c9/945b44493fbe7af26ffe1e43c506b376daeab070fc1a0923bef9bbf572eb/multidict-6.9.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6e6b7e3a1520c39a2772f414cd9ddf5995f77e4e823dfa1522af383addd465a3", size = 294751, upload-time = "2026-09-21T17:57:19.631Z" }, { url = "https://files.pythonhosted.org/packages/24/79/0969a9b38b814e5fa3af1b0a57708c49b3d9b3d55c208bac6e98b87bb028/multidict-6.9.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ca65cced0d67a9039e93bcd98a369920e499bf98296844ff56ead01c9085321f", size = 284523, upload-time = "2026-09-21T17:57:21.313Z" }, { url = "https://files.pythonhosted.org/packages/1d/1e/c32ddf53234c228b1a374ddabca19f542ac7f32752f5155982abe2304833/multidict-6.9.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:372c063f37480f62c1ae32dc3a1a0a5942b180c883f99789075a8a8ec3c4e709", size = 290275, upload-time = "2026-09-21T17:57:23.206Z" }, { url = "https://files.pythonhosted.org/packages/5d/09/7a99a3daafd987330abb865a2d36e432fd8bf18e790ddda55c5de15eb98f/multidict-6.9.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:827c92145b3b976b39430129c89d213b250dacbe5fce678e9a03940e6e848983", size = 303350, upload-time = "2026-09-21T17:57:25.086Z" }, { url = "https://files.pythonhosted.org/packages/2e/09/7e6ebaecc4e226d7fe27973c271d58b75c14c24e84c23736d21dc39997dd/multidict-6.9.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:9b0124b9c17e9890f0819b2e7a5f65ec9a2f5aabd6c8e7ad090c1675cf70dee6", size = 272232, upload-time = "2026-09-21T17:57:27.193Z" }, { url = "https://files.pythonhosted.org/packages/1f/6e/420e9e879b21cbdb5036214ee238efabcfab7bf262d3fc124132050f79ff/multidict-6.9.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:23136f5a564654eb61061ec6d5620a4c1ea32c8f552b65e9982a12b72bff601b", size = 300525, upload-time = "2026-09-21T17:57:29.004Z" }, { url = "https://files.pythonhosted.org/packages/b7/7f/c6b0896850b3ceb190ca302760d63148843371f027c5dde3d1d732343a08/multidict-6.9.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f2524ec55b3e65cbe235a8b3c36e2af3b635be02ed05e20c94e70c5c943c009e", size = 295933, upload-time = "2026-09-21T17:57:30.837Z" }, { url = "https://files.pythonhosted.org/packages/29/f2/f0ff3384a756227741c72f4ee94cadf1f9900df0ae961e48639f96115b1c/multidict-6.9.1-cp314-cp314t-win32.whl", hash = "sha256:35ba0263bae5dd3ad5aad767cc9afc01a8598c7dae30f1b3b2de98b1b32c28bd", size = 57776, upload-time = "2026-09-21T17:57:32.835Z" }, { url = "https://files.pythonhosted.org/packages/8d/87/013e1eed30ad46bec1eb68947bc9caf55ba2fd0d6aed04fe964e3b44c23b/multidict-6.9.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7c8d5882ba25ac0282258be435d8a05aa0cbcacfce15799154a338f847f159b9", size = 64881, upload-time = "2026-09-21T17:57:34.489Z" }, { url = "https://files.pythonhosted.org/packages/7f/95/64dd049bb4c53426e2fee7afb874daaa8b4426c41878785883a94511ecf8/multidict-6.9.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ac348379cf4de5538a0a213be1532d289aa801ec5d267c5909446b9ea2f8e2c3", size = 58582, upload-time = "2026-09-21T17:57:36.19Z" }, { url = "https://files.pythonhosted.org/packages/a2/2c/26bc1723592cfe965b575652a7749026474b0625b04b04564e1fba265dc5/multidict-6.9.1-cp315-cp315-android_24_x86_64.whl", hash = "sha256:f04551dce5a7db8c9659f2e4245494c182d0663b83661803e08d46bfcae5eda1", size = 62339, upload-time = "2026-09-21T17:57:37.967Z" }, { url = "https://files.pythonhosted.org/packages/bd/c5/614d2eb9995232d066aae7f17d1d454060b6b5d882b22fbaf6150b05ef1d/multidict-6.9.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c0c88085affd35c33e124e36930c5ad96aff9294ce195eaa0fd9cec962b64a82", size = 55009, upload-time = "2026-09-21T17:57:39.746Z" }, { url = "https://files.pythonhosted.org/packages/7f/24/f7179ecdf9cdd3581ace3292c3b883a1a500a374aa4e619bf6eb521c62e5/multidict-6.9.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a8b75dd3d3638d9a19f23e84af4ffab3b8940422c0df2da2a77005ef5aa3d7ea", size = 56273, upload-time = "2026-09-21T17:57:41.544Z" }, { url = "https://files.pythonhosted.org/packages/ac/72/c9dffcecccd1f8b9a58f960ec4eaecab62c4edb08d5bb984d4de67fe3fdb/multidict-6.9.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:006c4478de0a1876f4834e14255776286f09b9846b505fe63f67f9d173a9487c", size = 96390, upload-time = "2026-09-21T17:57:43.385Z" }, { url = "https://files.pythonhosted.org/packages/e9/3f/76542360e655cecdb7c158c7af88c69c4db0d2aac3421659d41214927229/multidict-6.9.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:2784090c30a586d5b45197bd9c32f87fb927216cde302f6cfd76d76577e90f08", size = 58900, upload-time = "2026-09-21T17:57:45.105Z" }, { url = "https://files.pythonhosted.org/packages/04/15/f28c71af461dd4ff15653a165cfd8ddaf01f3387ff80d2dd0de057d527d0/multidict-6.9.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:7c708566da8014b120a64b1eb6d200c6c0c8cb36296383723cdb6fc82038270b", size = 56540, upload-time = "2026-09-21T17:57:46.913Z" }, { url = "https://files.pythonhosted.org/packages/19/de/5ee4050c3ddad88c605a89fc1cd18e89abd40a64f183e6fb5276b6eb7f52/multidict-6.9.1-cp315-cp315-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:042fb0196047e786936a730bd302de83143950da45f2c16078da8f35e1cf7919", size = 331818, upload-time = "2026-09-21T17:57:48.825Z" }, { url = "https://files.pythonhosted.org/packages/70/e2/2d4e3c3ee70f583334ff236fe6a3babf7a05ce56169a15df65453af647c0/multidict-6.9.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2212a0c842c723d919ea4a22a9296cb6b244b386e4e6ea92adfc7fbf3095519", size = 329800, upload-time = "2026-09-21T17:57:50.91Z" }, { url = "https://files.pythonhosted.org/packages/d6/14/f64bd05667343a7250cdd99f81bda2270e3959f00832441f10c7f904e1c1/multidict-6.9.1-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fd882aa29bf402b62bf1fd7c19fd5df4b6528cf468a908864b39368572b662a9", size = 312048, upload-time = "2026-09-21T17:57:53.053Z" }, { url = "https://files.pythonhosted.org/packages/64/bf/a91256ce00199e7544e3eda79c4e9544eb7388c38fa22f11638c2e357ccb/multidict-6.9.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c57541034d12b215ab0a2bfa371d1a8a198da18176d0426c27105b9161a6862d", size = 343676, upload-time = "2026-09-21T17:57:55.069Z" }, { url = "https://files.pythonhosted.org/packages/14/48/628b978413518159228c2b2ec38645362ceb6427e935d83d716e26490a49/multidict-6.9.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d73fed4e37158ff00cd271871170b79e40138db51e625fd352fa17c6acb34f67", size = 347028, upload-time = "2026-09-21T17:57:57.157Z" }, { url = "https://files.pythonhosted.org/packages/8c/73/5f9dc2b5ac7dd7a82458ae9cd283b99c185834f23442ace295362dcc5e45/multidict-6.9.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:79da2491348b30810728050b4a8ec0416f85884125c2fd44655b3d01150d9c3e", size = 335670, upload-time = "2026-09-21T17:57:59.126Z" }, { url = "https://files.pythonhosted.org/packages/f8/84/6f319ee000f5ab3fa0055b194ad706fddd552c60c76a21d581c87db1d46c/multidict-6.9.1-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b4da208a63434d21a3df64d29758e650fc4aa8cb05848554b76949c296539cca", size = 307804, upload-time = "2026-09-21T17:58:01.2Z" }, { url = "https://files.pythonhosted.org/packages/3f/24/61be5fe1616d033532e8d63dc789c0f35b9793917834713eefe050589ba9/multidict-6.9.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:877ca17fdcfdf5c397493a71e5ff97a87bb181417fe717fdadc77c08c09301ac", size = 323409, upload-time = "2026-09-21T17:58:03.383Z" }, { url = "https://files.pythonhosted.org/packages/22/a6/eab8aaf59892169a0f4e8e78a5516a6caa6abd8e96e6410111258f4ef0ef/multidict-6.9.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:40aec5299e1ed71fbb988389059da381c3c1a60e0c649acceb2a35d9b128848e", size = 315746, upload-time = "2026-09-21T17:58:05.372Z" }, { url = "https://files.pythonhosted.org/packages/42/b1/d1e14bab80a74babf6e2ac971a878b84fdcf3a972f3ec3400ca12260fac5/multidict-6.9.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:4a409ccc42aefec904038695d5d7fd6d8f3af2721b7b6401d55135ec7d6d298e", size = 334529, upload-time = "2026-09-21T17:58:07.35Z" }, { url = "https://files.pythonhosted.org/packages/f5/50/a05b1d36f6d3363a5a334cebbf11624c9c7a69d66fd23345eb53b1d43b7e/multidict-6.9.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:0a5769559e3312dd96731fbe15b4abb6033368ac1cad5a98dadd21946a4c7d6c", size = 336455, upload-time = "2026-09-21T17:58:09.112Z" }, { url = "https://files.pythonhosted.org/packages/cd/0d/bf33cb097380252b08e1177b5ae0cab822c47ce80d236f3f724953c76721/multidict-6.9.1-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:03fac50ddfd8302175b77863a015eccfae76767cda5506eef86df559ba861e1f", size = 305865, upload-time = "2026-09-21T17:58:10.921Z" }, { url = "https://files.pythonhosted.org/packages/ba/ee/a2bd133391204b1cec5a5caf327dc9a06710439b7403220860a8fb164560/multidict-6.9.1-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:5ecace251ccfa705bf3d7d35c5032cf750f5c629809740405697bce5c118c4a4", size = 337531, upload-time = "2026-09-21T17:58:12.846Z" }, { url = "https://files.pythonhosted.org/packages/a8/32/c0972486f81a06c3bbb23063c85afaac53c51608f7e35bcb101a1006408b/multidict-6.9.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:ecc89dbd4155b2f8a47f4bbd89242a35ed15e8e0ec581cab5ac65fa38407329d", size = 332464, upload-time = "2026-09-21T17:58:14.673Z" }, { url = "https://files.pythonhosted.org/packages/69/4f/5cbbe852b45d8a7f0ab3ed042386e4894e463010eb230603596cb15d0fee/multidict-6.9.1-cp315-cp315-win32.whl", hash = "sha256:a3ffe881246d28a862f1985824f484cb7361f44d6b99c4c620436ef56462f38d", size = 51714, upload-time = "2026-09-21T17:58:16.409Z" }, { url = "https://files.pythonhosted.org/packages/d4/77/1fa118386deeae4e7a48870fa5cb073e748b0ac5265a6990651cefc48068/multidict-6.9.1-cp315-cp315-win_amd64.whl", hash = "sha256:59c123d0e948d760a5f930f316cfefa07e8d632ab84327c0693ee6a88171154f", size = 58619, upload-time = "2026-09-21T17:58:18.369Z" }, { url = "https://files.pythonhosted.org/packages/dd/ee/51c45748acfce9cb69d89dcfcd09c874b385407fd6993e885d6905659865/multidict-6.9.1-cp315-cp315-win_arm64.whl", hash = "sha256:31199204b3ced121ff5407a2c342326a5d27e3870abbf94bd80dbd2451b7bc8f", size = 53969, upload-time = "2026-09-21T17:58:20.102Z" }, { url = "https://files.pythonhosted.org/packages/aa/57/89d2d3dedfae2558853c1770c7cbcdfd5211eb4c4090b1e8cb570544a490/multidict-6.9.1-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:8879510a76940670517ea1cb589978da44b86e286ec3e50d664ef330817afce7", size = 107679, upload-time = "2026-09-21T17:58:21.98Z" }, { url = "https://files.pythonhosted.org/packages/c8/df/0c2a28870181c1762f90952b897f6db46c934ce073efe0442604cbc276db/multidict-6.9.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:aec65b53a07f580606593f877eefbb29a45939bfc0d3fe6e6d9f42b41b749f68", size = 64753, upload-time = "2026-09-21T17:58:23.622Z" }, { url = "https://files.pythonhosted.org/packages/43/7e/9c6a3e7619459407d8958f7e56ab3dda7f0738f5aed9bb4ad48076e8de83/multidict-6.9.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:14c56f73e78faa1f68bbb826197cd5871994e70e841b8590829c35912ece5c64", size = 62732, upload-time = "2026-09-21T17:58:25.46Z" }, { url = "https://files.pythonhosted.org/packages/38/eb/fd56c9d83ba0cdc3cf66f9acb278b24ef1f043b620b96d145685a3b57338/multidict-6.9.1-cp315-cp315t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:38c9986f9ce50c459b10de216a05f4bd7ed5ac63887d56e500a52bb464b861ce", size = 288648, upload-time = "2026-09-21T17:58:27.115Z" }, { url = "https://files.pythonhosted.org/packages/df/ef/51e79c2442b0f56ac4ac580108d713078b31297e028d468fc220971a305d/multidict-6.9.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:15db6a102cbaf1949cf028ecf080aac76d20bcd29ad4e092574db6c6b7af78a5", size = 304919, upload-time = "2026-09-21T17:58:29.143Z" }, { url = "https://files.pythonhosted.org/packages/6f/e4/774700cc5739353fe88a7204ed30e78a9403d98d4bb2496cc91f096d2e71/multidict-6.9.1-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:bd82c4977196681a499bb6ca9e462afbc5c91c1c15b6a991dfbd72733fea4dd2", size = 289094, upload-time = "2026-09-21T17:58:30.938Z" }, { url = "https://files.pythonhosted.org/packages/ca/41/0bec327bc3a8271825376b6d8a469d1f80e3ba1e2e730e019836e29663c3/multidict-6.9.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bb58ba73a3f96f9a3e46b1fab69929d7edbbddb3133ee74b5c3c54074a53c4f1", size = 312007, upload-time = "2026-09-21T17:58:32.964Z" }, { url = "https://files.pythonhosted.org/packages/c3/7a/9906bf5f1fe20e8e755affccc1a69ecd8aebb079ac65386e5ffc8eefbe75/multidict-6.9.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7c6eecfab7ce4cd9487ff8ba936fe38cdfd68c04faf3d5c710363c6a8e695659", size = 314657, upload-time = "2026-09-21T17:58:35.285Z" }, { url = "https://files.pythonhosted.org/packages/33/92/cf6d665a45663bbe216420fe632da10e9f4f842b07ce02bc3ddf8eb882b8/multidict-6.9.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c62e71e6289d78c0108d8aeb495f8bd3cad4bc1632fedddc9297ddf287ecc20", size = 301563, upload-time = "2026-09-21T17:58:37.472Z" }, { url = "https://files.pythonhosted.org/packages/97/4c/9f1b7945526905d58b4b2d422e2ea81326dda71f9bdc9c32c8002639234a/multidict-6.9.1-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:539c2cd5fed0947c135cd7eabaaac55f48300dfa1de0f3ca4edb5efa6606f471", size = 278540, upload-time = "2026-09-21T17:58:39.551Z" }, { url = "https://files.pythonhosted.org/packages/5c/d9/133aa472fc1be181de29a0c2e1c4cd778280882f532b404395f646742e46/multidict-6.9.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:3100d169ceb7bc8f05f89a6db11d1b21f119975fc26f29dc45a472e3569f0879", size = 294858, upload-time = "2026-09-21T17:58:41.791Z" }, { url = "https://files.pythonhosted.org/packages/bd/38/21f2305dd20ac5038fa581b5d7e0d8dbbd70dc6b50231c22ec173dacddda/multidict-6.9.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:696477ad71385c4795e3b8e4cf10b0d2c28c2a1ca6a955e031cb1e62993e9ee3", size = 285461, upload-time = "2026-09-21T17:58:43.844Z" }, { url = "https://files.pythonhosted.org/packages/ad/21/465a43c214d1d7e2d0c974e727d4b5935f8a0868838b5df9d0984c83c1dc/multidict-6.9.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:f5844e7befc707367807586f550fa97e23dcfef0728f02b98c7bc498a961a5df", size = 288964, upload-time = "2026-09-21T17:58:46.16Z" }, { url = "https://files.pythonhosted.org/packages/2d/58/72a3e8d56c1e05146d2d1841dbaacf8986375eded1e5a58d30eef191a580/multidict-6.9.1-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:abc7c2e4b47bfe6a9aea434d3fdebb9597ee636e914e92ba352f2068b9142f4c", size = 303924, upload-time = "2026-09-21T17:58:48.395Z" }, { url = "https://files.pythonhosted.org/packages/6f/b9/911084c04530a682a54536b4f738d3992a593b5ff6fea1d6a1f02f3acc6b/multidict-6.9.1-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:7ab379f95caee071a37cbd8be86d4c65accc651d391f9f97748fef006f38769c", size = 278479, upload-time = "2026-09-21T17:58:50.494Z" }, { url = "https://files.pythonhosted.org/packages/45/eb/9a333346c02c928a9e82534638bce85697c10c46b4901fcd70af19357356/multidict-6.9.1-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:557a4e1708df428ebe6c3081c83a273d275dad2446ce0d81fc648ac71afdb18d", size = 301650, upload-time = "2026-09-21T17:58:52.554Z" }, { url = "https://files.pythonhosted.org/packages/67/2f/1d936c4e45c8199db6082b6836e9182e31aa393de0de652637790831ca4b/multidict-6.9.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:475d04d5192eba487a3e2f935976340baa24529046e9c1c9c7a3b7bf80445ae1", size = 298024, upload-time = "2026-09-21T17:58:55.03Z" }, { url = "https://files.pythonhosted.org/packages/a4/0c/ca62ae2882b89a4cfeb0bbca873a4afdb5fc32994c4f8a37a7cb52ede7a0/multidict-6.9.1-cp315-cp315t-win32.whl", hash = "sha256:10083a8a0f4e1b26b599889e90b9802504ce5d3f7722f925bbb7ca47dd22a7c1", size = 57810, upload-time = "2026-09-21T17:58:57.141Z" }, { url = "https://files.pythonhosted.org/packages/02/8b/b2c2805c3eb6fa4cdf5512fb95ac6174dde71e34dd601c5be52f4df5dc82/multidict-6.9.1-cp315-cp315t-win_amd64.whl", hash = "sha256:e96ca64383efa107262ee3949f047af5ee4f1845ba09463466c04d35a83bd3ec", size = 64984, upload-time = "2026-09-21T17:58:59.446Z" }, { url = "https://files.pythonhosted.org/packages/fd/34/03f1d204d698c408f4754c969882609b6b7a902c0e3d16a7033f219cdeda/multidict-6.9.1-cp315-cp315t-win_arm64.whl", hash = "sha256:501ed8b02a5990c67a91c732843609d43a6be1f7576fcdfc867331239f37fbd3", size = 58742, upload-time = "2026-09-21T17:59:01.726Z" }, { url = "https://files.pythonhosted.org/packages/be/59/e26cb779be4c591d1a910f59d29aca9fba4de70349840a833beba2652371/multidict-6.9.1-py3-none-any.whl", hash = "sha256:7bf6478188f4e47bf5686e8a33da4ae28bf43b1b2528d9ee144d28492bfac60b", size = 19176, upload-time = "2026-09-21T17:59:03.501Z" }, ] [[package]] name = "nbclient" version = "0.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jupyter-client" }, { name = "jupyter-core" }, { name = "nbformat" }, { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/28/a5/b3bae4b590c0cbcada2c63a34f7580024e834a8ba213e949a2f906705787/nbclient-0.11.0.tar.gz", hash = "sha256:04a134a5b087f2c5887f228aca155db50169b8cd9334dee6942c8e927e56081a", size = 62535, upload-time = "2026-06-05T07:52:41.746Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/36/c9/94d73e5a01c5b926c3fa2496e97d7a8dc28ed5a77c0b2ed712f1a62e6694/nbclient-0.11.0-py3-none-any.whl", hash = "sha256:ef7fa0d59d6e1d41103933d8a445a18d5de860ca6b613b87b8574accdb3c2895", size = 25288, upload-time = "2026-06-05T07:52:40.115Z" }, ] [[package]] name = "nbformat" version = "5.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fastjsonschema" }, { name = "jsonschema" }, { name = "jupyter-core" }, { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b2/fd/80f407a9525bc5bd9865e5c37db3b78867fa43217f8aac5eab22b5f028b3/nbformat-5.11.0.tar.gz", hash = "sha256:7dbaed4a69cae28c2b4d44ab7430a6af4544fb89455023f6f21550be757b60c8", size = 151822, upload-time = "2026-08-06T12:29:55.597Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/be/4a/0eece9dad5e73230ca972f7bc29456ce7d74772f123d401fcf67379008f7/nbformat-5.11.0-py3-none-any.whl", hash = "sha256:f70a17f591a9ccd1c601d5e61a4b20972703926df0ba42458ce14bf575766bb6", size = 79820, upload-time = "2026-08-06T12:29:54.178Z" }, ] [[package]] name = "nest-asyncio2" version = "1.7.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/b4/73/731debf26e27e0a0323d7bda270dc2f634b398e38f040a09da1f4351d0aa/nest_asyncio2-1.7.2.tar.gz", hash = "sha256:1921d70b92cc4612c374928d081552efb59b83d91b2b789d935c665fa01729a8", size = 14743, upload-time = "2026-02-13T00:34:04.386Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c5/3c/3179b85b0e1c3659f0369940200cd6d0fa900e6cefcc7ea0bc6dd0e29ffb/nest_asyncio2-1.7.2-py3-none-any.whl", hash = "sha256:f5dfa702f3f81f6a03857e9a19e2ba578c0946a4ad417b4c50a24d7ba641fe01", size = 7843, upload-time = "2026-02-13T00:34:02.691Z" }, ] [[package]] name = "numcodecs" version = "0.16.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/44/bd/8a391e7c356366224734efd24da929cc4796fff468bfb179fe1af6548535/numcodecs-0.16.5.tar.gz", hash = "sha256:0d0fb60852f84c0bd9543cc4d2ab9eefd37fc8efcc410acd4777e62a1d300318", size = 6276387, upload-time = "2025-11-21T02:49:48.986Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/75/cc/55420f3641a67f78392dc0bc5d02cb9eb0a9dcebf2848d1ac77253ca61fa/numcodecs-0.16.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:24e675dc8d1550cd976a99479b87d872cb142632c75cc402fea04c08c4898523", size = 1656287, upload-time = "2025-11-21T02:49:25.755Z" }, { url = "https://files.pythonhosted.org/packages/f5/6c/86644987505dcb90ba6d627d6989c27bafb0699f9fd00187e06d05ea8594/numcodecs-0.16.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:94ddfa4341d1a3ab99989d13b01b5134abb687d3dab2ead54b450aefe4ad5bd6", size = 1148899, upload-time = "2025-11-21T02:49:26.87Z" }, { url = "https://files.pythonhosted.org/packages/97/1e/98aaddf272552d9fef1f0296a9939d1487914a239e98678f6b20f8b0a5c8/numcodecs-0.16.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b554ab9ecf69de7ca2b6b5e8bc696bd9747559cb4dd5127bd08d7a28bec59c3a", size = 8534814, upload-time = "2025-11-21T02:49:28.547Z" }, { url = "https://files.pythonhosted.org/packages/fb/53/78c98ef5c8b2b784453487f3e4d6c017b20747c58b470393e230c78d18e8/numcodecs-0.16.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad1a379a45bd3491deab8ae6548313946744f868c21d5340116977ea3be5b1d6", size = 9173471, upload-time = "2025-11-21T02:49:30.444Z" }, { url = "https://files.pythonhosted.org/packages/1c/20/2fdec87fc7f8cec950d2b0bea603c12dc9f05b4966dc5924ba5a36a61bf6/numcodecs-0.16.5-cp312-cp312-win_amd64.whl", hash = "sha256:845a9857886ffe4a3172ba1c537ae5bcc01e65068c31cf1fce1a844bd1da050f", size = 801412, upload-time = "2025-11-21T02:49:32.123Z" }, { url = "https://files.pythonhosted.org/packages/38/38/071ced5a5fd1c85ba0e14ba721b66b053823e5176298c2f707e50bed11d9/numcodecs-0.16.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:25be3a516ab677dad890760d357cfe081a371d9c0a2e9a204562318ac5969de3", size = 1654359, upload-time = "2025-11-21T02:49:33.673Z" }, { url = "https://files.pythonhosted.org/packages/d1/c0/5f84ba7525577c1b9909fc2d06ef11314825fc4ad4378f61d0e4c9883b4a/numcodecs-0.16.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0107e839ef75b854e969cb577e140b1aadb9847893937636582d23a2a4c6ce50", size = 1144237, upload-time = "2025-11-21T02:49:35.294Z" }, { url = "https://files.pythonhosted.org/packages/0b/00/787ea5f237b8ea7bc67140c99155f9c00b5baf11c49afc5f3bfefa298f95/numcodecs-0.16.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:015a7c859ecc2a06e2a548f64008c0ec3aaecabc26456c2c62f4278d8fc20597", size = 8483064, upload-time = "2025-11-21T02:49:36.454Z" }, { url = "https://files.pythonhosted.org/packages/c4/e6/d359fdd37498e74d26a167f7a51e54542e642ea47181eb4e643a69a066c3/numcodecs-0.16.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84230b4b9dad2392f2a84242bd6e3e659ac137b5a1ce3571d6965fca673e0903", size = 9126063, upload-time = "2025-11-21T02:49:38.018Z" }, { url = "https://files.pythonhosted.org/packages/27/72/6663cc0382ddbb866136c255c837bcb96cc7ce5e83562efec55e1b995941/numcodecs-0.16.5-cp313-cp313-win_amd64.whl", hash = "sha256:5088145502ad1ebf677ec47d00eb6f0fd600658217db3e0c070c321c85d6cf3d", size = 799275, upload-time = "2025-11-21T02:49:39.558Z" }, { url = "https://files.pythonhosted.org/packages/3c/9e/38e7ca8184c958b51f45d56a4aeceb1134ecde2d8bd157efadc98502cc42/numcodecs-0.16.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:b05647b8b769e6bc8016e9fd4843c823ce5c9f2337c089fb5c9c4da05e5275de", size = 1654721, upload-time = "2025-11-21T02:49:40.602Z" }, { url = "https://files.pythonhosted.org/packages/a1/37/260fa42e7b2b08e6e00ad632f8dd620961a60a459426c26cea390f8c68d0/numcodecs-0.16.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3832bd1b5af8bb3e413076b7d93318c8e7d7b68935006b9fa36ca057d1725a8f", size = 1146887, upload-time = "2025-11-21T02:49:41.721Z" }, { url = "https://files.pythonhosted.org/packages/4e/15/e2e1151b5a8b14a15dfd4bb4abccce7fff7580f39bc34092780088835f3a/numcodecs-0.16.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49f7b7d24f103187f53135bed28bb9f0ed6b2e14c604664726487bb6d7c882e1", size = 8476987, upload-time = "2025-11-21T02:49:43.363Z" }, { url = "https://files.pythonhosted.org/packages/6d/30/16a57fc4d9fb0ba06c600408bd6634f2f1753c54a7a351c99c5e09b51ee2/numcodecs-0.16.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aec9736d81b70f337d89c4070ee3ffeff113f386fd789492fa152d26a15043e4", size = 9102377, upload-time = "2025-11-21T02:49:45.508Z" }, { url = "https://files.pythonhosted.org/packages/31/a5/a0425af36c20d55a3ea884db4b4efca25a43bea9214ba69ca7932dd997b4/numcodecs-0.16.5-cp314-cp314-win_amd64.whl", hash = "sha256:b16a14303800e9fb88abc39463ab4706c037647ac17e49e297faa5f7d7dbbf1d", size = 819022, upload-time = "2025-11-21T02:49:47.39Z" }, ] [[package]] name = "numpy" version = "2.5.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/22/fd/89965aa4ac08c74998539fcbf24fa3540f3e15237fbeb6bcf9c908f4aade/numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3", size = 20755553, upload-time = "2026-07-04T17:08:00.933Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/62/7b/14687aa674250e5e546f616f486b0d56d3631cd5b2415739141ce40bdcea/numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277", size = 16801574, upload-time = "2026-07-04T17:06:12.423Z" }, { url = "https://files.pythonhosted.org/packages/e1/19/cc5bb2a3f2913d27d6dbb2c78d25921fabaedc6741d4a5a615a11f3c5bf3/numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1", size = 11772250, upload-time = "2026-07-04T17:06:15.726Z" }, { url = "https://files.pythonhosted.org/packages/42/77/fdf34a71dd30f54979b18603bee915e0aaf825b07afe79acd60b04b691e2/numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0", size = 5331516, upload-time = "2026-07-04T17:06:17.913Z" }, { url = "https://files.pythonhosted.org/packages/ce/e2/eb7efa015b4cce41e2517bf182a7fce0d7d5b9d9ed76a29bfa0f4fe4505c/numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e", size = 6664863, upload-time = "2026-07-04T17:06:20.02Z" }, { url = "https://files.pythonhosted.org/packages/a9/4b/a2b32dd94ee9ffbeecb28152240042a3949db33b1c834d44090b80e1b3b8/numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75", size = 15167977, upload-time = "2026-07-04T17:06:21.621Z" }, { url = "https://files.pythonhosted.org/packages/b8/a9/6e73d68500f80773f65f0654ea932019d6694329a0eb0ed0533de38df376/numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca", size = 16672469, upload-time = "2026-07-04T17:06:24.064Z" }, { url = "https://files.pythonhosted.org/packages/24/7d/ad3e59015135f5261c95fd4cafeff159c955febd83a99a1d9250c4233815/numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3", size = 16527531, upload-time = "2026-07-04T17:06:26.69Z" }, { url = "https://files.pythonhosted.org/packages/83/d0/a39b2fbcde9cb17a1dac678f254b33a6336298af9df338824c685425d5e8/numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9", size = 18431940, upload-time = "2026-07-04T17:06:29.521Z" }, { url = "https://files.pythonhosted.org/packages/04/12/cff070947791c1ed425ff76413189adbdc2fbe215eba7ce7fa454a03c7f8/numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2", size = 6066764, upload-time = "2026-07-04T17:06:32.571Z" }, { url = "https://files.pythonhosted.org/packages/65/66/53f31807a48a750f9d748da273bc3fcedd12b27ff1f3e373bfec55ef2dc0/numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2", size = 12430966, upload-time = "2026-07-04T17:06:34.926Z" }, { url = "https://files.pythonhosted.org/packages/2b/2a/d1a88066b1c14186f5d3c0d18c94f17b064511982bab0578d49ee9d43c29/numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b", size = 10350488, upload-time = "2026-07-04T17:06:37.785Z" }, { url = "https://files.pythonhosted.org/packages/eb/07/ec2a3f0c91761581d4b7104a740791800025983f9a4dc4e73f91a99aeac4/numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1", size = 16796419, upload-time = "2026-07-04T17:06:40.37Z" }, { url = "https://files.pythonhosted.org/packages/ab/ab/ddb499fc4f8780354395face5b65c7fd107bcd6e1d667a5f07d046956f6f/numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6", size = 11765832, upload-time = "2026-07-04T17:06:42.768Z" }, { url = "https://files.pythonhosted.org/packages/88/b3/3c28c558a09fc72100c646dac6d2fce8e834c471b0edca01a29996706117/numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d", size = 5325143, upload-time = "2026-07-04T17:06:45.466Z" }, { url = "https://files.pythonhosted.org/packages/5e/0e/ce19b985bb15c596f4f05954e76cccc77c845083b3b8f938a6c68e523128/numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1", size = 6659749, upload-time = "2026-07-04T17:06:47.288Z" }, { url = "https://files.pythonhosted.org/packages/2e/20/1ee6614d64332a1bba6411f38e68cb79eec1b2459e20a623777c5c5492a2/numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd", size = 15164716, upload-time = "2026-07-04T17:06:49.494Z" }, { url = "https://files.pythonhosted.org/packages/ed/a7/2bcd3fdbb87804755c35b729bf8709d62025c5f4cfd7d5b2415997097515/numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a", size = 16661440, upload-time = "2026-07-04T17:06:52.061Z" }, { url = "https://files.pythonhosted.org/packages/fc/d7/a41e3310c886fe457d36e670bbf24fae411aca8a7b6ad92a32afd924077c/numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7", size = 16526305, upload-time = "2026-07-04T17:06:54.605Z" }, { url = "https://files.pythonhosted.org/packages/53/75/4333a9a707c1edd3a4e1a0c58eca52c0f31e55089fa80db02b5565b24df7/numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6", size = 18423008, upload-time = "2026-07-04T17:06:57.54Z" }, { url = "https://files.pythonhosted.org/packages/ee/90/e314a32b1c11a2ffe818ddad3a57b50b4b6e1b6c487192eb50cdef0415d0/numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9", size = 6063885, upload-time = "2026-07-04T17:07:00.14Z" }, { url = "https://files.pythonhosted.org/packages/10/70/800b3fca480af32df9e8ea9f3d4a0c8feb4b32d7f195d174eabbda4829ad/numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74", size = 12425674, upload-time = "2026-07-04T17:07:02.387Z" }, { url = "https://files.pythonhosted.org/packages/8b/0b/196350c122f50f6ca56846f2d71efd5e0d24b7b2e07355e019b2e2c7a11e/numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107", size = 10350256, upload-time = "2026-07-04T17:07:04.878Z" }, { url = "https://files.pythonhosted.org/packages/db/f4/731b6085a83faf6ca843394cbd5e217280c214399f7e8b21b9f552af0ae2/numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8", size = 16795063, upload-time = "2026-07-04T17:07:07.374Z" }, { url = "https://files.pythonhosted.org/packages/bf/64/0e215f2048dd11a55bb989ed41b3585ef57452404e638d703a211a3e4157/numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75", size = 11776652, upload-time = "2026-07-04T17:07:09.907Z" }, { url = "https://files.pythonhosted.org/packages/b5/59/2b844c7a6e9deff69b404a66221e1542937734f65d5e6e39411876053862/numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2", size = 5335944, upload-time = "2026-07-04T17:07:12.227Z" }, { url = "https://files.pythonhosted.org/packages/86/51/9bf7cb2cabcebc9e017e4ec7e6322b378317a542c08b4cb68479c1efc716/numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b", size = 6656266, upload-time = "2026-07-04T17:07:14.368Z" }, { url = "https://files.pythonhosted.org/packages/83/3e/fb7615b211b82a32f44d5180a6d421b61f84d4fadd578b48ba4ac34e189f/numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95", size = 15179720, upload-time = "2026-07-04T17:07:16.272Z" }, { url = "https://files.pythonhosted.org/packages/41/5f/0f992cb24560673496c5d68de61913b57166ce530ffda07c1f280e0cc464/numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21", size = 16664835, upload-time = "2026-07-04T17:07:19.021Z" }, { url = "https://files.pythonhosted.org/packages/a2/2f/97d6475ee91afe2587797d09446f9d3e475ad4cb681662d824809327b75a/numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373", size = 16539135, upload-time = "2026-07-04T17:07:22.015Z" }, { url = "https://files.pythonhosted.org/packages/c4/5b/4db81e4ba0be7e2776b1de68c82aa862c7f8ec27e1b4927d4ae075e20678/numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438", size = 18426684, upload-time = "2026-07-04T17:07:24.941Z" }, { url = "https://files.pythonhosted.org/packages/1f/64/c0ba2d90724d450279a7df8f32057241070250a26a7e2b5337d77347f481/numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace", size = 6116103, upload-time = "2026-07-04T17:07:27.622Z" }, { url = "https://files.pythonhosted.org/packages/c1/1a/837f9ed7405adcd7a40538792eb169eddd8fa5630c16a1ef49dae71a30f4/numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a", size = 12562177, upload-time = "2026-07-04T17:07:29.887Z" }, { url = "https://files.pythonhosted.org/packages/22/ed/49707938b6dd0a78a9178dd93227dc89e4c11af47f5c798d70366e8d0483/numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0", size = 10627739, upload-time = "2026-07-04T17:07:32.568Z" }, { url = "https://files.pythonhosted.org/packages/a6/c7/bb4b882cfe7f299cbc8b66e42e7dd78cf9d14e40f9469fc5e3db7e15b3bd/numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22", size = 11894709, upload-time = "2026-07-04T17:07:34.941Z" }, { url = "https://files.pythonhosted.org/packages/40/3f/5af7f4a7f6224aef48017aa82bb6174c7a659d724be0c75017b7e64a55b4/numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7", size = 5453810, upload-time = "2026-07-04T17:07:37.495Z" }, { url = "https://files.pythonhosted.org/packages/20/c9/3474309bc94d634d3f9c3eddf03250ecb8c22cd948ef16fef69a77cc5d7b/numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d", size = 6761189, upload-time = "2026-07-04T17:07:39.563Z" }, { url = "https://files.pythonhosted.org/packages/90/8a/558ae39fdd55d7e7f7fef9a84a6e964ac6b23edbd2a07e52bb084500507d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09", size = 15225039, upload-time = "2026-07-04T17:07:41.682Z" }, { url = "https://files.pythonhosted.org/packages/63/27/ca7392b2d030277bdf0273e7d23255b3ee57d57a7c170a6f4fb3981e1e5d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4", size = 16701306, upload-time = "2026-07-04T17:07:44.611Z" }, { url = "https://files.pythonhosted.org/packages/02/42/03d53ae7996c44d4374a8262e9dc41671fd56cbb98f7d47ef85cf5da4c6b/numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1", size = 16589955, upload-time = "2026-07-04T17:07:47.694Z" }, { url = "https://files.pythonhosted.org/packages/7b/15/6c1784ae469640e65db111e9a34b3d0f14d91e8a38b9ce34810ced370dbb/numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077", size = 18464252, upload-time = "2026-07-04T17:07:50.684Z" }, { url = "https://files.pythonhosted.org/packages/94/a8/f98e50356cf167df656c526c2dfeec2d7dde182f2a3da4b458a5938e2776/numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf", size = 6263298, upload-time = "2026-07-04T17:07:53.445Z" }, { url = "https://files.pythonhosted.org/packages/72/ac/96ae880cdecad0b3275d9359fcec72667b49a4863c9f12942e43679dda02/numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af", size = 12748623, upload-time = "2026-07-04T17:07:55.384Z" }, { url = "https://files.pythonhosted.org/packages/a1/5a/4d2b1601df3602dba7a14f3348ba9bfe94a18adb428e693df6154c293831/numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb", size = 10697674, upload-time = "2026-07-04T17:07:58.506Z" }, ] [[package]] name = "packaging" version = "26.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] [[package]] name = "paginate" version = "0.5.7" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ec/46/68dde5b6bc00c1296ec6466ab27dddede6aec9af1b99090e1107091b3b84/paginate-0.5.7.tar.gz", hash = "sha256:22bd083ab41e1a8b4f3690544afb2c60c25e5c9a63a30fa2f483f6c60c8e5945", size = 19252, upload-time = "2024-08-25T14:17:24.139Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/90/96/04b8e52da071d28f5e21a805b19cb9390aa17a47462ac87f5e2696b9566d/paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591", size = 13746, upload-time = "2024-08-25T14:17:22.55Z" }, ] [[package]] name = "parso" version = "0.8.7" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/30/4b/90c937815137d43ce71ba043cd3566221e9df6b9c805f24b5d138c9d40a7/parso-0.8.7.tar.gz", hash = "sha256:eaaac4c9fdd5e9e8852dc778d2d7405897ec510f2a298071453e5e3a07914bb1", size = 401824, upload-time = "2026-05-01T23:13:02.138Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl", hash = "sha256:a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c", size = 107025, upload-time = "2026-05-01T23:12:58.867Z" }, ] [[package]] name = "pathspec" version = "1.1.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, ] [[package]] name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "ptyprocess" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" }, ] [[package]] name = "platformdirs" version = "4.11.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e5/98/0bf930c4f97d0266b58a89e36c015f56232c52b5d2f207215d48cca9e8f7/platformdirs-4.11.2.tar.gz", hash = "sha256:3a2ae5fca3520a01ab1be8b45613537f52ddf5b5f6f53d88233892dfbf0cd82d", size = 32716, upload-time = "2026-08-10T15:48:06.092Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/49/e2/4e6eee633809c376c024821b91ade709cbfd040ec53939ffbcc292aa7eee/platformdirs-4.11.2-py3-none-any.whl", hash = "sha256:7f89089b6ea71bda7962953edcf784b2e2d9d285b40ad88be2bb75c6e9d82ab4", size = 23361, upload-time = "2026-08-10T15:48:04.855Z" }, ] [[package]] name = "pluggy" version = "1.6.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] [[package]] name = "prompt-toolkit" version = "3.0.53" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "wcwidth" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7d/ea/39b988c938f75cb75d7045b5c69f8bfed47ee2152c8837fb403de29d6fb8/prompt_toolkit-3.0.53.tar.gz", hash = "sha256:9ec8a0ad96d5c56148b3f914aa79c1564c3fde5d2e6b876e7bc327e353cf8fa6", size = 435492, upload-time = "2026-07-26T20:56:14.758Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/54/6f/84908cad2d6aa5144abcf7b42709fe4fdb459bc640ec7ac5786e7693dabc/prompt_toolkit-3.0.53-py3-none-any.whl", hash = "sha256:01c0891d7f9237d5e339f7d3e42cdae80b7534abb1c7c0e3352efba6231492f2", size = 392288, upload-time = "2026-07-26T20:56:12.512Z" }, ] [[package]] name = "propcache" version = "0.5.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/4a/cb/e27bc2b2737a0bb49962b275efa051e8f1c35a936df7d5139b6b658b7dc9/propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba", size = 95887, upload-time = "2026-05-08T21:00:11.277Z" }, { url = "https://files.pythonhosted.org/packages/e6/13/b8ae04c59392f8d11c6cd9fb4011d1dc7c86b81225c770280300e259ffe1/propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a", size = 54654, upload-time = "2026-05-08T21:00:12.604Z" }, { url = "https://files.pythonhosted.org/packages/2c/7d/49777a3e20b55863d4794384a38acd460c04157b0a00f8602b0d508b8431/propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf", size = 55190, upload-time = "2026-05-08T21:00:13.935Z" }, { url = "https://files.pythonhosted.org/packages/44/c7/085d0cd63062e84044e3f05797749c3f8e3938ff3aeb0eb2f69d43fafc91/propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144", size = 59995, upload-time = "2026-05-08T21:00:15.526Z" }, { url = "https://files.pythonhosted.org/packages/9c/42/32cf8e3009e92b2645cf1e944f701e8ea4e924dffde1ee26db860bcbf7e4/propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9", size = 63422, upload-time = "2026-05-08T21:00:16.824Z" }, { url = "https://files.pythonhosted.org/packages/9e/1b/f112433f99fc979431b87a39ef169e3f8df070d99a72792c56d6937ac48b/propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42", size = 64342, upload-time = "2026-05-08T21:00:18.362Z" }, { url = "https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476", size = 61639, upload-time = "2026-05-08T21:00:19.692Z" }, { url = "https://files.pythonhosted.org/packages/cc/da/4d775080b1490c0ae604acda868bd71aabe3a89ed16f2aa4339eb8a283e7/propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba", size = 61588, upload-time = "2026-05-08T21:00:21.155Z" }, { url = "https://files.pythonhosted.org/packages/04/ac/f076982cbe2195ee9cf32de5a1e46951d9fb399fc207f390562dd0fd8fb2/propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a", size = 60029, upload-time = "2026-05-08T21:00:22.713Z" }, { url = "https://files.pythonhosted.org/packages/70/60/189be62e0dd898dce3b331e1b8c7a543cd3a405ac0c81fe8ee8a9d5d77e1/propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64", size = 56774, upload-time = "2026-05-08T21:00:24.001Z" }, { url = "https://files.pythonhosted.org/packages/ea/9e/93377b9c7939c1ffae98f878dee955efadfd638078bc86dbc21f9d52f651/propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913", size = 63532, upload-time = "2026-05-08T21:00:25.545Z" }, { url = "https://files.pythonhosted.org/packages/14/f9/590ef6cfb9b8028d516d287812ece32bb0bc5f11fbb9c8bf6b2e6313fec8/propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1", size = 61592, upload-time = "2026-05-08T21:00:27.186Z" }, { url = "https://files.pythonhosted.org/packages/b4/5e/70958b3034c297a630bba2f17ca7abc2d5f39a803ad7e370ab79d1ecd022/propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33", size = 64788, upload-time = "2026-05-08T21:00:28.8Z" }, { url = "https://files.pythonhosted.org/packages/12/fd/77fe5936d8c3086ca9048f7f415f122ed82e53884a9ec193646b42deef06/propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a", size = 62514, upload-time = "2026-05-08T21:00:30.098Z" }, { url = "https://files.pythonhosted.org/packages/cf/74/66bd798b5b3be70aa1b391f5cc9d6a0a5532d7fd3b19ec0b213e72e6ad9d/propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031", size = 39018, upload-time = "2026-05-08T21:00:31.622Z" }, { url = "https://files.pythonhosted.org/packages/61/7c/5c0d34aa3024694d6dcb9271cdbdd08c4e47c1c0ad95ec7e7bc74cdea145/propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42", size = 42322, upload-time = "2026-05-08T21:00:32.918Z" }, { url = "https://files.pythonhosted.org/packages/4d/91/875812f1a3feb20ceba818ef39fbe4d92f1081e04ac815c822496d0d038b/propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84", size = 38172, upload-time = "2026-05-08T21:00:35.124Z" }, { url = "https://files.pythonhosted.org/packages/c5/09/f049e45385503fe67db75a6b6186a7b9f0c3930366dc960522c312a825b1/propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a", size = 94457, upload-time = "2026-05-08T21:00:36.355Z" }, { url = "https://files.pythonhosted.org/packages/6b/65/83d1d05655baf63113731bd5a1008435e14f8d1e5a06cbe4ec5b23ad7a31/propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117", size = 53835, upload-time = "2026-05-08T21:00:38.072Z" }, { url = "https://files.pythonhosted.org/packages/a9/12/a6ba6482bb5ea3260c000c9b20881c95fa11c6b30173715668259f844ed7/propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098", size = 54545, upload-time = "2026-05-08T21:00:39.319Z" }, { url = "https://files.pythonhosted.org/packages/a9/19/7fa086f5764c59ec8a8e157cd93aa8497acc00aba9dcdec56bfffb32602d/propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4", size = 59886, upload-time = "2026-05-08T21:00:40.621Z" }, { url = "https://files.pythonhosted.org/packages/a1/e4/5d7663dc8235956c8f5281698a3af1d351d8820341ddd890f59d9a9127f2/propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e", size = 63261, upload-time = "2026-05-08T21:00:41.775Z" }, { url = "https://files.pythonhosted.org/packages/4a/4a/15a03adee24d6350da4292caeac44c34c033d2afe5e87eb370f38854560f/propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7", size = 64184, upload-time = "2026-05-08T21:00:43.018Z" }, { url = "https://files.pythonhosted.org/packages/8b/c6/979176efdaa3d239e36d503d5af63a0a773b36662ed8f52e5b6a6d9fd40e/propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d", size = 61534, upload-time = "2026-05-08T21:00:44.507Z" }, { url = "https://files.pythonhosted.org/packages/c8/22/63e8cd1bae4c2d2be6493b6b7d10566ddafad88137cfbc99964a1119853c/propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a", size = 61500, upload-time = "2026-05-08T21:00:45.796Z" }, { url = "https://files.pythonhosted.org/packages/60/5a/28e5d9acbac1cc9ccb67045e8c1b943aa8d79fdf39c93bd73cacd68008ea/propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2", size = 59994, upload-time = "2026-05-08T21:00:47.093Z" }, { url = "https://files.pythonhosted.org/packages/f3/40/db650677f554a95b9c01a7c9d93d629e93a15562f5deb4573c9ee136fed2/propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa", size = 56884, upload-time = "2026-05-08T21:00:48.376Z" }, { url = "https://files.pythonhosted.org/packages/80/45/70b39b89516ff8b96bf732fa6fded8cef20f293cb1508690101c3c07ec51/propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853", size = 63464, upload-time = "2026-05-08T21:00:49.954Z" }, { url = "https://files.pythonhosted.org/packages/f9/e2/fa59d3a89eac5534293124af4f1d0d0ada091ce4a0ab4610ce03fd2bdd8d/propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a", size = 61588, upload-time = "2026-05-08T21:00:51.281Z" }, { url = "https://files.pythonhosted.org/packages/0b/97/efb547a55c4bc7381cfb202d6a2239ac621045277bc1ea5dfd3a7f0516c0/propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704", size = 64667, upload-time = "2026-05-08T21:00:52.602Z" }, { url = "https://files.pythonhosted.org/packages/92/56/f5c7d9b4b7595d5127da38974d791b2153f3d1eae6c674af3583ace92ad3/propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4", size = 62463, upload-time = "2026-05-08T21:00:54.303Z" }, { url = "https://files.pythonhosted.org/packages/bd/3b/484a3a65fc9f9f60c41dcd17b428bace5389544e2c680994534a20755066/propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d", size = 38621, upload-time = "2026-05-08T21:00:55.808Z" }, { url = "https://files.pythonhosted.org/packages/1c/fd/3f0f10dba4dabad3bf53102be007abf55481067952bde0fdddff439e7c61/propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757", size = 41649, upload-time = "2026-05-08T21:00:57.061Z" }, { url = "https://files.pythonhosted.org/packages/90/ec/6ce619cc32bb500a482f811f9cd509368b4e58e638d13f2c68f370d6b475/propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f", size = 37636, upload-time = "2026-05-08T21:00:58.646Z" }, { url = "https://files.pythonhosted.org/packages/1b/82/c1d268bbbf2ef981c5bf0fbbe746db617c66e3bcefe431a1aa8943fbe23a/propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d", size = 98872, upload-time = "2026-05-08T21:00:59.889Z" }, { url = "https://files.pythonhosted.org/packages/f4/d4/52c871e73e864e6b34c0e2d58ac1ec5ccd149497ddc7ad2137ae98323a35/propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa", size = 56257, upload-time = "2026-05-08T21:01:01.195Z" }, { url = "https://files.pythonhosted.org/packages/67/f0/9b90ca2a210b3d09bcfcd96ecd0f55545c091535abce2a45de2775cfd357/propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94", size = 56696, upload-time = "2026-05-08T21:01:02.941Z" }, { url = "https://files.pythonhosted.org/packages/9d/0e/6e9d4ba07c8e56e21ddec1e75f12148142b21ca83a51871babce095334f4/propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164", size = 62378, upload-time = "2026-05-08T21:01:04.475Z" }, { url = "https://files.pythonhosted.org/packages/65/19/c10badaa463dde8a27ce884f8ee2ec37e6035b7c9f5ff0c8f74f06f08dac/propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f", size = 65283, upload-time = "2026-05-08T21:01:05.959Z" }, { url = "https://files.pythonhosted.org/packages/b0/b6/93bea99ca80e19cef6512a8580e5b7857bbe09422d9daa7fd4ef5723306c/propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c", size = 66616, upload-time = "2026-05-08T21:01:07.228Z" }, { url = "https://files.pythonhosted.org/packages/83/e4/5c7462e50625f051f37fb38b8224f7639f667184bbd34424ec83819bb1b7/propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc", size = 63773, upload-time = "2026-05-08T21:01:08.514Z" }, { url = "https://files.pythonhosted.org/packages/ca/b6/99238894047b13c823be25027e736626cd414a52a5e30d2c3347c2733529/propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f", size = 63664, upload-time = "2026-05-08T21:01:09.874Z" }, { url = "https://files.pythonhosted.org/packages/85/1e/a3a1a63116a2b8edb415a8bb9a6f0c34bd03830b1e18e8ce2904e1dc1cf4/propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb", size = 62643, upload-time = "2026-05-08T21:01:11.132Z" }, { url = "https://files.pythonhosted.org/packages/e4/03/893cf147de2fc6543c5eaa07ad833170e7e2a2385725bbebe8c0503723bb/propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751", size = 59595, upload-time = "2026-05-08T21:01:12.387Z" }, { url = "https://files.pythonhosted.org/packages/86/3b/04c1a2e12c57766568ba75ba72b3bf2042818d4c1425fab6fc07155c7cff/propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836", size = 65711, upload-time = "2026-05-08T21:01:13.676Z" }, { url = "https://files.pythonhosted.org/packages/1c/34/80f8d0099f8d6bacc4de1624c85672681c8cd1149ca2da0e38fd120b817f/propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f", size = 64247, upload-time = "2026-05-08T21:01:14.936Z" }, { url = "https://files.pythonhosted.org/packages/f3/1a/8b08f3a5f1037e9e370c55883ceeeee0f6dd0416fb2d2d67b8bfc91f2a79/propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55", size = 67102, upload-time = "2026-05-08T21:01:16.281Z" }, { url = "https://files.pythonhosted.org/packages/34/68/8bdb7bb7756d76e005490649d10e4a8369e610c74d619f71e1aedf889e9c/propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568", size = 64964, upload-time = "2026-05-08T21:01:17.57Z" }, { url = "https://files.pythonhosted.org/packages/0a/aa/50fb0b5d3968b61a510926ff8b8465f1d6e976b3ab74496d7a4b9fc42515/propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191", size = 42546, upload-time = "2026-05-08T21:01:18.946Z" }, { url = "https://files.pythonhosted.org/packages/ae/4c/0ddbae64321bd4a95bcbfc19307238016b5b1fee645c84626c8d539e5b74/propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7", size = 46330, upload-time = "2026-05-08T21:01:20.162Z" }, { url = "https://files.pythonhosted.org/packages/00/d9/9cddc8efb78d8af264c5ec9f6d10b62f57c515feda8d321595f56010fb23/propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96", size = 40521, upload-time = "2026-05-08T21:01:21.399Z" }, { url = "https://files.pythonhosted.org/packages/e2/ea/23ee535d90ce8bcc465a3028eb3cc0ce3bd1005f4bb27710b30587de798d/propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999", size = 94662, upload-time = "2026-05-08T21:01:22.683Z" }, { url = "https://files.pythonhosted.org/packages/b5/06/c5a52f419b5d8972f8d46a7577476090d8e3263ff589ce40b5ca4968d5be/propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e", size = 53928, upload-time = "2026-05-08T21:01:23.986Z" }, { url = "https://files.pythonhosted.org/packages/63/b1/4260d67d6bd85e58a66b72d54ce15d5de789b6f3870cc6bedf8ff9667401/propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539", size = 54650, upload-time = "2026-05-08T21:01:25.305Z" }, { url = "https://files.pythonhosted.org/packages/70/06/2f46c318e3307cd7a6a7481def374ce838c0fe20084b39dd54b0879d0e99/propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e", size = 59912, upload-time = "2026-05-08T21:01:26.545Z" }, { url = "https://files.pythonhosted.org/packages/4c/29/fe1aebec2ce57ab985a9c382bded1124431f85078113aa222c5d278430d4/propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979", size = 63300, upload-time = "2026-05-08T21:01:27.937Z" }, { url = "https://files.pythonhosted.org/packages/b4/18/2334b26768b6c82be8c69e83671b767d5ef426aa09b0cba6c2ea47816774/propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80", size = 64208, upload-time = "2026-05-08T21:01:29.484Z" }, { url = "https://files.pythonhosted.org/packages/2b/76/7f1bfd6afff4c5e38e36a3c6d68eb5f4b7311ea80baf693db78d95b603c4/propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825", size = 61633, upload-time = "2026-05-08T21:01:31.068Z" }, { url = "https://files.pythonhosted.org/packages/c4/46/b3ff8aba2b4953a3e50de2cf72f1b5748b8eca93b15f3dc2c84339084c09/propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39", size = 61724, upload-time = "2026-05-08T21:01:32.374Z" }, { url = "https://files.pythonhosted.org/packages/c5/01/814cfcafbcff954f94c01cf30e097ddc88a076b5440fbcf4570753437d40/propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4", size = 60069, upload-time = "2026-05-08T21:01:33.67Z" }, { url = "https://files.pythonhosted.org/packages/da/68/5c6f7622d510cc666a300687e06fd060c1a43361c0c9b20d284f06d8096a/propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5", size = 57099, upload-time = "2026-05-08T21:01:34.915Z" }, { url = "https://files.pythonhosted.org/packages/55/27/9cb0b4c679124085327957d42521c99dba04c88c90c3e55a6f0b633ebccc/propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702", size = 63391, upload-time = "2026-05-08T21:01:36.231Z" }, { url = "https://files.pythonhosted.org/packages/f0/9d/7258aaa5bdf60fc6f27591eef6fe52768cb0beda7140be477c8b12c9794a/propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3", size = 61626, upload-time = "2026-05-08T21:01:37.545Z" }, { url = "https://files.pythonhosted.org/packages/8e/0d/41c602003e8a9b16fe1e7eadf62c7bfba9d5474370b24200bf48b315f45f/propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5", size = 64781, upload-time = "2026-05-08T21:01:38.83Z" }, { url = "https://files.pythonhosted.org/packages/8b/f3/38e66b1856e9bd079deea015bc4a55f7767c0e4db2f7dcf69e7e680ba4ce/propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4", size = 62570, upload-time = "2026-05-08T21:01:40.415Z" }, { url = "https://files.pythonhosted.org/packages/95/ca/bbfe9b910ce57dde8bb4876b4520fc02a4e89497c10de26be936758a3aaa/propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0", size = 39436, upload-time = "2026-05-08T21:01:41.654Z" }, { url = "https://files.pythonhosted.org/packages/61/d2/45c9defbaa1ea297035d9d4cce9e8f80daafbf19319c6007f157c6256ea9/propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c", size = 42373, upload-time = "2026-05-08T21:01:43.041Z" }, { url = "https://files.pythonhosted.org/packages/44/68/9ea5103f41d5217d7d6ec24db90018e23aebec070c3f9a6e54d12b841fd8/propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0", size = 38554, upload-time = "2026-05-08T21:01:44.336Z" }, { url = "https://files.pythonhosted.org/packages/8a/81/fadf555f42d3b762eea8a53950b0489fdc0aa9da5f8ed9e10ce0a4e01b48/propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb", size = 99395, upload-time = "2026-05-08T21:01:45.883Z" }, { url = "https://files.pythonhosted.org/packages/f5/c9/c61e134a686949cf7971af3a390148b1156f7be81c73bc0cd12c873e2d48/propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078", size = 56653, upload-time = "2026-05-08T21:01:47.307Z" }, { url = "https://files.pythonhosted.org/packages/cb/73/daf935ea7048ddd7ec8eec5345b4a40b619d2d178b3c0a0900796bc3c794/propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa", size = 56914, upload-time = "2026-05-08T21:01:48.573Z" }, { url = "https://files.pythonhosted.org/packages/79/9f/aba959b435ea18617edd7cf0a7ad0b9c574b8fc7e3d2cd55fb59cb255d33/propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917", size = 62567, upload-time = "2026-05-08T21:01:49.903Z" }, { url = "https://files.pythonhosted.org/packages/6c/a1/859942de9a791ff42f6141736f5b37749b8f53e65edfa49638c67dd67e6a/propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe", size = 65542, upload-time = "2026-05-08T21:01:51.204Z" }, { url = "https://files.pythonhosted.org/packages/b5/61/315bc0fd6c0fc7f80a528b8afd209e5fc4a875ea79571b91b8f50f442907/propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03", size = 66845, upload-time = "2026-05-08T21:01:52.539Z" }, { url = "https://files.pythonhosted.org/packages/47/f7/9f8122e3132e8e354ac41975ef8f1099be7d5a16bc7ae562734e993665c0/propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335", size = 63985, upload-time = "2026-05-08T21:01:53.847Z" }, { url = "https://files.pythonhosted.org/packages/c8/54/c317819ec157cbf6f35df9df9657a6f82daf34d5faf15948b2f639c2192e/propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285", size = 63999, upload-time = "2026-05-08T21:01:55.179Z" }, { url = "https://files.pythonhosted.org/packages/5a/56/387e3f7dfce0a9233df41fb888aa1c30222cb4bbbf09537c02dd9bd85fe2/propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837", size = 62779, upload-time = "2026-05-08T21:01:57.489Z" }, { url = "https://files.pythonhosted.org/packages/a1/9c/596784cb5824ed61ee960d3f8655a3f0993e107c6e98ab6c818b7fb92ccb/propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8", size = 59796, upload-time = "2026-05-08T21:01:58.736Z" }, { url = "https://files.pythonhosted.org/packages/c2/3d/1a6cfa1726a48542c1e8784a0761421476a5b68e09b7f36bf95eb954aaba/propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366", size = 66023, upload-time = "2026-05-08T21:02:00.228Z" }, { url = "https://files.pythonhosted.org/packages/e4/0e/05fd6990369477076e4e280bcb970de760fddf0161a46e988bc95f7940ec/propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56", size = 64448, upload-time = "2026-05-08T21:02:01.888Z" }, { url = "https://files.pythonhosted.org/packages/cd/86/5f8da315a4309c62c10c0b2516b17492d5d3bbe1bb862b96604db67e2a37/propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d", size = 67329, upload-time = "2026-05-08T21:02:03.484Z" }, { url = "https://files.pythonhosted.org/packages/da/d3/3368efe79ab21f0cdf86ef49895811c9cc933131d4cde1f28a624e22e712/propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2", size = 65172, upload-time = "2026-05-08T21:02:04.745Z" }, { url = "https://files.pythonhosted.org/packages/d5/07/127e8b0bacfb325396196f9d976a22453049b89b9b2b08477cc3145faa44/propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821", size = 43813, upload-time = "2026-05-08T21:02:06.025Z" }, { url = "https://files.pythonhosted.org/packages/88/fb/46dad6c0ae49ed230ab1b16c890c2b6314e2403e6c412976f4a72d64a527/propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370", size = 47764, upload-time = "2026-05-08T21:02:07.353Z" }, { url = "https://files.pythonhosted.org/packages/e7/c4/a47d0a63aa309d10d59ede6e9d4cff03a344a79d1f0f4cd0cd74997b53e0/propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6", size = 41140, upload-time = "2026-05-08T21:02:09.065Z" }, { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, ] [[package]] name = "psutil" version = "7.2.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" }, { url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" }, { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, { url = "https://files.pythonhosted.org/packages/16/ba/0756dca669f5a9300d0cbcbfae9a4c30e446dfc7440ffe43ded5724bfd93/psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b", size = 139893, upload-time = "2026-01-28T18:15:06.378Z" }, { url = "https://files.pythonhosted.org/packages/1c/61/8fa0e26f33623b49949346de05ec1ddaad02ed8ba64af45f40a147dbfa97/psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9", size = 135589, upload-time = "2026-01-28T18:15:08.03Z" }, { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, ] [[package]] name = "ptyprocess" version = "0.7.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/20/e5/16ff212c1e452235a90aeb09066144d0c5a6a8c0834397e03f5224495c4e/ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220", size = 70762, upload-time = "2020-12-28T15:15:30.155Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993, upload-time = "2020-12-28T15:15:28.35Z" }, ] [[package]] name = "pure-eval" version = "0.2.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/cd/05/0a34433a064256a578f1783a10da6df098ceaa4a57bbeaa96a6c0352786b/pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42", size = 19752, upload-time = "2024-07-21T12:58:21.801Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] [[package]] name = "pycparser" version = "3.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, ] [[package]] name = "pygments" version = "2.20.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] [[package]] name = "pymdown-extensions" version = "11.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown" }, { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/21/a9/5f0c535ba3b08fe09270c16808e053a968868242ecbd5676d4e3a488bf28/pymdown_extensions-11.0.1.tar.gz", hash = "sha256:dd2905ae6fc5b75582fafb139a1266ffc754705efa902aa50067fa7ff4f94ec0", size = 857113, upload-time = "2026-07-02T17:59:22.955Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d6/54/da572c98c0b77626a91b5d3b89f0231d8bff5125c225420908632f8b342d/pymdown_extensions-11.0.1-py3-none-any.whl", hash = "sha256:db3943a62bab7e03af1364f0c4083e64b91fb097675a4b6cceccfbe9a77e5eb2", size = 269455, upload-time = "2026-07-02T17:59:21.271Z" }, ] [[package]] name = "pytest" version = "9.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] [[package]] name = "python-dateutil" version = "2.9.0.post0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "six" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] [[package]] name = "pyyaml" version = "6.0.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] [[package]] name = "pyyaml-env-tag" version = "1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/2e/79c822141bfd05a853236b504869ebc6b70159afc570e1d5a20641782eaa/pyyaml_env_tag-1.1.tar.gz", hash = "sha256:2eb38b75a2d21ee0475d6d97ec19c63287a7e140231e4214969d0eac923cd7ff", size = 5737, upload-time = "2025-05-13T15:24:01.64Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/04/11/432f32f8097b03e3cd5fe57e88efb685d964e2e5178a48ed61e841f7fdce/pyyaml_env_tag-1.1-py3-none-any.whl", hash = "sha256:17109e1a528561e32f026364712fee1264bc2ea6715120891174ed1b980d2e04", size = 4722, upload-time = "2025-05-13T15:23:59.629Z" }, ] [[package]] name = "pyzmq" version = "27.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "implementation_name == 'pypy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/04/0b/3c9baedbdf613ecaa7aa07027780b8867f57b6293b6ee50de316c9f3222b/pyzmq-27.1.0.tar.gz", hash = "sha256:ac0765e3d44455adb6ddbf4417dcce460fc40a05978c08efdf2948072f6db540", size = 281750, upload-time = "2025-09-08T23:10:18.157Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/92/e7/038aab64a946d535901103da16b953c8c9cc9c961dadcbf3609ed6428d23/pyzmq-27.1.0-cp312-abi3-macosx_10_15_universal2.whl", hash = "sha256:452631b640340c928fa343801b0d07eb0c3789a5ffa843f6e1a9cee0ba4eb4fc", size = 1306279, upload-time = "2025-09-08T23:08:03.807Z" }, { url = "https://files.pythonhosted.org/packages/e8/5e/c3c49fdd0f535ef45eefcc16934648e9e59dace4a37ee88fc53f6cd8e641/pyzmq-27.1.0-cp312-abi3-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1c179799b118e554b66da67d88ed66cd37a169f1f23b5d9f0a231b4e8d44a113", size = 895645, upload-time = "2025-09-08T23:08:05.301Z" }, { url = "https://files.pythonhosted.org/packages/f8/e5/b0b2504cb4e903a74dcf1ebae157f9e20ebb6ea76095f6cfffea28c42ecd/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3837439b7f99e60312f0c926a6ad437b067356dc2bc2ec96eb395fd0fe804233", size = 652574, upload-time = "2025-09-08T23:08:06.828Z" }, { url = "https://files.pythonhosted.org/packages/f8/9b/c108cdb55560eaf253f0cbdb61b29971e9fb34d9c3499b0e96e4e60ed8a5/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43ad9a73e3da1fab5b0e7e13402f0b2fb934ae1c876c51d0afff0e7c052eca31", size = 840995, upload-time = "2025-09-08T23:08:08.396Z" }, { url = "https://files.pythonhosted.org/packages/c2/bb/b79798ca177b9eb0825b4c9998c6af8cd2a7f15a6a1a4272c1d1a21d382f/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0de3028d69d4cdc475bfe47a6128eb38d8bc0e8f4d69646adfbcd840facbac28", size = 1642070, upload-time = "2025-09-08T23:08:09.989Z" }, { url = "https://files.pythonhosted.org/packages/9c/80/2df2e7977c4ede24c79ae39dcef3899bfc5f34d1ca7a5b24f182c9b7a9ca/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_i686.whl", hash = "sha256:cf44a7763aea9298c0aa7dbf859f87ed7012de8bda0f3977b6fb1d96745df856", size = 2021121, upload-time = "2025-09-08T23:08:11.907Z" }, { url = "https://files.pythonhosted.org/packages/46/bd/2d45ad24f5f5ae7e8d01525eb76786fa7557136555cac7d929880519e33a/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f30f395a9e6fbca195400ce833c731e7b64c3919aa481af4d88c3759e0cb7496", size = 1878550, upload-time = "2025-09-08T23:08:13.513Z" }, { url = "https://files.pythonhosted.org/packages/e6/2f/104c0a3c778d7c2ab8190e9db4f62f0b6957b53c9d87db77c284b69f33ea/pyzmq-27.1.0-cp312-abi3-win32.whl", hash = "sha256:250e5436a4ba13885494412b3da5d518cd0d3a278a1ae640e113c073a5f88edd", size = 559184, upload-time = "2025-09-08T23:08:15.163Z" }, { url = "https://files.pythonhosted.org/packages/fc/7f/a21b20d577e4100c6a41795842028235998a643b1ad406a6d4163ea8f53e/pyzmq-27.1.0-cp312-abi3-win_amd64.whl", hash = "sha256:9ce490cf1d2ca2ad84733aa1d69ce6855372cb5ce9223802450c9b2a7cba0ccf", size = 619480, upload-time = "2025-09-08T23:08:17.192Z" }, { url = "https://files.pythonhosted.org/packages/78/c2/c012beae5f76b72f007a9e91ee9401cb88c51d0f83c6257a03e785c81cc2/pyzmq-27.1.0-cp312-abi3-win_arm64.whl", hash = "sha256:75a2f36223f0d535a0c919e23615fc85a1e23b71f40c7eb43d7b1dedb4d8f15f", size = 552993, upload-time = "2025-09-08T23:08:18.926Z" }, { url = "https://files.pythonhosted.org/packages/60/cb/84a13459c51da6cec1b7b1dc1a47e6db6da50b77ad7fd9c145842750a011/pyzmq-27.1.0-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:93ad4b0855a664229559e45c8d23797ceac03183c7b6f5b4428152a6b06684a5", size = 1122436, upload-time = "2025-09-08T23:08:20.801Z" }, { url = "https://files.pythonhosted.org/packages/dc/b6/94414759a69a26c3dd674570a81813c46a078767d931a6c70ad29fc585cb/pyzmq-27.1.0-cp313-cp313-android_24_x86_64.whl", hash = "sha256:fbb4f2400bfda24f12f009cba62ad5734148569ff4949b1b6ec3b519444342e6", size = 1156301, upload-time = "2025-09-08T23:08:22.47Z" }, { url = "https://files.pythonhosted.org/packages/a5/ad/15906493fd40c316377fd8a8f6b1f93104f97a752667763c9b9c1b71d42d/pyzmq-27.1.0-cp313-cp313t-macosx_10_15_universal2.whl", hash = "sha256:e343d067f7b151cfe4eb3bb796a7752c9d369eed007b91231e817071d2c2fec7", size = 1341197, upload-time = "2025-09-08T23:08:24.286Z" }, { url = "https://files.pythonhosted.org/packages/14/1d/d343f3ce13db53a54cb8946594e567410b2125394dafcc0268d8dda027e0/pyzmq-27.1.0-cp313-cp313t-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:08363b2011dec81c354d694bdecaef4770e0ae96b9afea70b3f47b973655cc05", size = 897275, upload-time = "2025-09-08T23:08:26.063Z" }, { url = "https://files.pythonhosted.org/packages/69/2d/d83dd6d7ca929a2fc67d2c3005415cdf322af7751d773524809f9e585129/pyzmq-27.1.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d54530c8c8b5b8ddb3318f481297441af102517602b569146185fa10b63f4fa9", size = 660469, upload-time = "2025-09-08T23:08:27.623Z" }, { url = "https://files.pythonhosted.org/packages/3e/cd/9822a7af117f4bc0f1952dbe9ef8358eb50a24928efd5edf54210b850259/pyzmq-27.1.0-cp313-cp313t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f3afa12c392f0a44a2414056d730eebc33ec0926aae92b5ad5cf26ebb6cc128", size = 847961, upload-time = "2025-09-08T23:08:29.672Z" }, { url = "https://files.pythonhosted.org/packages/9a/12/f003e824a19ed73be15542f172fd0ec4ad0b60cf37436652c93b9df7c585/pyzmq-27.1.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c65047adafe573ff023b3187bb93faa583151627bc9c51fc4fb2c561ed689d39", size = 1650282, upload-time = "2025-09-08T23:08:31.349Z" }, { url = "https://files.pythonhosted.org/packages/d5/4a/e82d788ed58e9a23995cee70dbc20c9aded3d13a92d30d57ec2291f1e8a3/pyzmq-27.1.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:90e6e9441c946a8b0a667356f7078d96411391a3b8f80980315455574177ec97", size = 2024468, upload-time = "2025-09-08T23:08:33.543Z" }, { url = "https://files.pythonhosted.org/packages/d9/94/2da0a60841f757481e402b34bf4c8bf57fa54a5466b965de791b1e6f747d/pyzmq-27.1.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:add071b2d25f84e8189aaf0882d39a285b42fa3853016ebab234a5e78c7a43db", size = 1885394, upload-time = "2025-09-08T23:08:35.51Z" }, { url = "https://files.pythonhosted.org/packages/4f/6f/55c10e2e49ad52d080dc24e37adb215e5b0d64990b57598abc2e3f01725b/pyzmq-27.1.0-cp313-cp313t-win32.whl", hash = "sha256:7ccc0700cfdf7bd487bea8d850ec38f204478681ea02a582a8da8171b7f90a1c", size = 574964, upload-time = "2025-09-08T23:08:37.178Z" }, { url = "https://files.pythonhosted.org/packages/87/4d/2534970ba63dd7c522d8ca80fb92777f362c0f321900667c615e2067cb29/pyzmq-27.1.0-cp313-cp313t-win_amd64.whl", hash = "sha256:8085a9fba668216b9b4323be338ee5437a235fe275b9d1610e422ccc279733e2", size = 641029, upload-time = "2025-09-08T23:08:40.595Z" }, { url = "https://files.pythonhosted.org/packages/f6/fa/f8aea7a28b0641f31d40dea42d7ef003fded31e184ef47db696bc74cd610/pyzmq-27.1.0-cp313-cp313t-win_arm64.whl", hash = "sha256:6bb54ca21bcfe361e445256c15eedf083f153811c37be87e0514934d6913061e", size = 561541, upload-time = "2025-09-08T23:08:42.668Z" }, { url = "https://files.pythonhosted.org/packages/87/45/19efbb3000956e82d0331bafca5d9ac19ea2857722fa2caacefb6042f39d/pyzmq-27.1.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:ce980af330231615756acd5154f29813d553ea555485ae712c491cd483df6b7a", size = 1341197, upload-time = "2025-09-08T23:08:44.973Z" }, { url = "https://files.pythonhosted.org/packages/48/43/d72ccdbf0d73d1343936296665826350cb1e825f92f2db9db3e61c2162a2/pyzmq-27.1.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1779be8c549e54a1c38f805e56d2a2e5c009d26de10921d7d51cfd1c8d4632ea", size = 897175, upload-time = "2025-09-08T23:08:46.601Z" }, { url = "https://files.pythonhosted.org/packages/2f/2e/a483f73a10b65a9ef0161e817321d39a770b2acf8bcf3004a28d90d14a94/pyzmq-27.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7200bb0f03345515df50d99d3db206a0a6bee1955fbb8c453c76f5bf0e08fb96", size = 660427, upload-time = "2025-09-08T23:08:48.187Z" }, { url = "https://files.pythonhosted.org/packages/f5/d2/5f36552c2d3e5685abe60dfa56f91169f7a2d99bbaf67c5271022ab40863/pyzmq-27.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01c0e07d558b06a60773744ea6251f769cd79a41a97d11b8bf4ab8f034b0424d", size = 847929, upload-time = "2025-09-08T23:08:49.76Z" }, { url = "https://files.pythonhosted.org/packages/c4/2a/404b331f2b7bf3198e9945f75c4c521f0c6a3a23b51f7a4a401b94a13833/pyzmq-27.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:80d834abee71f65253c91540445d37c4c561e293ba6e741b992f20a105d69146", size = 1650193, upload-time = "2025-09-08T23:08:51.7Z" }, { url = "https://files.pythonhosted.org/packages/1c/0b/f4107e33f62a5acf60e3ded67ed33d79b4ce18de432625ce2fc5093d6388/pyzmq-27.1.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:544b4e3b7198dde4a62b8ff6685e9802a9a1ebf47e77478a5eb88eca2a82f2fd", size = 2024388, upload-time = "2025-09-08T23:08:53.393Z" }, { url = "https://files.pythonhosted.org/packages/0d/01/add31fe76512642fd6e40e3a3bd21f4b47e242c8ba33efb6809e37076d9b/pyzmq-27.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cedc4c68178e59a4046f97eca31b148ddcf51e88677de1ef4e78cf06c5376c9a", size = 1885316, upload-time = "2025-09-08T23:08:55.702Z" }, { url = "https://files.pythonhosted.org/packages/c4/59/a5f38970f9bf07cee96128de79590bb354917914a9be11272cfc7ff26af0/pyzmq-27.1.0-cp314-cp314t-win32.whl", hash = "sha256:1f0b2a577fd770aa6f053211a55d1c47901f4d537389a034c690291485e5fe92", size = 587472, upload-time = "2025-09-08T23:08:58.18Z" }, { url = "https://files.pythonhosted.org/packages/70/d8/78b1bad170f93fcf5e3536e70e8fadac55030002275c9a29e8f5719185de/pyzmq-27.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:19c9468ae0437f8074af379e986c5d3d7d7bfe033506af442e8c879732bedbe0", size = 661401, upload-time = "2025-09-08T23:08:59.802Z" }, { url = "https://files.pythonhosted.org/packages/81/d6/4bfbb40c9a0b42fc53c7cf442f6385db70b40f74a783130c5d0a5aa62228/pyzmq-27.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:dc5dbf68a7857b59473f7df42650c621d7e8923fb03fa74a526890f4d33cc4d7", size = 575170, upload-time = "2025-09-08T23:09:01.418Z" }, ] [[package]] name = "referencing" version = "0.37.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, { name = "rpds-py" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, ] [[package]] name = "requests" version = "2.34.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, { name = "charset-normalizer" }, { name = "idna" }, { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] [[package]] name = "rpds-py" version = "2026.6.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" }, { url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" }, { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, { url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" }, { url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" }, { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, { url = "https://files.pythonhosted.org/packages/a4/9e/b818ee580026ec578138e961027a68820c40afeb1ec8f6819b54fb99e196/rpds_py-2026.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223", size = 343012, upload-time = "2026-06-30T07:15:36.005Z" }, { url = "https://files.pythonhosted.org/packages/f3/6b/686d9dc4359a8f163cfbbf89ee0b4e586431de22fe8248edb63a8cf50d49/rpds_py-2026.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f", size = 338203, upload-time = "2026-06-30T07:15:37.462Z" }, { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, { url = "https://files.pythonhosted.org/packages/85/ed/adab103321c0a6565d5ae1c2998349bc3ee175b82ccc5ae8fc04cc413075/rpds_py-2026.6.3-cp313-cp313-win32.whl", hash = "sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0", size = 201710, upload-time = "2026-06-30T07:15:53.894Z" }, { url = "https://files.pythonhosted.org/packages/7b/ed/a03b09668e74e5dabbf2e211f6468e1820c0552f7b0500082da31841bf7b/rpds_py-2026.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80", size = 219454, upload-time = "2026-06-30T07:15:55.25Z" }, { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, ] [[package]] name = "ruff" version = "0.16.8" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ba/78/449cb84790bd5cc3823b2652ee405a4558856e5c4195aee3a16bf7b3eb5d/ruff-0.16.8.tar.gz", hash = "sha256:9247bf92b5f04d825c8639a4fe423ec2e4222acd9222e58412b0dab7e442798b", size = 4938814, upload-time = "2026-09-16T15:54:46.688Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ac/25/6071aabc530e9be7e2c195e8fe3f7aea2735405b6cf447212832d7811831/ruff-0.16.8-py3-none-linux_armv6l.whl", hash = "sha256:6ffbd6d87383c1edf5f6fa890f10200950240d7c1a16052a19a09d3a2307dd38", size = 10048966, upload-time = "2026-09-16T15:53:57.605Z" }, { url = "https://files.pythonhosted.org/packages/54/98/07f90ecbc74dd5fb5764f11f2bc774d6a7cffef92d2ff5f5b4e9e23c754e/ruff-0.16.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:42ed6b878ed61e3acca92f2730a17acff39286944ea82398544696366a6f925e", size = 10165498, upload-time = "2026-09-16T15:54:01.14Z" }, { url = "https://files.pythonhosted.org/packages/fe/1f/e6a712e3b47cad4a40600134105ed193cb773f618a42eb7ba323cb812cc0/ruff-0.16.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7ea781c7f2afba8c6a505ea0fb3f994020249e0c450635f5381286fea6b46170", size = 9830004, upload-time = "2026-09-16T15:54:03.998Z" }, { url = "https://files.pythonhosted.org/packages/23/f2/311a08776d75d81c7676e20b6b020ae63cbe881fcdc7a8dd64e6e18bdd93/ruff-0.16.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8efeae3bbe414a5efefda11a792dfb51ef90ac48d50c4830de2f644caf3e8659", size = 9986558, upload-time = "2026-09-16T15:54:06.804Z" }, { url = "https://files.pythonhosted.org/packages/f3/ed/37b6cb3d3ba8c73e68ae3eb1d502383beb5aa05a582bb7bb3a922f929f54/ruff-0.16.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a79b795469fef7fc6e908b218eed2eb17332afd85031db6480dc864560e69b2", size = 9877332, upload-time = "2026-09-16T15:54:09.552Z" }, { url = "https://files.pythonhosted.org/packages/22/cc/40873a8f36ad084cc540d55fcca7077264d5b13b24659e9180c176fb2b08/ruff-0.16.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3fdc5563cdc50555e6fba39322850860e9267c1b3d12c26a74729d8604c3c812", size = 10507125, upload-time = "2026-09-16T15:54:12.152Z" }, { url = "https://files.pythonhosted.org/packages/c3/e4/fc91a642b78ccbab6b9477720f3644ae7a10a9bcce69a934679cd64f62bc/ruff-0.16.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:34508983c70665578dab88f5223d8e6228307e1135398ca8bfc8b7e9501e282b", size = 11336694, upload-time = "2026-09-16T15:54:15.489Z" }, { url = "https://files.pythonhosted.org/packages/c2/3d/bbd2a9a600a4e73dc3e7548a249c8d1671273464b55822c6fae50f602dff/ruff-0.16.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:644bb578569e0ffc575741232bd385dacdd6fbe123f1a729e7a225f54aa3957f", size = 10774448, upload-time = "2026-09-16T15:54:18.16Z" }, { url = "https://files.pythonhosted.org/packages/1a/41/d83af9879a7b6e8bf5fe16b1da0b134049d2f5d3afac12defb0897cb84bd/ruff-0.16.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15e7d226246961db9235098333caa13063906d3851136b84c2900b82f5daa1df", size = 10323796, upload-time = "2026-09-16T15:54:20.743Z" }, { url = "https://files.pythonhosted.org/packages/f5/2c/cefd07bfe914b84943ea769ade8d607bd22750b965d3228eefd7cebd15d0/ruff-0.16.8-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:a2bf6bc3e9ebdd4449abc6f06cf64b98051a2c61cf94d2fe9596518c881f1a1e", size = 10514115, upload-time = "2026-09-16T15:54:23.497Z" }, { url = "https://files.pythonhosted.org/packages/f3/9d/76a2e26c79a23be6e6e3664c57bec9e9fc8de155cfb9e4b67ea91b64f9d7/ruff-0.16.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6ca111ba0849539165e9e59d2b442542f3c1e8060ebbdea82494f1ffbccb1e1f", size = 10072582, upload-time = "2026-09-16T15:54:26.185Z" }, { url = "https://files.pythonhosted.org/packages/2e/d4/f42edddb39668af1a559ceafa3823aedd65633a48dc9768e775485faa2c1/ruff-0.16.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:359a1e5b495448ee1e91018064382ebc86f90e8aac2fed222c7d0e4e8df85fd2", size = 9879644, upload-time = "2026-09-16T15:54:29.278Z" }, { url = "https://files.pythonhosted.org/packages/f8/d4/913e3195d95e0378786c6656945c865f534a3560e29139da4882aff630d1/ruff-0.16.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:59e8f5681349474110b24d62e93cfda6593f5fa3473446ca3705200cac1a08b9", size = 10231569, upload-time = "2026-09-16T15:54:32.036Z" }, { url = "https://files.pythonhosted.org/packages/2b/c4/8aa6ea0bdcedbd1bf87397e2fc4ed8406448ea5842f8660bc6e5f163039d/ruff-0.16.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:efa3e7a16d1baaa79957888dfdf8be9ef2e44db81cb032af06d76632ab59e773", size = 10663666, upload-time = "2026-09-16T15:54:34.838Z" }, { url = "https://files.pythonhosted.org/packages/3d/02/7f10ef4700bc223c30a3fdd10631a29830c45524b810a3c7ed947af64591/ruff-0.16.8-py3-none-win32.whl", hash = "sha256:55793ba85c69921e89be061426d91a78652d6e50317c962240922747a4eb713f", size = 10093472, upload-time = "2026-09-16T15:54:37.47Z" }, { url = "https://files.pythonhosted.org/packages/1e/5d/a509c07d714b6da88f2c518b4637cf6f1d46b074be8f0f1e5fb9ff5126fe/ruff-0.16.8-py3-none-win_amd64.whl", hash = "sha256:a6b85621fd3c81e31fc5f5add09c9c078b430db3595ca632efafdec9e64ebfaa", size = 10586899, upload-time = "2026-09-16T15:54:40.488Z" }, { url = "https://files.pythonhosted.org/packages/fe/a0/50787329e4f20bf9dc9f6230015d46ec69c51a97ace5bc202dae4755365d/ruff-0.16.8-py3-none-win_arm64.whl", hash = "sha256:d075e820af612102ce217f07cc93e69f9490b10ec13ea85fa87bd03d996cef8a", size = 10386316, upload-time = "2026-09-16T15:54:43.332Z" }, ] [[package]] name = "six" version = "1.17.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] [[package]] name = "sortedcontainers" version = "2.4.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, ] [[package]] name = "stack-data" version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "asttokens" }, { name = "executing" }, { name = "pure-eval" }, ] sdist = { url = "https://files.pythonhosted.org/packages/28/e3/55dcc2cfbc3ca9c29519eb6884dd1415ecb53b0e934862d3559ddcb7e20b/stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", size = 44707, upload-time = "2023-09-30T13:58:05.479Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, ] [[package]] name = "starlette" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, ] [[package]] name = "tornado" version = "6.5.9" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/22/15/ca7aaebb77f850493cba71ace508aeffb896d1ad8976417b48b79823dbd2/tornado-6.5.9.tar.gz", hash = "sha256:4d868544ebdf2fc155239a74397f65ebfea9b4d3635296c96b5dfb0701c354f5", size = 536145, upload-time = "2026-09-14T18:08:37.434Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a0/4a/4d3459fd2f360dd99233f69f22b36012a42856ace526da2c6da3cb0e6df2/tornado-6.5.9-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:dea3bee433b73d6312d993ce89ad5f2b8d3ed9fde56e79ba5342b9edb78d342a", size = 464279, upload-time = "2026-09-14T18:08:20.909Z" }, { url = "https://files.pythonhosted.org/packages/26/4b/5af2f7dcd674e2f1cacef763456f79823404f50954789d6e1ec7a70cd58a/tornado-6.5.9-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:22bed49cdb55292d1d58f46008ba2cc3020e5b84995ef6ae749676c8b5309514", size = 462440, upload-time = "2026-09-14T18:08:22.711Z" }, { url = "https://files.pythonhosted.org/packages/9e/0f/38f8d16011de3b1ce2babb4c3bcefb126d2dc47d04878ae3ad5c39a00caa/tornado-6.5.9-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cc0876a58eee9143476f24901b1b8eb4d52443283ade2b6bca650dd1da04f84", size = 465496, upload-time = "2026-09-14T18:08:24.407Z" }, { url = "https://files.pythonhosted.org/packages/8c/e9/3888f265d1e287fc63cd61a6696f96a650024e32f9e89c0b1817ac5c40f4/tornado-6.5.9-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc0d703cc7ec92b47ebfe236c97cd765c1f2e45744a2e156e6677e54c948274d", size = 466468, upload-time = "2026-09-14T18:08:25.928Z" }, { url = "https://files.pythonhosted.org/packages/d6/8b/d2bcf417981fa6f7698ab84a52394a76cf608e6704d266e2e0e29f1fd9d0/tornado-6.5.9-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:93f9ab0226ef625b94cacb7981a5af104f1735ea02f92282abe5c389109fb31d", size = 466303, upload-time = "2026-09-14T18:08:27.566Z" }, { url = "https://files.pythonhosted.org/packages/7e/a5/c5cfab11420c3f908238a6c861b8f3864fdbd2b2b87e30c9c16f36a11512/tornado-6.5.9-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5801bbf94dfde8d21087b6aa0b17f3a5dab6110dfe44e7e4492df5de393ac15b", size = 465706, upload-time = "2026-09-14T18:08:29.182Z" }, { url = "https://files.pythonhosted.org/packages/ab/36/5a5da0aa15b7afdd46ece09c004a75af7d00f6a96b692bf696ff94bc2a55/tornado-6.5.9-cp39-abi3-win32.whl", hash = "sha256:dca9f377e80efe5dd4d52a2776b67492a81a39b2d931de169b6b36d0c1da444c", size = 466750, upload-time = "2026-09-14T18:08:30.836Z" }, { url = "https://files.pythonhosted.org/packages/34/d7/2446cf3125372e2b4be6483b4b93f360dc50ce2493864755484b23bb50f2/tornado-6.5.9-cp39-abi3-win_amd64.whl", hash = "sha256:f810730ac71eadf009e6e5cb6d534d096887310f5649cff19d7e0d19c0bf4ee7", size = 467191, upload-time = "2026-09-14T18:08:32.521Z" }, { url = "https://files.pythonhosted.org/packages/39/70/f1024364bf78fa921c63ef2883d36a3b3b705750c98ae174593afaab7a65/tornado-6.5.9-cp39-abi3-win_arm64.whl", hash = "sha256:b5f4d92798337260c6d9f9f0e400c7539dd5e55cf93479a3f8508c9aa09c2b3b", size = 466211, upload-time = "2026-09-14T18:08:35.18Z" }, ] [[package]] name = "traitlets" version = "5.16.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/2c/2e/a7fbfe268c8a3b32546930c0297c101d65a4a14c304ad5790a9f478f0e4e/traitlets-5.16.1.tar.gz", hash = "sha256:ed900c2b631aa3a112811139fa97b8d2c3bad5e989656bba4b7e52c7852c18c1", size = 166137, upload-time = "2026-08-03T08:32:36.848Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ad/66/0d785f0bc5e4315a96c989bb476d0fc07ea4f85132550c7b156ca2035d52/traitlets-5.16.1-py3-none-any.whl", hash = "sha256:f775618166caa0396c8e337099240f2bd3e5e917d203b2e6fbe21a58d3cb1f6b", size = 86211, upload-time = "2026-08-03T08:32:34.48Z" }, ] [[package]] name = "truststore" version = "0.10.4" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, ] [[package]] name = "typing-extensions" version = "4.16.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] [[package]] name = "urllib3" version = "2.8.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, ] [[package]] name = "uvicorn" version = "0.51.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "h11" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a2/65/b7c6c443ccc58678c91e1e973bbe2a878591538655d6e1d47f24ba1c51f3/uvicorn-0.51.0.tar.gz", hash = "sha256:f6f4b69b657c312f516dd2d268ab9ae6f254b11e4bac504f37b2ab58b24dd0b0", size = 94412, upload-time = "2026-07-08T10:59:05.962Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/45/ec/dbb7e5a6b91f86bfb9eb7d2988a2730907b6a729875b949c7f022e8b88fa/uvicorn-0.51.0-py3-none-any.whl", hash = "sha256:5d38af6cd620f2ae3849fb44fd4879e0890aa1febe8d47eb355fb45d93fe6a5b", size = 73219, upload-time = "2026-07-08T10:59:04.44Z" }, ] [[package]] name = "watchdog" version = "6.0.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220, upload-time = "2024-11-01T14:07:13.037Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/39/ea/3930d07dafc9e286ed356a679aa02d777c06e9bfd1164fa7c19c288a5483/watchdog-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdd4e6f14b8b18c334febb9c4425a878a2ac20efd1e0b231978e7b150f92a948", size = 96471, upload-time = "2024-11-01T14:06:37.745Z" }, { url = "https://files.pythonhosted.org/packages/12/87/48361531f70b1f87928b045df868a9fd4e253d9ae087fa4cf3f7113be363/watchdog-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c7c15dda13c4eb00d6fb6fc508b3c0ed88b9d5d374056b239c4ad1611125c860", size = 88449, upload-time = "2024-11-01T14:06:39.748Z" }, { url = "https://files.pythonhosted.org/packages/5b/7e/8f322f5e600812e6f9a31b75d242631068ca8f4ef0582dd3ae6e72daecc8/watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0", size = 89054, upload-time = "2024-11-01T14:06:41.009Z" }, { url = "https://files.pythonhosted.org/packages/68/98/b0345cabdce2041a01293ba483333582891a3bd5769b08eceb0d406056ef/watchdog-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:490ab2ef84f11129844c23fb14ecf30ef3d8a6abafd3754a6f75ca1e6654136c", size = 96480, upload-time = "2024-11-01T14:06:42.952Z" }, { url = "https://files.pythonhosted.org/packages/85/83/cdf13902c626b28eedef7ec4f10745c52aad8a8fe7eb04ed7b1f111ca20e/watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:76aae96b00ae814b181bb25b1b98076d5fc84e8a53cd8885a318b42b6d3a5134", size = 88451, upload-time = "2024-11-01T14:06:45.084Z" }, { url = "https://files.pythonhosted.org/packages/fe/c4/225c87bae08c8b9ec99030cd48ae9c4eca050a59bf5c2255853e18c87b50/watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a175f755fc2279e0b7312c0035d52e27211a5bc39719dd529625b1930917345b", size = 89057, upload-time = "2024-11-01T14:06:47.324Z" }, { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079, upload-time = "2024-11-01T14:06:59.472Z" }, { url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078, upload-time = "2024-11-01T14:07:01.431Z" }, { url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076, upload-time = "2024-11-01T14:07:02.568Z" }, { url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077, upload-time = "2024-11-01T14:07:03.893Z" }, { url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078, upload-time = "2024-11-01T14:07:05.189Z" }, { url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077, upload-time = "2024-11-01T14:07:06.376Z" }, { url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078, upload-time = "2024-11-01T14:07:07.547Z" }, { url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065, upload-time = "2024-11-01T14:07:09.525Z" }, { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070, upload-time = "2024-11-01T14:07:10.686Z" }, { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, ] [[package]] name = "wcwidth" version = "0.8.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/34/74/c6428f875774288bec1396f5bfcbc2d925700a4dad61727fd5f2b12f249d/wcwidth-0.8.2.tar.gz", hash = "sha256:91fbef97204b96a3d4d421609b80340b760cf33e26da123ff243d76b1fda8dda", size = 1466253, upload-time = "2026-06-29T18:11:11.601Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/96/42/3e5985a0a7e57de470b320c6d6a1a67c844f6737a587f3d44dd13d1819e7/wcwidth-0.8.2-py3-none-any.whl", hash = "sha256:d63947694a0539a1d51e01eda7caf800c291020e6cdd7e28ad7b14dd33ad4f85", size = 323166, upload-time = "2026-06-29T18:11:09.888Z" }, ] [[package]] name = "yarl" version = "1.25.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "idna" }, { name = "multidict" }, { name = "propcache" }, ] sdist = { url = "https://files.pythonhosted.org/packages/75/16/e8be8e2fb175bbf41a0680381a319f1199fae256588241a2ac8677eafb49/yarl-1.25.1.tar.gz", hash = "sha256:03dd38de09bc213e9a8b29761eec33ee1d5318dac0e49d8af36e4d27830e23a7", size = 246245, upload-time = "2026-09-15T19:35:02.264Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/75/b3/cd32ac66ae622b854c2df0ac52106dda220d361b65a64fde7d5b3684aa3f/yarl-1.25.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:94d7aa6debf92a1dd14cb5280b083a764169a13cfb23a452111160274ed989f4", size = 144798, upload-time = "2026-09-15T19:31:01.821Z" }, { url = "https://files.pythonhosted.org/packages/61/fb/a2c52a8007c2051ba74662afb112ecf3d00346af4c25e33df9d80fd14fb8/yarl-1.25.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:83d4a37e4b95da4d8bda930d6d35b75b4cdadbacbb4980cae290ea3100b5d51d", size = 104583, upload-time = "2026-09-15T19:31:04.05Z" }, { url = "https://files.pythonhosted.org/packages/be/dd/ee38aec8e09fdf957e50d4085453fbe202f56c6c3b4cf07b81cdb4f09ee9/yarl-1.25.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e029648f9c951db30e98a7d7ec90835db88ec4b32820efe2a9bdc2287e032eb6", size = 104325, upload-time = "2026-09-15T19:31:06.338Z" }, { url = "https://files.pythonhosted.org/packages/1e/b3/058dbfb1857b484c9cf9cc135659f50b85ce66e03c99e44dc2f7b6161f55/yarl-1.25.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d781294bb815ecb5ea57ff6bbf8038e0a31a95fdf3e1788f66e0dc100d64b58", size = 115358, upload-time = "2026-09-15T19:31:08.593Z" }, { url = "https://files.pythonhosted.org/packages/db/39/29693446cf0cf6b15a0e2f75a5d40f93c56819b05b0622196f45e95b5cc0/yarl-1.25.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e12c538e00e7c1b286a07061046b90e8124e6a9793efae2c70db6a4aad07faad", size = 107658, upload-time = "2026-09-15T19:31:10.802Z" }, { url = "https://files.pythonhosted.org/packages/86/b3/3c4dd7e1af43b931fba95e0a722737f2ea94a6d199c802585282831d7abd/yarl-1.25.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7e4de3ac4adbad3d0bc7c6f4360a7dbff5de2f15e3b723be3198074e17fd9c40", size = 122660, upload-time = "2026-09-15T19:31:12.84Z" }, { url = "https://files.pythonhosted.org/packages/bd/b5/1b60dbc3cfc9c5712b15148c206748f2bc93953ffdbe25ea75b63dfc89c9/yarl-1.25.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:419f392a1da624877975709e3864dfe833af6cc7671b39318086d456e288380c", size = 126506, upload-time = "2026-09-15T19:31:15.088Z" }, { url = "https://files.pythonhosted.org/packages/bc/7b/ca212cbe170ac8b96e45317ecbcf9c3c3ecf0cdec98d5b088a9c4088929b/yarl-1.25.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6f117789d22dce188e5754e8bc65b7e6ebf8cb73963b9fa761f672a5883769d", size = 117050, upload-time = "2026-09-15T19:31:17.241Z" }, { url = "https://files.pythonhosted.org/packages/cb/c3/72b4938cdbe619ad71ac156182faef4908846b84dc3ca4dbb4c4e6f84014/yarl-1.25.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:80e47012e730da131c9f059c80936783f9659aae22dc31c03c0595590d11ed54", size = 114174, upload-time = "2026-09-15T19:31:19.294Z" }, { url = "https://files.pythonhosted.org/packages/e8/43/268717870f9ba0cc9701a95181587f6dc8c5f387aab4aeecc83158f38a79/yarl-1.25.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e80f557716fd765439577131e526b8942ffc2c07bdbc5e39fa62f660ba1e963f", size = 114944, upload-time = "2026-09-15T19:31:21.414Z" }, { url = "https://files.pythonhosted.org/packages/da/84/baa5bf504d51fe062c4bcaf62936da97fffb43285978d0b39984824231fd/yarl-1.25.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f61964f235a43738bfac50da46fc4254943a7eea3051aeb0b6fc7c992c29fadc", size = 108263, upload-time = "2026-09-15T19:31:23.388Z" }, { url = "https://files.pythonhosted.org/packages/a4/28/779a2ed9e0152a601a27039bed9aead3f0b79797a67e2c44bfa444622dd8/yarl-1.25.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e546fe1d4a93ebc2910f0d768baff19faa09843ab3f2036a67ed6e69fae4419d", size = 122184, upload-time = "2026-09-15T19:31:25.343Z" }, { url = "https://files.pythonhosted.org/packages/f8/1f/118e9e5b8f07694d63fd3222e801d7782270003f1a222aa798df3f8d5933/yarl-1.25.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:cce0727fd5ac04d372fa9bbfde9febc2bcf209aadfcf0468e45dec72719895d1", size = 114001, upload-time = "2026-09-15T19:31:27.465Z" }, { url = "https://files.pythonhosted.org/packages/0f/ae/a4cf1cf372313734b17996d4007f9f73596e7a178b9485802e5494ecf484/yarl-1.25.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af4ea5b37403ef4e30f3927eaed540db942bde01d8d3ff083527c0704d1c9c68", size = 120565, upload-time = "2026-09-15T19:31:29.47Z" }, { url = "https://files.pythonhosted.org/packages/05/79/ad94f93ca731bc9e44d321833ab96b82a4f9f5f63cf773f81a4aeea5ecc1/yarl-1.25.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:68782fdb4027b8d1eee25ec35e9a6db05e863b899eb0310b3a33b6c3fef55707", size = 117060, upload-time = "2026-09-15T19:31:31.367Z" }, { url = "https://files.pythonhosted.org/packages/bb/cc/51a7b4abf4ac593b8e7eb3794b28e5a35ae26eed8bc04787628d215af82f/yarl-1.25.1-cp312-cp312-win_amd64.whl", hash = "sha256:7d575b54cb3863ef9bc290ea4b009999d55dc237326131e4853cf33e888fee03", size = 102593, upload-time = "2026-09-15T19:31:33.329Z" }, { url = "https://files.pythonhosted.org/packages/9d/21/0941a6b93a58b59a1ec75e5333bf06929b671309c43c0cd201c172d9c39f/yarl-1.25.1-cp312-cp312-win_arm64.whl", hash = "sha256:bc3ac7bf569f6b64dad04dd7808c7872dae8a97df657856eac05e9b7e3614a85", size = 97697, upload-time = "2026-09-15T19:31:35.855Z" }, { url = "https://files.pythonhosted.org/packages/7b/ed/2f3129bbcc9a5c8ba12cc2b29d8060a3bab9c8043c456cfd4b5ca3188890/yarl-1.25.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:25868beca8b6765f8f7d0e11fe6dd7c66dd4b0793b9500286d20cc92352126a5", size = 143623, upload-time = "2026-09-15T19:31:37.966Z" }, { url = "https://files.pythonhosted.org/packages/17/e1/f1bc3390fdca352826676b531d0712736f156919090206700421d46b2c37/yarl-1.25.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:10b2fd95332f0d716d5eee3c9fb2ce8eada19082de7fee83d32e37992fd75c26", size = 104011, upload-time = "2026-09-15T19:31:40.25Z" }, { url = "https://files.pythonhosted.org/packages/a8/aa/50acc5c3e5da04172ae3c281c75405af4d2ca911e16120ab0563f4dffb66/yarl-1.25.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0f12afda4eea8c8994a76d4df1875c765194f5fbe8a9d197929ea303caee29ec", size = 103677, upload-time = "2026-09-15T19:31:42.46Z" }, { url = "https://files.pythonhosted.org/packages/30/d2/7d1e0ab9f8390e1fbcede5a6dbf70d23c96ad09b8c5567f3a514d1ddb0e2/yarl-1.25.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:14b79a30a93a3ce2e8832603fd0ab780ada281b0ba5110b519a634f2d7d7d1fc", size = 115392, upload-time = "2026-09-15T19:31:44.371Z" }, { url = "https://files.pythonhosted.org/packages/71/e1/5ba1e3a2a22139213655e760919038e8ed7e2d4a99826d0bbddb3beb96e5/yarl-1.25.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4bd6340d20ae2c7ca719b87b426e808e90743b676d05d4c26c4fb5ca71f41184", size = 107493, upload-time = "2026-09-15T19:31:46.273Z" }, { url = "https://files.pythonhosted.org/packages/f5/53/780653d5e0f73831f467cf13548912e5eec97f21dc49fc8daf21da027df4/yarl-1.25.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:126a2533570c554719ca40a1288fdee1700b6bc82e7131aa69fa85252d92e651", size = 122537, upload-time = "2026-09-15T19:31:48.654Z" }, { url = "https://files.pythonhosted.org/packages/03/92/d54fa70236c6036271c9c9c09fd978df5cbe3ef49ef6c46e9b833476d215/yarl-1.25.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a3faadac7d812ddac258feb57b9846b60c1b437c4f4b9ad42595c6f6fe4390df", size = 126170, upload-time = "2026-09-15T19:31:50.872Z" }, { url = "https://files.pythonhosted.org/packages/0e/b7/a82a49bf88340b837ef6972b508a1604ae377b9e6904b46b10cf5f1cf925/yarl-1.25.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be80550d9bfe83d9b62398a37081a90434e6df2d978ec345c3d2820de6beddab", size = 117012, upload-time = "2026-09-15T19:31:53.189Z" }, { url = "https://files.pythonhosted.org/packages/ef/78/5d684b411e3f3602464ee9b538db48205038f8605872985f61efb809ced0/yarl-1.25.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e07595c7d6f4db270ceede356a1bd1c07a34f1c26f958d1ed0cd7b48e0d2bba3", size = 114950, upload-time = "2026-09-15T19:31:55.694Z" }, { url = "https://files.pythonhosted.org/packages/2f/11/51d82b852c64f7fad0fc7a7ff3031517204887e874c722bbca839c0b23ac/yarl-1.25.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:eb96ed1ae6c7d072d60840c0434aef07a2df611812810807fbc54263a6053e9a", size = 115428, upload-time = "2026-09-15T19:31:57.966Z" }, { url = "https://files.pythonhosted.org/packages/e4/49/9d1978049bf646b9ea918313926453c6901b71c92f097467777d47d36a88/yarl-1.25.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3feb99222553a8cbedfa52c2f59dd84c3f50d5b582c728d522caf8d72769a54b", size = 108428, upload-time = "2026-09-15T19:32:00.048Z" }, { url = "https://files.pythonhosted.org/packages/43/35/7b8f1ebb45d7ec3dda7d1909bf44f458de41ef91e2937f107733582a5166/yarl-1.25.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a2ed0ba415ccdf08f14bf544cb78346d0f76086707ffee24921a2c84dbf1305a", size = 121961, upload-time = "2026-09-15T19:32:02.436Z" }, { url = "https://files.pythonhosted.org/packages/63/d6/d8b689ab7ca26edeb85f6ff28812aac7a25376eefc1780e303a7bfbaceff/yarl-1.25.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2b49375d22299b0a834c2bca72f39aaecc270d96fb24c30424899676f487b22a", size = 114961, upload-time = "2026-09-15T19:32:04.456Z" }, { url = "https://files.pythonhosted.org/packages/cf/d5/1a1798ea4dc6b7ee3260010a27907ebc697c95dae99817d817ed446d24aa/yarl-1.25.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ef74070ac553c59eb4f04258722066d6c6135b7baa03b2e9f2da65c096e96d98", size = 120036, upload-time = "2026-09-15T19:32:06.5Z" }, { url = "https://files.pythonhosted.org/packages/91/8d/b1b35ed7903da6669b1d367cb2c09436acd4ff508029b4f39a0c0c2058fc/yarl-1.25.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0a66db89ea473abeac4b70523cafd94db3772380e565f9d28af7a179b7af71fa", size = 117276, upload-time = "2026-09-15T19:32:09.401Z" }, { url = "https://files.pythonhosted.org/packages/3a/8f/4db01cef62caff0d7a4593ed694fb8a41a27a11158cab80d290221f13e57/yarl-1.25.1-cp313-cp313-win_amd64.whl", hash = "sha256:1f51020b2eb8a003c84925638ec63c21a750a4bddd3a22ec8eac6a742dadf1b9", size = 101945, upload-time = "2026-09-15T19:32:11.545Z" }, { url = "https://files.pythonhosted.org/packages/c0/5e/3ce00497c5c0babb74d4130c10c3828ccd215b4819d12020c42429f991ac/yarl-1.25.1-cp313-cp313-win_arm64.whl", hash = "sha256:b10dd0557ba422715b5206b3743192135a6022acca8baec51aa127d0a75db8fe", size = 97270, upload-time = "2026-09-15T19:32:14.127Z" }, { url = "https://files.pythonhosted.org/packages/80/cf/54023edfab7aa773b860503db0c56e962ccab0922803ee97988c176ea090/yarl-1.25.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:a9ca696eb02e5c02a8afd872ada510eba9b7fe6e68b9572c2e9a9b1941e31e2e", size = 143975, upload-time = "2026-09-15T19:32:16.416Z" }, { url = "https://files.pythonhosted.org/packages/d7/a8/e6c1be0e6761d0f2d10bbf33a3e1e02b99dc83874d92945d7b461a72481e/yarl-1.25.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a5877f2255aab518ebe528289037699201d5dc5f045f2396cb30aa02db22f57f", size = 104018, upload-time = "2026-09-15T19:32:18.364Z" }, { url = "https://files.pythonhosted.org/packages/6e/bb/dda344765ffd3430afe1a1c66c866a57fae67786537d4f14607df6505ac1/yarl-1.25.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7a5c3115595995779ee21f2567035793911c3802a43c74f3fbb0314929ec67ac", size = 104156, upload-time = "2026-09-15T19:32:20.459Z" }, { url = "https://files.pythonhosted.org/packages/e5/5f/ed1538bcd06009fe990d6d283dd7667f639e62a81e35c6d8c6ef6c08fb3c/yarl-1.25.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77e5099b99b37f3cf79c246998ca9f7313a78054cd1809ec46bc1afad47e1c4c", size = 116025, upload-time = "2026-09-15T19:32:22.766Z" }, { url = "https://files.pythonhosted.org/packages/a2/af/2185daf56b99830d3356ecfada46faaa49945de6626e842b7728088d4980/yarl-1.25.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6efaf45df6a849cef613a03a94c845647456662f85438c886bb67a9c027c8c2c", size = 106985, upload-time = "2026-09-15T19:32:24.749Z" }, { url = "https://files.pythonhosted.org/packages/c1/65/bc1ae564fb4b04a30b6a8f250e787772581c57e4c3d5cf07ac3359de3103/yarl-1.25.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d5f90e44653c4e0f78501ed9bb7d3fce835a8d62b7c6ed0cb16557534087e743", size = 123030, upload-time = "2026-09-15T19:32:27.084Z" }, { url = "https://files.pythonhosted.org/packages/6a/3e/e2afcde10d74e53b3fa889960991efb3019beda2b1682a01de720a302056/yarl-1.25.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:632da579b2d879f6bad20f2cfa35ded1efe2f4f77f8abb26a6234a5b236acd2f", size = 126765, upload-time = "2026-09-15T19:32:29.332Z" }, { url = "https://files.pythonhosted.org/packages/a2/be/415b00c0fe5a0615b062a456b26623d7ec91c2bee20faea1a14045aa0469/yarl-1.25.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:30eec96e8a91bd588ce897c9543f6d5d8d34b28fbcba28a4dedf20ebeae9fe57", size = 117199, upload-time = "2026-09-15T19:32:31.49Z" }, { url = "https://files.pythonhosted.org/packages/97/27/3d8c63ddd3e8bcfd033748ab93876678ce59bacd66e4cb1ed851c9c5b37e/yarl-1.25.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:12b6bc4906e11f5e1a1cdcb12296e7afbd366c783cc8073403cd2fb74334e453", size = 115187, upload-time = "2026-09-15T19:32:34.137Z" }, { url = "https://files.pythonhosted.org/packages/39/b7/7a81d0be1a502a26a0d4326c6f2ecb736c824f570ea1c6529f2b0b227b50/yarl-1.25.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9d6ed3d17bccce4c05343e1ca8da13bc5c02c812a4e7282ddd05e8769322d3fc", size = 116085, upload-time = "2026-09-15T19:32:36.438Z" }, { url = "https://files.pythonhosted.org/packages/f0/69/39fff459916aa0fab42215dc47b759586fd80f94aa56dfc4a7c15ba6e0dc/yarl-1.25.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:f38a70074041d3b7e138e452799f5174198bae5bd5ab2000917badf403908c5f", size = 107996, upload-time = "2026-09-15T19:32:38.959Z" }, { url = "https://files.pythonhosted.org/packages/c0/39/80b9a55a3335590451d9ecf3eb593a8c635351f4c905ef056d7e8a8fd9e7/yarl-1.25.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ca89e4e21854ed27ec753297dde84b16c9f8e53b14a4866fb44457d643c19f8", size = 122549, upload-time = "2026-09-15T19:32:41.151Z" }, { url = "https://files.pythonhosted.org/packages/42/7d/a179c6757818bb59372a4adafd09f7f26a3b4a0f04c3ae404b544c0b0c82/yarl-1.25.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:1ab7618921a93767387a4b83776f751588f5b5ae9bb5bc96620e2e2e00bca868", size = 115107, upload-time = "2026-09-15T19:32:43.072Z" }, { url = "https://files.pythonhosted.org/packages/32/2b/a773ac867e4ab53a98ed98e5cefe3bae31e6f550252ca9d1de266f1a40c5/yarl-1.25.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0ae12ff2b805fa02c4dab838005caef735e39986322698c48588d3beacb65c62", size = 120666, upload-time = "2026-09-15T19:32:45.061Z" }, { url = "https://files.pythonhosted.org/packages/bc/41/52be6505e85b0f76b4f85b01b5de7e06a0512201abc2c95e14e099549174/yarl-1.25.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:90c30ed53546da833c700115c0064c22120d1b1560f474699fd31f22dd668233", size = 117505, upload-time = "2026-09-15T19:32:47.177Z" }, { url = "https://files.pythonhosted.org/packages/f5/01/349c0386caedbbe488d519f252df54efac8a1459282d466c474bdd84a620/yarl-1.25.1-cp314-cp314-win_amd64.whl", hash = "sha256:acfa7e22aa6c6e7a5996a41d275bfa01efa7ea56ab890590280e9063e2cf5c1b", size = 103446, upload-time = "2026-09-15T19:32:49.615Z" }, { url = "https://files.pythonhosted.org/packages/5c/f0/8ec63180f77912f0dc4e5a42760cb8c08d20da1d5ace3578a01b84d1f3d8/yarl-1.25.1-cp314-cp314-win_arm64.whl", hash = "sha256:8e7d98cdbb6d71e726f7d525952867096053d1f290dd4e3c50d7d313a136f414", size = 99159, upload-time = "2026-09-15T19:32:51.686Z" }, { url = "https://files.pythonhosted.org/packages/47/7d/92d2220d6886b70ab1ed8579533ac2af2dfac716d5d929001daff7986df9/yarl-1.25.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d21f0fa80a02d05299207eeaafef345d812ace96d5306e4ef265e1d419a615fa", size = 150071, upload-time = "2026-09-15T19:32:53.911Z" }, { url = "https://files.pythonhosted.org/packages/64/fc/b245e448124bcda9340df38e3553fa222b50260fca027a84095e9bd8642d/yarl-1.25.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:17c9877a89fb6e2bca6f9087eb24cd7fb434653946ef5075e470d23d49b52287", size = 106780, upload-time = "2026-09-15T19:32:56.443Z" }, { url = "https://files.pythonhosted.org/packages/51/e2/9a6ce2e334ebf218a30335ae76fb1696459430d42f733b8cb0d7d65b84d3/yarl-1.25.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:29273edf1530e397bd07cb784db1fbe0d2590b77569f2e24679a9c0a2d763b94", size = 107361, upload-time = "2026-09-15T19:32:58.827Z" }, { url = "https://files.pythonhosted.org/packages/ed/70/66e8c76b569b450d16e190f15071c916c3df70b0e33927e415ac497cf0c2/yarl-1.25.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b7abffdf37af1cec6a2ad69b827aa84320db5894791bc8ed932dc93fb274b7e9", size = 114396, upload-time = "2026-09-15T19:33:02.24Z" }, { url = "https://files.pythonhosted.org/packages/73/23/0d82838a05c57fdc05bc8b66e8c92dcc0df15e27463a5f163142d521c682/yarl-1.25.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2239a02249d9326655419e0168a28ca9008938eaab31dc29fc875c217927a6c0", size = 104882, upload-time = "2026-09-15T19:33:04.494Z" }, { url = "https://files.pythonhosted.org/packages/86/d4/ea08615c4edaa6049a13a2f1128944d068d1893abda7d708d4d7ea01599a/yarl-1.25.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:664ec6a520b74a1df2810666eb67695fcb77fa663e6ea0a25aaf2e529cb24dfa", size = 119485, upload-time = "2026-09-15T19:33:06.583Z" }, { url = "https://files.pythonhosted.org/packages/1a/82/0898bdce9b1ae403b308b9c733d0d24af4a3464270c2c081f457b16c3e0d/yarl-1.25.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4f1c91f5a5980a937ff8e238e98e6897e1ad74a4b1e2c0d68c73b5ffbb3f5c0b", size = 122490, upload-time = "2026-09-15T19:33:08.653Z" }, { url = "https://files.pythonhosted.org/packages/d1/38/97d79b81c342b78246cfedb74809e68841f3198d21653e10d3232bd9c622/yarl-1.25.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c88edaec8c349ad4c5ad4c486a3defcc4b80ceb2f074436ffa0a87caf5e76a6", size = 115336, upload-time = "2026-09-15T19:33:11.056Z" }, { url = "https://files.pythonhosted.org/packages/8e/9d/2577896554cd310dc470adb6da0b7dd0b435cb63e2565204a7ac240e504c/yarl-1.25.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:35dcbea443fafb3eece757ad4e514560ddeb6c34cfae1582c620d7b293d7feee", size = 111825, upload-time = "2026-09-15T19:33:13.204Z" }, { url = "https://files.pythonhosted.org/packages/29/6b/7ac49d8ba84a5c4bd73415a4c949d22c749cb3762579b3d50e48019a78aa/yarl-1.25.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:882569ff613758cac762a457a5d72d6e211b28d4bcfea89d1d71ea942b02eac0", size = 114655, upload-time = "2026-09-15T19:33:15.553Z" }, { url = "https://files.pythonhosted.org/packages/e5/18/e5942a16723f5b72f9b1297fd5a85a54f6300cd15c0dcb5005b90cd89156/yarl-1.25.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d0f1489233a254bb3643d2f05de7d59019254d81daeca6b9162fe9edef57e0c7", size = 106395, upload-time = "2026-09-15T19:33:17.599Z" }, { url = "https://files.pythonhosted.org/packages/7b/2d/549fa46240781513ebc47ae7eb418df428a163a2a3d644cc9cbb3ecb7846/yarl-1.25.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f41753a76f4f63927d03a0d8ba8f5ce0f2083bec29a8cfaccc55371b1564b96b", size = 119277, upload-time = "2026-09-15T19:33:19.973Z" }, { url = "https://files.pythonhosted.org/packages/76/16/4763f78dcdc0b3b9fb3842b04afe72b9320857c6a69300c62a0eab03d119/yarl-1.25.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8fb0eb4955adf0579001581f2f71a126e8781ba61bcd120f127b0401163c6c2d", size = 112504, upload-time = "2026-09-15T19:33:22.464Z" }, { url = "https://files.pythonhosted.org/packages/ae/b4/974e3edfe0d188393ce1cb9de400111c63fe61f4eb3b772a500d84c970d1/yarl-1.25.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a1e32763e641a1566507d90a8d3b19bfc3cc04a9d4e5ae3e32189874ed4b58a3", size = 116243, upload-time = "2026-09-15T19:33:24.788Z" }, { url = "https://files.pythonhosted.org/packages/b0/aa/157b940428da80c104ca09666a740e51c94963df65d5b112e06b52e4d7a8/yarl-1.25.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65b5b2066651b7432d389e9799d979c703bcc6ef44266bb8153ef54e91e4aab3", size = 115822, upload-time = "2026-09-15T19:33:26.886Z" }, { url = "https://files.pythonhosted.org/packages/7e/af/19fbdce41412e1b96825544cc52cd7029d3724655d0988237972f078bd29/yarl-1.25.1-cp314-cp314t-win_amd64.whl", hash = "sha256:734f6e5400352ac4254456003d462866c684703570929cff7a7bde015d0cb371", size = 107386, upload-time = "2026-09-15T19:33:29.009Z" }, { url = "https://files.pythonhosted.org/packages/2a/99/f6431c8968e89be608d74b28ae2d024521b2953f27dd44e0dece5e04f67a/yarl-1.25.1-cp314-cp314t-win_arm64.whl", hash = "sha256:287e99ff5aa4dc1c7630bfc683ded6f106d756c99dec432a2d7f197a784f51c6", size = 102094, upload-time = "2026-09-15T19:33:31.151Z" }, { url = "https://files.pythonhosted.org/packages/c7/3b/4f51eab40c2eabea6c3d5b121dff4b8988dc35087732ffede12d2be8b8dd/yarl-1.25.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:9b1bdaae98bc016825dd3c9d8ee1832f829b3341f9cc6ebd1a1b0a7fef7367cc", size = 143875, upload-time = "2026-09-15T19:33:33.52Z" }, { url = "https://files.pythonhosted.org/packages/41/05/bbd58fc063f5f299a883f810760b265ca26c8167c91cb9a494d0fe2387e1/yarl-1.25.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:e7011b8fb8c4054bf0c12e5edc6cd83778b0028e99ce59b18586ed036f92cfdc", size = 104028, upload-time = "2026-09-15T19:33:36.22Z" }, { url = "https://files.pythonhosted.org/packages/12/ee/2fba0aecb52e7020e189f684148783aa0b9cfa3b3bfb0b400646eef70ad4/yarl-1.25.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:f074e8d4aa0a5798920ddb6de3d08b228c614ff3724c3e8bd7577f4bafea867b", size = 104041, upload-time = "2026-09-15T19:33:38.86Z" }, { url = "https://files.pythonhosted.org/packages/38/35/884beab53ed88c7247d1671972b5ef116f7351fe0c7e6de8c3558372cb16/yarl-1.25.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7d42e7e3ca399555578b4d617e3a6ecf13371b3743a115995fa010c7bf341459", size = 116018, upload-time = "2026-09-15T19:33:44.265Z" }, { url = "https://files.pythonhosted.org/packages/e2/cc/1a91b685afb55cc18608443ace95280e96e263a97565811732b6788d3269/yarl-1.25.1-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:aa4ed3dd308548f9e707d9caaf005d2d7f8c1e7868f858dfeb47fe76e16b391d", size = 107033, upload-time = "2026-09-15T19:33:46.45Z" }, { url = "https://files.pythonhosted.org/packages/c5/c1/58b379fcb1d68d907b7fcf75200c44321896509b2a6a74abbb4b19d864d2/yarl-1.25.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:42a66563d8cc056ee32e6191e05097a7b2b3bc302e0bc3133daf8710eb18bd26", size = 123257, upload-time = "2026-09-15T19:33:48.631Z" }, { url = "https://files.pythonhosted.org/packages/d7/2d/1fe96cf5c2aeab10095e48f38585cf5a8451fb7253234822398e52aa5336/yarl-1.25.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:98d370568f393215d605304cdb77b3d5539bd192c75b623c7304c42c8d6d8273", size = 126745, upload-time = "2026-09-15T19:33:50.999Z" }, { url = "https://files.pythonhosted.org/packages/ad/60/8674394ce43f4dadae573a1d6f451716438e9eab7d7fe8d643c673a32d85/yarl-1.25.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23bf5b403c879a54964e0feac7285688e04bb220074878d737d331522da0a5bf", size = 117255, upload-time = "2026-09-15T19:33:53.456Z" }, { url = "https://files.pythonhosted.org/packages/2f/72/0faa30e02605d56127d42bb987dcc97da3863b7bf70b9bfbf5f739c05e30/yarl-1.25.1-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d45673badd08456d0340e9364eddafe1c53a9d2896424294de4d7dd71ad3ee57", size = 115166, upload-time = "2026-09-15T19:33:55.669Z" }, { url = "https://files.pythonhosted.org/packages/8c/90/9a46eac564c437e128285c5c1d7bb385d268394e9209d84f6bf4a14471ef/yarl-1.25.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:0136d640dfa9b0523853e411430a99f8a91eca85774c6420285a33b755bc6de3", size = 116082, upload-time = "2026-09-15T19:33:57.78Z" }, { url = "https://files.pythonhosted.org/packages/ad/38/4b1a686a3758878f93d2f1ea943f5a165f3555769cd16e761cfd0efdca17/yarl-1.25.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:59ba3a6e1aa8cfe5adf4bd270fd965db21955401b7ca6f1696010c55ed4daec2", size = 108048, upload-time = "2026-09-15T19:34:00.25Z" }, { url = "https://files.pythonhosted.org/packages/da/14/f348eb967f31a58348612a2b93bd8a2ab664548e2b5e879cac7f592f7201/yarl-1.25.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:87796fedc3ba97ec14fab55acb48584276e6c1e4c1e89c422bda62c838e754a9", size = 122775, upload-time = "2026-09-15T19:34:02.455Z" }, { url = "https://files.pythonhosted.org/packages/ab/e9/4f7b79700f88cb9e8bb66f8b54f9bce1844c013a2c39fdc47112e9334c95/yarl-1.25.1-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:bd0912757081f89b107d6c00b2ff8a194401b0b87eadcf4481de2b865a8fd44f", size = 115096, upload-time = "2026-09-15T19:34:05.283Z" }, { url = "https://files.pythonhosted.org/packages/cf/37/f9cb020331997d3eb887bd28d5410ecfd3d80bf163c23d7cec490d78dade/yarl-1.25.1-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:b51c159a9794633f5e0db7ecec7b2b6e3734eca1f5d17dc989ff3552a43ff78b", size = 120655, upload-time = "2026-09-15T19:34:07.382Z" }, { url = "https://files.pythonhosted.org/packages/12/83/52fceb22891a41f168db7ec22fd1d81e06b6a0b8d9f70921bd3e785defd0/yarl-1.25.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:319e070a01db9920fb63761843f96a104c8e2b9427266731810dc1e22595b17c", size = 117488, upload-time = "2026-09-15T19:34:09.988Z" }, { url = "https://files.pythonhosted.org/packages/f5/5c/ce6c4ff1247fcbe4b33d462c23a097106d909b173fde7042bc52290466e2/yarl-1.25.1-cp315-cp315-win_amd64.whl", hash = "sha256:a2059a2d891bd156bc5184e7ab7a56e78a84dfcfdeac8c501b552533ad1c36ee", size = 103434, upload-time = "2026-09-15T19:34:12.56Z" }, { url = "https://files.pythonhosted.org/packages/3c/a1/766a906b0704fb26d52b19dc22bed48a8ba0544203b70dcf44da350e8194/yarl-1.25.1-cp315-cp315-win_arm64.whl", hash = "sha256:a78b50b4f7918a3de71105d5c0b93bbc57bb8339a4d03a9dfd449f9068e76f3d", size = 99155, upload-time = "2026-09-15T19:34:15.132Z" }, { url = "https://files.pythonhosted.org/packages/bd/d3/a1d09b32cb6ab14f66b44939f5b4255b8b9e747aef3974af1d5d80ccc2fd/yarl-1.25.1-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:b5402a340723fa7da00b5cff987ddab61276be6d11251ea71ae02bcac54890d8", size = 149284, upload-time = "2026-09-15T19:34:17.452Z" }, { url = "https://files.pythonhosted.org/packages/7f/3b/fe554d879692650bca70bfbc0df124e82e4d2bb7456f698c7756f1279a96/yarl-1.25.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:eda19ea5ee88742f47a2340816e6f2d40b53bed3ab5b69794769f36af9f35bb4", size = 106399, upload-time = "2026-09-15T19:34:21.474Z" }, { url = "https://files.pythonhosted.org/packages/e1/4b/e7af56177ac8d40094c82d7728224c0b8472157d50d362e5fb3b014b2bc8/yarl-1.25.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:75baa6cf9b6d1c52f3e111a130e202fd8cf0a5b3a066c3f73d615e885092e4ec", size = 106968, upload-time = "2026-09-15T19:34:23.619Z" }, { url = "https://files.pythonhosted.org/packages/da/4f/2df41fd738d46f23ef829ae8b4468d94bb6070038fc6dfab6165ed44fea8/yarl-1.25.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbcef5a9119ef653653132cccaf999b30a0af6f33bb0a4ba80bec30056868487", size = 114717, upload-time = "2026-09-15T19:34:25.879Z" }, { url = "https://files.pythonhosted.org/packages/69/ea/002b66df53bbd1aed1c23358ff99c9bdc744fe3f4d2740e1b2fdd7192885/yarl-1.25.1-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7efc9f082dfed77c316edffa9deb52888e1bc6789171887cc1f68e06d65465c8", size = 105198, upload-time = "2026-09-15T19:34:28.204Z" }, { url = "https://files.pythonhosted.org/packages/b1/7c/95c8bc0c8f97d71e59c94525ad60d76f5c57d3f2820f08137ca8b9f0542a/yarl-1.25.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe01645169a2112aa1d4ebc3e4c5f029c5c8f97adfc32e5d37c993b39a994d75", size = 120271, upload-time = "2026-09-15T19:34:30.5Z" }, { url = "https://files.pythonhosted.org/packages/5d/7a/6fe9da56ec77927baa669fd86c39c567ce6205bab53d581082c6744c8ae7/yarl-1.25.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1c557dfd5e3db046053a0bdc72261ade790ebe8e2c7a41b36b0ca1f14cb95f3", size = 123572, upload-time = "2026-09-15T19:34:32.73Z" }, { url = "https://files.pythonhosted.org/packages/8b/83/35f222d17fa70a14c7c74fdf112ccf5515e0c2a87082b1f9b99f7693bf57/yarl-1.25.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ce4d6ccafb33d39bd78444612d14938ead674c25702ded2ee9c54a47735d225", size = 115228, upload-time = "2026-09-15T19:34:35.344Z" }, { url = "https://files.pythonhosted.org/packages/da/6f/fbaaf619423578a7d898d0f226ae47bc1906c293865c416d83c17b828b0e/yarl-1.25.1-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:80a063f8297fc796296f00f100be520f209b23dc98f93ce8eba6ee7122598209", size = 111618, upload-time = "2026-09-15T19:34:37.656Z" }, { url = "https://files.pythonhosted.org/packages/ba/78/7383278f1b3cf8e0496bd95b3281a7b09b89217b6b428db24c6b99b3deca/yarl-1.25.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:7cb414a73e21a7ab58254926073f2930cb22f5b4314ea4260a687e2b3fd4dce3", size = 114839, upload-time = "2026-09-15T19:34:40.099Z" }, { url = "https://files.pythonhosted.org/packages/62/49/5506e5b6d29aab91bd845cc9016d88c3d3f81b81bc242b8100bdd5737825/yarl-1.25.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:85a18376073f8a39aa07be34f9fc77e2869aa72c55c441efdd2cf79a0407504d", size = 106212, upload-time = "2026-09-15T19:34:42.768Z" }, { url = "https://files.pythonhosted.org/packages/46/8a/19877c193b7c5929f4b07118c18bbe390f3fadd0f59dd98c0cd12b31fa5c/yarl-1.25.1-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:77716e245c90f058466a05e6a465bb8600f767a8f4b18b4d40f3aff958e5f73c", size = 119985, upload-time = "2026-09-15T19:34:44.976Z" }, { url = "https://files.pythonhosted.org/packages/4e/6c/0a46fbbf9ecbcbd0cc20d2193394254b9e19817f22c814aab60f99847400/yarl-1.25.1-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:1e80dcf1446e1b080b1932b0d103c464a04112f5bc31f0f983ad418172063cde", size = 112081, upload-time = "2026-09-15T19:34:47.45Z" }, { url = "https://files.pythonhosted.org/packages/ef/30/93f5d471230c74ccd06255d0842739f86551f937f9e63a5e947853c6244a/yarl-1.25.1-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:bdc8d8b8c22e9e43ac68316b5e6cf083dec537f4ec213cb4aa967b583bc3fa64", size = 116995, upload-time = "2026-09-15T19:34:49.972Z" }, { url = "https://files.pythonhosted.org/packages/2b/80/c386593035ee3f9c6c6af0847b5578f2830c674794a9d7701b744a3ebd42/yarl-1.25.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:dfbf531053a0935f2e871bcd4753f90313688772ff8c017f5ea402e315a78c1f", size = 115570, upload-time = "2026-09-15T19:34:52.628Z" }, { url = "https://files.pythonhosted.org/packages/38/02/eef443559563ef8f2e10469387b8b1e97cb5efee95b288a56da60801f7ee/yarl-1.25.1-cp315-cp315t-win_amd64.whl", hash = "sha256:b13b88747769537f3d32e89e3a735da10c0a9e35d7322928c701b5f93d3afffd", size = 106811, upload-time = "2026-09-15T19:34:54.935Z" }, { url = "https://files.pythonhosted.org/packages/88/91/41e284ca2cf5211e05dae031d126a3668aea88fa759df56e7e35c6ad25ba/yarl-1.25.1-cp315-cp315t-win_arm64.whl", hash = "sha256:783dd1467083f4d3f7722ad6a313f24c173e7571372738fcb7a6e6d1ba48df25", size = 101804, upload-time = "2026-09-15T19:34:57.231Z" }, { url = "https://files.pythonhosted.org/packages/54/22/318c7980066769c6bcd9221ed2248294f5698811da099013098c670565ed/yarl-1.25.1-py3-none-any.whl", hash = "sha256:681c758b0490f9e96b78e5fa8e8dc6e648e9185bb6eaebe73183c33ea0c445f3", size = 63617, upload-time = "2026-09-15T19:34:59.616Z" }, ] [[package]] name = "zarr" version = "3.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "donfig" }, { name = "google-crc32c" }, { name = "msgspec" }, { name = "numcodecs" }, { name = "numpy" }, { name = "packaging" }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ae/29/6330391bab511e2428c2409fc15f6f32e4def9b2ee5eec4e95b8300304be/zarr-3.4.0.tar.gz", hash = "sha256:b68676576a6fc42683dfcc0daeb3c0e994f71e1680696990bde056bd3acf3542", size = 829693, upload-time = "2026-09-15T20:58:38.216Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/2b/6c/7c9cf79cb2122f7c94a36c0456b476515fa01c1343705d5411b3e8c1a1e9/zarr-3.4.0-py3-none-any.whl", hash = "sha256:0a5e6c9b5c4139759e612556c7fb32c8ee4e4877a46b3556244093413eb72395", size = 376931, upload-time = "2026-09-15T20:58:36.565Z" }, ] [[package]] name = "zarr-http-server" source = { editable = "." } dependencies = [ { name = "starlette" }, { name = "uvicorn" }, { name = "zarr" }, ] [package.dev-dependencies] docs = [ { name = "griffe-inherited-docstrings" }, { name = "mkdocs" }, { name = "mkdocs-material" }, { name = "mkdocstrings" }, { name = "mkdocstrings-python" }, { name = "pymdown-extensions" }, { name = "ruff" }, ] examples = [ { name = "fsspec", extra = ["http"] }, { name = "httpx" }, ] test = [ { name = "httpx" }, { name = "httpx2" }, { name = "hypothesis" }, { name = "ipykernel" }, { name = "nbclient" }, { name = "nbformat" }, { name = "pytest" }, ] [package.metadata] requires-dist = [ { name = "starlette", specifier = ">=1.0" }, { name = "uvicorn", specifier = ">=0.29" }, { name = "zarr", specifier = ">=3.1" }, ] [package.metadata.requires-dev] docs = [ { name = "griffe-inherited-docstrings", specifier = "==1.1.3" }, { name = "mkdocs", specifier = "==1.6.1" }, { name = "mkdocs-material", specifier = "==9.7.7" }, { name = "mkdocstrings", specifier = "==1.0.6" }, { name = "mkdocstrings-python", specifier = "==2.0.8" }, { name = "pymdown-extensions", specifier = "==11.0.1" }, { name = "ruff", specifier = "==0.16.8" }, ] examples = [ { name = "fsspec", extras = ["http"] }, { name = "httpx" }, ] test = [ { name = "httpx" }, { name = "httpx2" }, { name = "hypothesis" }, { name = "ipykernel" }, { name = "nbclient" }, { name = "nbformat" }, { name = "pytest" }, ] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/000077500000000000000000000000001527014137600240245ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/.readthedocs.yaml000066400000000000000000000052471527014137600272630ustar00rootroot00000000000000# Read the Docs configuration for the zarr-indexing docs site, separate from # the zarr-python site configured by the repo-root .readthedocs.yaml. The RTD # project for zarr-indexing must set its configuration-file path to # packages/zarr-indexing/.readthedocs.yaml. version: 2 build: os: ubuntu-22.04 tools: python: "3.12" jobs: post_checkout: # Cancel pull request builds that do not touch this package. Exit code # 183 cancels the build and reports success to the Git provider. Scoped # to PR builds ("external" versions) because origin/main is only a # meaningful diff base there. The shared docs config and header script # live outside the package directory but change this site too. # The diff is taken against the merge base (origin/main...HEAD), not # against the tip of main: a two-dot diff also reports everything that # landed on main after the PR branched, which made every PR that was # behind main build every site. The checkout is shallow, so fetch the # full history first; without a merge base the diff fails and the build # runs, which is the safe default. - | if [ "$READTHEDOCS_VERSION_TYPE" = "external" ]; then git fetch --quiet --unshallow origin main pull/$READTHEDOCS_VERSION_NAME/head || true; if git diff --quiet origin/main...HEAD -- packages/zarr-indexing packages/mkdocs-base.yml packages/source-version.js; then exit 183; fi fi install: - pip install --upgrade pip - pip install ./packages/zarr-indexing --group packages/zarr-indexing/pyproject.toml:docs build: html: # Build from inside the package rather than pointing `-f` at its config # from the repo root. mkdocs resolves some settings relative to the # current working directory rather than to the config file, so building # from elsewhere looks for them in the wrong place -- and silently, since # the paths are valid, just wrong. zarr-indexing hit this: with # `pymdownx.snippets` and a relative `base_path`, its snippets were # searched for under the repo-root docs/ and the build failed with # SnippetMissingError, while `just docs-check` passed because it runs # from here. Building from the package directory makes this identical to # the local and CI invocations, so a green build there means a green # build here. # # $READTHEDOCS_OUTPUT is absolute, so the cd does not affect it. - cd packages/zarr-indexing && mkdocs build --strict --site-dir $READTHEDOCS_OUTPUT/html mkdocs: configuration: packages/zarr-indexing/mkdocs.yml zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/CHANGELOG.md000066400000000000000000000521231527014137600256400ustar00rootroot00000000000000# Release notes ## 0.3.0 (2026-09-29) ### Features - Chunk plans now have a factored, columnar form. `ChunkPlan.partition()` returns a `GridPartition`: one `StridedSet` or `IndexedSet` table per output dimension the transform reads independently, plus `joint_sets`, one `JointSet` per connected component of index arrays. Arrays that share request axes are sorted into chunks together; independent ones stay in separate tables, so `(u, v) -> (a[u], b[u], c[v])` stores 3,000 index values rather than 3,000,000. A `ChunkProjection` is one row of each table and is derived on demand, so planning costs the sum of the touched chunks per axis rather than their product, and a vectorized consumer can read the tables (`chunk_coords()`, the CSR columns, the memoized read-only `local` coordinates) without materializing a projection per chunk. The projections a plan yields map the same cells to the same storage as before. Correlated projections now keep residual slices symbolic and index arrays compact along axes they do not vary over, and `vindex` compilation preserves singleton axes for the same reason. A hand-built transform in which two output maps read one input axis (a diagonal, which no selection produces) is rejected with `ValueError`; the whole-transform walk that previously served it produced wrong projections. ([#4310](https://github.com/zarr-developers/zarr-python/pull/4310)) - Make every `LazyArray` read supply a chunk projection, including unpartitioned reads and independently executed partition views. Partition views retain the source grid and full source base shape, so indexing and repartitioning use the same coordinate frame as other views. ([#4349](https://github.com/zarr-developers/zarr-python/pull/4349)) - Make `LazyArray` indexing lazy by default: use `view[...]`, `view.oindex[...]`, and `view.vindex[...]` directly, without a `.lazy` accessor. Iteration yields lazy views, so arithmetic over iterated elements no longer works on values; `result()` and NumPy conversion materialize. Add synchronous `write(values)` and assignment through composed views, and an explicit `EagerArrayAdapter` for consumers such as Dask that require eager indexing. Writes are planned against the source's write grid, discovered from `write_chunk_sizes` or `chunks`: an affine selection is one basic assignment, and any other selection reads, updates, and rewrites each touched cell once, so storage round trips scale with touched chunks rather than selected elements. A source with no advertised grid is written one element at a time. Writes bypass the reader, so a caching reader is not invalidated. Views keep their literal domain instead of re-zeroing after every selection: `a[10:20]` has domain `[10, 20)` and `a[10:20][2:5]` has domain `[12, 15)`, as in TensorStore, while NumPy keys stay positional. An `IndexDomain` key restricts a view to literal coordinates and an `IndexTransform` key composes onto it. Box partitions keep the request's coordinates, so a part view's domain is a sub-domain of its parent's. ([#4350](https://github.com/zarr-developers/zarr-python/pull/4350)) ### Bugfixes - Direct `IndexTransform.oindex` and `IndexTransform.vindex` selections now reject integer array coordinates outside the `np.intp` range before conversion. Previously, oversized `uint64` values could wrap to negative coordinates and silently select a different location in a domain containing negative coordinates. ([#4333](https://github.com/zarr-developers/zarr-python/pull/4333)) - Reject invalid wire index-array values and unrepresentable normalized bounds/ranks, and raise an explicit error for unsupported intersections sharing an affine and lookup input axis instead of returning incorrect coordinates. Group negative chunk-coordinate tuples without merging distinct chunks. ([#4345](https://github.com/zarr-developers/zarr-python/pull/4345)) - Validate inclusive `index_array_bounds` against all raw index values when loading transforms and output maps from JSON. Accept valid finite and one-sided constraints, and reject out-of-bounds values eagerly before offset, stride, or map simplification. Validated immutable maps need not retain the constraints; message normalization preserves the original bounds. ([#4347](https://github.com/zarr-developers/zarr-python/pull/4347)) - Delegate LazyArray source tokenization to Dask, honoring its registered normalizers, source hooks, and deterministic-token requirements. Remove local content-hashing and UUID fallbacks. Dask remains optional for indexing and reading. ([#4348](https://github.com/zarr-developers/zarr-python/pull/4348)) ### Improved Documentation - Rewrote the guide's prev/next navigation links as markdown so mkdocs validates them at build time; the rendered pages are unchanged. ([#4291](https://github.com/zarr-developers/zarr-python/pull/4291)) - The docs now render the release notes as a page instead of linking to `CHANGELOG.md` on GitHub, so each docs version shows its own changelog. ([#4311](https://github.com/zarr-developers/zarr-python/pull/4311)) - Correct indexing, reader, serialization, cache, and integration descriptions to match supported behavior; qualify NumPy/TensorStore compatibility and performance claims. Describe current contracts in source and test docstrings instead of narrating prior implementations. Clarify that immutable index coordinates do not snapshot source values. ([#4345](https://github.com/zarr-developers/zarr-python/pull/4345)) ### Misc - Expand planner property tests across signed origins and chunk IDs, custom grids, mixed affine and lookup dependencies, duplicate coordinates, and empty domains. Verify exact request coverage and storage mapping with an independent pointwise oracle. ([#4346](https://github.com/zarr-developers/zarr-python/pull/4346)) - Removed every `assert` statement from the package's runtime code and dropped the `zarr-indexing` exemption from the repo-wide ruff `S101` rule, so none can return. Asserts are stripped under `python -O`; the ones here narrowed types or guarded internal invariants rather than validating input, so most were restructured away (branching on the map type, carrying the narrowed value in a local) and the remainder became explicit `RuntimeError`s for states the public API cannot reach. No user-visible behavior changes. ([#4364](https://github.com/zarr-developers/zarr-python/pull/4364)) - The test suite now runs in the package's own environment instead of the zarr-python root environment, so `just test` works from an unpacked sdist, where the suites that need `zarr` skip. Inside the repository, the justfile recipes add the in-repo `zarr`. ([#4406](https://github.com/zarr-developers/zarr-python/pull/4406)) ## 0.2.1 (2026-08-12) ### Improved Documentation - The documentation builds from the package directory, so snippet includes resolve against this package's docs instead of zarr-python's; the 0.2.0 docs rendered some examples from the wrong files. ([#4252](https://github.com/zarr-developers/zarr-python/pull/4252)) ## 0.2.0 (2026-08-12) ### Features - `LazyArray` has an explicit reader boundary: an `IndexTransform` decides which values belong in a result, and a `Reader` decides how one backend obtains them, preserving the transform exactly. `LazyArray(source)` is conservative and assumes only basic indexing; `LazyArray.from_numpy(array)` selects the optimized NumPy reader; `with_reader` selects any other. Readers do not define indexing semantics, partitioning, scheduling, or result ownership. Both built-in readers lower through NumPy system memory, so a device array needs a custom reader that transfers into the supplied output buffer. ([#4222](https://github.com/zarr-developers/zarr-python/pull/4222)) - Added the `zarr_indexing.testing` subpackage, behind a `testing` extra (`pip install zarr-indexing[testing]`), carrying the Hypothesis machinery this package tests itself with. `ChainedIndexingStateMachine` composes basic, orthogonal and vectorized selections onto a `LazyArray` wrapping an array you supply, then checks every view's shape, `result()`, and assembled `parts()` against NumPy; `zarr_indexing.testing.strategies` exports the selection strategies alone, for a project with its own harness. Nothing outside the subpackage imports Hypothesis. ([#4222](https://github.com/zarr-developers/zarr-python/pull/4222)) - Added `UnitStepReader` / `unit_step_reader`: a backend adapter for sources whose basic indexing accepts only ascending step-1 slices (FFI bindings, HTTP range endpoints). Every key it presents is `slice(start, stop, 1)` per axis; strides, reversals, and gathers are applied to the in-memory block by the residual lowering. The integrations guide documents the companion dense-box re-partition idiom — resolving a unit-stride rectangular view as one backend slab read while keeping partitioned reads for strided and fancy selections. ([#4222](https://github.com/zarr-developers/zarr-python/pull/4222)) - Negative-step slices are supported, following merged ndsel 1.0-draft.2 and TensorStore 0.1.84: `arr[::-1]`, `arr[5:1:-2]`, and reversal composed over an already-strided or already-gathered view. One desugaring rule covers both signs — omitted bounds resolve on the side the traversal starts and stops, and the origin is `trunc(start / step)` — while a reversed interval is an error rather than a silently empty selection. A reversing slice normally yields a negative domain origin, since the result stays anchored to the source coordinate frame; `LazyArray` re-bases every view to origin 0, so its positional dialect is unaffected. ([#4222](https://github.com/zarr-developers/zarr-python/pull/4222)) - Fancy selections compose without restriction, on both `LazyArray` and `IndexTransform`: a second `oindex`/`vindex`/mask step may land on any axis of an already-fancy view, including axes an existing index array merely broadcasts along. Array-carrying transforms are chained through `compose`, and resolution handles the resulting mixed, correlated and diagonal index-array structures on one shared pointwise path, classified by `IndexTransform.index_array_structure`. Only hand-built affine diagonals — an index array and a slice map bound to the same axis — remain unsupported. `__dask_tokenize__` digests a view's canonical transform body rather than embedding it, so tokens stay small for large fancy selections. ([#4222](https://github.com/zarr-developers/zarr-python/pull/4222)) - Added `LazyArray`, which grafts the full NumPy indexing dialect onto any source exposing `shape`, `dtype`, and basic integer/slice `__getitem__` — a chunked store, an FFI binding, an HTTP endpoint. `view[...]`, `view.oindex[...]` and `view.vindex[...]` each compose an `IndexTransform` and return a new view without reading anything; `result()` and `numpy.asarray(view)` materialize. Selections use positional NumPy semantics (negatives wrap, scalars drop their axis, coordinate arrays keep order and duplicates), which the new `zarr_indexing.boundary` module translates into the algebra's literal coordinates. Consumers that need indexing to produce data, such as `dask.array.from_array`, wrap a view in `EagerArrayAdapter`. A read is divided along a **partitioning** — discovered from the wrapped array, or chosen with `with_parts` / `with_parts_per_axis` / `unpartitioned`. `parts()` yields one `Partition` per box, pairing a resolvable sub-view with where its cells belong in the result; `result()` is the assembly of that walk, and re-partitioning never changes what it returns. `base_shape` says which shape a partitioning is expressed in. `is_box`, `bounding_box()` and `strides()` report whether a selection is rectangular, so a consumer can dispatch a slab read against a gather. See the [guide](https://zarr-indexing.readthedocs.io/en/latest/guide/) for the model and the [design notes](https://zarr-indexing.readthedocs.io/en/latest/design-notes/) for the box/query distinction, the relationship to TensorStore, and current scope limits. ([#4222](https://github.com/zarr-developers/zarr-python/pull/4222)) - Added source-independent chunk planning. `plan_chunks(transform, grids)` returns a lazy, reusable `ChunkPlan` whose `ChunkProjection`s each pair a chunk-local transform with a transform back to the request, over one shared cell domain, so a consumer can read a chunk and place its values without re-deriving either. The same representation covers basic, orthogonal and vectorized indexing, and carries global chunk bounds plus conservative full/partial/unknown coverage. I/O, buffering and scheduling stay with the consumer. `zarr_indexing.grid` gained `EdgeDimensionGrid` and `dimension_grids_from_chunks` for building the per-axis grids it takes. ([#4222](https://github.com/zarr-developers/zarr-python/pull/4222)) ### Bugfixes - An adversarial review of the whole package found, and this fixes, several defects at its boundaries: `result()` and `__array__(copy=True)` could hand back a live view of a source that merely stored its data in NumPy; the wire format emitted a document nothing could load for a selection that selects nothing, and its domain loader validated nothing; chunk-selection lowering described a transposed block in two separate cases; a map derived from a vectorized selection carried a stale `input_dimension`, which made one view's answer depend on how it was partitioned; and `oindex` over a correlated view applied NumPy's vectorized rule instead of the outer product. ([#4222](https://github.com/zarr-developers/zarr-python/pull/4222)) - Correctness fixes to indexing and resolution, all reachable from 0.1.0: - An integer index applied to an axis a previous `oindex`/`vindex` step had already indexed left an all-singleton `ArrayMap` still naming the axis the integer removed, which after renumbering aliased a different one. Such a map now collapses to a `ConstantMap` at composition time. - A `vindex` selection whose coordinate arrays are not on the leading axes (`vindex[..., i, j]`, `vindex[..., mask]`) laid out its result incorrectly and raised a shape mismatch on a partitioned read. Gathered dimensions now follow NumPy's placement rule, and the per-part gather is realigned to the scatter. - An `oindex`/`vindex` step whose entries are all slices, applied to a view with a fancy-indexed axis, applied those slices positionally to every axis of the existing index array — including broadcast singletons — truncating it to size 0, so `result()` returned an unwritten buffer. Reindexing is now dependency-aware. - `parts()` raised on a view emptied by a slice over an axis of extent 1; an empty domain now yields no parts, matching `result()`. - Negative-stride chunk projection swapped the endpoints while keeping the step negative, selecting nothing where the reversed axis was meant. Composition evaluated an inner index array over `range(size)` rather than the outer domain's own range, resolving every coordinate wrongly whenever that domain did not start at 0 — which both step-1 and negative-step slices produce. - A domain dimension no output map depends on, left behind when a later basic index consumes the axis a `vindex` array varied over, was miscounted in three places: the partition walk's out-selection rank, the lowering engine's axis restoration, and the correlated gather's broadcast. - The parts of a correlated view narrowed to a single point came back rank 1 where the view was rank 0, so the documented `out[part.out_selection] = part.view.result()` assembly raised `ValueError`. - `result()` could return memory shared with the wrapped array: an unpartitioned read of a basic selection lowered to plain slicing and handed back a view of the source, and `numpy.array(view, copy=True)` inherited the alias. It now always allocates, and verifies the parts covered the output before returning. - `IndexTransform.from_json` rejects a non-integer `index_array` with an `NdselError` carrying `invalid_json`, rather than truncating a float array, coercing booleans, or leaking NumPy's conversion error for strings. - `result(parts=...)` raises `ValueError` rather than `AssertionError` when the supplied parts do not tile the view, and `with_parts` / `with_parts_per_axis` raise the documented `ValueError` for non-iterable input. ([#4222](https://github.com/zarr-developers/zarr-python/pull/4222)) ### Deprecations and Removals - The canonical JSON converters are now methods on the types that own the serialization: `IndexTransform.to_json()` / `IndexTransform.from_json()`, `IndexDomain.to_json()` / `IndexDomain.from_json()`, and `to_json()` on each output map kind. `output_index_map_from_json` remains a function, in `zarr_indexing.output_map`, because the wire form is a tagged union and loading it dispatches rather than belonging to any one kind. The free functions they replace — `transform_to_canonical`, `transform_from_canonical`, `index_domain_to_json`, `index_domain_from_json`, `output_index_map_to_json`, and the historical aliases `index_transform_to_json` / `index_transform_from_json` — are removed. There had been two spellings of each conversion; there is now one. ([#4222](https://github.com/zarr-developers/zarr-python/pull/4222)) - `ArrayMap` no longer has an `input_dimension` field: what a map depends on is read from its full-rank index array's shape (its non-singleton axes), the single source of truth. A selection narrowed to a single coordinate is now built as the `ConstantMap` it equals (`array_map_or_constant`), so a length-1 fancy selection classifies as a box; hand-built all-singleton or shared-axis `ArrayMap`s resolve through the pointwise path. The wire format is unaffected — it never carried the field. The provisional tuple resolver and selector bridge are gone with it: `iter_chunk_transforms` and `sub_transform_to_selections` are removed, their role taken by `plan_chunks` and the paired projections it returns. ([#4222](https://github.com/zarr-developers/zarr-python/pull/4222)) - Operations moved onto the types that own them, following the arrangement TensorStore uses (public headers are the types; every transform operation lives in `internal/` and surfaces as a method): - `compose(outer, inner)` is now `outer.compose(inner)`, and the algorithm moved to the private `zarr_indexing._composition`. - `selection_to_transform(selection, transform, mode)` is now `transform.select(selection, mode)`. - `index_array_structure(transform)` is now the `transform.index_array_structure` property. - `array_map_dependent_axis(m)` is now the `ArrayMap.dependent_axis` property, alongside a new `ArrayMap.dependency_axes` giving every axis a map varies over. `zarr_indexing.affine` is now the private `zarr_indexing._affine`; it was never exported or documented. ([#4222](https://github.com/zarr-developers/zarr-python/pull/4222)) - `with_parts` is now three named methods — `with_parts`, `with_parts_per_axis` and `unpartitioned` — instead of one parameter whose meaning was decided by the type of what it was given. `Partition.array` is `Partition.view`, no longer the inverse of `LazyArray.array`. `ArrayMap`, `IndexTransform` and `Partition` can be compared and hashed, which `frozen=True` had implied and neither could do. `LazyArray.base_shape` says which shape a partitioning is expressed in. ([#4222](https://github.com/zarr-developers/zarr-python/pull/4222)) ### Misc - Restructured the documentation-contract tests: the snippet include graph is now discovered by scanning the rendered markdown instead of hand-maintained registries, prose and navigation assertions moved out of CI, and `pymdownx.snippets` now sets `check_paths: true` so an unresolvable include fails `mkdocs build --strict` instead of silently rendering nothing. ([#4222](https://github.com/zarr-developers/zarr-python/pull/4222)) ## 0.1.0 (2026-07-31) ### Features - First release of `zarr-indexing`: TensorStore-style index transforms and the ndsel wire format. ([#4196](https://github.com/zarr-developers/zarr-python/pull/4196)) - Reworked the JSON layer to conform to the [ndsel](https://github.com/zarr-developers/ndsel) draft wire format, which adapts TensorStore's `IndexTransform`. A new `zarr_indexing.messages` module (`parse_ndsel`, `normalize_ndsel`, `NdselError`) is a pure JSON-to-JSON layer that accepts all five message kinds (`point`/`box`/`slice`/`points`/`transform`) and normalizes them to the canonical transform body, enforcing the full ndsel error taxonomy. The package is checked against the vendored, language-agnostic ndsel conformance corpus. Serialization produces and consumes the canonical body (`IndexTransform.to_json`/`from_json`, and the `IndexDomain` pair). On serialization, orthogonal (`oindex`) `index_array` maps no longer emit `input_dimension` alongside `index_array` (a combination both ndsel and TensorStore reject), and degenerate all-singleton index arrays collapse to constant maps; the in-memory `input_dimension` is reconstructed from the array's dependency axes on load. ([#4196](https://github.com/zarr-developers/zarr-python/pull/4196)) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/CONTRIBUTING.md000066400000000000000000000016721527014137600262630ustar00rootroot00000000000000# Contributing to zarr-indexing Package-scoped development commands live in the [`justfile`](./justfile) (requires [just](https://github.com/casey/just)): ``` just test # run the test suite (extra args go to pytest) just lint # ruff, same invocation as CI just typecheck # pyright, same invocation as CI just docs-check # strict build of the docs site just check # the checks above plus TensorStore parity just docs-serve # serve the docs site locally ``` Run them from this directory, or from the repository root as `just packages/zarr-indexing/`. The test recipe layers this package into the repository-root environment. Chunk-resolution tests use this package’s own grids; the Dask example also uses `zarr`, which is not a dependency of the base indexing package. ## License MIT The package lives at `packages/zarr-indexing` inside the [zarr-python](https://github.com/zarr-developers/zarr-python) repository. zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/LICENSE.txt000066400000000000000000000021441527014137600256500ustar00rootroot00000000000000The MIT License (MIT) Copyright (c) 2015-2025 Zarr Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/README.md000066400000000000000000000057371527014137600253170ustar00rootroot00000000000000# zarr-indexing Composable, lazy coordinate transforms for Zarr array indexing. Documentation: This package implements TensorStore-inspired index transforms. The core idea: each supported indexing operation (slicing, fancy indexing, etc.) produces a coordinate mapping from user space to storage space. These mappings compose without reading selected source values. `LazyArray` materializes them on request; the transform algebra itself performs no source I/O. Key types: - `LazyArray` — wraps a system-memory/basic-indexing source with lazy indexing: `LazyArray.from_numpy(numpy_array)[10:50, ::2].oindex[[3, 1, 1], :]`. The key's type picks the frame: NumPy keys are positions relative to the view, while an `IndexDomain` or `IndexTransform` key addresses the view's absolute domain, which every view keeps. composes a transform and returns a new view without reading data, and `result()` materializes it into owned system memory. `LazyArray(source)` uses the basic reader; `from_numpy` selects `numpy_reader`, which currently uses the same slab-and-gather implementation. Device sources that refuse NumPy conversion need a custom reader to transfer values into the output buffer. - `Reader` — the explicit backend execution boundary: transforms say which values belong in the result, while readers say how a backend obtains them - `IndexDomain` — a rectangular region of integer coordinates - `IndexTransform` — maps input coordinates to storage coordinates - `ChunkPlan` and `ChunkProjection` — lazily partition a selection over a caller-selected grid and pair each chunk-local transform with its placement in the request, without binding a storage backend or scheduler - `GridPartition` — the plan's factored, columnar form: one table per axis (`StridedSet`, `IndexedSet`) plus `joint_sets` for connected index-array components, from which projections are derived on demand - `ConstantMap`, `DimensionMap`, `ArrayMap` — the three ways a single output dimension can depend on the input - `compose` — chain two transforms into one The base package depends on NumPy and the standard library; its optional testing module also requires Hypothesis. The package does not import `zarr`. It is developed in the [zarr-python](https://github.com/zarr-developers/zarr-python) repository, and its examples include reading Zarr arrays through Dask. Installing it does not replace Zarr's indexing implementation. ## Installation ```bash pip install zarr-indexing ``` ## Examples - [Lazy indexing a NumPy array](examples/lazy_indexing_numpy/README.md) - [Lazy indexing with Dask](examples/lazy_indexing_dask/README.md) ## Contributing Development commands, the test suite and the docs build are described in [CONTRIBUTING.md](https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-indexing/CONTRIBUTING.md) in the repository. Issues and pull requests go to [zarr-developers/zarr-python](https://github.com/zarr-developers/zarr-python). zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/benchmarks/000077500000000000000000000000001527014137600261415ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/benchmarks/README.md000066400000000000000000000044111527014137600274200ustar00rootroot00000000000000# Chunk planning benchmarks Run from the repository root, with `zarr-indexing` installed in the selected Hatch environment: ```sh hatch run test.py3.12-minimal:python packages/zarr-indexing/benchmarks/chunk_planning.py --repeats 9 ``` `chunk_planning.py` measures construction, projection traversal, local-column access, and chunk-coordinate enumeration. See the script for workloads and output fields. Use the same interpreter, dependencies, inputs, and script revision when comparing checkouts. Put the intended checkout's package source on `PYTHONPATH`; installed package metadata is also required for version lookup. Record both git revisions and the interpreter/dependency versions alongside saved output. Compare complete operations, not only constructor times: plans can defer work until iteration. Measure allocation separately from elapsed time. Distinguish streaming consumption from retaining all projections or coordinates, and report whether input arrays and transform construction are included. Repeated local column access measures cache reuse, which trades allocation against retained memory. Bounded coordinate batches avoid constructing the full coordinate array. The current script reports `peak_mib` as the incremental peak tracked by `tracemalloc` during a separate invocation after the timed calls. It is not process RSS or total retained memory. Inputs and most transforms are constructed before measurement. The case walks construct plans and consume projections without retaining them; `all_coordinates` materializes the complete coordinate array. There is no bounded-batch coordinate workload in this script. `local_rows` reuses one prepared table: its first timed invocation populates the local-coordinate cache, while later timed invocations and the allocation probe reuse that cache. Its median mixes a cold first call with warm calls, and its reported peak excludes the already-retained cache. A separate fresh-table measurement would be needed to quantify cold cache construction. These scripts measure planning rather than codec or storage throughput. Repeat measurements with alternating operation order before interpreting small timing differences. Preserve raw benchmark output as an experiment artifact rather than accumulating successive result tables in this README. zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/benchmarks/chunk_planning.py000066400000000000000000000114431527014137600315140ustar00rootroot00000000000000"""Reproducible planning measurements; no storage I/O or timing assertions. Run with the package on PYTHONPATH in the repository's Hatch test environment. Use the same interpreter and this script with the old/new package paths to compare. The connected-components probe is deliberately not a production planner: it measures independent table construction, without defining a new public row API. """ from __future__ import annotations import argparse import json import statistics import time import tracemalloc from typing import TYPE_CHECKING, Any import numpy as np from zarr_indexing import ArrayMap, IndexDomain, IndexTransform, plan_chunks from zarr_indexing.grid import dimension_grids_from_chunks if TYPE_CHECKING: from collections.abc import Callable def measure(operation: Callable[[], Any], repeats: int) -> dict[str, float]: """Time repeated calls, then trace one additional call's incremental peak. State retained by earlier calls can affect the extra call; this is not a measurement of process RSS or all memory owned by the operation's inputs. """ samples = [] for _ in range(repeats): start = time.perf_counter() operation() samples.append(time.perf_counter() - start) tracemalloc.start() result = operation() _, peak = tracemalloc.get_traced_memory() tracemalloc.stop() del result return {"median_ms": statistics.median(samples) * 1000, "peak_mib": peak / 2**20} def consume(values: Any) -> int: return sum(1 for _ in values) def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--repeats", type=int, default=5) args = parser.parse_args() results: dict[str, Any] = {} indices = np.arange(1000) cases = [ ("box", IndexTransform.from_shape((1000, 1000)), (10, 10), (1000, 1000)), ( "sorted", IndexTransform.from_shape((1_000_000,)).oindex[np.arange(1_000_000)], (100_000,), (1_000_000,), ), ( "correlated", IndexTransform.from_shape((10_000, 10_000)).vindex[indices * 9, indices * 9], (10, 10), (10_000, 10_000), ), ( "correlated_slab", IndexTransform.from_shape((1000, 1000, 1000)).vindex[indices, indices, :], (1000, 1000, 1000), (1000, 1000, 1000), ), ] for name, transform, chunks, shape in cases: grids = dimension_grids_from_chunks(chunks, shape=shape) results[name] = measure( lambda transform=transform, grids=grids: consume(plan_chunks(transform, grids)), args.repeats, ) transform = IndexTransform.from_shape((10_000,)).oindex[np.arange(10_000)] grids = dimension_grids_from_chunks((10,), shape=(10_000,)) (table,) = plan_chunks(transform, grids).partition().sets def read_local_rows() -> None: for row in range(len(table)): table.local[table.run(row)] # The first timed call fills table.local; later calls, including the traced # allocation probe, reuse it. The table and retained cache are not re-created. results["local_rows"] = measure(read_local_rows, args.repeats) part = plan_chunks( IndexTransform.from_shape((100, 100, 100)), dimension_grids_from_chunks((1, 1, 1), shape=(100, 100, 100)), ).partition() results["all_coordinates"] = measure(part.chunk_coords, args.repeats) # TensorStore factors connected components in the input/grid dependency # graph. Here outputs 0 and 1 depend on u, while output 2 depends on v. # A single joint block unnecessarily expands these independent components. i = np.arange(1000) transform = IndexTransform( IndexDomain.from_shape((1000, 1000)), (ArrayMap(i[:, None]), ArrayMap(i[:, None]), ArrayMap(i[None, :])), ) grids = dimension_grids_from_chunks((1000, 1000, 1000), shape=(1000, 1000, 1000)) results["component_partition"] = measure( lambda: plan_chunks(transform, grids).partition(), args.repeats ) results["component_walk"] = measure( lambda: consume(plan_chunks(transform, grids)), args.repeats ) base = IndexTransform.from_shape((1000, 1000, 1000)) results["component_vindex_compile"] = measure( lambda: base.vindex[i[:, None], i[:, None], i[None, :]], args.repeats ) left = IndexTransform(IndexDomain.from_shape((1000,)), (ArrayMap(i), ArrayMap(i))) right = IndexTransform(IndexDomain.from_shape((1000,)), (ArrayMap(i),)) results["independent_components_probe"] = measure( lambda: ( plan_chunks(left, grids[:2]).partition(), plan_chunks(right, grids[2:]).partition(), ), args.repeats, ) print(json.dumps(results, indent=2)) if __name__ == "__main__": main() zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/changes/000077500000000000000000000000001527014137600254345ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/changes/README.md000066400000000000000000000016711527014137600267200ustar00rootroot00000000000000Writing a changelog entry for `zarr-indexing` ----------------------------------------------- Fragments in **this** directory are release notes for the `zarr-indexing` package only — kept separate from the parent zarr-python `changes/` directory so a PR touching only `packages/zarr-indexing/` produces a release note for this package only. Please put a new file in this directory named `xxxx..md`, where - `xxxx` is the pull request number associated with this entry - `` is one of: - feature - bugfix - doc - removal - misc Inside the file, please write a short description of what you have changed, and how it impacts users of `zarr-indexing`. A `zarr-indexing` release runs `towncrier build` in `packages/zarr-indexing/`, which consumes the fragments here and updates `CHANGELOG.md`. Fragments that describe parent zarr-python changes (not the transforms package) belong in the top-level `changes/` directory, not here. zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/000077500000000000000000000000001527014137600247545ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/_static/000077500000000000000000000000001527014137600264025ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/_static/favicon-96x96.png000066400000000000000000000306521527014137600313460ustar00rootroot00000000000000PNG  IHDR_`{` pHYsodtEXtSoftwarewww.inkscape.org< IDATx}y|Uι7&lU* Ep\@[kTlUYVZpׯ-l, [s}~$dAk|>瓛s3g33̼!_L;[wG4kڽ3>B~I Nj4!oPD1۔Ec.:Џ[O觎/B}J1 8]Q?~jkPp_zkg-~޽)1 Ժ\Q6e`HJܗ6s\3_|77=el0"d~z"oeOU05uम8?:ՑLs\E!A @*G`EG3nyv,Ƽy``JZ{SNJOV:sϏbgav LI++?j]-HL2 @m4Pe?  W=SRG*>s~RL+Wzh##yx(f'Ҥf3Uu(w"T<ʍ1qsR*ں|QQI]g̘Ƭxibhӭ 5ulɭOqJ[;;Ң*& g@TYU2yj,P>x{ xɩcֵʚ`?}6x )鷉Ӳ+\ [Ϙ:D$bU)+[0{qwE7pg#;HNCP@.' Y譠XKʜ/P/B8Am (9ɩ=a 8m/SJP@䌽i臦 ,ŰaN~RMI @9Tp&Ӧ4pĽPP6JP@: 5a Q2X qYsABj:X<.+cFK5*U(bjpWc?!}t'SNNnZOBL،!\*PxkO (,VhUTJAIDêKTh=]OO0j܂{Rn+f q/' ܝ:Vw *DX`ЗWǧ6i,}nS{{0* / UU (B 0ķ'VZר@AțT^ Zma ~-RYok{"|ܓr;STp8M1= N]xԩ^DJ4OǍkC~)ό,E6p nV+WT,b*5By D:Ú aJw05q1QO ;~q僸d51}b,VG0J??Dl7.2 ot@⪺ZyKlJ!U-zXBc}(A@8|FͣJKߕ:2Q [<6()2N8` sZ,)eq)缚;kȖJs|Zeו7*n2o7.7.E-lϩW%aŷ64 Hxq5 z8Фp>tQ$ d47&DLݿx4?)DZV "y6Xg( ~e<@ߙRYR "H FZwqoѲ*t|2W|Ci>F?0{vwzB7~/0*9h+0C ͡##)OhNT >w`h#% '`( DD!Pט+cP( 'xT*a_Z5e x'Zע;y6D4- ϪЮL|rOmiЂ;ҴᢶOM-/6?ߞziB9P*04S Eu[EG'T 6"ʩɇ4RBѾ@c |d!,T]CEd)W>Z/17=S.ʽl^pGڈcǛwFA `KڠȦd]?9 e< ($!.wΚy`9nP MAαl>Xt6)4(!BԪqwP0ɢʐ29R/0yPp|YhV9֬TV4)hZS FE,VGZ^ aè7rN6@S@h~"꧍JE&oP4!olʱFbTI`w礼a"bEzͅbN~ b(dTJZuN`IQ%~Im1aӞmܜ'x0[Ӯ0/9Pg:B՚MH*:٬c O|>TjoMPgO~NMy$5PGřlmo+w4gE:gPz*QxǧaQ((/pTLe!7ՎP&ݒr[ȴUWCT,#FNu8~3?EřTD<6:oTUNLFk>7W5`ѨuY[9OzaFaNC_B y<  "T=ϟ"F>2U`(+Ð!bJEWK@ ^PbaO Fv!haM1~sj]vuK[?X?2'h kB-nB4Ӓ⛩R0ECi,Fbj a: Tx9 ŹUv@nr1}N`JDOl']Ee+P@ c^\(^v*|V֌V[ ot8Qv-(1#(ڏtQTWVy^H{5,n9 _hcn"332 iDR +AѼTO+EDs?Ӳ-9EUcmeybǍV1< ࣇ]-b!D(1fsS GԀ } \!hy!^֎@?cRD!!1<:-$({“YK/]/ynq A5\c%a͑Yn44հxZ_ +jpt q`&irOF:w/~*\ 2WXz9M-^*pݣQȱy_?@zcf^N0hr86\)x[‚ugXZ_(e~ kS{Ѱ#do OCo4/{Mפ-=={>+n-I >6v̈́Nt|P : z>('9"*!  KoJHd)GE\S9fŽ YT: 9b~9;}Æ9y^N`EG']s6Wj%7匵Y;44Vϻ6/+ŏ Jc"JƊ3&K:pgm˽&9/lkTEIR\.}UCRA _5aW( RϘm]?lﶻl;ImVtd ux÷uO ax$r!jY2Hv5`DK(o—o.E_'dP!lTQ2kjw%{$6cg%W"WGp:aPdCؓsXSH a5*Q<Ab1mJ|E_0. r,Q$?P`(R_m DQbuTk0jḡř g &Lrx0P_!1jʹ&&&Ð(xD,C·X|*FQo AH>[N1Q ^-[FLO`̵ KcJAVN:q*_F~qu) [~?BcwxhI]q.B,(ixsyE9Z=[Rb|@C9(HX[l_&)]Ȁ7aGoi,:U\|dgH3Q =GV7{tP5 7~E֢WCAi ֬R;8A#Ty>MZ1s. |ǮFq}qYv qlr`с|EnRs!Q:Xf+xEx~u\1FH8(bǯ-iJIdR'D++prx8GI):.{ K__xXرaK"6 X9K^w߻ӰмE(AhXDT mOcbRQ|@8MD$ @ @b\$bt8״Pk& fOtBە~ո6-}W$_0&@1VeίJKSCŪhZJ.IJM5r3 U6F"ʭ\h`!?PDLZ_("\͸[gToP}KHKT嶫fm-zn;#6v@Ҫ̑'>r/NYrq1RUw|_Օ(P\t= H;Ϫ)Zճ1lS+Ix'ňK#]}6u6e;!du_C pX**kK! ⥊ CT.\/ f Ljoʏ+δ5u+z*H@L\xәG[gVgV铲5Y=>^1B!K|?MH^仔)iܾq 5B .O|?=}Kڛ蚤)\PĔ x8!ѫG@(3h ^!r}y@LB0 @O(e @  &nF0a]:^0bcdJ@e8u'^A+rW6볖$&R>ԛhfv^ \UZJU/۩Ԭ|%u A_cؔ r)aIiD4% ;J\[<_Ghx| aY1G@IBX,p։׋h*^A;aY̱'>䁹|7`3qmf6벖" & #2J %oj$|9Ǹq8m<}k3tBPgkάWi&~[ˡ ~`\[mȮm? 4ZE%RQyD-y+ ˡӫ19B0"==/#ȦUt4]G*e/XDZuygUP-&l=9cLwOy\y f'3I0y&;7_ ~ygϲI\T­q"T+`Z8*ēˡ;ib ]4ŠsP#ė;;ܳ'~!Z8B\2VnkCNipeE‡Y*9gJ>=8+쉷>kbg1/I,*-D\@=v;{mZ;*Uv/?g9,ļ'aFV\C :ʤL6Q1miZ dhaJ<,GYb5;_X ⨴X*VvM^(EPq|3b( @a/vgc"_fTšt= H_%m^ `uNKA?kXk7V T3_d|ܮ/hQ:6Uv[2Ƽ.)'M@FtMF?jqgАٱ <F@\<6$Y/,J ,!Ey%,σEZW><[Q"-l"X4wŚQ 5C{Q+v:g25dlX)DXv>9ּNą3/rD?P*th/9cmpVmV,PLcIb@ E),h Og2@b1w">*+iCz2h7lYxfb ,$]Nu$ V&^tGtvfΧE;%/NT^M{[pUcM=UxNW(QM(ׯM+@5>I 8Aܤ1՛UpQJ'C` g7 J#!JhL[(1 p !&7cs˹CL*NH9@nSwN,}hy92XptxP͵`PkRxl(( p˜/`uE=%bVLObBkZbU>AOg2@M>6SyQ-,z(-[Z|V8S_hKKP U`)[O򛤼&5f썟\U62 6qbU>"%kN [ZΟZ&7rZ{ZOnTR*]7N-OIO=c]sZyШ+IDATC/ez0jk&Q1Xl'7œpuPϱchx$i<#=LLbn5`F\*MdkU1ݱfHӊp6f= yֈ@?fTW=T8Y|V!6C }%mfX/]*QlX@oqB6]~~!"cU 9J6튲'9 E^''|0?x4I1K|7"'69wMy6)IۃY𼈳F;|o29 1EN y@rH845r@9MdQǻ ]y쨈l-%1nͤRxai}R R N1*qn1eNN;u<2wǤ& 3<#|#Ltb '#IK`Mvib"'q@>oS43J׭QEW $x-0 !v&LwERQ"tbα4_y[B^ IROF8eNns |3m ^~~G L}5' ؽ>|Jq\[¬`u7I0bgl&`˕>%1)n>s%?d.|hS]Q\z9|(no|a-9f+ʚK1,q՞;ɗU%*_Ew t]PV' C Sꞕp6-$ \-*;j! &8CWp=4ri6aM>Pó%c*h jΘ.|Q۵=9-YYs-o$ulC+e"¸=FĵPTx>l@a I^<I#K;2/uIjn,=&(XW)F<|xڢHkt8\wJ7w(̚-:#"먦l06"ozL]CKk!kO㼢YA7$|8^ jMLr.gԵ b8AN}pEYaszulI/;\U@JҫZf#\Xs$:f?4|`-Q,B޼Nز%6h#y}UU#ו7"hֹFQW8%"$h4 OA]㳏Յ}K@ Եp|;~=:Ʀ@5\-ƴH+Q-:N?@H:75z":Ŷش3}ug$n "5rК֯gj&.TDU\C]5o -άʲ56 @vDo8r8if4[KH*B:8`-6"\E]JH)(hSTAYu^ Z\$EꉾBy񱣱җ:ޢWu$Mgͨ1]2>V ^ojr#&K7 4II &]=}w5Zlp|C;(Dt й£a@m롚s(X `@zqf:8FL6*3[F8:KB+_$1\'|z90\}t_QK.qW* N)] JjTf, V䁮p&$h6G^/" o|yƷ3RR0oGt"sLrM| "za/xOD2IċJ0ӎ՘JFu}]4.< K57L#o B):N͖ `w](́J\aoZl} # M;9~{!}| (G+ Њ4mE00/Qrt zi3\ߤqYUfSlZ4D`rb ÜM1F V VpХh[HuG6ŤjU݋BPū$k(-(kec =\}a da"N*.2˻ɊP " "(pw?xe%Fw<:iPk.6.S0ad=•K闱G2ĹVZwK}@;7Lic<l;Q{wX%A_@ML ŀB]xc'FkTOAVSo M49ENUO1KC9f}t; /kf>wUo~MiK,"po $*ec&jTzhJ,Ö˻MIfa}6 }#qB,{F߂_&xUŴ<0@@Ǵu|{CɁ5V'Tn~/.K꿬롩l$BdgC@9@HK)>(oSG]m09_//l`ZAUC{i[3lAl`vEE3=~GĔ,":jƝ@@VO2.HLD\rNԢ#~'$Q8@|v56/Üc:'E;)籓Y='Y-O&9}yF~ $ho$J߳MZe10*b{p2NG /7G{?occ7V#Qğ퐠(3*D8GoI* G{C Ю_>V̥a@sϟs:+"Aa"U;~1AnkL~0Muwc$W}Chsw0ۧ}raCB A.@yWS?AdژJoRT>pMO'#ek"Hȫ :PRO)J!K޷Afm $xQRϢ{ߣ>Tom _WXڻhNN:Ax*)zʚ?ڷW!ekYн42E"T>?U>F~l/+ LS1:F+}KNKh:P XtQ| ?X;^A~(V/Wf%SVg7و_U y9D>wpIIk~FޟX;/[ˊ욹?[+*vZ_ %S^8P= 0+/-ӷ^Jч >i}r$S_Oi*2ފ"z%~qWbE>PP|EStEGIENDB`zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/_static/logo_bw.png000066400000000000000000001302301527014137600305370ustar00rootroot00000000000000PNG  IHDR;' pHYsodtEXtSoftwarewww.inkscape.org< IDATxweſ'*z/JE$XAPDP,"EcB'$T@swݛ[vw&a3;3LfO;Mm| i`S} p x #:~O\I@G>"lO}0&!3@҂fk-mhA$_9|\VOg;$D/V}8bʬےfOFO!\F?pgRFDaڡfm-mLhA=%\ '`U%m/nJNnCkafFDd~XAZgI Mmt IHpq&َdugr1{Jr "aljTN :5†>`;*02j;Hon* e{퇳6Npm' a n "h|mVCl-=cI>&96@4@((؜ WCIH&@5l?j{o"Ev Q5N~~ĹDk!=^3*ʮ6ʇ6 %]O [mOg֖t%Q}Av2;W$%%V"%Q{y=*Kfj-mm"#ͪ$|emDEp'HZ]߁ZnU ؾ gˀ%)ٽL'b oo!ۻ"@[ʹҖ6ʅ?֐4wh{vw%$b½;U*ؾvD@e"PbV:f2 83Q0hq@{D>eHn+(Ǡz? m_ߑ_J?Hپ6DK*%sA;rj>&V3O2ydžO؟HZS?Aw( 1nD CI%͞a?Dr$/i1= 4z=牂zPjm"zpJ9QEڶi; `" lntl?b{wo?ptw"42+ߵճr۷؜eETM3$ S/ؾ$G(AO$*r? կ/}H׏/IkBfD?{%}Նھd_9&m"A#]2T]?ARC ukcny J "69)#m"!ipJ o0gٞ i>I?&V'ja*y,~Ib:v[;+$ݚ ]}uG`:%]@^˒Z"`yg`I)'j$yo=IPҠt̵D#]@Ds"p /k$),mm" izY/CϘWTϰ$ 'V+'hc~MJTz?.bI;p*bg &TؖxOP^M KT<t0տhNopgyG%#iDWph.o V++$N;~&#|rVvtLt2MaDLpe6;F؏StES϶oYVi/b;p$mJ[=DPRHZJ9 @~JO耤Y$} s% |pyu o 2:Hŝ$4I ?Ib[mD^B- Fm@JT\+b`:rֿKo }ۯf;$C+)Ⱦ}SmN̾ W1"sm+>̹S#B[moDc}"%]/iVƴ&@b$*fWZgyߵRcV%ϲ}4%N_y8C$R;X81<.\I-%+QDTUA+mi>DP0rյ2W}nyIRbFU"N?+N+UO=Is~=O'dAV3Ոt笪*IҖ6Mv΢ !f f߰ tمh\7hi Y]7t 04/iY\Dp`%}w{,Dq&I:"rޱ}*q?!fAVgcۗ+ɃehA!iy>&WOf;$KQy=B&nm?퐴BfOw8-N9H!NeH*[ 矑SCp:jپ4.:E4 PY|&Բֵ}IHP-Ⱦ*nAu3q5?afrE~Y"|BY*ꨡ~IIJ*oG*})ؖO Dd(hv*Pwkjl͒6bfYf6{%%_TTz !OA%imD<.o"S(< 54ۯ8<*n &qDѽFJZ@[>hA 3N Oy\g{wۏdJP("fQ[logCIJ xhΘ,NE' }๞aOT֕ڣHGnKR"XI^X|xc$]*ilTM Fr  2 CII1nS6}[%ܓ&Ni$A[?7ڎО Bc%Yۇ߉zBB)7D 췉>V%JYhD (:l~ SkMmoo{tB=_̸Z@G Wтm*%%i۴ k 7\PC |pYΫm#^E:jHBkc&~B) άJ@eFD)^%T" M !.!(t"eKJ5諔 5tY!oßTu/Wo&{]^:"[.HL"X{I[iˌ6Gi?Jv9we*$,/ 'S>TN%~PJ@'!I>w:A#Jӻ7!ZRMg+K=sKWeu7qDr6;=̃B`8i:`=W#`SSvrhAp.!(#ђsmܷ%P5`:۟%rnǁnJDo%-J[g`P&1%1+#:e9~$&*yW};gY)G#. f$"iṱ7'Խ>tߊ|!;hSNJ#D $-H(OYҢt/-%rć>Q{q"B(C])TJ^7"gsٴHC]fqؾM[ vj*}o-DPI`s'$zìl?(WzxH\9,nN)ˣOaT%z.e90%eFHZ" "Chs._}:9꩒pE~S Y{nM~Wbu71* _}4B8f{Jfhfx"HѾ$\e†i|OFPzMbt<#neZuM&+TvqH좟P?]U1^F-!HE[Typ K"ݭ,%|@K ZLGei{gCxa.xT?R8%."bGS$+i4>3qJRDgÈ(eJex 8P҅` 3?N$mlPQɹ;AP:28G_ee|uv7&k]s~o)7DJw9s&ί׼8?~؏-DBND3E\ߧ{%Dc[1r+܀H F L,IS`Ԓu쾩A\~k4|D aʄAD /%GwbaG;P"ȝw!6uoRA-k5r]X!  M:"yJ6L%s(V x A5g3޹sOlݍ&1+wMO={I:G/4r_9KYryTk4r? {O:>"3/SYU/֋7r[d[Mt#PZd-ˀmSI;/gaBd-~,!iI)δ^?ƶwtgq%U( ?ۇI $!,m<{r:WRpG1Ht>BDږm_'ދ&+OL\h#>!ҫ>l$ \IyVm_6` ؾ3PjFk;X< ih"L~%:"s%[5Y~&GDҟȉKHc}oJ]_''EދU 7F%(HFҹEQBkI[H h[f\Omqu$]In`;ۛپ%PJ.%{ucI:D[\m\8{?LRJiU؅8#W9l]VW&4A%j5?$7D"<{K*~FHRҭ(`âJ`{}(icI!^M gWt9ک0z/CED ɾ}fu pOCtZ, jwf+R},WsM OHru4yG,%X,Nؤ;DNe Dζ>BvJqY~%IQnmqDKZFA(qf!tn8{};fBZ!T\߈Ⱦa´+˄2fDCgNn#'%f%𴤣LkD!"ZÍ&}O$]AUu=h`9z iDcLR fu(Akؾ̎tIK@p3>K<!\JX{.Ljb1$$͞Kt(n/iaIlߢY 8ʹ}+s7)`AW+Ժzί*cVLdD{&E%]Md{l_-2 >#-JA,nx=O$RJL ?hZ,nNIJ*itL^Kθ"~vI)ؾǡE|lWK!ym'lo)j=Jk $B^JURv,†:C;usI-" Db\L8Z/?S .$ t*TcnF[nVIrt1/?$zť&Ky#b<[%m^uDZZ] }ǒ]㊲eIJ2ɝ~'eUA@#'왻FZLda"PV;#[LdW{Zr_N IYG[#iBd:h"xh-Хg `{h85pAlе㬈:c5uL w ծHj=1pHزi8T=磓}˥:fC(Lvԋk4r2&\BhkHʒ.#L9S*/q%I3)>[cb0Ii qm_DpE |䟳klGZ%mYdI7B{~/͟*2UB.gyNI!{|^]={ A)wWqL(x8h=. KWZ 8/$-JjWqI!<9ϓl:"go]yk%_ #ϑ=i$ T4K Ot=N1ˊwslCʶ~:ܟ>W%~ؙP޳K*]v#N퓈GkRtL̾7&ۑeq{ *y({+{$"b#Enۛ1C}]!jW4&JS"f%ճY%}hA0.+'V)t=?FgCl`{ tD+m !(#IG*T;O"N DrJDs~$iUΑ6ȗ/r!TV9%2V9;꟱}-NI#"4HBuf3K2[IXYk}G@$DR@*)>'>O6t RN lL\OΚuaDjLz8I~? Srg5өr~Ba:۟#3)t4zv-(:SOu?Um i&I_"rEClf0Nˈ[0 %m-p|aM17PQ:D,_}<1PJJG[#cG ߓ4WZpH[!2&K:9V9W}} DȆڙ00m7DxbWvShy'$>Db++>" H :f5[oZhT6= z晴hڞh7}',>? +&1XpLmI~|'Jsg5 <漶fq0"+y6IwJƮDOHE`ݹyieOo_~'}@WD]'JodKysߚ56Zg!8Թ˛`3zB&C3N`ihV"?&ɇP"ȝwM\תڿ Y`>^D;`"po Rq{NuN5103dh/"ȝgh6TM8iM W\N'\RELxֶuV&*Eonp31{9q6!Te}Cۏ;J+8L~3=:Thzn3$m,?RXOT6 b87cx*]ew+AdRyλH9y%*LyDϨ$( ˕yqys\Ūp:4J-+܊cV:a74Օ$*1CCv%^Jc SWzHPBQ Yn];"M4.YP^ }9R{pZ"2ȶ=I+4 Rs&n]Lص59X0& qZxX`7,DpE'`00pTt]Ev'n&B܊ n~9T q*D68b%Qpc( Rg̻(6L+ad-DH-s/KhI;lHrnݒ=LqʥaP @{^L/Yhd!wEm>I?&ϣ e"|PZG?i(bvZ/QU#(:v71`ygvG5bBL8lEbQķ}Ʊ"=+Dͫ%DйfHHMUԂl?nv3i= Nq y@7=,L;&v-!r_&dų+ON :jM=r.^",?1X ݥhx>O_爕ڜ=vB't|&1AP,Nm9{4p^/128QO"s,O#]> gMwVMlε7]L'^%bKk7s bD!{ N3`f[ N3E`:캿h6Nvď0krߝ*HN&~'RKQh԰gX7Ib;[zY~DPu?q]]++,Wڅ{mH:^}^È`<1뙩EFg[3f|105A}Mgs YDP1nԵFsGtAdX>%zve.@ -vcܱ?=ǵFUe:fӯQ)MB'8f#edb%17L!zg\ubQG[#ۆ\4h~k$NU͆\DDo4Ca p"ȝkA |؟!QǞ@ gc{Ufn TV9 'ܹf%T͒/t̿DPtnLO lߋܗ` Um_`#I+zМԇs׃ eGPM 6l!n)s*kW_Q_},`."qay2^ j>ù4VAl%V;9̒ftBcۗ_%R鉖y(s = ¾+"Gv_xr:ۛ^dA瞠7DQ9WRthZVO>q% 'f?iel!yQ  > C:^s" /0ۥ?kFS`I$_z$a,TI :d@dJ+JXRf&'[IK%B;^IT(/uTմ!tI&=9".olWwt,A'K߲-b2I߂qC /)H^xך=Ol/ ʊw኶ t.}$>]B\GdVђ" `NIG+B DI?V`{Qkq _km7UgIZQ@x)Qu.|PHIe{'Idժi(FLntvwz>W~*l_:kBY~Mmb\fI߭oBY~M Sg[DTM>T;T(3|L ٶoT(3 a Sz/Qʍp '#M69aBCTbf<jԴA:_]a{q4J>;[8+w7Mss2NdqQzu]Q.KdAS?U)5@?u9Y"d QP]Ki-M@ɛteљޢB1^G#yrˆ wשLa2B7A9k<%sQDl IOV$Ot!\{x"#D;ih"EBT҃kvJkfp?,P]}D.yt&7FhOqy#e%[BƎZ J@c]1 {FJ}tB wluMt& WFjYQ!oP ʔ^ FD1yp][{z"f^9rT/QJ"CZHxM" "}gc#jU1JB`UޔsTִ 2?Ϫ +?nk%Gq{:z&jCOg[Dpף~nzw[XUOde:_`bN`/Oo z"'>/rͷy1Ι۟/T=矦fz?ـê_|%[_VAe5ZI\@=Wo$ eMKwUm bUѝy@ Ltܶ͏ "ȝcUBѩp4D;Dzw̞?o{3T=_7]>ӥEfoV5:& zkᏤmEQĤHVV 8b `abV]-x<F w]Od0y9=~WY"ȝkyCGD5~oQ]Jo"ȝkq"e!uκ5jk|BH>,"YNg\5qC7:$T1.:} `Qn`~^'O"ȝs}B*SJ7w.1~ȕkιדI~pyA>\aD;T@~d&uM4nΏxE "ةqzm5'J5[?Z-,I 0:BBk8Ν|jy>\D;jD|u~gM!{xA܇Cz mdê*2@snc{ wϫNmE969T ө}IKX-[ݗ́ԖHJxgJߦ5jsw4 7d!J'>p5BO5G~,7,imI"~!=.=PNy[v>ˁ+3Ǹ!١,$i sّ _eNIKH:Xz,[e%w_.6t qw_"N$eR[tm&P"᭪G( =D5&,#Fc$!IqoTWY<$ T m2TfϷe*4\Bvܳ燉fP8Uz =O$3VWW9tf" 2ysE3 /$af0 "$":[!&bf}-0YDP tlQ ;pG>*i?Ie2L< ih"'m:(PIݖq91fDǿy qZxN(z6N !A\!y!8 IDAT `DEVt1ޗ?loݠ0"0 . o"c1+cms{آ쭅1ߙ(+rKnj=Qk<熢D+Ξt 1{>(R)+'f$?{87 $ϿI9:-_4S."fx \H}C4^@}@t %DBI1#[UnVO娵ؚHIoM ~l!cdBY_1F/[XU0 B{BY_s|{I,JFmD3S2E8 Y*_o5Gޑ#eF bJS sܫJ^5sݺ e}ݺO7zof֙a6B# aof;(?O/h6LJ )]+k؝DDO=#VKec#OKI*]FÄi좣UTTmA~BZ7TB`53b?w+y?o@5.Oa6rhdT?3=ښA)isfk|@F#_k{pIQi\Wk{H\({y?y"x&}}z FʈX]^ :DE8Sh$UY9YzRtQo"x`ߺM(^~Z[)iK9맵M~MNj&,K6üt/iր؃Ӧt6X(x_] /&Hy\hT%]gJ>&ӆM,N*Mf?gI! {zaзs2xw#xxʢ);#ʻW#x2gd=}A4l27f@ۓ>&:~nPY7ǁTCJHRM|d2m -5p+in>I4R$~pt@%$i~G|D|йh8һ lIK8RkB FKR_[>)R~Zov{@'y'"TxLܾ<j˰&XH/X i*Yq^:E;yߟ2(v :k*U6:STabB,%F;'ǷI)\"xG;qsz-zswc4`*) z"e" j_0@TKgD0 eDK Bx2#!bpz a~&0"4%I}rNFgEm`"s/o,t!>مXd(+ؚh"ڳӾi Dp `U"]`7 9,A)t#UILt'r$\,SeR"ڲϚ??p21~p,J | NԎRks>AaSB7(C Y8Dp7}{XwsLEsL7<;.L2s?/*v=,&N=Jϋӿ@$7L/꼫Sk*g #˃ҾB|$W9ྡFu $D+KD:5sm_j" آ=FD{+n_ϋ:+OGׄ,0g6^%ēzyaU.'ܗYm`bJ‰ Ust],JGɈs0ߤVpK$ZIԸu込'[+KӞ=Okk%lFR{g$Q4DW Ù{q/˧OyA#.IzLuƔ/T4D;ZTzFZ,IrGS!.>AsSSAyl"K]ql"حt[F5aIyjBwZ/o36"@op(U}ʶٞDYqԮ 5Ν2lWHnݏ{7ΕuYc6o"ȝ+Br{AşרYDpU[+}=ښE}˳,pgϽݚI 2&Y5m I+{q/u>F?ݪ:6zAg"]cGt. t~(Џ5r׼Z@%*}w^z #:y< $L`:7?{5R{]^h"k칯[3` UAt;uW4dySIcmѽETdךsJW?_7dY*j:% ?Շk5rޜ jW.M^"ȝ{з^CCYeJ+h-HW2aL]H'u*I Ve mI`(Z`U@'Ś/ΡT$q>l= $]At_ݝRwMzYl^%/9&lY6s/wƃv)-TzL 2ۤ?$&ƏJb{סia18c gSi$=PzlKYg>_y=sDG@Ҽk2yJ)a0{R^D*qAlV HXҵ|(1F?q?.C""Ë;HF|#%]5LstmK̾ Hڱ$"i3=Ah <"iIKaDB\s}RB~|;۟LQ#h&nmON򜓉xh"C"Ck 9%};+"h]q;j" M-$HvGٞ؏kOW]߯;Ɂ$Mjo9E!p;w ,pK"OůIڭUN _-n8Q"]VdXC칖Z=pTLevwT%ixLjk/#k߱A&n WFzف HY# 9Ջńnm$NloDI[fmmlAL>C~Q "")kt_0ř8MdxuWqc{, B^X1-Cd$qCO}uqtvt7QXXk@{ppr#N%G4Y+/<!/#Dђ)it@ڊhY1K6"x=dyÞHIY͞>"xL9ED̯:î^A `8 ,\1ll`$Jخ$jIt^ ؙhi]4lHmeb2&D;u56x>:x;RK5+B6ğԓ_FDd܍I@1[=*I!VOa*fu]L`FM[ݷc+󋘚f/R2._:70ϡk.l5=Ap" N U){6 P sx@58 o)0 ~l=#Y}d*dFR'*'J\>Ͽ cVagc4E<4 :3`0D5?c &`6< 1 < )y .;PyK œ,`s@0ػK ˍ;Z ؤ뀍n;œxX`t5ɘ | $a>󆑰t X$1 F=s,my[cj/ONtṖ޽Ԓ 6sA"`YnJ/"7vJ/92>[XKD~h94~;ƹAֶzY0UʁԯQNu}#nKkf}Մ,]Ǒp,_31Kle^=j:smD`הv}#:N}M@45ihVe/2?^=EkWyj] O`klJQv`8aJ-hb;gǩ:5!'y)&(C ߺotsA=,TOP< /y. :i @$fǥzkT4OW`-&u%vR ?vExڤ>y'mf}t;m\xAAY|3{pNF193asDuw Sɳne"ۙ<&%o;0۸hV:ҙs5 - g]C+gx:y}@(J>w6LM{ֹWT\G61zbS*9oM^]=gL/8B3:  O1&/#4#? ZNYscҙ|ys]֍F;0Vݻ)ST6?7-4`j0aXI?갎4 Ob!m3ԬS/^#=G]7:( sݎZr}o^1X6 trfeq.|=:@pSDѨHhU`zQ0ؤ>o ;+>^ 1 @:]Jjd1;Q"j,-Ldqձt~ ̍#R{QI7Kq&@ib!&mOk`+fe;Sa%*I[Ž3hBE[?DFaoKuQO6>JR#mܗǗ 8gu9a69 I1cA9T{G W`v>F`D=A٪>`˔{hܾ@uNب~/1}7fg-O~\zc]V!,k㫐LcowJ!f7|)y? _i;$R΢nE=gi@0:à"9{U0wKv^~T*QGs F`;bQ\`#2T5&_FD.G9Xؚ1hζM{Rnrvt[\¾Cqrgk`:PW2ԹfeOUy{*jrƃ-UD-"sBrэk+NTuOq"rl?a^{Ն>{3|‹Yѭe[D#KAۯs&Qh-=ϰҾXBxDd]}Z(|9܋l*_ZU/P眘A8/S+38@SS|3hHYrܩDn*%"wzMXUw8KU.U<.66?!"r)zn**ZU$P~rm"%1-RED6aδ$lt݉7cYdLwE1)\Tum/"W#`"rl6U@w[q܄ƞ#!ߦ(.XTVU=;X*`5Uc?ueEp5~sY;rgv" ni1/ߡ\ȳ?i=owwb~uebϻ`ܤFN%VDn t$nW̑n&8^Un?_D\DDUoP 1­˼xN]YUx\DB2S~rFg)쎥- xՐ|ADnż?6M9R/bdI-`goϩOs`7"2n>_cQ;aEDVMSUט ,0Yȝ/̠5E&^D6(uv9`}DD>)"sPa}|wM%(*G!zݬ, "ce70eU4M\e@#܅%ZrۘzNcc.L $MKw*g`n^&'߉ )]Tu *PDd-8p,{f(c?J0cm\)HWs AD1[}6]FUG"2 S̕ee#pu,՘671ak~EGUC}ܔI_]T u~  evl?^DVO%"lmP< "r|"RDlVYA2yLD~#"PkUj"" @S_d[Dʧ> =L׶{3l"f8K\63JOv"2DD+6=װ~|XW1<9/~o\K)qvEhWU0я1{S"r,EO-`!"GF睱\[IQ~`m0PZ >z~3'beHeT("+O4e)X׃JYEd{؄˘}foL[lE-sgU}3b襃v&Y3!pt3TVeb_YeǕU|9H\-V|cq$b*Tɍdmjwϐ}^#矩\C>vI*bgEd]U)e"" w.qҹ)\*qy}XҒMV%ލx P;NúvMly4WI1gȍT`nآy'$aoߐˠy4r0xύ \qQlBI@-9FݱI:Wi1sN uR?Qad{L}r!v^VN^QUwW`XIQE= dłnR]"6KAv"2W`':tH n6뤝@NTջ6U[OSTS2Tf|TTuoˆW]DdV]}jrv^E)j#10+[z`p3EUvrz}ʹy`ێ}|$n;:X,=D9_Ի2 m ^.Ŏ آ2 p"o\i00[]9}p.R똼UC̰HNa+4ͨR'̻ظ`/lXjձNVP,yMj\s;E}iדL#yAFHXEΙSx9sxV`;˝u+C۫{؂Pl15$o܀3O`zsh#,0I|!IE^W@p-ٔ{X [=y@mn༅&܁}\+ar}Nd)kD̹ KFuٯp;k 4Բt- MTu];C+>,y&$oPU{> 9Ey. BF.%o XA?у+MF 0Y=7*uX Mx0ZتzNKDm|Té7OQO9vC:uWot#MQUb;ϰ8rswCU͘3N͂ǿ~8a݌:"Ra~2 6<xπ hV`? , r?c`m(Ẹ(Z(6^t5XTu[U}b~G_&h>wEdu+F?WakO~JTm(p<:FUl wajDdj9L|CDDU_R=1OIb-ȡEdg .vD(⤥'N 8Kڬ펠9E^I'N5d'oU^YUN1P``gJ-x+Q^:ק^MtN?vioo!ro4 MO;Q׻ uѹESzr1A ɓ1!hsGp=nEd-ڽ2N{iz~8UTې'w93|SUM7vqW/Wv [zW:7qK\P\0j(eiK64;9{K}}|@UǪ6,3"qM1 Q# q$)Fv&?[h/Fn T0z^O."b.rSnvgU3m "Ww0X-f;ޫSռs`;c/lFap(ˋHsyj0[곆'@UP/Ӏ47XGx6sǟR`1/'Ed)Xr%zX=9lHDf @p'nEd%6nC 9iyd|U V8]jUU&ȩy#TXt7{<^y ˅dn軬s jIrԵ1p$"u[0oTjx4-"2^FI}('@USo`;'.X)4 s~skw|&7`jرW 0o Tz_y>y `ҟJVkD珹X \=)~%UDvEDl3>&/aY䗘q<9DdI5 [ٕFx.ƀOl+sF ە螳{5<z9T:ф~Kla1IU6[?hX:)Zg|NyROƂ[^{ =Lu1Ot0p{6:^5F\07УgI:}~ |,8l6{ kȍ1GD +߂5vQl]L0Kñzt&S khAws܉cvu/K5fo+h {Sa4KNPLB6?\?l\ Surl)ob.g<+#'[@1[INԄi՗$FQ=/nK-~1u[QףwFZf>]ds),eXm^(II{ Q3˯BoM {~FybapM\6h'@?.t] |6z0u#>-> e,z 3FzTsyUGU_la6>-LR0{8\Px6Deչ$"si-i11]Tml?D$YHGQqXD|VEI*FN,kGLm7;9}+)G"UT]*"ct?V 0X%q6A{K>--duy|LVS3T#j5':?P|؊Y_DZX9]D84DX_lr/a RG2 SȢjRrIDV.P x8#U-."u.T}71/ae^EUwVXqrYr>I9CUn.+"{0U}GUD"" n^r5f=#DdUg`p8.LY׮=]Gz!s>*"8HߤH nd˫mEԧ4. {Kl% ],( 9:K/~#LQlS^#rN;܆IԅᣱLyDs fG#Ryck ωߝTuo#Hujap4?ʢ1x[US!9/v9`ivOȟEl@`YU=Z={ß\Vnc\DV.m֪QD61~M\ӛ&"},/d?Ezy9oZnݱ03"X7Tl2= MTLPvϊNhvAgbk8=slLLe~[UDd# PPڕU˪zQD>-/qY6U1CmY,$7q=bǼ\t9 S IVՋQՇׯ9^\馷st˥4J!~$aڙ<n}fC8zQWTul¿|VDK1j@oL?"r'N7>dk6:MD>+`zמCYV~0ض9O(716w6똧G9$uIT>`x&"ϩ.)Ɋ:U}BU=H0RDfqc;;T^xex+"?tиmr.3MdѠ51pR^JPD=svR¤sQ>aYLwbmq SsCuPNC_3SUbovtX;׀2WgfJjɧ'_׾Lm U%c !; mӷnem)cWx+܄ֈ!"TY[S;U}-眡RUB0]jn""C05f^4INS6j-tMIU?ݕcDj,] 'I_d8Xs, 9cމr v9ޙC,-7鐜er(0F51yWj4̟<<2feU㨝ȸΉ|SYpۤ 3֟V{=<1,dHm/? 4`]w`A;P}:u~$uӁ"  pNH9gL߶i㯍r0qNJir. ލǗvS4ߩ :ԟI5Mu[zo,lg|Pl UBH15)=g^UO)* nƶƹ V\UO:B?TZ5_ñ08*lz[t/sB+~4йn7")!/}AYdXqwMBé^ $cgqr/ȭ"RLwIZ_Hܷz r1Ꝫ%$d :YD@/GWxڀ^D vo^,:'uCOE%\l4>>4էM?|!A Ys<: e_U}.ώ-hבX,w:8F;@ΡC )p\."hy*ޯK&`1"7u">[rH,.'0yZON+,A͌&ǰTuFgYʭuj)瓱ޔ5ZrE`lpɄ1X`ap^9uZ?EUPʩzx( F(ux#1qsbtʐ\mOlGX~TzUY.`y[U g鿈ccٶDp,DYr5Xn!͈8fHBΠxHc SSLI^X<r>XY{,S0='ՠ«U=cmU'ۖ*P19oay޾'4i7ƿ77XkJLy`q 7zj,vu$q&yu~c 0IJu8Z/EA; kV A\=̉yDq) >|OEm}MlK\>SfsbiI?ckn6np$`; +^Ԓzr 'Q|R0Db'l0 s_%:$^H>W)gs5)@,F1kt"X NWUr Qm( x3Xvv SiFQ }04l낼>1sh:SǶz0{a/:gg6s"S!vf.@?ph4&n~w ͎Ss C7Lt/.54i=c%o Ixɩ_)6ű'0% IDAT#ð߷9KDFtoo<*m2N\QC0/'iJ}bf2"Iw}n@1lE=x8 :>'5wΙSo\ 7BHZϩՕ z~~} &aa 1ف`{ÛCvcx$n zFEmu.Y'OֹSN^vaʶv4ZM+1>w $fd{X`nI/1a_ѹ6z\@!]"_ ႣR8j2Ap:pF/-dHiܤGyyfǢ:̡~7mȦF<rG@$rǜXv\ 5 5 c-ژ` oC4,J)|sMQ/CY;@Jd=$U P{Y^_%\S:'!;z3ECbF= C15 9fR@1\sc7HMQ$@V֘,<@0ծ З75c5k>Π%I|L.FotYww5s}c1Liiw^?B-m"+(`a [?A,n|65L~ҹ/ >2Z)>$횢]8t3Z DɬVIiCRELHzy3x 7gӤ_+`oܗrcc 8cI$,]qZ`ie1q .'1:LWY {ݢmD|b>SkحY$b`6IrxqB .)=4X5vPQ>8rфB_\tB^aWz&$j+qG$Ĥ6{~puh|)F#Ra .V`I,tfkyXXӀM3ֹ%mQY$]{ S;7*Apl]ȭÝ2UJP j;"b~0H. ^baEmL&۸@5Z$DL8vyqژ sZ:>b65y Koy\Ӄ?P  B?@=9rXG$;X@\YƦcIj % ?6Hi|-L=MIYaܾo@d,%m"e;Sj<U5 9$zOw:9H7|AV}M6uj`Tu)ouHlG! ,/px@jn>iT u4Lw1f^wm8:>L.-,~8l܃vVvJN:=hɓm=cN'"{ xG=dԍ- RWMQ?ek{'M`.|a5Lډ]ʩS--!y@P }KMJJw\A$;Sˆ19HP5-Ap|V,CxX%^w-8m>8υ:otð#(v`y,l[خlF`WUB:b❽@ϳo ys`>-{*(?!a+lhuX1/i)5$wn׻ygLp/&M}raicrmh6ha,[Е^gWy[`m:סqAC,Emw_ G_m9^:ݍ:w  rP IR/,V (ՂI7Y8Y1?nM^M̨=_/? 5ޝ]ڣvA:5KD ߐѣ@6~l!ݮVGiIxpDM-9} z}"8>.T|0lgZ,+@zp7},Km &c9Buq\Fo`hK s Em#NX[ F) `.yjMٝ׷)yR}G;O`ޯr/ ׇc|,&^# nG6}- <|Z z3{%9v?IIkaPj>>R@пL@3f OM;? 1 <oos<\D~Zs!!q`nH1^,ݜl( hVƬ~BzSו/zN県̓y:3:h&Lr}g_H/6vBR[+C<isb>!ߙdd#g2j126y AҞ_f Y_ǼVe-BJs XYg~dm@e!5?;QU!$S>`*Ȯ>U%@azFP4x> n$6>4NmJC>rަ_Z=J8bǀp-s@01c19O]JNF] Jfyh a!Do>{H_:s!jc4S9(,Z ز< 4I=(|!gGa`~rg&0u,#V6`TZ/6&$/zޙUO$[a_Ƒn̍vRAb>B do8KӔ>G6>SMcr"`zL*azLͲ,ALAbAEa;es u,.MkŔTf\ A!HVvCj3!!OD,@pNJ/ 7a{|nCo2^V=a.|jAp4̮/OsChH2w yv1,khHNS1oq Kz4+azC?``M,㢟 _P}<^TY fd ?̉ޚ'R,Z5|<灎KSf !# q=H>΋r]Ӏ;jwoApFM27va`);%xka^eJ֠aA|\zţk8|Fie6*Qv>T[ Ga\tt}p;nL-92@;6DEXyTxhStY99jxtX.Z_CByBi˧LQbI`B"p0{%mPql}F?AÈ}% --,|6vgs`ѫ{NMoppZ@i ~>t ^< qtިu09xNT^d2ps6Ki'F@qz/3yM'ȟC;sQ-vu2o >8i"ӱE:E3=z..qYf@ Gb}xՑFN?kM>ϲkbrLnp*nhev`>1=qoQ;gE\Q7b0}z:D<`ǐFFB & eJ;S$ ZtNHN ]f%' xK6A#e 4gt}O_B}@`.҄:F?97~dVxT*RH8c'28=1w@lW+fk(ރ@")cev`]}e*KVb =PSDlQ;!Xj a@N, ac@1 WƖ@9gǂT[:~2 hLbhL=jҨ27;< \rkdBw2,12Ņg:0ƂI&%ȨQyѹ濍@@&!& PeX,Ѧ/{1Cw6jΞ#c">lr ziF*[r֜|g*6~,UtkjIb-^vϔl2Sk._ |AmO4_tlԇp|lkq >hp?fy5`v0%-t6i!@O (cpR=@:25c+Iks7_@sWB8JI2c0OxB!Ā:-݇Es  N>WjaG[߁ 8/$X880nA }pw_3Z<C~@ijjwyo$jv>N@1vEu~tWƂEOƈ3E`W/{9@?NM#rOGG%@pf<UQ09m)s\:q^-y1;X8]@p׬mv؏PSK6[78>;>wtaMLv,6rtk$ح*3۷g?뻡0+M>-aғw'ΦN [q*G,ƍgC;+'=Rv`tpͳ5@>NmꁏβԦ3V*2NR,Vt?g@ [ITs <BV Y3} iP(yD7l׌v(LG5yK,nX3p^2]F:'{hU-Bmd76= /|)&$ ?vAOꝌsd]"Va}7!}.{OTgKm +;^K4! IM 6vyAP$Aot\@#7V@KF3c}"ى@~#N,U>R)91a 3 \}ׇ(l}1JP{d1,˽-ޙ//:P( `/I7f 6 ˛Xg?U;^ ޻+5C؝Z`h6z/j =yn ~_ s+ *wk[!d Pl 3v6aX^wϩATOa:a0AKqS,yxUji@$p6qF?$벯0Vh&`.|Ϫt ,mz$Ir=GCXm0/,A=Ob*@+x޷q1_Un H*`T! 4eHLI!Vtd˫!z|ј`L;NU3NVꙪ111K;eTOٺY~ EjleyUN{+)LTUK2/𜈜'"KDOQ1ϜK#hY/\fT%JI|szVRQg),a/39QU uIh<!;}rKOva"܉YVSߧ]SI$T|U; C8]D5S^@0DtY "܅0|X^?TPtjS2\gCy3/gw#!Q pS},)hG!ZDҗ@/>!he(RzVq|+ y Vt; *i[T5U=R3b@p;`po@Iq*(?/ (zJ!)8$`vB@)Kt bK2Cy-þwA+ ,ͷ8@X<: NN=$cTPvvxȻ6^+)KΨ6B]@ N3T0@0l"2|::۵*/f1) 4 niPy_fm{.ʮUIi$ J @/ fxm&"bju#҃:K#YEd$XP([WI+a솒NC@_GD2&IDAT~R X5X[U.L%v"ҿ3ƅ6RտVc2>{b w; 1*X#C| u^x*K hPv"2qY\mCKf5+h/; u~pB.!zT obpv?Fp^U},B@DȞl =<:.`KU]WUt%YT9Uй8 /"NjȂSU"`a ~UX"p vc1PJ*UFc*aO3mG]2vn1PU{+lb6oC8@X;?=8A-F "sXʇ0Pn6QT2I`gsc0NDYP0k܉`B%=#p/f9F Tu3U_̅9Dl~R,╪Q͊PUoWխ1k01&+@t2 @Dv0Gr;u*-NuTuku<UZxsœJz. Dd9GAznU?*S 7Cpۋo2 l ,r=nw~% 9;/mMu+DdVU鸾<DdX{XD(U}+sL`?3ډ+[,%w"ӱ-Ѫd+d0 RJ ,?cT3^LDvf%S|FKDdj=ŗ\7uLmM4TvJ fߋaY!x@\)$7KDdML=}'h0ފKUu|vK?:qlT QI%^RK<`F; !"ñ~wL>p!pڳ+EDd>,oQ,ӁX;EERHWP3|ʇW(U}]#.}nc * _DrQYs6aLtǩޡJ:@m/LwSU})UR`pIT޾,Ӂ׈7V6 HW.J*) ORXU}ȾUҾ 1o1ATVI%e~h,2NRY%K534UPp*PIl>lhR* "" cʷJ*pR@,/".MTէ[%TRI#i1SnKSJJ(ITYZ2 ̃%G;M_%$J ^x g<C%TRI^2ˀ/m,Z-#_%TRI4#"]/ epl/sTRI%386 hY/\$:ZTR ^xiuPJ*?""CP c6 X2(K/ TT*rKGaia40܎!._TRIdf/3x |K>ZUo(cTRI2 /H۱ WֱJ*T Xe/ 8X\B% rVfj UX|'p){%T26`/qJ燀+@)4 R$c'PB% i` 4[aQT2U^T*R@gRA""|.\3w:URI%TНT@Ёpq*@¤`mH]l?~zET7iUhRAFqpN_EJz%K* (Xpׁ_;EJ )LDd5 ,Yv}nrK%Ys;.pTPɀfG* (*۩0(U}ȾURI#iW@I"",CNQ7[%x``JLDdla"pZd*y- P0}B̆Jq=dfppbG* ""KPT"V T@0HDDvf>.VЧ p`,w'?aǿA̴y!MN lH1/;knc bah ;*14mj166hC?{S:~哪[om7s {OK+`58$k}u 5pLw.ҤFJ>A @3( WM>D'.}smR] @(-af7w l".}smR @;)-cf7 Zw,$Bƀ3Xh r-A.E`o˹633b$3ES0\OI> vSZF!lz}FAx8Cؒsm; LG!I AX,w7d Lc@@J~5 Qw'd @fdfo<=nkr ̅B Rr4Vl IJx7 RdNl/*`8:ǁ!w+Z`I+2+ ̋ \ Ǥqk6 B R*4 Cx5׺N^R'R.'K .l k 3XG7Rd^)38C8>'ӴqF B }QpBN쩬 AIv]&H_]o6l5@E!(| ա.m(2h T!ci'^?wgG=(g dQV>Zv!ϸ&HV%w.4@HU(R ӽ6 UHLsxɶRt\ F_AxNAH9i  HR9+  uHm FQwߙk]S)R7 N (pI}ls?3KZRRwl3 L]'ĥ+@jA!3 =;AxUi 3;xbـ3z@F!1D.crtK\F 4B ef+G`6ϙ O( 4F Owuߛ Hk٩2J`wg!`RT6Ruqx0=`p~mD6 @L!2SN;gxH(zfv @?v[()[ IENDB`zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/_static/source-version.js000077700000000000000000000000001527014137600361002../../../source-version.jsustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/api/000077500000000000000000000000001527014137600255255ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/api/boundary.md000066400000000000000000000000641527014137600276720ustar00rootroot00000000000000--- title: boundary --- ::: zarr_indexing.boundary zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/api/chunk_resolution.md000066400000000000000000000001041527014137600314350ustar00rootroot00000000000000--- title: chunk_resolution --- ::: zarr_indexing.chunk_resolution zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/api/domain.md000066400000000000000000000000601527014137600273120ustar00rootroot00000000000000--- title: domain --- ::: zarr_indexing.domain zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/api/eager.md000066400000000000000000000005361527014137600271360ustar00rootroot00000000000000--- title: Eager adapter --- # Eager adapter `LazyArray` indexing returns views. Consumers that require indexing to produce data, such as `dask.array.from_array`, wrap a view in `EagerArrayAdapter`: the adapter reads each indexed block through the view's reader and partitioning, while the view itself keeps lazy indexing. ::: zarr_indexing.eager zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/api/errors.md000066400000000000000000000000601527014137600273570ustar00rootroot00000000000000--- title: errors --- ::: zarr_indexing.errors zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/api/grid.md000066400000000000000000000022421527014137600267740ustar00rootroot00000000000000--- title: grid --- `zarr_indexing.grid` owns compact chunk-grid metadata so indexing plans can be constructed without importing Zarr. `FixedDimension(size, extent)` represents regular chunks in constant memory, including a clipped final data region; `VaryingDimension(edges, extent)` represents explicit rectilinear chunk edges. `ChunkGrid(dimensions=...)` combines these dimensions and returns `ChunkSpec` objects whose `shape` is the valid data size and whose `codec_shape` preserves the full codec-buffer size at a regular-grid boundary. `dimension_grids_from_chunks` returns these compact dimensions: integer chunk shapes become `FixedDimension` instances and positive per-axis edge sequences become `VaryingDimension` instances. An empty-axis sequence of zeros (including an empty sequence) becomes `FixedDimension(size=0, extent=0)`. `DimensionGridLike` remains the narrow protocol used by the chunk planner, while `EdgeDimensionGrid` is kept for explicit edge-based and coordinate-origin examples. Zarr's array implementation can later import these compact grid types from `zarr_indexing`; this package intentionally has no import dependency on Zarr. ::: zarr_indexing.grid zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/api/index.md000066400000000000000000000103231527014137600271550ustar00rootroot00000000000000--- title: API reference --- # API reference Choose the guide stopping point that matches your job before following module links: - **Use lazy indexing:** finish [Lazy views compose](../guide/index.md#lazy-views-compose), then open [`zarr_indexing.lazy_array`](lazy_array.md) for `LazyArray`. - **Integrate a chunked source:** finish [A plan is a product of per-axis tables](../guide/index.md#a-plan-is-a-product-of-per-axis-tables), then open [`zarr_indexing.chunk_resolution`](chunk_resolution.md) for `plan_chunks` and `GridPartition`. Start with [Coordinates are addresses](../guide/index.md#coordinates-are-addresses) if literal coordinates are unfamiliar. The modules are layered: the transform algebra at the bottom, chunk resolution and the wire format built on top of it. **The transform algebra** - [`zarr_indexing.domain`](domain.md) — `IndexDomain`, a rectangular region of integer coordinates with an explicit (possibly non-zero) origin - [`zarr_indexing.output_map`](output_map.md) — `ConstantMap`, `DimensionMap`, and `ArrayMap`: three coordinate mappings that preserve order and duplicates, one per storage dimension - [`zarr_indexing.transform`](transform.md) — `IndexTransform`, which pairs a domain with output maps, plus the indexing (`[...]`, `.oindex`, `.vindex`), `intersect`, `translate`, and `compose` operations, and `selection_to_transform` **Chunk resolution** - [`zarr_indexing.chunk_resolution`](chunk_resolution.md) — `plan_chunks`, which lazily projects a request through a caller-selected grid, the reusable `ChunkPlan` and paired-transform `ChunkProjection` values, and the plan's factored form: `GridPartition` (from `ChunkPlan.partition`), holding one `StridedSet` or `IndexedSet` table per axis and `joint_sets` for connected index-array components - [`zarr_indexing.grid`](grid.md) — `DimensionGridLike`, the Protocol describing the narrow chunk-grid surface chunk resolution consumes, so that nothing here imports `zarr`, plus `EdgeDimensionGrid` and `dimension_grids_from_chunks`, a concrete per-axis grid for callers with no zarr grid to hand **Lazy arrays** - [`zarr_indexing.lazy_array`](lazy_array.md) — `LazyArray`, a wrapper for system-memory/basic-indexing sources with lazy indexing for TensorStore-style deferred indexing, plus `Partition` and `parts()` / `with_parts()`, which determine the boxes a read is broken into. Device sources require an explicit custom reader that transfers into the supplied system-memory output - [`zarr_indexing.reader`](reader.md) — `Reader`, the backend execution boundary that obtains the values described by a complete transform; `basic_reader` serves conservative duck arrays and `numpy_reader` is selected explicitly by `LazyArray.from_numpy` - [`zarr_indexing.boundary`](boundary.md) — the translation between NumPy's positional dialect and the transform algebra's literal coordinates **The ndsel wire format** (see [the guide](../ndsel.md)) - [`zarr_indexing.messages`](messages.md) — `parse_ndsel` / `normalize_ndsel`, the pure JSON→JSON message layer, and `NdselError` - [`zarr_indexing.json`](json.md) — lowering between canonical ndsel bodies and in-memory transforms **Errors** - [`zarr_indexing.errors`](errors.md) — the index-error types this package raises, also exported at the top level. `zarr.errors` defines classes of the same names, which are different objects; both subclass `IndexError` **Test support** (needs the `testing` extra) - [`zarr_indexing.testing.stateful`](testing_stateful.md) — `ChainedIndexingStateMachine`, a Hypothesis state machine that composes indexing steps onto a `LazyArray` wrapping your array and checks every step against NumPy, plus `apply_selection`, the NumPy model it checks against - [`zarr_indexing.testing.strategies`](testing_strategies.md) — the selection strategies the machine draws from, for a project that has its own harness Every name listed in `zarr_indexing.__all__` is re-exported at the top level, so `from zarr_indexing import IndexTransform` and `from zarr_indexing.transform import IndexTransform` are equivalent. `zarr_indexing.testing` is deliberately not among them: it imports `hypothesis`, which the rest of the package does not. zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/api/json.md000066400000000000000000000000541527014137600270170ustar00rootroot00000000000000--- title: json --- ::: zarr_indexing.json zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/api/lazy_array.md000066400000000000000000000053401527014137600302260ustar00rootroot00000000000000--- title: lazy_array --- `LazyArray[...]` is metadata-only: every derived view keeps the same reader and composes its transform without reading data. `result()` allocates owned system memory, then calls that reader once for each projected part. Rectangular parts write directly into their final slices; advanced placement may first use an owned dense temporary. `LazyArray(source)` assumes only basic indexing, while `LazyArray.from_numpy(array)` selects `numpy_reader`. Both currently use the same slab-and-gather implementation. The built-in readers lower through NumPy system memory and support sources whose basic reads can be converted there. A device source that refuses NumPy conversion needs a custom reader that transfers into the output buffer. Derived views and parts share their reader and part views may be materialized concurrently, so stateful readers must synchronize their own mutable state. The key's type picks the frame: NumPy keys are relative positions, `IndexDomain` and `IndexTransform` keys are absolute, and every view keeps its absolute domain. See [the guide](../guide/index.md#the-keys-type-picks-the-frame). Every public `Partition.view.transform` directly maps that view's own domain coordinates into its raw `Partition.view.array`, including for non-first partitions; a box part's domain is a sub-domain of the parent view's literal domain. `Partition.projection.chunk_transform` intentionally stays local to the selected chunk. During parent materialization (`view.result(parts=parts)`) the reader receives both frames in one `ReadContext`: the public global transform in `context.transform` and the local plan in `context.projection`. Every view retains the source grid and plans its reads, so `part.view.result()` also supplies both frames. Its projection's result placement is relative to that part view, rather than the parent output. Further indexing and repartitioning use the same source-global coordinate frame. Even `unpartitioned()` reads carry a projection for the single source-wide cell. `view[key]`, `view.oindex[key]`, and `view.vindex[key]` return lazy views, and iteration yields lazy first-axis views. `result()` and `numpy.asarray(view)` materialize values. `view.write(values)` synchronously writes to the original source through the composed transform and returns `None`; `view[key] = values` writes a selected sub-view. Writes require a writable source. For consumers requiring eager indexing, import `EagerArrayAdapter` from `zarr_indexing` and wrap the view. The adapter delegates shape, rank, dtype, NumPy conversion, and tokenization to the view, but materializes each `adapter[key]`. Use it with `dask.array.from_array`; direct lazy indexing is not a reliable Dask block-read interface. ::: zarr_indexing.lazy_array zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/api/messages.md000066400000000000000000000000641527014137600276560ustar00rootroot00000000000000--- title: messages --- ::: zarr_indexing.messages zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/api/output_map.md000066400000000000000000000000701527014137600302410ustar00rootroot00000000000000--- title: output_map --- ::: zarr_indexing.output_map zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/api/reader.md000066400000000000000000000046121527014137600273140ustar00rootroot00000000000000--- title: Readers --- # Readers An `IndexTransform` defines which source value belongs at every result position. A `Reader` defines how a particular backend obtains those values. Readers do not define indexing semantics, partitioning, scheduling, or result ownership. `Reader.read_into(source, context, out)` receives a `ReadContext` whose `transform` maps zero-origin output-buffer coordinates to global coordinates in `source`, with `context.transform.domain.shape == out.shape`. A view's transform keeps its literal domain; `ReadContext` re-bases it to origin zero on construction, so readers never see a view's coordinates. Its optional `projection` describes one planned read. `LazyArray.result()` always supplies it, including for partition views and unpartitioned reads. Direct callers of the reader protocol may omit it when their reader supports that. The projection's `chunk_transform` remains chunk-local, its `cell_transform` places cells in the zero-origin result buffer of the view that planned the read, which is what result placement, and its `chunk_domain` describes the grid cell. The global read transform and the projection's chunk transform deliberately use different coordinate frames. An implementation must fill every cell of `out` in place, preserve the global transform's exact values, order, and dtype, and return `None`. It must neither replace nor retain `out`, which may be a strided writable view. Backend exceptions propagate unchanged. Derived part views share their reader and may be resolved concurrently, so a stateful reader owns its own synchronization. Reader wrappers compose by intercepting this one operation and forwarding the same source, context, and output buffer to an inner reader: ```python class RecordingReader: def __init__(self, inner): self.inner = inner self.calls = [] def read_into(self, source, context, out, /): self.calls.append((source, context, out.shape, out.dtype)) self.inner.read_into(source, context, out) inner = RecordingReader(numpy_reader) outer = RecordingReader(inner) view = LazyArray.from_numpy(array).with_reader(outer) values = view.result() ``` Both wrappers observe the same arguments, in outer-to-inner order, and log output metadata without retaining the output buffer. This delegation pattern supports policies such as logging and caching without library-defined wrapper primitives. ::: zarr_indexing.reader zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/api/testing_stateful.md000066400000000000000000000001041527014137600314260ustar00rootroot00000000000000--- title: testing.stateful --- ::: zarr_indexing.testing.stateful zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/api/testing_strategies.md000066400000000000000000000001101527014137600317460ustar00rootroot00000000000000--- title: testing.strategies --- ::: zarr_indexing.testing.strategies zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/api/transform.md000066400000000000000000000012651527014137600300660ustar00rootroot00000000000000--- title: transform --- An `IndexTransform` is a function between coordinate spaces, and its field names follow the function, not the data: | the API says | in array terms | | --- | --- | | input space (`domain`, `input_rank`) | request coordinates — the result being built | | output space (`output`, one map per dimension) | source coordinates — where values are read | `output` is not data: it is the rule, per source dimension, for producing coordinates. Values flow source → request, against the arrow. The [guide](../guide/index.md#a-transform-points-from-the-request-to-the-source) demonstrates each output map form against its NumPy counterpart. ::: zarr_indexing.transform zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/api/writer.md000066400000000000000000000012161527014137600273630ustar00rootroot00000000000000--- title: Writers --- # Writers `LazyArray.write(values)` and assignment through a view are implemented by `write_into`, which writes through a transform using only basic integer/slice assignment on the source. Independent affine selections become one basic assignment. Other selections are scattered against the source's write grid: each touched cell is read once, updated in memory, and written back, so storage round trips are bounded by touched cells rather than selected elements. NumPy sources receive one fancy assignment instead, and a source that advertises no grid is written one element at a time without reading. ::: zarr_indexing.writer zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/design-notes.md000066400000000000000000000410751527014137600277040ustar00rootroot00000000000000--- title: Design notes --- # Design notes This page records advanced rationale that the API does not state directly: how this library relates to TensorStore, why rectangular selections are a category rather than a fast path, and what is deliberately not implemented yet. The visual guide owns the mechanics of [literal coordinates](guide/index.md#coordinates-are-addresses), [view composition](guide/index.md#lazy-views-compose), [chunk plans](guide/index.md#a-request-becomes-a-chunk-plan), and [their paired projections](guide/index.md#one-cell-domain-two-projections). ## Relationship to TensorStore The core is [TensorStore's](https://google.github.io/tensorstore/index_space.html) index-transform model, implemented here in Python against NumPy. The visual guide introduces the shared model in [Coordinates are addresses](guide/index.md#coordinates-are-addresses) and [Lazy views compose](guide/index.md#lazy-views-compose); the comparison here is about the deliberately matching semantics: - **The model.** Both use an `IndexTransform` made of an input domain and one output index map per storage dimension, in constant, affine, and index-array forms. - **Slice semantics.** Slice bounds are literal domain coordinates: no clamping, no negative wrapping, non-empty intervals must be contained in the domain, and a strided slice's domain origin is `trunc(start/step)` rounded toward zero. `tests/test_tensorstore_parity.py` compares the enumerated cases with TensorStore when that optional dependency is installed. - **The wire format.** [ndsel](ndsel.md) uses TensorStore's domain and output-map field names for transform bodies, with an additional `kind` discriminator. `tests/test_ndsel_tensorstore.py` checks interoperability for the tested cases. Their validation rules differ, and loading and re-emitting a message through the engine can normalize or discard metadata; see [lowering to a transform](ndsel.md#lowering-to-a-transform). - **Chunk partitioning.** Both factor a transform over a grid before visiting any cell, rather than intersecting the whole transform with each chunk. TensorStore's `IndexTransformGridPartition` holds strided sets and index array sets and derives a per-cell transform on iteration; [`GridPartition`](api/chunk_resolution.md) also partitions index-array dependency components. Its `StridedSet` is per storage axis, where TensorStore's is per input dimension and spans every grid axis reading it (which is how TensorStore factors diagonals). `joint_sets` holds one `JointSet` per connected index-array component, including independent single-array components within a mixed request. Both derive the per-chunk transforms from the partition ([the guide](guide/index.md#a-plan-is-a-product-of-per-axis-tables) shows the tables). TensorStore keeps strided sets implicit, while this library materializes their per-axis rows for vectorized consumers. Pure affine diagonals need grouping by input dimension; mixed affine/index-array dependencies need joint partitioning. TensorStore classifies a connected component containing index-array edges as an index-array set ([source](https://github.com/google/tensorstore/blob/66b2ce5290fa2ec5c8019682391421062ce767a2/tensorstore/internal/grid_partition.h#L58-L67)). Four deliberate differences: | | TensorStore | `zarr-indexing` | | --- | --- | --- | | Dialect | Coordinate indices are literal; negative coordinates do not wrap | The algebra keeps that dialect; each public boundary picks its own. [`LazyArray`](api/lazy_array.md) speaks positional NumPy, `IndexTransform` speaks literal. [`zarr_indexing.boundary`](api/boundary.md) is the translation | | Scheduling | An internal C++ scheduler owns concurrency and chunk ordering | [`parts()`](api/lazy_array.md) exposes the partition structure so the caller's own scheduler — dask, a thread pool, a task queue — drives it | | Wire format | [Documented JSON schema](https://google.github.io/tensorstore/index_space.html#index-transform) | [ndsel](ndsel.md) is spec-first, with a vendored conformance corpus exercised by this implementation | | Backends | A driver ecosystem (zarr, N5, neuroglancer, GCS, …) built into the library | No drivers. The default reader needs `shape`, `dtype`, basic integer/slice indexing, and selected slabs convertible to NumPy system memory; other backends use explicit custom readers. A device reader owns transfer into the supplied system-memory output | The mechanics of a [chunk plan](guide/index.md#a-request-becomes-a-chunk-plan) and its [paired projections](guide/index.md#one-cell-domain-two-projections) belong to the visual guide. The relevant comparison is that both libraries use the paired-transform boundary rather than a read key plus scatter indices, so slices, outer products, and correlated gathers remain ordinary transforms that a consumer can lower to its own execution vocabulary. The ownership boundary differs. `plan_chunks` retains only the logical request and caller-supplied grid; it does not own reads, writes, buffers, locks, or scheduling. Zarr can therefore plan reads against an inner codec-chunk grid and writes against an atomic shard grid; napari or dask can turn the same projections into tasks without putting a dask dependency in this package. `coverage` describes selection coverage relative to that grid. A `full` classification can help a writer avoid reading old values, but does not by itself prove that a write is safe: encoding requirements, conflicts, duplicate semantics, and concurrency remain consumer responsibilities. `unknown` means the planner has not established complete or partial coverage. This implementation performs Python-level bookkeeping over NumPy. This page provides no benchmark establishing a general performance ordering against TensorStore; costs depend on the selection and execution backend. The partition tables can be consumed without constructing a `ChunkProjection` for each chunk. Materializing projections adds Python object construction. ## Bounding-box selections vs query selections Every selection this library can express falls into exactly one of two categories. The boundary between them is structural, not a heuristic: **A box** is a transform whose output maps are all `ConstantMap` or `DimensionMap` — no `ArrayMap`. Such a map is affine and monotone: storage coordinate `offset + stride * i` for `i` running over an interval. The whole selection is therefore described by `O(ndim)` integers — an interval and a stride per dimension — composition and intersection are interval arithmetic, and the coordinates it touches form a regular lattice. Basic indexing produces one, and composing basic indexing with basic indexing keeps one. **A query** is a transform with at least one `ArrayMap` — an explicit lookup table of coordinates. Its stored coordinate arrays cost space proportional to their stored size. Coordinates may repeat or scatter, but can also be contiguous and local. Current query-resolution paths inspect these arrays. Fancy indexing can produce a query, but singleton or constant selections can collapse to `ConstantMap`; subsequent indexing can therefore make a query affine again. A second query composes onto any axis of an existing one — including the axes it merely broadcasts along — by evaluating the existing lookup tables at the new coordinates. Those coordinate arrays are ordered sequences, never mathematical sets. Their order and duplicate entries are part of the indexing semantics and must survive planning and materialization. [ndsel](ndsel.md) encodes the same split in its message kinds: `point`, `box`, and `slice` desugar to constant and affine output maps and are always boxes; `points` desugars to `index_array` maps. A transform without index-array maps is a box in the engine’s structural classification. Loading can further simplify degenerate index arrays to constants, so an arbitrary incoming body with `index_array` fields need not remain a query. For example: ```python from zarr_indexing import IndexTransform IndexTransform.from_shape((100, 80))[10:50, ::4].to_json()["output"] # [{'offset': 0, 'stride': 1, 'input_dimension': 0}, # {'offset': 0, 'stride': 4, 'input_dimension': 1}] import numpy as np gather = IndexTransform.from_shape((100, 80)).oindex[np.array([90, 3, 3]), slice(None)] gather.to_json()["output"][0] # {'offset': 0, 'stride': 1, 'index_array': [[90], [3], [3]], # 'index_array_bounds': ['-inf', '+inf']} ``` The representation helps a consumer choose a lowering strategy. Independent affine axes can often be read with slices plus reversal, permutation, or broadcasting. Arbitrary affine maps can also express diagonals, so the absence of `ArrayMap` alone is not proof of a rectangular slab. Queries may be lowered through gathers or covers, and can sometimes simplify to slices. Chunk plans group selected coordinates by chunk while preserving result placement; repeated coordinates do not imply repeated visits to the same chunk. [`LazyArray`](api/lazy_array.md) exposes the category directly: ```python import numpy as np import zarr from zarr_indexing import LazyArray arr = zarr.create_array({}, shape=(100, 80), chunks=(30, 40), dtype="int32") arr[:] = np.arange(8000).reshape(100, 80) lazy = LazyArray(arr) slab = lazy[10:50, ::4] slab.is_box # True slab.bounding_box() # ((10, 50), (0, 77)) slab.strides() # (1, 4) slab.shape # (40, 20) gather = lazy.oindex[[90, 3, 3], :] gather.is_box # False gather.bounding_box() # ((3, 91), (0, 80)) gather.strides() # None gather.shape # (3, 80) ``` `bounding_box()` is defined for both: it is the hull, the smallest interval per storage dimension containing every coordinate the selection reaches. `strides()` is defined only for a box and gives the step per dimension. These summaries omit traversal direction, input-axis correspondence, and result layout. For example, forward and reversed views have identical bounds and stride magnitudes but different ordered results. Use the transform for the complete selection. For independent axes with multiple selected coordinates, a stride magnitude greater than one leaves gaps in the hull. Singleton axes are an exception, and a query can also cover every cell of its hull. The slab above spans a 40x77 hull over the 40x20 cells it selects, so a consumer that issued one rectangular read of the hull and discarded the rest would transfer 3.85x the data. A query's hull is looser still and carries no stride at all: 88 rows of hull over three selected rows. An empty *box* touches no coordinate to report an interval around, so `bounding_box()` is `None` while `strides()` still answers — the step is a property of the selection's shape, not of the region it reaches. An empty query returns `None` from both. There is deliberately no separate `BoxView` type today. A statically-typed rectangular-only view is a plausible next step, but it should be introduced by a consumer that needs the guarantee in its signatures rather than speculatively; `is_box` is the runtime check until then. ## Negative-origin domains and prependable grids Literal coordinates let a domain grow at its lower end without changing the identity of anything already present. Prepending three cells extends `[0, 6)` to `[-3, 6)`: the new cells receive addresses `-3`, `-2`, and `-1`, while the old cells keep addresses `0` through `5`. Coordinate `0` does not become coordinate `3`. The adjacent intervals `[-3, 0)`, `[0, 3)`, and `[3, 6)` follow the half-open adjacency rule: each stopping boundary is included exactly once as the next interval's starting boundary. ```text before [0, 6): | 0 1 2 | 3 4 5 | chunk coordinate | 0 | 1 | after [-3, 6): | -3 -2 -1 | 0 1 2 | 3 4 5 | | -1 | 0 | 1 | chunk coordinate ``` The same holds for chunk grids. `EdgeDimensionGrid` is the convenient concrete grid for a zero-origin array: its chunk offsets are prefix sums starting at zero. `DimensionGridLike` is the more general protocol consumed by chunk planning, so it admits grids with negative chunk and cell coordinates, including this prependable example: ```python --8<-- "snippets/coordinate_origins.py:prepend-grid" ``` Here the literal cell domain `[-3, 0)` belongs to chunk `-1`. Both public projection transforms share the same synthetic input cell domain `[0, 3)`. Evaluating its three points shows the two distinct outputs: `chunk_transform` produces zero-origin chunk-local coordinates `0, 1, 2`, while `cell_transform` produces the literal request coordinates `-3, -2, -1`. The shared input domain is not itself the chunk-local coordinate frame. ## Related work TensorStore is the prior art for the transform algebra, as described above. At the execution boundary, this package instead gives each backend a `ReadContext` through a `Reader`. Its global transform answers **which values?**; the reader answers **how does this backend obtain them?** A partition view's transform directly addresses the raw source in global coordinates. Its optional projection retains the paired planning transforms, of which only `chunk_transform` addresses zero-origin chunk-local coordinates. The reader must preserve the global transform exactly, but it does not participate in indexing semantics, partitioning, scheduling, or result ownership. Earlier versions used a capability taxonomy modeled on historical indexing dialects. That model required deciding which fragment of a request a backend could accept and finishing the rest elsewhere. A reader lowers the complete transform and can compose through delegation instead. This resembles [zarrita.js store extensions](https://zarrita.dev/packages/zarrita.html), where storage-specific behavior is an explicit extension point rather than an inferred array capability. This is an architectural analogy, not a claim of API or implementation compatibility. ## Current scope Negative steps are supported as of ndsel 1.0-draft.2: `a[::-1]` reverses, one desugaring rule covers both signs, and a reversed interval is an error rather than a silently empty selection. One consequence: a negative step normally produces a negative domain origin. Reversing a length-20 zero-origin axis gives the domain `[-19, 1)`, because the result stays anchored to the source coordinate frame and a reversing map traverses that frame backwards. `LazyArray` views keep that literal domain, so `view.transform.domain` shows it; positional keys are normalized against the domain's origin, so the NumPy dialect never requires typing it. A caller wanting zero-origin coordinates re-bases explicitly with `translate_domain_to`. Supported fancy selections compose across already-fancy views: a second `oindex`/`vindex`/mask step may land on any axis of an already-fancy view, including axes an existing index array merely broadcasts along, so `lazy.oindex[[2, 0], :].oindex[:, [1, 3]]` selects the outer product it spells. An array-carrying transform is composed — the new selection is applied to an identity transform over the current domain and chained on with `compose`, which evaluates the existing lookup tables at the new coordinates — rather than rewritten in place. Resolution classifies the result by structure (`index_array_structure`): pure per-axis outer products keep the orthogonal resolvers, and everything else — correlated maps, mixtures, index arrays sharing an input axis (as in paired vectorized coordinates) — takes the general reader/intersection path. Chunk planning factors index arrays into connected dependency components before flattening, so independent groups do not expand one another. Vectorized selection preserves broadcast singletons to retain those dependencies. Some current limits are: - **Shared affine dependencies.** Planning rejects two affine output maps sharing an input axis with `ValueError`. An index array sharing a varying input axis with an affine map takes the general classification and raises `NotImplementedError`. Pure affine diagonals would need grouping dependent storage axes by input dimension; mixed components need joint partitioning. - **Finite explicit bounds only.** `IndexDomain` has no implicit or unbounded dimensions; the message layer will normalize a body with `"-inf"`/`"+inf"` bounds, but the engine layer refuses to lower one into a transform. TensorStore supports both. - **Labels are carried, not propagated.** `IndexDomain` holds optional dimension labels and the wire format round-trips them, but indexing operations build new domains without them, so a label does not survive a slice. ## Selection to chunk operations [The selection-flow guide](guide/selection-flow.md) traces normalization, partitioning, and paired chunk-local/request projections with executable examples. zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/examples/000077500000000000000000000000001527014137600265725ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/examples/lazy_indexing_dask.md000066400000000000000000000001671527014137600327660ustar00rootroot00000000000000--8<-- "lazy_indexing_dask/README.md" ## Source Code ```python --8<-- "lazy_indexing_dask/lazy_indexing_dask.py" ``` zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/examples/lazy_indexing_numpy.md000066400000000000000000000011401527014137600332040ustar00rootroot00000000000000--8<-- "lazy_indexing_numpy/README.md" `LazyArray(source)` uses the conservative built-in reader: `source` must expose `shape`, `dtype`, and basic integer/slice indexing, and every selected slab must be convertible to NumPy system memory. Coordinate arrays passed through `oindex` or `vindex` are ordered and duplicate-preserving; they are not sets. When a view is partitioned, each `Partition.view.transform` addresses the raw source globally while `Partition.projection.chunk_transform` stays zero-origin and chunk-local. ## Source Code ```python --8<-- "lazy_indexing_numpy/lazy_indexing_numpy.py" ``` system_memory_chunk_cache.md000066400000000000000000000005231527014137600342640ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/examples--8<-- "system_memory_chunk_cache/README.md" `LazyArray` owns the indexing-derived result shape and assembly, while the example's `SystemMemoryChunkReader` owns synchronous system-memory cache state and chunk reads for each materialized part. ## Source Code ```python --8<-- "system_memory_chunk_cache/system_memory_chunk_cache.py" ``` zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/guide/000077500000000000000000000000001527014137600260515ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/guide/index.md000066400000000000000000000611101527014137600275010ustar00rootroot00000000000000# Visual guide The whole model in one sentence: indexing through `LazyArray` builds a view, chunk planning partitions its coordinates, and `result()` materializes the view. This page follows one familiar NumPy selection, `source[2:5]`, through those stages. The first four sections are for anyone indexing arrays: coordinates, transforms, composition, and result axes. **If you are using lazy indexing rather than building a storage backend, you can stop after section four.** The last three sections are for integrators: they turn a request into a chunk plan, pair each chunk read with its place in the result, and show the per-axis tables the plan is built from. Throughout, one division of labor holds: the transform answers **which values?** and is independent of the backend; the reader answers **how do I obtain them?** and must preserve the transform exactly. ## An index selects coordinates {#an-index-selects-coordinates} Begin with an ordinary NumPy array. `source` contains the values 10 through 15. The selection `source[2:5]` takes source coordinates 2, 3, and 4, containing the values 12, 13, and 14. ```text source coordinate | 0 1 2 3 4 5 source value | 10 11 12 13 14 15 selection | [12 13 14] source[2:5] result coordinate | 0 1 2 source coordinate | 2 3 4 result value | 12 13 14 ``` The result defines its own coordinates: `0`, `1`, and `2`. The aligned rows make the correspondence explicit: those result coordinates receive values `12`, `13`, and `14` from source coordinates `2`, `3`, and `4`. The wrapper below gives the same familiar selection a lazy spelling. Indexing with `[...]` creates `view`; the last line asks for its values and checks the observable NumPy result. ```python --8<-- "snippets/canonical_slice.py:canonical-slice" ``` The important first step is simply that an index describes which source values fill a result in a particular order. The next section gives the numbers on both sides of that description a precise meaning. ## Coordinates are addresses {#coordinates-are-addresses} ### How to read a half-open interval `[0, 1)` is a **half-open interval**: start at 0, inclusive, and stop at 1, exclusive. The `[` includes the lower boundary, while the `)` excludes the upper boundary. For integer coordinates, `[0, 1)` therefore enumerates the ordered sequence `[0]`. Half-openness lets adjacent slices and chunks meet without a gap or overlap. Concatenation is ordered: the first interval is followed by the second. When the first interval's exclusive stop matches the second interval's inclusive start, the shared boundary coordinate appears exactly once. - `[0, 1) -> [0]` — Start at 0 and stop before 1, so the sequence contains only 0. - `[1, 3) -> [1, 2]` — Start at 1 and stop before 3, so the sequence contains 1 and 2. - `concat([0, 1), [1, 3)) = [0, 3)` — Append the second interval after the first. Their matching exclusive/inclusive boundary produces one continuous interval without a gap or duplicated coordinate. Explicit coordinates, including coordinate arrays, are always ordered sequences rather than mathematical sets. Their order is semantic, and repeated coordinates remain repeated in the result. In the transform algebra, **coordinates are just integers**. A negative coordinate is a real address in a domain, with the same status as zero or a positive coordinate; it is not automatically shorthand for counting backward from an array's end. ```text domain [-2, 3) coordinate | -2 -1 0 1 2 status | address address address address address ``` `IndexDomain` makes those bounds explicit. In the example below, narrowing the domain at `-1` selects the literal address `-1`; the wrapper at the end treats `-1` the way NumPy does — as the last position. ```python --8<-- "snippets/coordinate_origins.py:coordinate-origin" ``` Why carry literal coordinates at all? They let independently described regions keep stable addresses — a domain can even grow at its lower end without renumbering what is already there. The [design notes](../design-notes.md#negative-origin-domains-and-prependable-grids) work through that prepending example; nothing else in this guide depends on it. The literal model and NumPy's positional model are both useful, but they answer different questions: | Surface | Meaning of an integer index | Meaning of `-1` | | --- | --- | --- | | `IndexDomain` and `IndexTransform` | A literal coordinate in the current domain | The actual address `-1`, if the domain contains it | | `LazyArray` with a NumPy key | A NumPy-style position in the current view | The last position, normalized against the domain's origin before it reaches the transform algebra | | `LazyArray` with an `IndexDomain` or `IndexTransform` key | A literal coordinate in the view's domain | The actual address `-1`, if the domain contains it | `LazyArray` reads NumPy keys as positions because it is an array-like wrapper: negative indices wrap exactly as they do in NumPy. The view itself keeps its literal domain, as a TensorStore view does: `source[10:20]` has domain `[10, 20)`, and a further `[2:5]` on it has domain `[12, 15)`. A domain or transform key addresses those literal coordinates directly. ### The key's type picks the frame {#the-keys-type-picks-the-frame} **The type of a key decides whether it is relative or absolute.** This is the one rule to remember about indexing a `LazyArray`: | Key type | Read as | Example | | --- | --- | --- | | slice, integer, `...`, `None`, index array, mask | **Relative**: positions in the current view, NumPy-style | `view[2:5]`, `view[-1]`, `view.oindex[[3, 1]]` | | `IndexDomain` | **Absolute**: coordinates of the view's domain, restricted to a box | `view[IndexDomain((12,), (15,))]` | | `IndexTransform` | **Absolute**: composed onto the view; the key's domain becomes the new view's domain | `view[IndexTransform.identity(view.transform.domain)[12:15]]` | No key type is valid in both readings, so a key never has two meanings. That is the property pandas lost with value-based `ix` dispatch and removed in favor of `loc` and `iloc`; TensorStore keeps it by giving each key type one fixed reading, and so does this wrapper. The only difference from TensorStore is which reading the NumPy key gets: absolute there, relative here. Whichever key produced a view, **its domain is always absolute**: `view.transform.domain` reports literal coordinates after `view[2:5]` exactly as it does after `view[IndexDomain(...)]`. The relative reading exists only at the moment a NumPy key is interpreted, so relative and absolute steps compose freely. ```python --8<-- "snippets/key_types.py:key-types" ``` ### A transform points from the request to the source An `IndexTransform` records how every coordinate in a request finds its source coordinate. For the slice from the first section, request coordinate `i` maps to source coordinate `i + 2`. This direction is deliberate: request to source, not source to request. ```text request coordinate | 0 1 2 | | | | i + 2 | v v v source coordinate | 2 3 4 source value | 12 13 14 ``` A transform speaks function vocabulary while this guide speaks array vocabulary. The two line up like this: | the API says | this guide says | | --- | --- | | input space (`domain`, `input_rank`) | request coordinates — the result being built | | output space (`output`, one map per dimension) | source coordinates — where values are read | `output` names the output side of the coordinate *function*, not the data: values flow source → request, against the arrow. The neutral names exist because transforms compose — in a chain, an interior transform's output space is just the next transform's input space, neither a request nor a source. ### The three map kinds, in NumPy terms Every output dimension is produced by one of three map forms. Each has a NumPy counterpart, shown executably below. The examples share one helper — and it doubles as the answer to how a bare transform meets data at all: a reader materializes it into a buffer. ```python --8<-- "snippets/output_maps.py:resolve-helper" ``` `DimensionMap` is an arithmetic rule — the slice above is one, mapping request `i` to source coordinate `i + 2`: ```python --8<-- "snippets/output_maps.py:dimension-map" ``` `ArrayMap` stores explicit source coordinates for irregular or fancy indexing; order and repeats survive into the result: ```python --8<-- "snippets/output_maps.py:array-map" ``` `ConstantMap` fixes one source coordinate for every request cell. Whether an axis appears in the result is decided by the **domain**, never by the map: `image[2, :]` compiles to a `ConstantMap(2)` with no corresponding domain axis (the axis is dropped), while pairing a constant map with a length-`n` domain axis that no map consumes yields `n` cells all reading one coordinate — a broadcast, the one arrangement with no NumPy index counterpart: ```python --8<-- "snippets/output_maps.py:constant-map" ``` Together, the request domain and these per-source-dimension maps are the complete reusable description of an index. ## Lazy views compose {#lazy-views-compose} A lazy view can be indexed again. Each step changes the request-to-source description, but it does not read an intermediate array. The chain is reduced to one direct transform from the newest request to the original source. ```text source[2:5][::-1][1:] new request | intermediate view | original source ------------+-------------------+---------------- 0 | 1 | 3 1 | 2 | 2 direct map: request i -> source (3 - i) ``` The executable example first selects `source[2:5]`, then reverses that view and trims its first element: ```python --8<-- "snippets/lazy_composition.py:lazy-composition" ``` Immediately after `composed` is created—and before the final `result()` call—its metadata is ready to inspect: | Available without reading | Value in this example | | --- | --- | | `composed.shape` | `(2,)` | | `composed.transform` | One transform over the literal domain `[-3, -1)`, mapping request `i` to source `-i` | Neither property needs source values. Composition works only on the coordinate description; the assertion's call to `result()` is the first operation in the example that materializes the selected data. !!! warning "Stop here: the materialization boundary" Indexing through `[...]` composes a selection without reading source values. These operations request values: - `result()` - `numpy.asarray(view)` and NumPy operations that convert the view (`numpy.add(view, 1)` does so; `numpy.shape(view)` and `numpy.ndim(view)` can use metadata without reading values) Dask tokenization may also inspect values, depending on the wrapped source and tokenization path. Python arithmetic such as `view + 1` raises `TypeError` instead: this wrapper defers indexing, not a general compute graph. Iteration yields lazy first-axis views; call `result()` on each to read it. `view.write(values)` writes through the composed transform to the original writable source, synchronously, and returns `None`. `view[key] = values` writes the selected sub-view in the same way. These calls do not create futures or transactions. Read-only sources remain usable for reads; writes require source assignment support. Storage atomicity and concurrent writer coordination remain the backend's responsibility. ## An index defines a result array {#an-index-defines-a-result-array} An index chooses source points and also defines how those points are arranged in the result. In the 3-by-4 image below, `image[1, :]` and `image[1:2, :]` choose the same four source points: values `4`, `5`, `6`, and `7`. ```text same selected source cells source coordinate | (1, 0) (1, 1) (1, 2) (1, 3) value | 4 5 6 7 image[1, :] result coordinate | 0 1 2 3 value | 4 5 6 7 shape | (4,); source axis 0 is omitted image[1:2, :] result coordinate | (0, 0) (0, 1) (0, 2) (0, 3) value | 4 5 6 7 shape | (1, 4); source axis 0 is retained with length 1 ``` The integer in `image[1, :]` fixes source axis 0. No result coordinate varies along that axis, so it is omitted and the result shape is `(4,)`. The slice in `image[1:2, :]` preserves source axis 0 as a length-one result axis, so the result shape is `(1, 4)`. ```python --8<-- "snippets/axis_manipulation.py:axis-shape-comparison" ``` `None` inserts a new length-one axis without selecting different source points. Here it produces the shape `(4, 1)`: ```python --8<-- "snippets/axis_manipulation.py:axis-insertion" ``` ## A request becomes a chunk plan {#a-request-becomes-a-chunk-plan} Continue with the 3-by-4 image and `image[1, :]` introduced above. Giving the image a 2-by-2 chunk shape does not change the four selected values or their order. It changes only how the work is divided: columns 0 and 1 come from chunk `(0, 0)`, while columns 2 and 3 come from chunk `(0, 1)`. ```text column 0 1 | 2 3 ----------+---------- row 0 0 1 | 2 3 row 1 [4] [5]| [6] [7] <- image[1, :] ----------+---------- row 2 8 9 | 10 11 left part right part chunk_coords (0, 0) (0, 1) global chunk_domain [0,2) x [0,2) [0,2) x [2,4) selected global cells (1,0), (1,1) (1,2), (1,3) chunk-local cells (1,0), (1,1) (1,0), (1,1) request coordinates 0, 1 2, 3 ``` Every planned chunk keeps three coordinate frames distinct: - `chunk_coords` identifies a cell in the chunk grid. Chunk coordinates are literal integers, so a grid that grows at its lower end can hold a chunk whose coordinate really is `-1` — not an alias for the final chunk (see the [design notes](../design-notes.md#negative-origin-domains-and-prependable-grids)). - `chunk_domain` gives that chunk's bounds in **global source coordinates**. Here the two domains are `[0, 2) × [0, 2)` and `[0, 2) × [2, 4)`. - Chunk-local positions start from zero inside each chunk. Global column 2 is therefore local column 0 in chunk `(0, 1)`. This zero-origin local frame is separate from both the global `chunk_domain` and the possibly negative chunk coordinate. `plan_chunks` needs only a transform and the chunk layout: one grid object per source dimension. A per-dimension grid answers four questions — which chunk contains a source index, where a chunk starts, how long it is, and the vectorized form of the first (`index_to_chunk`, `chunk_offset`, `chunk_size`, `indices_to_chunks`). The library builds these from chunk sizes via `dimension_grids_from_chunks`; the executable example hand-rolls one instead, to show that the whole contract is those four answers. It plans the canonical request over 2-by-2 chunks, and iterates the same plan again to show that planning is reusable. (The two transforms it inspects on each projection are the next section's subject.) ```python --8<-- "snippets/chunk_projection.py:chunk-projection" ``` The plan describes work but does not perform it. It contains no array source, storage backend, codec pipeline, buffer, or scheduler. A Zarr reader, a task queue, or a viewport can consume the same logical plan and decide independently how and when to fetch its two chunks. On the wrapper, this partitioning is called **parts**: `with_parts(shape)` gives a `LazyArray` a grid of uniform boxes to divide its reads along (re-partitioning is a pure setter — it changes how a read is divided, never what `result()` returns), and a wrapped array advertising its own `chunks` is partitioned that way automatically. A zero-length source axis has no chunks. `LazyArray` accepts a positive uniform part shape for that axis, or explicit per-axis spellings `()`, `(0,)`, and `(0, 0)`; each produces no parts and the same empty result. Zero-sized parts remain invalid on a nonempty axis. ## One cell domain, two projections {#one-cell-domain-two-projections} A chunk read has to answer two questions at once: which cells belong to this chunk, and where does each of those cells belong in the requested result? Think of a projection as a small table with one row per selected cell. For each row, `chunk_transform` gives the cell's zero-origin address inside the chunk, and `cell_transform` gives the position in the requested result that receives its value. The row numbers of that table are the shared **cell domain** — a synthetic input space both transforms accept, which is why one input point can be evaluated on both sides. ```text left chunk (0, 0) shared cell coordinate | 0 1 cell_transform | v v request coordinate | 0 1 shared cell coordinate | 0 1 chunk_transform | v v chunk-local coordinate | (1, 0) (1, 1) right chunk (0, 1) shared cell coordinate | 0 1 request coordinate | 2 3 chunk-local coordinate | (1, 0) (1, 1) ``` The directions are exact: **shared synthetic input cell domain → request via `cell_transform`**, and **shared cell domain → chunk-local via `chunk_transform`**. Neither arrow starts at the request or maps one output space into the other. On the wrapper, `view.parts()` returns one `Partition` per planned chunk; each bundles a sub-view of the request (`.view`), that chunk's projection (`.projection`), and the NumPy selection placing its values in the result (`.out_selection`). Within one `Partition`, the frames divide: `Partition.view.transform` is a different, global transform — it maps the part view directly into the raw wrapped source — while only `Partition.projection.chunk_transform` uses zero-origin chunk-local coordinates. Parent assembly passes both frames to the reader. Independently scheduled `part.view.result()` calls plan against the same source grid and supply both frames too. Their result placement is relative to the part view being read. A partition view can be indexed or repartitioned like any other view; its base shape remains the full source shape. | Projection field | What its output coordinates mean | | --- | --- | | `cell_transform` | Literal coordinates in the original request; its output rank is the request rank | | `chunk_transform` | Zero-origin coordinates in the selected chunk's local frame; its output rank is the source rank | The cell domain enumerates corresponding cells; it is not itself either output coordinate space. The canonical row selection has a one-dimensional request and a two-dimensional source, so its paired projections have request rank one and source rank two. ### Order and duplicates need the request-side projection Orthogonal indexing (`.oindex`) applies each axis's indexer independently, like `numpy.ix_` — an outer product; the [pattern reference](patterns.md) develops the dialects. It can visit source cells in an order that does not match chunk order, and it can visit one source cell more than once. In the request below, row 4 comes first and row 1 appears twice. ```text request position | 0 1 2 source row | 4 1 1 result row | row 4 row 1 row 1 ``` A source bounding box cannot reconstruct this result. The box spanning rows 1 through 4 also includes unrequested rows 2 and 3, and its increasing coordinate order does not record that row 4 comes first. Narrowing the read to just rows 1 and 4 still does not record the second use of row 1. For the same reason, a chunk-local selector alone says which cells to read inside a chunk but cannot say which request positions receive them, especially when the chunks are processed in a different order. The executable example assembles the 3-by-4 request from a 6-by-8 source with 3-by-4 chunks. Each `Partition` resolves its own sub-view — the global transform addressing the raw source — and `out_selection` places those values at their request-side positions; the paired projection stays available on `part.projection` for consumers that read chunks directly. The assertion checks the reordered, duplicated result against direct NumPy indexing. ```python --8<-- "snippets/chunk_projection.py:advanced-projection" ``` The paired representation preserves information that a bounding box or local selector discards: exact request order, duplicate destinations, and the correspondence between every request position and its chunk-local source cell. ## A plan is a product of per-axis tables {#a-plan-is-a-product-of-per-axis-tables} Look again at the two projections of `image[1, :]`. Each chunk transform has one map per source axis, and every one of those maps came from restricting the request's map for *that axis alone* to *that axis's* chunk: the fixed row `ConstantMap(1)` lands in row-chunk 0 whatever the column chunk is, and the column slice meets column-chunk 0 as local columns `0:2` and column-chunk 1 as local columns `0:2` whatever the row chunk is. Restricting a transform to a chunk distributes over axes whenever each output map reads its own request axis — which every basic and orthogonal selection satisfies. So the plan does not intersect the whole transform with every chunk. It resolves each axis once, into a table with one row per chunk that axis touches, and a projection is one row of each table combined. ```text image[1, :] over 2-by-2 chunks axis 0 (rows): ConstantMap(1) axis 1 (columns): DimensionMap row | chunk start local extent row | chunk start local_start extent origin full 0 | 0 0 1 1 0 | 0 0 0 2 0 yes 1 | 1 2 0 2 2 yes row_shape (1, 2): 1 x 2 = 2 projections projection (0, 0) = axis-0 row 0 x axis-1 row 0 -> chunk (0, 0), local (1, 0:2), request columns 0:2 projection (0, 1) = axis-0 row 0 x axis-1 row 1 -> chunk (0, 1), local (1, 0:2), request columns 2:4 ``` `ChunkPlan.partition()` returns this factored form, a `GridPartition`. Its `sets` hold one table per source axis the request reads independently, in axis order, and `joint_sets` one table per connected group of index arrays; `row_shape` is the number of rows in each; and the plan walks the rows in row-major order over it. The executable example reads the two tables above off the plan, checks that the plan's projections are exactly the partition's rows, and evaluates the second row on both of its transforms: ```python --8<-- "snippets/grid_partition.py:strided-tables" ``` There are three kinds of table, matching the three map kinds and the one arrangement that does not factor. A `StridedSet` holds a `ConstantMap` or `DimensionMap` axis; an `IndexedSet` holds an orthogonal `ArrayMap` axis (`.oindex`) with its coordinates grouped by chunk; and each `JointSet` holds one connected group of index arrays. Arrays sharing request axes are sorted together; independent groups are separate tables in `joint_sets`. Singleton broadcast axes retain this independence. The [API reference](../api/chunk_resolution.md) documents every column. The gather from the previous section, `oindex[[4, 1, 1], 2:6]` over 3-by-4 chunks, groups rows `1, 1` into chunk 0 and row `4` into chunk 1 while remembering that row `4` fills request position 0. A `vindex` selection keeps its points paired in the joint table: ```python --8<-- "snippets/grid_partition.py:indexed-and-joint" ``` Building independent strided tables costs the *sum* of touched chunks per axis. Index-array grouping also scales with the selected point count; the planner flattens each connected component separately. Independent components do not expand into their Cartesian product until their rows are iterated. And projections are derived from rows only when asked for: `chunk_coords()` lists every chunk the plan touches without materializing a row, and a consumer can read the columns directly, as [Integration boundaries](integrations.md#reading-the-tables-directly) shows. For example, `(u, v) -> (a[u], b[u], c[v])` has two independent components. The first two storage axes share `u`; the third reads `v`. With 1,000 values per input axis, planning stores 3,000 index values rather than 3,000,000. Each projection has one synthetic axis per nonconstant component, followed by residual request axes, including axes no output reads. ```python --8<-- "snippets/grid_partition.py:components" ``` A hand-built affine diagonal — two `DimensionMap`s reading one request axis, which no selection produces — would need a table spanning several storage axes; the plan rejects it with `ValueError`. --- zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/guide/integrations.md000066400000000000000000000310311527014137600310770ustar00rootroot00000000000000# Integration boundaries For the complete path from indexing syntax to chunk coordinates, local selectors, and result positions, start with [From a selection to chunk operations](selection-flow.md). The core package supplies indexing plans and synchronous readers. It does not provide a general scheduler, codec pipeline, or async execution engine. A synchronous cache is included as an example. A consumer decides when projections run, how decoded chunks are obtained, and where completed values are retained. An `IndexTransform` says which source values belong in a result; a `Reader` lowers that complete transform for one backend. The reader does not choose indexing semantics or result ownership. ## Zarr chunk dispatch A Zarr-oriented reader can consume each public `ChunkProjection` and use its `chunk_coords` to obtain one decoded chunk from its own storage and codec layers. This tiny source keeps four in-memory chunks keyed by their global chunk coordinates and records the exact reads. For each projection, the consumer enumerates the shared synthetic input cell domain, evaluates `chunk_transform` to read zero-origin chunk-local coordinates, and evaluates `cell_transform` to place each value at its literal request coordinate. ```python --8<-- "snippets/integrations.py:zarr-consumer" ``` The two reads are exactly `(0, 0)` and `(0, 1)`; untouched chunks `(1, 0)` and `(1, 1)` are never read. The assembled request is `[4, 5, 6, 7]`. The example intentionally begins with already decoded in-memory chunks: storage keys, codecs, scheduling, caching, and asynchronous orchestration remain the consumer's policy rather than responsibilities of the plan. ## Reading the tables directly {#reading-the-tables-directly} A consumer that fetches many chunks per request — a codec pipeline, a prefetcher — does not need a `ChunkProjection` object per chunk. The plan's [factored form](index.md#a-plan-is-a-product-of-per-axis-tables) is a few NumPy arrays per axis, and everything a chunk copy needs is a row of each: the chunk index, the chunk-local start and extent, and where the cells land in the request. `chunk_coords()` alone answers "which chunks?" for a prefetch, without materializing anything. The example assembles a strided box from its `StridedSet` tables, one slice per chunk. Three things a real consumer also has to get right are checked at the end: a descending stride needs the same care with a negative stop that any NumPy slice does, a chunk slab comes out in storage-axis order while the result is in request-axis order, and a request axis no map reads (`None` in the selection) is a length-one axis of the result. ```python --8<-- "snippets/grid_partition.py:table-consumer" ``` The four reads are the four chunks the two axis tables multiply out to, and no intermediate transform, domain, or projection was built. A gather reads its `IndexedSet` rows the same way — `index[pointer[i]:pointer[i + 1]]` and the matching `positions` — and a general selection its `joint_sets` rows, whose `local` coordinates already have the chunk origin subtracted. Both tables compute `local` once on first access and return the same read-only array afterward, so accessing a row does not recalculate every point. ## One slab read or many part reads A backend with an efficient native subset operation may benefit from receiving one complete dense selection so it can choose its own chunk dispatch. Other backends may benefit from partitioning, including for strided or fancy selections. The tradeoff depends on the backend, chunk layout, latency, memory, and selection; a single read is not universally fastest. A read's **cover** is the smallest unit-step slab enclosing its coordinates. With this package's basic readers, partitioning limits each source read to the part's selected cover. This may reduce over-reading, but a partition that spans the source can still require the entire hull. Custom readers choose their own source operations under the reader contract. The composed view carries enough to make that call at materialization time, and `with_parts()` returns a view with a new partitioning. This example uses unit strides as a sufficient condition for its independently mapped selections: ```python --8<-- "snippets/integrations.py:dense-box-repartition" ``` The corner gather reads four single cells instead of the 10-by-10 hull, and the dense box becomes exactly one backend call. Both regimes go through `result()`; only the partitioning in force differs. ### Sources that accept only unit-step slices For affine selections, `basic_reader` uses positive-step slices and applies reversal or layout changes in memory. Fancy selections can require reading a cover containing unselected values. The source must accept the emitted steps. Use [`unit_step_reader`][zarr_indexing.reader.UnitStepReader] for a source that accepts only unit steps. Every key it receives is an ascending unit-step slice per axis, with strides, reversals, and gathers applied to the in-memory block instead: ```python view = LazyArray(source).with_reader(unit_step_reader) ``` For strided selections, the ratio of cover cells to selected cells depends on stride, length, and endpoint alignment. Partitioning can reduce that cover. ## napari-like consumer This is a **napari-like consumer**, not a napari integration. It models the boundary a viewport could use without importing or claiming support for napari. `RecordingArray` exposes a chunked, basic-indexing source — and its `chunks` attribute is why the reads below split along `(2, 2)` boxes: `LazyArray` discovers a partitioning from the wrapped array at construction (`read_chunk_sizes`, then `chunks`), with `with_parts` as the explicit override. Composing the visible slice records no reads. Only `result()` materializes it, with the exact source selectors `1:2, 0:2` and `1:2, 2:4`; neither selector crosses into an untouched neighboring chunk. ```python --8<-- "snippets/integrations.py:viewport-consumer" ``` The viewport owns its interaction loop and any cancellation, caching, or background execution. `LazyArray` contributes the composable selection and the partition plan, then resolves only when the consumer asks for the result. ### A system-memory chunk cache Napari accepts NumPy-like array objects and can defer materialization until an image region is displayed. The indexing plan still deliberately owns no cache or scheduler. A viewport adapter can place that policy around the plan, as the executable reference below demonstrates. This remains a **napari-like consumer, not a napari integration**. It models only decoded chunks resident in system memory, synchronously. For setup instructions and the complete executable, see the [system-memory chunk cache example](../examples/system_memory_chunk_cache.md). ```text read succeeds NEW -> QUEUED -> LOADING -------------> READY -> EVICTED ^ | | | read fails | v +--------- FAILED retry EVICTED -> QUEUED reload ``` The example keeps the lifecycle records and transitions explicit: ```python --8<-- "system_memory_chunk_cache/system_memory_chunk_cache.py:chunk-cache-types" ``` Its source represents already decoded chunks and records each read: ```python --8<-- "system_memory_chunk_cache/system_memory_chunk_cache.py:chunk-cache-source" ``` `LazyArray` converts a cache selection into transforms and partitions, then allocates and assembles the result. The facade constructs exactly one tuple from `view.parts()`: it derives the chunk coordinates to pin from that tuple, then passes the same owned parts to `view.result(parts=parts)`. Planning is therefore performed once for the request rather than repeated during materialization. Neither pinning nor the result call rebuilds the plan; both reuse those prepared `Partition` objects. `SystemMemoryChunkReader` receives one `ReadContext` for each materialized part. Its global `context.transform` directly addresses the raw source, while `context.projection.chunk_transform` addresses the already identified chunk locally. The reader consumes that supplied projection directly; it never calls the chunk planner. `LazyArray` retains responsibility for the projection's result placement and final assembly. The reader owns only cache state and source reads, while `SystemMemoryChunkCache` remains the thin NumPy-style facade that prepares and pins the one plan: Its indexing dialects remain explicit: `cache[key]` accepts basic indexing (integers, slices, ellipsis, and new axes), while `cache.oindex[key]` combines per-axis index arrays as an outer product. Array keys are not silently treated as orthogonal by plain square brackets; callers choose that behavior through the named accessor. ```python --8<-- "system_memory_chunk_cache/system_memory_chunk_cache.py:chunk-cache-wrapper" ``` ### Follow one viewport through the cache ```python --8<-- "system_memory_chunk_cache/system_memory_chunk_cache.py:chunk-cache-worked-example" ``` The worked example uses a 6-by-8 image, 3-by-4 chunks, and capacity for two decoded chunks. Every read delta follows directly from the viewport request: | Step | Viewport | New reads | Resident afterward | Why | | --- | --- | --- | --- | --- | | 1 | `image[1:5, 2]` | `(0, 0)`, `(1, 0)` | `(0, 0)`, `(1, 0)` | Both projected chunks are loaded and assembled as `[10, 18, 26, 34]`. | | 2 | `image[3:5, 2]` | None | `(0, 0)`, `(1, 0)` | The ready buffer for `(1, 0)` is reused and becomes most recently used. | | 3 | `image[0:2, 5]` | `(0, 1)` | `(0, 1)`, `(1, 0)` | Placement returns `[5, 13]`, then LRU pressure evicts `(0, 0)`. | | 4 | `image[1:5, 2]` | `(0, 0)` | `(0, 0)`, `(1, 0)` | The evicted chunk is reloaded while the required ready chunk is retained. | | 5 | `image[3:5, 4:6]` | `(1, 1)` fails; no repeated read; `(1, 1)` succeeds after retry | `(0, 0)`, `(1, 1)` | Failure is retained until explicit retry; the repaired source then returns `[[28, 29], [36, 37]]`. | The example defers eviction until a successful outermost request finishes, so it can temporarily exceed capacity during assembly. Capacity counts decoded chunks, not bytes, event records, or temporary arrays. Failed requests skip that eviction step. The example assumes an unchanged source and one source/grid per reader; it does not implement invalidation or synchronization for concurrent requests. The prepared tuple supplies the projections used for each read. The event log makes the failure boundary equally explicit: | Chunk | Transition | Reason | | --- | --- | --- | | `(1, 1)` | `NEW -> QUEUED` | requested | | `(1, 1)` | `QUEUED -> LOADING` | queue drained | | `(1, 1)` | `LOADING -> FAILED` | source read failed | | `(1, 1)` | `FAILED -> QUEUED` | explicit retry | | `(1, 1)` | `QUEUED -> LOADING` | queue drained | | `(1, 1)` | `LOADING -> READY` | source read completed | A repeated request while the record is `FAILED` creates no event and performs no source read. The retained failure forces the caller to choose when retry is appropriate. A real viewport adapter could drain the queue in workers and invalidate its canvas when chunks become ready without changing the selection or projection semantics shown here. [Napari's image-layer documentation](https://napari.org/dev/howtos/layers/image.html) describes its NumPy-like array boundary. Neuroglancer's [`ChunkState`](https://github.com/google/neuroglancer/blob/master/src/chunk_manager/base.ts) is conceptual prior art for making residency explicit. This example is a smaller, independently authored, synchronous teaching model; it does not copy that implementation or reproduce its full worker/GPU lifecycle. --- ## Dask tokenization `LazyArray.__dask_tokenize__()` combines Dask's token for the wrapped source with the serialized view transform. Dask owns source hashing, registered normalizers, custom source hooks, and deterministic-token requirements. Tokenization may read or hash source values. Dask is optional for indexing and reading, but required when requesting a Dask token. The reader and partitioning are omitted because they must preserve values. Changing a source after graph construction does not update existing Dask keys. For consumers that require eager indexing, such as `dask.array.from_array`, wrap the view in `EagerArrayAdapter` (importable from `zarr_indexing`). The adapter materializes each indexed block while the wrapped view keeps lazy indexing, and its token derives from the view's. See Dask's [from_array documentation](https://docs.dask.org/en/stable/generated/dask.array.from_array.html). zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/guide/patterns.md000066400000000000000000000225331527014137600302400ustar00rootroot00000000000000# Indexing pattern reference Every NumPy indexing idiom is modeled by an `IndexTransform`: a domain (the result's coordinates) and one output map per source dimension. This page builds that model **by hand for each idiom**, so the anatomy is explicit — which map kind an idiom needs, where the offset and stride go, and how an index array's shape spells outer-product versus pointwise. Each model is then proven equal to what the selection compiler derives, and its values are checked against NumPy. ## The idiom-to-model matrix Each idiom over a 6-by-8 `image`, shown two ways: the Python construction, and the same transform in **wire form** — the [ndsel](../ndsel.md) canonical body `to_json` produces. Both spell the whole object: a domain whose extent is the result shape, then one output map per source dimension. The index-array variables (`rows`, `columns`, `mask_rows, mask_columns = np.nonzero(mask)`, and friends) are defined in the executable matrix at the end of the page. **`image[1:5, ::2]`** — box. The offset picks where cell 0 reads; the stride skips: === "Python" ```python IndexTransform( domain=IndexDomain.from_shape((4, 4)), output=( DimensionMap(input_dimension=0, offset=1), DimensionMap(input_dimension=1, stride=2), ), ) ``` === "JSON" ```json { "input_rank": 2, "input_inclusive_min": [0, 0], "input_exclusive_max": [4, 4], "input_labels": ["", ""], "output": [ {"offset": 1, "stride": 1, "input_dimension": 0}, {"offset": 0, "stride": 2, "input_dimension": 1} ] } ``` **`image[2, :]`** — box. A rank-1 domain with two output maps: the dropped axis survives as the wire's constant form, a bare `{"offset": 2}`: === "Python" ```python IndexTransform( domain=IndexDomain.from_shape((8,)), output=(ConstantMap(2), DimensionMap(input_dimension=0)), ) ``` === "JSON" ```json { "input_rank": 1, "input_inclusive_min": [0], "input_exclusive_max": [8], "input_labels": [""], "output": [ {"offset": 2}, {"offset": 0, "stride": 1, "input_dimension": 0} ] } ``` **`image[::-2, :]`** — box. Reversal is nothing but a negative stride, and the offset is where cell 0 reads (row 5): === "Python" ```python IndexTransform( domain=IndexDomain.from_shape((3, 8)), output=( DimensionMap(input_dimension=0, offset=5, stride=-2), DimensionMap(input_dimension=1), ), ) ``` === "JSON" ```json { "input_rank": 2, "input_inclusive_min": [0, 0], "input_exclusive_max": [3, 8], "input_labels": ["", ""], "output": [ {"offset": 5, "stride": -2, "input_dimension": 0}, {"offset": 0, "stride": 1, "input_dimension": 1} ] } ``` **`image[2:2, :]`** — box. Emptiness lives in the domain (`input_exclusive_max[0]` equals the minimum); the maps are ordinary: === "Python" ```python IndexTransform( domain=IndexDomain.from_shape((0, 8)), output=( DimensionMap(input_dimension=0, offset=2), DimensionMap(input_dimension=1), ), ) ``` === "JSON" ```json { "input_rank": 2, "input_inclusive_min": [0, 0], "input_exclusive_max": [0, 8], "input_labels": ["", ""], "output": [ {"offset": 2, "stride": 1, "input_dimension": 0}, {"offset": 0, "stride": 1, "input_dimension": 1} ] } ``` **`image[mask]`** — query. A mask is its nonzero coordinates: two correlated index arrays over one flat axis, entry `i` of each pairing into one cell: === "Python" ```python IndexTransform( domain=IndexDomain.from_shape((10,)), output=(ArrayMap(mask_rows), ArrayMap(mask_columns)), ) ``` === "JSON" ```json { "input_rank": 1, "input_inclusive_min": [0], "input_exclusive_max": [10], "input_labels": [""], "output": [ {"offset": 0, "stride": 1, "index_array": [0, 0, 1, 1, 2, 3, 3, 4, 5, 5], "index_array_bounds": ["-inf", "+inf"]}, {"offset": 0, "stride": 1, "index_array": [0, 5, 2, 7, 4, 1, 6, 3, 0, 5], "index_array_bounds": ["-inf", "+inf"]} ] } ``` **`image[np.ix_(rows, columns)]`** — query. The outer product is spelled by nesting: `[[4], [1], [1]]` varies down the first axis, `[[2, 5]]` across the second, each singleton along the other: === "Python" ```python IndexTransform( domain=IndexDomain.from_shape((3, 2)), output=(ArrayMap(rows.reshape(3, 1)), ArrayMap(columns.reshape(1, 2))), ) ``` === "JSON" ```json { "input_rank": 2, "input_inclusive_min": [0, 0], "input_exclusive_max": [3, 2], "input_labels": ["", ""], "output": [ {"offset": 0, "stride": 1, "index_array": [[4], [1], [1]], "index_array_bounds": ["-inf", "+inf"]}, {"offset": 0, "stride": 1, "index_array": [[2, 5]], "index_array_bounds": ["-inf", "+inf"]} ] } ``` **`image[vector_rows, vector_columns]`** — query. Pointwise: two flat arrays over one shared axis: === "Python" ```python IndexTransform( domain=IndexDomain.from_shape((3,)), output=(ArrayMap(vector_rows), ArrayMap(vector_columns)), ) ``` === "JSON" ```json { "input_rank": 1, "input_inclusive_min": [0], "input_exclusive_max": [3], "input_labels": [""], "output": [ {"offset": 0, "stride": 1, "index_array": [4, 1, 1], "index_array_bounds": ["-inf", "+inf"]}, {"offset": 0, "stride": 1, "index_array": [2, 5, 2], "index_array_bounds": ["-inf", "+inf"]} ] } ``` **`image[broadcast_rows, broadcast_columns]`** — query. NumPy broadcasting, kept compact: singleton axes retain the independent dependencies: === "Python" ```python IndexTransform( domain=IndexDomain.from_shape((2, 3)), output=( ArrayMap(broadcast_rows.reshape(2, 1)), ArrayMap(broadcast_columns.reshape(1, 3)), ), ) ``` === "JSON" ```json { "input_rank": 2, "input_inclusive_min": [0, 0], "input_exclusive_max": [2, 3], "input_labels": ["", ""], "output": [ {"offset": 0, "stride": 1, "index_array": [[0], [3]], "index_array_bounds": ["-inf", "+inf"]}, {"offset": 0, "stride": 1, "index_array": [[1, 4, 6]], "index_array_bounds": ["-inf", "+inf"]} ] } ``` **`image[rows, 2:6]`** — query. One lookup table (order and repeats kept) beside one ordinary affine map — one index array makes the whole selection a query: === "Python" ```python IndexTransform( domain=IndexDomain.from_shape((3, 4)), output=( ArrayMap(rows.reshape(3, 1)), DimensionMap(input_dimension=1, offset=2), ), ) ``` === "JSON" ```json { "input_rank": 2, "input_inclusive_min": [0, 0], "input_exclusive_max": [3, 4], "input_labels": ["", ""], "output": [ {"offset": 0, "stride": 1, "index_array": [[4], [1], [1]], "index_array_bounds": ["-inf", "+inf"]}, {"offset": 2, "stride": 1, "input_dimension": 1} ] } ``` Two structural rules do all the work: - **Category**: `ConstantMap` and `DimensionMap` entries keep a selection a box at any composition depth; one `ArrayMap` makes it a query permanently. See the [design notes](../design-notes.md#bounding-box-selections-vs-query-selections) for why consumers dispatch on this. - **Fancy flavor is spelled by shape**: index arrays varying over distinct axes (singleton elsewhere) form an outer product; arrays sharing their non-singleton axes pair pointwise. ## The executable matrix Each case hand-builds the model, checks shape, category, and NumPy values (resolved through the public reader), then proves the selection compiler derives the same transform. One wrinkle the last assert documents: compiled *basic* selections keep literal domains (`t[1:5, ...]` starts at coordinate 1 — see [Positions vs literal coordinates](#positions-vs-literal-coordinates)), so they equal the zero-origin models after `translate_domain_to`: ```python --8<-- "snippets/indexing_patterns.py:indexing-patterns" ``` `LazyArray` exposes the transform machinery through a positional array-like API. Its `[...]`, `.oindex[...]`, and `.vindex[...]` operations return views and normalize positions before composition. This boundary differs from the literal coordinate semantics of `IndexTransform`, as the following table shows. The executable matrix checks the documented cases, not every possible NumPy expression. ## Positions vs literal coordinates | Surface | Meaning of an integer index | Meaning of `-1` | | --- | --- | --- | | `IndexDomain` and `IndexTransform` | A literal coordinate in the current domain | The address `-1`, when the domain contains it | | `LazyArray` | A NumPy-style position in the current view | The last position, normalized before transform composition | The wrapper's three indexing modes all use positions in the current view. Each derived view begins at position zero, while the transform algebra underneath retains literal coordinates. The [Coordinates are addresses](index.md#coordinates-are-addresses) section develops that distinction with non-zero and negative-origin domains. --- zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/guide/selection-flow.md000066400000000000000000000245271527014137600313370ustar00rootroot00000000000000# From a selection to chunk operations A selection describes values in an array. A chunk operation identifies a chunk, which values to select inside its decoded buffer, and where those values belong in the result. This page follows that conversion through the current planner and explains where the proposed sharding integration would fit. The public planning API is `plan_chunks(transform, grids)`. The plan describes coordinate mappings; a consumer decides how to turn them into selectors for its buffer implementation. Zarr's default indexers have not been replaced. ## The flow ```text selection + source domain + indexing dialect | normalize / compose | request-to-storage mapping | + caller-selected chunk grid v factor and partition the request | +---------+-----------------+ | | declarative projections execution lowering chunk coordinates chunk coordinates chunk_transform chunk_selection cell_transform out_selection coverage selected-value shape / coverage flag | | +------------+--------------+ | consumer fetches, decodes, gathers/scatters, encodes ``` The right-hand branch describes consumer responsibilities, not a second public planning API. A consumer can lower table rows directly without first allocating a transform pair for every chunk. ## 1. Interpret the selection The indexing boundary chooses the dialect before chunk planning begins: - A positional boundary, such as `LazyArray`, interprets NumPy-style negative indices and slice clipping against the current view. - `IndexTransform` uses literal coordinates. Bounds are addresses; they do not wrap or clip like NumPy bounds. - Basic, orthogonal, and vectorized indexing specify different mappings. Orthogonal arrays form an outer product; vectorized arrays select paired, broadcast coordinates. Normalization resolves omitted dimensions and slice bounds and checks the selection. Composing another selection onto a view produces a mapping directly to the original source, rather than reading the intermediate view first. See [Indexing patterns](patterns.md) for the dialect examples. An `IndexTransform` represents this mapping with a request domain and one output map per storage dimension: a constant, an affine function of a request axis, or an index-array lookup. An integer selection can therefore remove a result axis while retaining a constant coordinate for that storage dimension. ## 2. Keep the coordinate spaces distinct | Space | Meaning | Example for `a[1:7:2, 2:6]` | | --- | --- | --- | | Global storage | Addresses in the source array | Rows `1, 3, 5`; columns `2, 3, 4, 5` | | Chunk grid | Which grid cell contains an address | With `(4, 4)` chunks, address `(5, 4)` belongs to chunk `(1, 1)` | | Chunk local | Address relative to that chunk's origin | `(5, 4) - (4, 4) = (1, 0)` | | Request domain | Coordinates accepted by the request transform | May have a nonzero origin under literal slice semantics | | Result buffer | Zero-based positions in the materialized result | Source `(5, 4)` goes to result `(2, 2)` | For each storage dimension, the grid supplies `index_to_chunk`, `chunk_offset`, and `chunk_size`. Uniform grids can use integer division; irregular grids use their actual boundaries. The planner consumes this grid protocol rather than assuming that all chunks have the same size. A declarative `cell_transform` returns **request-domain coordinates**, not necessarily zero-based buffer positions. A consumer placing values into a NumPy result subtracts the request domain origin. An execution consumer must translate these into zero-based buffer positions. ## 3. Partition without expanding the entire request Planning groups selected values by the chunks that contain them, retaining the mapping back to their original request positions. For affine selections, the planner finds each run that falls inside one chunk. A run records the chunk coordinate, chunk origin, local start, number of selected values, and starting request position. For a rectangular request, the multidimensional chunk operations are products of these axis runs. `StridedSet` tables materialize these per-axis rows, without materializing their full Cartesian product. For index arrays, planning groups coordinates by chunk while retaining their request positions. Arrays that share request axes must be grouped together: those coordinates are correlated. Independent connected components stay separate until chunk operations are visited. This avoids expanding an outer product into one coordinate tuple per requested value during preparation. `ChunkPlan.partition()` memoizes the factored `GridPartition`. Its tables include strided sets, indexed sets, and joint sets for connected index-array components. Plan iteration builds paired projections from those tables. Affine diagonals (two slice maps reading one request axis) are rejected with `ValueError`; see [the current limitations](../design-notes.md#current-scope). An empty request produces no chunk operations. Repeated coordinates remain repeated request positions for reads: the selection is an ordered mapping, not a mathematical set of source points. ## 4. Describe each chunk's contribution The declarative representation is a `ChunkProjection`: - `chunk_coords` identifies the grid cell, and `chunk_domain` gives its global storage bounds. - `chunk_transform` maps a shared synthetic domain to chunk-local coordinates. - `cell_transform` maps the same synthetic domain to request-domain coordinates. The shared domain pairs each source value with its destination. Its coordinates are neither automatically chunk-local nor automatically result positions. For every point `p` in that domain, the defining relationship is: ```text request_transform(cell_transform(p)) == chunk_origin + chunk_transform(p) ``` A consumer can evaluate these transforms, consume the factored tables directly, or lower them to its own selection vocabulary. For a NumPy reader, the eventual operation has this meaning: ```text result[result_selection] = decoded_chunk[chunk_selection] ``` The chunk coordinates identify which decoded chunk to obtain. For writes, values flow in the opposite direction. Basic selectors use slice/integer semantics; coordinate selectors are paired arrays that broadcast together. The consumer must preserve both the selected-value shape and the coordinate correspondence when choosing between these representations. ### A strided rectangle crossing four chunks For an `(8, 10)` source with `(4, 4)` chunks, `a[1:7:2, 2:6]` has result shape `(3, 4)`. The following slices describe the four contributions; slice stops are shown clipped to each chunk's extent, so equivalent generated stops may differ. | Chunk coordinates | Chunk origin | Chunk-local selection | Result selection | | --- | --- | --- | --- | | `(0, 0)` | `(0, 0)` | `[1:4:2, 2:4]` | `[0:2, 0:2]` | | `(0, 1)` | `(0, 4)` | `[1:4:2, 0:2]` | `[0:2, 2:4]` | | `(1, 0)` | `(4, 0)` | `[1:2:2, 2:4]` | `[2:3, 0:2]` | | `(1, 1)` | `(4, 4)` | `[1:2:2, 0:2]` | `[2:3, 2:4]` | This executable example checks both the touched chunks and the assembled values. The example enumerates the small synthetic domains to make both projection directions explicit. Production consumers can instead lower the factored tables to slices and coordinate arrays. Array slicing stands in for storage and codecs: ```python --8<-- "snippets/selection_flow.py:basic" ``` ### A gather with repeated, out-of-order coordinates For `a[[6, 1, 6, 4]]` and chunk size `4`, grouping by chunk must preserve these pairs: | Chunk | Chunk-local coordinates | Result positions | | --- | --- | --- | | `(0,)` | `[1]` | `[1]` | | `(1,)` | `[2, 2, 0]` | `[0, 2, 3]` | Chunk `(1,)` is fetched once, but the value at local coordinate `2` contributes to two result positions. Chunk visitation order need not be result order; `out_selection` restores the requested arrangement. ```python --8<-- "snippets/selection_flow.py:gather" ``` ## 5. Coverage and execution `StridedSet.full` and `ChunkProjection.coverage` describe coverage of the valid **data extent** of a grid cell, including a clipped boundary cell. A singleton cell selected once is full even when the request stride has magnitude greater than one. Repeated coverage is not full; fancy coverage remains conservative. Zarr's merge operation can skip a read for a full data-extent write and allocate a fill-valued codec buffer when the selected data is smaller than that buffer. That shortcut also requires the consumer's expected value layout and order; complete coverage by a reversal or reordered gather alone does not establish that the supplied buffer can be copied directly into codec order. Touching every chunk alone does not prove full coverage of each chunk. Planning does not fetch bytes, decode buffers, choose concurrency, or define write-conflict handling. Consumers must preserve request ordering where their write API requires it, and account for memory retained by emitted operations. ## 6. Applying the flow to sharding A sharded array has an outer shard grid and an inner codec-chunk grid. A consumer can partition the request first by outer shard, then map each contribution into the shard's coordinate frame and partition against the inner grid: ```text request -> global storage -> outer shard + request-to-shard-local mapping -> inner chunk + request-to-inner-local mapping -> inner selection + destination in the result ``` This is a proposed integration, not implemented by this PR. The existing sharding codec receives selectors and calls its own indexer against the inner grid. A future consumer could retain compact transforms through both levels, collect touched chunk coordinates for byte-range retrieval, and lower value selectors only when needed. Direct scattering into the final result would also require a pipeline interface carrying that buffer and its destination mapping. No on-disk format change is needed. Fill values, missing chunks, partial-write coverage, and the consumer's duplicate-write policy must survive both levels. See [Integration boundaries](integrations.md) for consumers of projections and tables, and [Design notes](../design-notes.md) for implementation constraints. zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/index.md000066400000000000000000000043731527014137600264140ustar00rootroot00000000000000# zarr-indexing This library is for modelling and transforming NumPy-style array indexing expressions. It separates the *declaration* of an array indexing expression from the result of that expression. Developed for use in [`zarr`](https://zarr.readthedocs.io). Inspired by [TensorStore's index-transform model](https://google.github.io/tensorstore/index_space.html). ## Install `zarr-indexing` is developed in the [zarr-python repository](https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-indexing) and released independently of `zarr` itself: ``` pip install zarr-indexing ``` ## Quickstart Wrap an array, compose a lazy view with `view[...]`, and call `result()` when you want its values: ```python --8<-- "snippets/canonical_slice.py:landing-quickstart" ``` Composing these selections does not read source values; the example reads them at `result()`. Construction inspects source metadata, and Dask tokenization can inspect source values. [Lazy views compose](guide/index.md#lazy-views-compose) shows how the chain stays one description, and where the materialization boundary is. ## Learn more - [Visual guide](guide/index.md) — one selection followed from coordinates to chunk plan. Using lazy indexing, start at [An index selects coordinates](guide/index.md#an-index-selects-coordinates); integrating a chunked backend, start at [A request becomes a chunk plan](guide/index.md#a-request-becomes-a-chunk-plan) and finish with the per-axis tables a plan is [built from](guide/index.md#a-plan-is-a-product-of-per-axis-tables). - [Indexing pattern reference](guide/patterns.md) — every selection form with its NumPy-verified result. - [Integration boundaries](guide/integrations.md) — what a reader, writer, or scheduler owns, and what the plan owns. - [Lazy indexing a NumPy array](examples/lazy_indexing_numpy.md) and [with Dask](examples/lazy_indexing_dask.md) — runnable examples. - [The ndsel wire format](ndsel.md) — the JSON form of a selection. - [Design notes](design-notes.md) — TensorStore lineage, box vs query, and deliberate limits. - [API reference](api/index.md) - [Release notes](release-notes.md) · [License (MIT)](https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-indexing/LICENSE.txt) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/ndsel.md000066400000000000000000000222151527014137600264050ustar00rootroot00000000000000--- title: The ndsel wire format --- # The ndsel wire format [ndsel](https://github.com/zarr-developers/ndsel) is a draft JSON representation of NumPy-style n-dimensional selections, adapted from TensorStore's `IndexTransform` model. This page documents the wire format, not the coordinate model: [Coordinates are addresses](guide/index.md#coordinates-are-addresses) introduces literal coordinates, and [Lazy views compose](guide/index.md#lazy-views-compose) shows how views combine before a transform is serialized. `zarr-indexing` implements ndsel in two layers: | Layer | Module | Depends on | Job | | --- | --- | --- | --- | | Message | [`zarr_indexing.messages`](api/messages.md) | stdlib only | Validates and desugars JSON, removing redundant constant-map fields. | | Engine | [`zarr_indexing.json`](api/json.md) | NumPy | Lowers a *canonical* body into an in-memory [`IndexTransform`](api/transform.md), and back. | Constraints that only make sense for a real array — finite bounds, index arrays as `ndarray`s — are checked during engine lowering. The message layer also limits input rank to 32 and checks affine input-dimension references. These checks are stricter than the draft's required message validation. As a result, `messages` normalizes a message with `"-inf"` bounds that `IndexTransform.from_json` refuses to lower. ## Two entry points [`parse_ndsel`](api/messages.md#zarr_indexing.messages.parse_ndsel) structurally validates a message of any kind and returns it unchanged. Use it to confirm that a message is well formed while keeping it in its compact shorthand form. [`normalize_ndsel`](api/messages.md#zarr_indexing.messages.normalize_ndsel) desugars a message into the single deterministic **canonical transform body** of the spec (section 4.3): a bare `IndexTransform` body without the `kind` discriminator. ```python from zarr_indexing import normalize_ndsel normalize_ndsel({"kind": "box", "inclusive_min": [10, 5], "shape": [40, 1]}) # {'input_rank': 2, # 'input_inclusive_min': [10, 5], # 'input_exclusive_max': [50, 6], # 'input_labels': ['', ''], # 'output': [{'offset': 0, 'stride': 1, 'input_dimension': 0}, # {'offset': 0, 'stride': 1, 'input_dimension': 1}]} ``` Normalization is idempotent: re-tag the output with `kind: "transform"` and normalizing it again returns the same body. The canonical body uses TensorStore's `IndexTransform` field vocabulary, but normalization does not guarantee acceptance by TensorStore. For example, the message layer leaves index-array content unchecked, whereas TensorStore validates it; TensorStore also requires unique nonempty labels and restricts finite index values to `[-(2**62 - 2), 2**62 - 2]`. See [TensorStore's index-space constraints](https://google.github.io/tensorstore/index_space.html). Both entry points raise [`NdselError`](api/messages.md#zarr_indexing.messages.NdselError), which carries the spec `reason` code (`unknown_kind`, `rank_mismatch`, `step_zero`, `output_map_conflict`, …) alongside a human-readable detail, so callers can branch on the code rather than on message text. ## The five message kinds Four are shorthands; the fifth is the canonical form itself. | `kind` | Fields | Selects | | --- | --- | --- | | `point` | `coords` | A single element. Normalizes to rank 0 with one `constant` output map per dimension. | | `box` | `inclusive_min`, one of `exclusive_max` / `inclusive_max` / `shape`, `labels` | A rectangular region. At most one upper-bound spelling may appear; omission gives implicit positive infinity. | | `slice` | `start`, `stop`, `step`, `labels` | A strided region, one Python-style slice per dimension. | | `points` | `coords` (a list of coordinate rows) | An explicit list of points — the `vindex` case. Normalizes to one `index_array` output map per dimension over a shared rank-1 input domain. | | `transform` | `input_rank`, `input_inclusive_min`, one of the three `input_*` upper bounds, `input_labels`, `output` | The full canonical form. | Value rules for validated fields (excluding verbatim `index_array` payloads and discarded constant-map fields): every integer is a 64-bit signed value; JSON booleans are **not** integers (Python's `isinstance(True, int)` is guarded against explicitly); the `"-inf"` / `"+inf"` sentinels are legal only in bound positions; and an implicit bound is the one-element `[n]`-bracket form, whose implicit/explicit flag survives normalization intact. ## Lowering to a transform The engine layer converts between canonical bodies and `IndexTransform`s: ```python from zarr_indexing import IndexTransform, normalize_ndsel canonical = normalize_ndsel({"kind": "box", "shape": [2, 3]}) t = IndexTransform.from_json(canonical) assert t.to_json() == canonical ``` `IndexDomain` carries the same pair for a bare domain body, and each output map kind has a `to_json`; `output_index_map_from_json` dispatches the wire's structurally discriminated union back to the right kind. Exact JSON equality in this example is not a general round-trip guarantee: implicit flags are removed and degenerate array maps are collapsed. `index_array_bounds` constrains raw index-array values before the map's offset and stride are applied. Both `IndexTransform.from_json` and `output_index_map_from_json` validate every supplied value against the inclusive bounds when loading. Finite and one-sided bounds are supported; omitted bounds and `["-inf", "+inf"]` impose no additional constraint. Values outside the bounds raise `NdselError("invalid_json", ...)`, including in singleton arrays and zero-stride maps. Empty arrays satisfy any well-formed, ordered bounds. Validation is eager: an invalid entry rejects the entire map even if a later selection would avoid that entry. After validation the engine owns immutable index coordinates, so it need not retain the bounds; serialization emits unbounded constraints for non-degenerate maps. Message normalization preserves the original bounds without checking array contents. This implementation does not defer bounds errors until individual positions are accessed. A canonical body carrying a `"-inf"` or `"+inf"` bound cannot be lowered — an `IndexDomain` addresses a finite array — so `IndexTransform.from_json` raises. And implicit bounds lower *by value*: the `[n]`-bracket flag is a message-layer concern, and the engine keeps only the integer. ### The `index_array` round trip ndsel and TensorStore both **reject** an output map that carries both `input_dimension` and `index_array`. The in-memory [`ArrayMap`](api/output_map.md#zarr_indexing.output_map.ArrayMap), though, records its dependency axes in its full-rank array shape, with no `input_dimension` field: - **On serialize**, a non-degenerate `index_array` map is emitted *without* `input_dimension`. - **On load**, dependency axes are the full-rank array's non-singleton axes. Maps sharing these axes describe correlated coordinates. The engine also accepts lower-rank nonempty arrays by prepending singleton axes; that convenience is not a guarantee of compatibility with other ndsel consumers. An all-singleton `index_array` — size 1 — selects the same coordinate regardless of the input, so it is collapsed to a `constant` map on serialize: ```python from zarr_indexing import IndexTransform IndexTransform.from_shape((100, 100)).oindex[[5], 0:2].to_json() # {'input_rank': 2, # 'input_inclusive_min': [0, 0], # 'input_exclusive_max': [1, 2], # 'input_labels': ['', ''], # 'output': [{'offset': 5}, # {'offset': 0, 'stride': 1, 'input_dimension': 1}]} ``` The size-1 input dimension stays in the domain, unconsumed by any output map. The transform is still valid and the output shape is unchanged. A length-1 `oindex` selection therefore round-trips behaviorally (an `ArrayMap` comes back as a `ConstantMap`) rather than by object identity. Empty index arrays also serialize as constant maps with offset zero: the empty input domain carries the fact that no coordinates are selected. This avoids losing trailing shape information in JSON when an array has a leading zero-length axis. ## Conformance The package is checked against the language-agnostic ndsel conformance corpus, vendored unmodified under [`tests/conformance/`](https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-indexing/tests/conformance) — one JSON file per message kind plus `errors.json`, with the source commit recorded in `PROVENANCE.md`. Each fixture is either a *success* case (`input` + expected `normalized` body) or an *error* case (`input` + expected reason code). Matching every fixture establishes corpus conformance; the fixtures do not prove correctness for every possible input or universal TensorStore compatibility. `tests/test_conformance.py` runs the whole corpus as one parametrized test per fixture, so a corpus update reports failures fixture by fixture rather than as a single opaque assertion. Do not edit the vendored files; to pick up spec changes, re-vendor from a newer ndsel commit and update the recorded SHA. A second, optional test (`tests/test_ndsel_tensorstore.py`, skipped unless `tensorstore` is installed) checks against TensorStore itself by loading canonical bodies into `tensorstore.IndexTransform` and re-loading TensorStore's own `to_json()` output back through the engine layer. zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/release-notes.md000066400000000000000000000010041527014137600300370ustar00rootroot00000000000000 --8<-- "CHANGELOG.md" zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/snippets/000077500000000000000000000000001527014137600266215ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/snippets/axis_manipulation.py000066400000000000000000000014431527014137600327210ustar00rootroot00000000000000"""Indexing defines result axes as well as selected source points.""" import numpy as np from zarr_indexing import LazyArray # --8<-- [start:axis-shape-comparison] image = np.arange(12).reshape(3, 4) lazy = LazyArray.from_numpy(image) integer_view = lazy[1, :] slice_view = lazy[1:2, :] INTEGER_RESULT = integer_view.result() SLICE_RESULT = slice_view.result() assert INTEGER_RESULT.tolist() == [4, 5, 6, 7] assert INTEGER_RESULT.shape == (4,) assert SLICE_RESULT.tolist() == [[4, 5, 6, 7]] assert SLICE_RESULT.shape == (1, 4) # --8<-- [end:axis-shape-comparison] # --8<-- [start:axis-insertion] inserted_view = lazy[1, :, None] INSERTED_RESULT = inserted_view.result() assert INSERTED_RESULT.tolist() == [[4], [5], [6], [7]] assert INSERTED_RESULT.shape == (4, 1) # --8<-- [end:axis-insertion] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/snippets/canonical_slice.py000066400000000000000000000012621527014137600323020ustar00rootroot00000000000000"""The canonical basic-selection example used throughout the guide.""" import numpy as np from zarr_indexing import LazyArray # --8<-- [start:landing-quickstart] import numpy as np from zarr_indexing import LazyArray source = np.array([10, 11, 12, 13, 14, 15]) view = LazyArray.from_numpy(source)[2:5] view.result() # array([12, 13, 14]) # --8<-- [end:landing-quickstart] LANDING_QUICKSTART_RESULT = view.result() assert LANDING_QUICKSTART_RESULT.tolist() == [12, 13, 14] # --8<-- [start:canonical-slice] source = np.array([10, 11, 12, 13, 14, 15]) lazy = LazyArray.from_numpy(source) view = lazy[2:5] assert view.result().tolist() == [12, 13, 14] # --8<-- [end:canonical-slice] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/snippets/chunk_projection.py000066400000000000000000000034351527014137600325440ustar00rootroot00000000000000"""Use public chunk projections without choosing a storage backend.""" import numpy as np from numpy.typing import NDArray from typing import cast from zarr_indexing import DimensionGridLike, IndexTransform, LazyArray, plan_chunks # --8<-- [start:chunk-projection] class RegularGrid: """A small parameterized implementation of the public grid protocol.""" def __init__(self, size: int) -> None: self.size = size def index_to_chunk(self, index: int) -> int: return index // self.size def chunk_offset(self, chunk: int) -> int: return chunk * self.size def chunk_size(self, chunk: int) -> int: return self.size def indices_to_chunks(self, indices: NDArray[np.intp]) -> NDArray[np.intp]: return np.floor_divide(indices, self.size).astype(np.intp) transform = IndexTransform.from_shape((3, 4))[1, 0:4] grids = cast( tuple[DimensionGridLike, DimensionGridLike], (RegularGrid(2), RegularGrid(2)), ) plan = plan_chunks(transform, grids) PROJECTIONS = tuple(plan) assert tuple(projection.chunk_coords for projection in plan) == ((0, 0), (0, 1)) PAIRED_DOMAINS = tuple( (projection.chunk_transform.domain, projection.cell_transform.domain) for projection in PROJECTIONS ) assert all(chunk_domain == cell_domain for chunk_domain, cell_domain in PAIRED_DOMAINS) # --8<-- [end:chunk-projection] # --8<-- [start:advanced-projection] image = np.arange(48).reshape(6, 8) advanced = LazyArray.from_numpy(image).with_parts((3, 4)).oindex[[4, 1, 1], 2:6] ADVANCED_EXPECTED = image[[4, 1, 1]][:, 2:6] ADVANCED_RESULT = np.empty_like(ADVANCED_EXPECTED) for part in advanced.parts(): ADVANCED_RESULT[part.out_selection] = part.view.result() np.testing.assert_array_equal(ADVANCED_RESULT, ADVANCED_EXPECTED) # --8<-- [end:advanced-projection] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/snippets/coordinate_origins.py000066400000000000000000000037321527014137600330610ustar00rootroot00000000000000"""Literal coordinate domains and a grid that supports prepending.""" import numpy as np from numpy.typing import NDArray from typing import cast from zarr_indexing import ( DimensionGridLike, IndexDomain, IndexTransform, LazyArray, plan_chunks, ) # --8<-- [start:coordinate-origin] domain = IndexDomain(inclusive_min=(-2,), exclusive_max=(3,)) assert domain.contains((-1,)) assert domain.narrow(-1).inclusive_min == (-1,) values = np.array([10, 20, 30, 40, 50]) assert LazyArray.from_numpy(values)[-1:].result().tolist() == [50] # --8<-- [end:coordinate-origin] # --8<-- [start:prepend-grid] class PrependableGrid: """A regular grid whose coordinates may extend below zero.""" def index_to_chunk(self, index: int) -> int: return index // 3 def chunk_offset(self, chunk: int) -> int: return chunk * 3 def chunk_size(self, chunk: int) -> int: return 3 def indices_to_chunks(self, indices: NDArray[np.intp]) -> NDArray[np.intp]: return np.floor_divide(indices, 3).astype(np.intp) projection, = plan_chunks( IndexTransform.identity(IndexDomain((-3,), (0,))), cast(tuple[DimensionGridLike], (PrependableGrid(),)), ) assert projection.chunk_coords == (-1,) assert projection.chunk_domain == IndexDomain((-3,), (0,)) assert projection.chunk_transform.domain == IndexDomain((0,), (3,)) assert projection.chunk_transform.domain == projection.cell_transform.domain PREPEND_SHARED_CELL_COORDS = ((0,), (1,), (2,)) prepend_shared_cell_points = np.asarray(PREPEND_SHARED_CELL_COORDS, dtype=np.intp) PREPEND_CHUNK_LOCAL_COORDS = tuple( tuple(point) for point in projection.chunk_transform.apply_many(prepend_shared_cell_points).tolist() ) PREPEND_REQUEST_COORDS = tuple( tuple(point) for point in projection.cell_transform.apply_many(prepend_shared_cell_points).tolist() ) assert PREPEND_CHUNK_LOCAL_COORDS == ((0,), (1,), (2,)) assert PREPEND_REQUEST_COORDS == ((-3,), (-2,), (-1,)) # --8<-- [end:prepend-grid] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/snippets/grid_partition.py000066400000000000000000000146051527014137600322170ustar00rootroot00000000000000"""Read a plan's factored form — its per-axis tables — instead of its projections.""" from typing import Any import numpy as np from zarr_indexing import IndexDomain, IndexedSet, IndexTransform, StridedSet, plan_chunks from zarr_indexing.output_map import DimensionMap from zarr_indexing.grid import dimension_grids_from_chunks # --8<-- [start:strided-tables] grids = dimension_grids_from_chunks((2, 2), shape=(3, 4)) plan = plan_chunks(IndexTransform.from_shape((3, 4))[1, :], grids) partition = plan.partition() rows, columns = partition.sets assert isinstance(rows, StridedSet) and rows.input_dimension is None # the fixed row assert isinstance(columns, StridedSet) and columns.input_dimension == 0 assert columns.chunk.tolist() == [0, 1] assert columns.local_start.tolist() == [0, 0] assert columns.extent.tolist() == [2, 2] assert columns.origin.tolist() == [0, 2] # request columns 0:2 and 2:4 assert columns.full.tolist() == [True, True] # One row of each table is one projection; the plan iterates exactly those rows. assert partition.row_shape == (1, 2) assert partition.chunk_coords().tolist() == [[0, 0], [0, 1]] rows = list(partition) assert rows == list(plan) assert rows[1].chunk_transform.apply((0,)) == (1, 0) # cell 0: chunk-local (1, 0) ... assert rows[1].cell_transform.apply((0,)) == (2,) # ... which is request column 2 # --8<-- [end:strided-tables] # --8<-- [start:indexed-and-joint] grids = dimension_grids_from_chunks((3, 4), shape=(6, 8)) gather = IndexTransform.from_shape((6, 8)).oindex[np.array([4, 1, 1]), 2:6] rows, columns = plan_chunks(gather, grids).partition().sets assert isinstance(rows, IndexedSet) assert rows.chunk.tolist() == [0, 1] # rows 1 and 1 land in chunk 0, row 4 in chunk 1 assert rows.pointer.tolist() == [0, 2, 3] # chunk 0 owns entries 0:2, chunk 1 owns 2:3 assert rows.index.tolist() == [1, 1, 4] # grouped by chunk, request order within a chunk assert rows.positions.tolist() == [1, 2, 0] # the request positions those entries fill assert rows.local.tolist() == [1, 1, 1] # chunk-local: 1 - 0, 1 - 0, 4 - 3 points = IndexTransform.from_shape((6, 8)).vindex[np.array([0, 5, 5]), np.array([7, 0, 1])] joint = plan_chunks(points, grids).partition().joint_sets[0] assert joint is not None assert joint.chunk.tolist() == [[0, 1], [1, 0]] # touched chunks, lexicographic assert joint.pointer.tolist() == [0, 1, 3] # point 0 alone; points 1 and 2 share a chunk assert joint.positions.tolist() == [0, 1, 2] assert joint.local.tolist() == [[0, 3], [2, 0], [2, 1]] # --8<-- [end:indexed-and-joint] # --8<-- [start:table-consumer] def read_box_through_tables( source: np.ndarray[Any, Any], chunks: tuple[int, ...], transform: IndexTransform ) -> tuple[np.ndarray[Any, Any], list[tuple[int, ...]]]: """Assemble a box selection from the per-axis tables, one slice per chunk. No paired projection objects are materialized. The consumer reads table columns, constructs selectors, and assembles each chunk contribution, including axis permutation and expansion where needed. """ partition = plan_chunks( transform, dimension_grids_from_chunks(chunks, shape=source.shape) ).partition() domain = transform.domain out = np.empty(domain.shape, dtype=source.dtype) tables = [table for table in partition.sets if isinstance(table, StridedSet)] assert len(tables) == len(partition.sets) # a box: strided tables only # A chunk slab comes out in storage-axis order; the result is in request-axis # order, and request axes no map reads (`None` in the selection) are length 1. read_axes = [table.input_dimension for table in tables if table.input_dimension is not None] to_request_order = tuple(np.argsort(read_axes)) unread_axes = [axis for axis in range(domain.ndim) if axis not in read_axes] reads: list[tuple[int, ...]] = [] for table_rows in np.ndindex(*partition.row_shape): chunk_key: list[int] = [] chunk_sel: list[int | slice] = [] out_sel: list[slice] = [slice(None)] * domain.ndim for table, row in zip(tables, table_rows, strict=True): chunk_key.append(int(table.chunk[row])) start = int(table.local_start[row]) count = int(table.extent[row]) if table.input_dimension is None: chunk_sel.append(start) continue stop: int | None = start + table.stride * count if table.stride < 0 and stop < 0: stop = None # a negative stop would count from the end chunk_sel.append(slice(start, stop, table.stride)) first = int(table.origin[row]) out_sel[table.input_dimension] = slice(first, first + count) chunk = source[tuple(slice(k * c, (k + 1) * c) for k, c in zip(chunk_key, chunks, strict=True))] values = np.transpose(chunk[tuple(chunk_sel)], to_request_order) for axis in unread_axes: values = np.expand_dims(values, axis) out[tuple(out_sel)] = values reads.append(tuple(chunk_key)) return out, reads image = np.arange(63).reshape(7, 9) box = IndexTransform.from_shape((7, 9))[1:6:2, 5:] values, reads = read_box_through_tables(image, (3, 4), box) np.testing.assert_array_equal(values, image[1:6:2, 5:]) assert reads == [(0, 1), (0, 2), (1, 1), (1, 2)] # A reversed axis, an inserted axis, and a transposed transform read the same way. reversed_box = IndexTransform.from_shape((7, 9))[6:0:-1, None, ::3] np.testing.assert_array_equal(read_box_through_tables(image, (3, 4), reversed_box)[0], image[6:0:-1, None, ::3]) transposed = IndexTransform( domain=IndexDomain.from_shape((9, 7)), output=(DimensionMap(input_dimension=1), DimensionMap(input_dimension=0)), ) np.testing.assert_array_equal(read_box_through_tables(image, (3, 4), transposed)[0], image.T) # --8<-- [end:table-consumer] # --8<-- [start:components] i = np.arange(1000) transform = IndexTransform.from_shape((1000, 1000, 1000)).vindex[ i[:, None], i[:, None], i[None, :] ] partition = plan_chunks( transform, dimension_grids_from_chunks((1000, 1000, 1000), shape=(1000, 1000, 1000)) ).partition() assert [group.output_dimensions for group in partition.joint_sets] == [(0, 1), (2,)] assert sum(group.index.size for group in partition.joint_sets) == 3000 assert partition.row_shape == (1, 1) (projection,) = partition assert projection.chunk_transform.apply((5, 9)) == (5, 5, 9) assert projection.cell_transform.apply((5, 9)) == (5, 9) # --8<-- [end:components] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/snippets/indexing_patterns.py000066400000000000000000000163301527014137600327230ustar00rootroot00000000000000"""Every indexing idiom hand-built as an IndexTransform, proven against the compiler.""" from typing import Any, Literal, TypedDict import numpy as np from zarr_indexing import ( ArrayMap, ConstantMap, DimensionMap, IndexDomain, IndexTransform, ReadContext, numpy_reader, ) # --8<-- [start:indexing-patterns] class PatternCase(TypedDict): """One indexing idiom: its hand-built transform model and its NumPy result.""" name: str mode: Literal["basic", "oindex", "vindex"] selection: Any transform: IndexTransform expected: Any shape: tuple[int, ...] category: Literal["box", "query"] image = np.arange(48).reshape(6, 8) rows = np.array([4, 1, 1], dtype=np.intp) columns = np.array([2, 5], dtype=np.intp) mask = image % 5 == 0 mask_rows, mask_columns = np.nonzero(mask) vector_rows = np.array([4, 1, 1], dtype=np.intp) vector_columns = np.array([2, 5, 2], dtype=np.intp) broadcast_rows = np.array([[0], [3]], dtype=np.intp) broadcast_columns = np.array([[1, 4, 6]], dtype=np.intp) PATTERN_CASES: tuple[PatternCase, ...] = ( { # image[1:5, ::2] — an offset picks where cell 0 reads; a stride skips. "name": "basic-slice", "mode": "basic", "selection": (slice(1, 5), slice(None, None, 2)), "transform": IndexTransform( domain=IndexDomain.from_shape((4, 4)), output=( DimensionMap(input_dimension=0, offset=1), DimensionMap(input_dimension=1, stride=2), ), ), "expected": image[1:5, ::2], "shape": (4, 4), "category": "box", }, { # image[2, :] — the dropped axis survives as a ConstantMap: the result # is rank 1, but there is still one output map per source dimension. "name": "integer-axis-removal", "mode": "basic", "selection": (2, slice(None)), "transform": IndexTransform( domain=IndexDomain.from_shape((8,)), output=(ConstantMap(2), DimensionMap(input_dimension=0)), ), "expected": image[2, :], "shape": (8,), "category": "box", }, { # image[::-2, :] — reversal is only a negative stride; the offset is # where result cell 0 reads (the last selected row, 5). "name": "negative-stride", "mode": "basic", "selection": (slice(None, None, -2), slice(None)), "transform": IndexTransform( domain=IndexDomain.from_shape((3, 8)), output=( DimensionMap(input_dimension=0, offset=5, stride=-2), DimensionMap(input_dimension=1), ), ), "expected": image[::-2, :], "shape": (3, 8), "category": "box", }, { # image[2:2, :] — emptiness lives in the domain; the maps are ordinary. "name": "empty-selection", "mode": "basic", "selection": (slice(2, 2), slice(None)), "transform": IndexTransform( domain=IndexDomain.from_shape((0, 8)), output=( DimensionMap(input_dimension=0, offset=2), DimensionMap(input_dimension=1), ), ), "expected": image[2:2, :], "shape": (0, 8), "category": "box", }, { # image[mask] — a mask is its nonzero coordinates: two correlated # ArrayMaps over one flat result axis, row i paired with column i. "name": "boolean-mask", "mode": "vindex", "selection": mask, "transform": IndexTransform( domain=IndexDomain.from_shape((10,)), output=(ArrayMap(mask_rows), ArrayMap(mask_columns)), ), "expected": image[mask], "shape": (10,), "category": "query", }, { # image[np.ix_(rows, columns)] — the outer product is spelled by shape: # each array varies over its own distinct axis, singleton on the other. "name": "orthogonal", "mode": "oindex", "selection": (rows, columns), "transform": IndexTransform( domain=IndexDomain.from_shape((3, 2)), output=(ArrayMap(rows.reshape(3, 1)), ArrayMap(columns.reshape(1, 2))), ), "expected": image[np.ix_(rows, columns)], "shape": (3, 2), "category": "query", }, { # image[vector_rows, vector_columns] — pointwise: both arrays share # the same axis, so entry i of each pairs into one coordinate. "name": "vectorized", "mode": "vindex", "selection": (vector_rows, vector_columns), "transform": IndexTransform( domain=IndexDomain.from_shape((3,)), output=(ArrayMap(vector_rows), ArrayMap(vector_columns)), ), "expected": image[vector_rows, vector_columns], "shape": (3,), "category": "query", }, { # image[broadcast_rows, broadcast_columns] — NumPy broadcasting, # kept compact: singleton axes encode independent dependencies. "name": "broadcasting", "mode": "vindex", "selection": (broadcast_rows, broadcast_columns), "transform": IndexTransform( domain=IndexDomain.from_shape((2, 3)), output=( ArrayMap(broadcast_rows.reshape(2, 1)), ArrayMap(broadcast_columns.reshape(1, 3)), ), ), "expected": image[broadcast_rows, broadcast_columns], "shape": (2, 3), "category": "query", }, { # image[rows, 2:6] — one lookup-table axis (repeats and order kept) # beside one ordinary affine axis: one ArrayMap makes the whole # selection a query. "name": "repeated-out-of-order", "mode": "oindex", "selection": (rows, slice(2, 6)), "transform": IndexTransform( domain=IndexDomain.from_shape((3, 4)), output=( ArrayMap(rows.reshape(3, 1)), DimensionMap(input_dimension=1, offset=2), ), ), "expected": image[rows, 2:6], "shape": (3, 4), "category": "query", }, ) base = IndexTransform.from_shape(image.shape) def resolve(transform: IndexTransform) -> np.ndarray[Any, Any]: """Materialize a transform against `image` through the public reader.""" out = np.empty(transform.domain.shape, dtype=image.dtype) numpy_reader.read_into(image, ReadContext(transform), out) return out for case in PATTERN_CASES: transform = case["transform"] # The model is the idiom: shape, values, and category all follow from it. assert transform.domain.shape == case["shape"] np.testing.assert_array_equal(resolve(transform), case["expected"]) is_query = any(isinstance(m, ArrayMap) for m in transform.output) assert ("query" if is_query else "box") == case["category"] # The selection compiler derives the same transform. Compiled basic # selections keep literal domains (t[1:5, ...] starts at 1, not 0); # re-zeroing exposes the equality with the NumPy-shaped model. compiled = base[case["selection"]] if case["mode"] == "basic" else ( getattr(base, case["mode"])[case["selection"]] ) assert compiled.translate_domain_to((0,) * compiled.input_rank) == transform # --8<-- [end:indexing-patterns] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/snippets/integrations.py000066400000000000000000000140451527014137600317050ustar00rootroot00000000000000"""Boundaries for chunked-source and viewport consumers.""" from typing import Any import numpy as np from zarr_indexing import IndexDomain, LazyArray # --8<-- [start:zarr-consumer] class RecordingChunkSource: """A decoded-chunk source keyed by public chunk coordinates.""" def __init__(self, chunks: dict[tuple[int, ...], np.ndarray[Any, Any]]) -> None: self.chunks = chunks self.reads: list[tuple[int, ...]] = [] def read(self, chunk_coords: tuple[int, ...]) -> np.ndarray[Any, Any]: self.reads.append(chunk_coords) return self.chunks[chunk_coords] def _domain_points(domain: IndexDomain) -> np.ndarray[Any, np.dtype[np.intp]]: """Enumerate a rectangular domain with a trailing coordinate axis.""" if domain.ndim == 0: return np.empty((1, 0), dtype=np.intp) points = np.moveaxis(np.indices(domain.shape, dtype=np.intp), 0, -1).reshape( -1, domain.ndim ) points += np.asarray(domain.inclusive_min, dtype=np.intp) return points def _gather_and_scatter( destination: np.ndarray[Any, Any], source: np.ndarray[Any, Any], source_points: np.ndarray[Any, np.dtype[np.intp]], destination_points: np.ndarray[Any, np.dtype[np.intp]], ) -> np.ndarray[Any, Any]: """Gather and scatter a flattened point batch, including rank zero.""" values = np.asarray(source[tuple(source_points.T)]).reshape(-1) if destination_points.shape[-1] == 0: destination[()] = values.reshape(destination.shape)[()] else: destination[tuple(destination_points.T)] = values return values zarr_image = np.arange(12).reshape(3, 4) zarr_chunks = { (chunk_row, chunk_column): zarr_image[ chunk_row * 2 : (chunk_row + 1) * 2, chunk_column * 2 : (chunk_column + 1) * 2, ] for chunk_row in range(2) for chunk_column in range(2) } zarr_source = RecordingChunkSource(zarr_chunks) zarr_view = LazyArray.from_numpy(zarr_image).with_parts((2, 2))[1, 0:4] ZARR_RESULT = np.empty(zarr_view.shape, dtype=zarr_image.dtype) shared_domains: list[tuple[IndexDomain, IndexDomain]] = [] chunk_local_coords: list[tuple[tuple[int, ...], ...]] = [] request_coords: list[tuple[tuple[int, ...], ...]] = [] read_values: list[tuple[int, ...]] = [] for part in zarr_view.parts(): projection = part.projection assert projection.chunk_transform.domain == projection.cell_transform.domain shared_domains.append( (projection.chunk_transform.domain, projection.cell_transform.domain) ) domain = projection.chunk_transform.domain cell_points = _domain_points(domain) local_points_array = projection.chunk_transform.apply_many(cell_points) result_points_array = projection.cell_transform.apply_many(cell_points) chunk = zarr_source.read(projection.chunk_coords) values_array = _gather_and_scatter( ZARR_RESULT, chunk, local_points_array, result_points_array ) local_points = tuple(tuple(point) for point in local_points_array.tolist()) result_points = tuple(tuple(point) for point in result_points_array.tolist()) values = tuple(int(value) for value in values_array) chunk_local_coords.append(local_points) request_coords.append(result_points) read_values.append(values) ZARR_SOURCE_KEYS = tuple(zarr_source.chunks) ZARR_SOURCE_READS = tuple(zarr_source.reads) ZARR_DISPATCHED_CHUNKS = ZARR_SOURCE_READS ZARR_SHARED_DOMAINS = tuple(shared_domains) ZARR_CHUNK_LOCAL_COORDS = tuple(chunk_local_coords) ZARR_REQUEST_COORDS = tuple(request_coords) ZARR_READ_VALUES = tuple(read_values) assert ZARR_SOURCE_KEYS == ((0, 0), (0, 1), (1, 0), (1, 1)) assert ZARR_SOURCE_READS == ((0, 0), (0, 1)) assert ZARR_CHUNK_LOCAL_COORDS == (((1, 0), (1, 1)), ((1, 0), (1, 1))) assert ZARR_REQUEST_COORDS == (((0,), (1,)), ((2,), (3,))) assert ZARR_READ_VALUES == ((4, 5), (6, 7)) assert ZARR_RESULT.tolist() == [4, 5, 6, 7] # --8<-- [end:zarr-consumer] # --8<-- [start:viewport-consumer] class RecordingArray: """An array-like source that records the basic reads it receives.""" def __init__(self, data: np.ndarray[Any, Any], chunks: tuple[int, ...]) -> None: self._data = data self.chunks = chunks self.keys: list[tuple[slice, ...]] = [] @property def shape(self) -> tuple[int, ...]: return self._data.shape @property def dtype(self) -> np.dtype[Any]: return self._data.dtype def __getitem__(self, key: tuple[slice, ...]) -> np.ndarray[Any, Any]: self.keys.append(key) return self._data[key] viewport_source = RecordingArray(np.arange(12).reshape(3, 4), chunks=(2, 2)) viewport = LazyArray(viewport_source)[1, 0:4] VIEWPORT_READS_BEFORE_RESULT = tuple(viewport_source.keys) assert VIEWPORT_READS_BEFORE_RESULT == () assert viewport.result().tolist() == [4, 5, 6, 7] VIEWPORT_SOURCE_KEYS = tuple(viewport_source.keys) VIEWPORT_SOURCE_CHUNKS = tuple( (key[0].start // 2, key[1].start // 2) for key in VIEWPORT_SOURCE_KEYS ) assert VIEWPORT_SOURCE_KEYS == ( (slice(1, 2, 1), slice(0, 2, 1)), (slice(1, 2, 1), slice(2, 4, 1)), ) assert VIEWPORT_SOURCE_CHUNKS == ((0, 0), (0, 1)) # --8<-- [end:viewport-consumer] # --8<-- [start:dense-box-repartition] def materialize(view: LazyArray) -> Any: """Read a dense box as one slab; resolve everything else per part.""" strides = view.strides() if view.is_box and strides is not None and all(s == 1 for s in strides): view = view.with_parts(view.base_shape) return view.result() slab_source = RecordingArray(np.arange(100).reshape(10, 10), chunks=(4, 4)) slab = LazyArray(slab_source) dense = slab[2:9, 1:8] # a dense box: every stride 1 assert materialize(dense).shape == (7, 7) assert len(slab_source.keys) == 1 # one slab read; the source dispatches slab_source.keys.clear() gather = slab.oindex[[0, 9], [0, 9]] # a query: keep the chunk parts assert materialize(gather).tolist() == [[0, 9], [90, 99]] assert len(slab_source.keys) == 4 # four covers, each inside one chunk assert all( (key[0].stop - key[0].start) * (key[1].stop - key[1].start) == 1 for key in slab_source.keys ) # --8<-- [end:dense-box-repartition] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/snippets/key_types.py000066400000000000000000000022131527014137600312050ustar00rootroot00000000000000"""The type of a key decides whether it is read relative to the view or as absolute coordinates.""" import numpy as np from zarr_indexing import IndexDomain, IndexTransform, LazyArray source = np.arange(30) # --8<-- [start:key-types] view = LazyArray(source)[10:20] assert view.transform.domain == IndexDomain((10,), (20,)) # NumPy keys are positions relative to the view: 0 is its first element. relative = view[2:5] assert relative.result().tolist() == [12, 13, 14] assert view[-1].result() == 19 # An IndexDomain key names absolute coordinates of the view's domain. absolute = view[IndexDomain((12,), (15,))] assert absolute.result().tolist() == [12, 13, 14] # An IndexTransform key composes onto the view; index the identity over the # view's domain to spell any selection in absolute coordinates. literal = IndexTransform.identity(view.transform.domain) assert view[literal[12:15]].result().tolist() == [12, 13, 14] assert view[literal[::-1]].result().tolist()[:2] == [19, 18] # Whichever key produced a view, its domain is absolute. assert relative.transform.domain == absolute.transform.domain == IndexDomain((12,), (15,)) # --8<-- [end:key-types] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/snippets/lazy_composition.py000066400000000000000000000005701527014137600325770ustar00rootroot00000000000000"""Composing views keeps indexing lazy until the final result call.""" import numpy as np from zarr_indexing import LazyArray # --8<-- [start:lazy-composition] source = np.array([10, 11, 12, 13, 14, 15]) view = LazyArray.from_numpy(source)[2:5] composed = view[::-1][1:] assert composed.result().tolist() == source[2:5][::-1][1:].tolist() # --8<-- [end:lazy-composition] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/snippets/output_maps.py000066400000000000000000000042711527014137600315570ustar00rootroot00000000000000"""The three output map kinds, each demonstrated against its NumPy counterpart.""" from typing import Any import numpy as np from zarr_indexing import ( ArrayMap, ConstantMap, DimensionMap, IndexDomain, IndexTransform, ReadContext, numpy_reader, ) # --8<-- [start:resolve-helper] source = np.array([10, 11, 12, 13, 14, 15]) def resolve(transform: IndexTransform, values: np.ndarray[Any, Any]) -> np.ndarray[Any, Any]: """Materialize `transform` against `values` through the public reader.""" out = np.empty(transform.domain.shape, dtype=values.dtype) numpy_reader.read_into(values, ReadContext(transform), out) return out # --8<-- [end:resolve-helper] # --8<-- [start:dimension-map] # DimensionMap is an affine rule: request i reads source offset + stride * i. # Its NumPy counterpart is a basic slice. sliced = IndexTransform( domain=IndexDomain.from_shape((3,)), output=(DimensionMap(input_dimension=0, offset=2, stride=1),), ) assert resolve(sliced, source).tolist() == source[2:5].tolist() # A negative stride walks the source backward, like a negative-step slice. reversed_view = IndexTransform( domain=IndexDomain.from_shape((3,)), output=(DimensionMap(input_dimension=0, offset=4, stride=-2),), ) assert resolve(reversed_view, source).tolist() == source[4::-2].tolist() # --8<-- [end:dimension-map] # --8<-- [start:array-map] # ArrayMap is an explicit list of source coordinates; order and duplicates # are semantic. Its NumPy counterpart is fancy indexing. gather = IndexTransform( domain=IndexDomain.from_shape((3,)), output=(ArrayMap(index_array=np.array([4, 1, 1])),), ) assert resolve(gather, source).tolist() == source[[4, 1, 1]].tolist() # --8<-- [end:array-map] # --8<-- [start:constant-map] # ConstantMap reads one source coordinate for every request cell. NumPy can # express the same values with source[[0, 0, 0, 0]] or with a broadcast. # The constant map represents the repeated coordinate without storing a list. repeat = IndexTransform( domain=IndexDomain.from_shape((4,)), output=(ConstantMap(offset=0),), ) assert resolve(repeat, source).tolist() == np.broadcast_to(source[0:1], (4,)).tolist() # --8<-- [end:constant-map] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/docs/snippets/selection_flow.py000066400000000000000000000050211527014137600322050ustar00rootroot00000000000000"""Follow selections into public chunk projections and assemble their values.""" from typing import Any import numpy as np from zarr_indexing import IndexTransform, plan_chunks from zarr_indexing.grid import dimension_grids_from_chunks # --8<-- [start:basic] def assemble(source: np.ndarray[Any, Any], transform: IndexTransform) -> np.ndarray[Any, Any]: grids = dimension_grids_from_chunks((4,) * source.ndim, source.shape) result = np.empty(transform.domain.shape, dtype=source.dtype) for projection in plan_chunks(transform, grids): bounds = tuple( slice(grid.chunk_offset(c), grid.chunk_offset(c) + grid.data_size(c)) for grid, c in zip(grids, projection.chunk_coords, strict=True) ) chunk = source[bounds] # A storage/codec layer supplies this in production. domain = projection.chunk_transform.domain for position in np.ndindex(domain.shape): point = tuple(a + b for a, b in zip(domain.origin, position, strict=True)) local = projection.chunk_transform.apply(point) request = projection.cell_transform.apply(point) destination = tuple( x - origin for x, origin in zip(request, transform.domain.origin, strict=True) ) result[destination] = chunk[local] return result source = np.arange(80).reshape(8, 10) selection = (slice(1, 7, 2), slice(2, 6)) transform = IndexTransform.from_shape(source.shape)[selection] grids = dimension_grids_from_chunks((4, 4), source.shape) assert [p.chunk_coords for p in plan_chunks(transform, grids)] == [(0, 0), (0, 1), (1, 0), (1, 1)] result = assemble(source, transform) np.testing.assert_array_equal(result, source[selection]) assert result.tolist() == [[12, 13, 14, 15], [32, 33, 34, 35], [52, 53, 54, 55]] # --8<-- [end:basic] # --8<-- [start:gather] values = np.arange(8) coordinates = np.array([6, 1, 6, 4]) gather = IndexTransform.from_shape(values.shape).vindex[coordinates] gathered = assemble(values, gather) np.testing.assert_array_equal(gathered, values[coordinates]) pairs: dict[tuple[int, ...], list[tuple[int, int]]] = {} for projection in plan_chunks(gather, dimension_grids_from_chunks((4,), values.shape)): domain = projection.chunk_transform.domain pairs[projection.chunk_coords] = [ (projection.chunk_transform.apply((i,))[0], projection.cell_transform.apply((i,))[0]) for i in range(domain.origin[0], domain.exclusive_max[0]) ] assert pairs == {(0,): [(1, 1)], (1,): [(2, 0), (2, 2), (0, 3)]} # --8<-- [end:gather] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/examples/000077500000000000000000000000001527014137600256425ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/examples/lazy_indexing_dask/000077500000000000000000000000001527014137600315105ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/examples/lazy_indexing_dask/README.md000066400000000000000000000054431527014137600327750ustar00rootroot00000000000000# Lazy Indexing with Dask This example demonstrates how to use `zarr_indexing.LazyArray` with Dask, both as an explicitly adapted array Dask can wrap and as a source of independent tasks, and compares the two ways of deferring an indexing operation. The example shows how to: - Wrap a `LazyArray` — over a Zarr array or over a view of one — in `EagerArrayAdapter` and pass the adapter to `dask.array.from_array` - Build one Dask task per partition from `parts()`, compute them in parallel, and place each result with the partition's `out_selection` - Read `is_complete` to inspect coverage of a partition cell - Inspect `__dask_tokenize__` for the example's equal source/selection pairs; token equality can support task deduplication but does not promise persistent caching - Measure what a task graph costs for indexing-only work, against composing the same selections into one transform The adapter exposes no `chunks` attribute, so `dask.array.from_array` chooses its own block size unless one is given. The partitioning that `parts()` reports is discovered from the wrapped array and is independent of Dask's blocks. `LazyArray[...]` returns another lazy view. `EagerArrayAdapter(view)[...]` returns materialized values, providing the block reads Dask expects. Import both classes from `zarr_indexing`. Use the adapter for `from_array`; passing a lazy view directly is not a reliable integration. ## Choosing Between Them If Dask is doing arithmetic across chunks, reductions, rechunking, or distributed execution, it is the right tool, and its task graph is what makes that work. For indexing-only workloads, graph construction and scheduling can be an additional cost. The example measures repeated leading slices and reports graph layers and timings for the selected Dask version. It does not establish general complexity bounds or a guaranteed speedup: Dask can optimize graphs, and costs depend on the selection, chunk layout, and scheduler. `LazyArray` stores one composed transform rather than retaining a wrapper for each prior selection. Applying a chain still costs work for every operation; index-array composition may process arrays whose size depends on earlier selections. Reading also incurs partition planning and source I/O. ## Running the Example The script declares its dependencies inline ([PEP 723](https://peps.python.org/pep-0723/)), so the easiest way to run it is with [uv](https://docs.astral.sh/uv/), which installs them automatically: ```bash cd packages/zarr-indexing uv run --with-editable . examples/lazy_indexing_dask/lazy_indexing_dask.py ``` Alternatively, run it with plain Python, in which case you must first install `zarr`, `zarr-indexing`, `dask[array]`, `numpy`, and `pytest` yourself: ```bash cd packages/zarr-indexing python examples/lazy_indexing_dask/lazy_indexing_dask.py ``` lazy_indexing_dask.py000066400000000000000000000146051527014137600356570ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/examples/lazy_indexing_dask# /// script # requires-python = ">=3.12" # dependencies = [ # "zarr @ git+https://github.com/zarr-developers/zarr-python.git@main", # "zarr-indexing>=0.1", # "dask[array]==2025.3.0", # "numpy==2.4.3", # "pytest==9.0.2" # ] # /// # """ Demonstrate using zarr_indexing.LazyArray with Dask """ import sys import time import dask import dask.array as da import numpy as np import pytest import zarr from dask.base import tokenize from zarr_indexing import EagerArrayAdapter, LazyArray @pytest.fixture def source() -> zarr.Array: """A chunked Zarr array to wrap.""" array = zarr.create_array(store={}, shape=(40, 30), chunks=(10, 10), dtype="i4") array[:] = np.arange(40 * 30).reshape(40, 30) return array def test_from_array(source: zarr.Array) -> None: """Adapt lazy selections for `dask.array.from_array`.""" lazy = LazyArray(source) # `from_array` expects eager block indexing, which `EagerArrayAdapter` # provides. Each Dask block reads its own region through the adapter. array = da.from_array(EagerArrayAdapter(lazy), chunks=(10, 10)) print(array) assert np.array_equal(array.compute(scheduler="threads"), source[:]) # A view works the same way, and its shape is the shape of the selection. view = LazyArray(source)[5:35, 3:27] array = da.from_array(EagerArrayAdapter(view), chunks=(10, 10)) assert array.shape == (30, 24) assert np.array_equal(array.compute(scheduler="threads"), source[5:35, 3:27]) def test_parts_as_tasks(source: zarr.Array) -> None: """Build one task per partition and compute them in parallel.""" view = LazyArray(source)[5:35, 3:27] # The partitioning is discovered from the wrapped array's chunks, so each # partition of the view lies within one stored chunk. parts = list(view.parts()) print(f"{len(parts)} parts for a {view.shape} view of a {source.shape} array") # A partition carries a sub-view to resolve and where its result belongs, so # reads can run concurrently with this source and reader; assembly below # places the returned blocks sequentially. @dask.delayed def read(part: object) -> np.ndarray: return part.view.result() blocks = dask.compute(*[read(part) for part in parts], scheduler="threads") result = np.empty(view.shape, dtype=view.dtype) for part, block in zip(parts, blocks, strict=True): result[part.out_selection] = block assert np.array_equal(result, source[5:35, 3:27]) # `is_complete` reports whether a partition covers its whole partition of # the base array. A writer also needs storage-unit alignment, value-order, # and concurrency checks before using coverage to skip a read. complete = [part.box for part in parts if part.is_complete] print(f"{len(complete)} of {len(parts)} parts cover their chunk completely") def test_tokenize(source: zarr.Array) -> None: """Check token equality for these unchanged source and selection pairs.""" lazy = LazyArray(source) # These wrappers have equal tokens despite being different Python objects. # Source mutation and token hooks affect whether cached results remain valid. assert tokenize(lazy) == tokenize(LazyArray(source)) assert tokenize(lazy[0:10]) == tokenize(LazyArray(source)[0:10]) # Different selections are different tasks. assert tokenize(lazy[0:10]) != tokenize(lazy[10:20]) # These two slice chains serialize to the same transform and token. assert tokenize(lazy[0:20][5:10]) == tokenize(lazy[5:10]) def test_indexing_only_workload() -> None: """Compare an accumulating task graph with a fused transform. This measures repeated leading slices at several depths. LazyArray retains one composed transform, but the loop still performs each selection. Dask's graph construction and execution costs depend on graph optimizations and chunk layout; these measurements do not establish general complexity bounds. Timings are printed rather than asserted, since they depend on the machine. """ data = np.zeros((2000, 4), dtype="i4") # 2000 chunks, one row each def dask_chain(depth: int) -> da.Array: array = da.from_array(data, chunks=(1, 4)) for _ in range(depth): array = array[1:] return array def lazy_chain(depth: int) -> LazyArray: view = LazyArray.from_numpy(data) for _ in range(depth): view = view[1:] return view # Read once through each path first, so the timings below exclude the cost # of importing and initializing the machinery. dask_chain(1)[:2].compute(scheduler="synchronous") lazy_chain(1)[:2].result() header = ( f"{'selections':>10} {'dask compose':>13} {'dask read':>10} {'layers':>7}" f" {'LazyArray compose':>18} {'LazyArray read':>15}" ) print(header) for depth in (1, 5, 20): start = time.perf_counter() chained = dask_chain(depth) dask_compose = time.perf_counter() - start start = time.perf_counter() from_dask = chained[:2].compute(scheduler="synchronous") dask_read = time.perf_counter() - start start = time.perf_counter() view = lazy_chain(depth) lazy_compose = time.perf_counter() - start start = time.perf_counter() from_lazy = view[:2].result() lazy_read = time.perf_counter() - start # Both paths describe the same selection, so they read the same data. assert np.array_equal(from_dask, from_lazy) layers = len(chained.__dask_graph__().layers) print( f"{depth:>10} {dask_compose * 1e3:>12.2f}ms {dask_read * 1e3:>9.2f}ms {layers:>7}" f" {lazy_compose * 1e3:>17.3f}ms {lazy_read * 1e3:>14.3f}ms" ) if __name__ == "__main__": # Run the example with printed output, and a dummy pytest configuration file specified. # Without the dummy configuration file, at test time pytest will attempt to use the # configuration file in the project root, which will error because Zarr is using some # plugins that are not installed in this example. sys.exit( pytest.main( [ "-s", __file__, f"-c {__file__}", # Suppress: "PytestAssertRewriteWarning: Module already imported so # cannot be rewritten; zarr" "-W", "ignore::pytest.PytestAssertRewriteWarning", ] ) ) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/examples/lazy_indexing_numpy/000077500000000000000000000000001527014137600317365ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/examples/lazy_indexing_numpy/README.md000066400000000000000000000032341527014137600332170ustar00rootroot00000000000000# Lazy Indexing a NumPy Array This example demonstrates how to wrap an array in `zarr_indexing.LazyArray` and index it without reading data. The example shows how to: - Wrap a NumPy array and read the forwarded `shape`, `dtype`, and `ndim` - Compose selections through `[...]`, `.oindex[...]`, and `.vindex[...]`, and materialize the composed view once with `result()` - Iterate lazy row views and write through a composed view with `write(values)` or indexed assignment; writes finish synchronously on the original source - Tell a box selection (slices and integers, described by an interval and a step per dimension) from a query selection (points gathered through an index array) using `is_box`, `bounding_box()`, and `strides()` - Declare a partitioning with `with_parts()`, iterate it with `parts()`, and assemble a result from the partitions `LazyArray` wraps compatible sources exposing `shape`, `dtype`, and basic slicing whose results can be converted to NumPy system memory. This includes a Zarr array, and the partitioning is then discovered from the array's chunks. The Dask example covers that case. ## Running the Example The script declares its dependencies inline ([PEP 723](https://peps.python.org/pep-0723/)), so the easiest way to run it is with [uv](https://docs.astral.sh/uv/), which installs them automatically: ```bash cd packages/zarr-indexing uv run --with-editable . examples/lazy_indexing_numpy/lazy_indexing_numpy.py ``` Alternatively, run it with plain Python, in which case you must first install `zarr-indexing`, `numpy`, and `pytest` yourself: ```bash cd packages/zarr-indexing python examples/lazy_indexing_numpy/lazy_indexing_numpy.py ``` lazy_indexing_numpy.py000066400000000000000000000126401527014137600363300ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/examples/lazy_indexing_numpy# /// script # requires-python = ">=3.12" # dependencies = [ # "zarr-indexing>=0.1", # "numpy==2.4.3", # "pytest==9.0.2" # ] # /// # """ Demonstrate lazy indexing over a plain NumPy array with zarr_indexing.LazyArray """ import sys import numpy as np import pytest from zarr_indexing import LazyArray def test_wrap_and_compose() -> None: """Wrap an array, compose selections without reading, then materialize once.""" data = np.arange(12 * 8).reshape(12, 8) lazy = LazyArray.from_numpy(data) # The wrapper forwards the attributes an array consumer expects. assert lazy.shape == (12, 8) assert lazy.dtype == data.dtype assert lazy.ndim == 2 # `[...]` returns another LazyArray. No element of `data` is read. view = lazy[2:10, ::2] print(view) assert view.shape == (8, 4) # Selections compose. Each step narrows the view; still nothing is read. smaller = view[1:5, 1:3] # `result()` performs the read. NumPy is the reference for the whole chain. assert np.array_equal(smaller.result(), data[2:10, ::2][1:5, 1:3]) # Selections use positional NumPy semantics: indices count from zero within # the current view, and negative indices count from the end. assert np.array_equal(lazy[-1].result(), data[-1]) assert np.array_equal(lazy[::-1].result(), data[::-1]) # Orthogonal and vectorized indexing also return lazy views. rows = np.array([9, 1, 4]) assert np.array_equal(lazy.oindex[rows, :].result(), data[rows, :]) cols = np.array([0, 3, 7]) assert np.array_equal(lazy.vindex[rows, cols].result(), data[rows, cols]) # Indexing remains lazy; np.asarray explicitly materializes the view. assert isinstance(lazy[2:4, 0], LazyArray) assert np.array_equal(np.asarray(view), data[2:10, ::2]) def test_iteration_and_writes() -> None: """Iterate lazy rows and synchronously write through a composed view.""" data = np.arange(24).reshape(6, 4) view = LazyArray.from_numpy(data)[1:5][::2] rows = list(view) assert all(isinstance(row, LazyArray) for row in rows) np.testing.assert_array_equal(rows[1].result(), data[3]) assert view[:, 1:3].write(-1) is None np.testing.assert_array_equal(data[[1, 3], 1:3], np.full((2, 2), -1)) view[1, 0] = 99 assert data[3, 0] == 99 def test_box_and_query_selections() -> None: """Distinguish selections that describe a region from selections that gather points.""" data = np.arange(12 * 8).reshape(12, 8) lazy = LazyArray.from_numpy(data) # This box selection is built from slices and integers. It is described # completely by an interval and a step per dimension, so a consumer can # serve it as one strided read. box = lazy[2:10, ::2] print(f"box: is_box={box.is_box} bounding_box={box.bounding_box()} strides={box.strides()}") assert box.is_box assert box.bounding_box() == ((2, 10), (0, 7)) assert box.strides() == (1, 2) # A query selection gathers points through an index array. Its coordinates # are a lookup table, so `strides()` is undefined and `bounding_box()` is # the hull of the points rather than an exact description. query = lazy.oindex[np.array([9, 1, 4]), :] print(f"query: is_box={query.is_box} bounding_box={query.bounding_box()}") assert not query.is_box assert query.strides() is None assert query.bounding_box() == ((1, 10), (0, 8)) # This slice retains an index-array map, so this particular view stays a query. # Singleton gathers and later scalar indexing can instead collapse to a box. assert not query[0:2, 0:2].is_box def test_parts() -> None: """Iterate the partitions a view covers, and assemble the result from them.""" data = np.arange(12 * 8).reshape(12, 8) # A plain NumPy array declares no partitioning, so `with_parts` states one. # Partitioning changes the granularity of reads, never the result. lazy = LazyArray.from_numpy(data).with_parts((4, 4)) view = lazy[2:10, ::2] parts = list(view.parts()) print(f"{len(parts)} parts") for part in parts[:2]: print(f" base_coords={part.base_coords} box={part.box} complete={part.is_complete}") # Each part carries a sub-view of its own, where that sub-view lands in the # result, and whether it covers its partition completely. Resolving the # parts and placing them is what `result()` does. assembled = np.empty(view.shape, dtype=view.dtype) for part in parts: assembled[part.out_selection] = part.view.result() assert np.array_equal(assembled, view.result()) # The partitioning is a read strategy, so a different one gives the same data. assert np.array_equal( LazyArray.from_numpy(data).with_parts((5, 3))[2:10, ::2].result(), assembled ) if __name__ == "__main__": # Run the example with printed output, and a dummy pytest configuration file specified. # Without the dummy configuration file, at test time pytest will attempt to use the # configuration file in the project root, which will error because Zarr is using some # plugins that are not installed in this example. sys.exit( pytest.main( [ "-s", __file__, f"-c {__file__}", # Suppress: "PytestAssertRewriteWarning: Module already imported so # cannot be rewritten; zarr" "-W", "ignore::pytest.PytestAssertRewriteWarning", ] ) ) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/examples/system_memory_chunk_cache/000077500000000000000000000000001527014137600330715ustar00rootroot00000000000000README.md000066400000000000000000000044131527014137600342730ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/examples/system_memory_chunk_cache# System-memory chunk cache This executable reference architecture demonstrates a small, synchronous system-memory chunk cache for a NumPy-like image consumer. It is inspired by Neuroglancer's explicit chunk lifecycle: ```text NEW -> QUEUED -> LOADING -> READY | v FAILED READY -> EVICTED FAILED -> QUEUED (explicit retry) ``` `RecordingChunkSource` owns decoded source-chunk reads and records them for the example. `LazyArray` converts NumPy-style indexing into transforms and partitions, then assembles the final result. `SystemMemoryChunkReader` intercepts each materialized part and owns the lifecycle records, queue draining, resident ready buffers, LRU eviction, retained load failures, and explicit retry. The reader owns cache state and source reads, but not result shape or assembly. Each request calls `view.parts()` once and keeps the resulting tuple. The cache queues the tuple's chunk coordinates and defers eviction, then materializes with `view.result(parts=parts)`, so scheduling and assembly reuse one plan. Every reader call consumes the exact projection attached to its `ReadContext`; the reader does not invoke the chunk planner again. The requests demonstrate lazy selections, paired chunk projections, overlapping viewport requests that reuse resident chunks, eviction under capacity pressure, a retained failure that does not retry implicitly, and an explicit retry after the source is repaired. The integration guide contains the detailed request table. Capacity counts resident chunks, not bytes, and is enforced after a successful request. A request can temporarily exceed it; a failed request skips that eviction step. Records, event history, and output/coordinate buffers are outside this count. The example assumes an unchanged source and one source/grid per reader; it has no cache invalidation for source mutation, and its mutable state is not thread-safe. This is synchronous system-memory reference architecture, not a production-ready cache, scheduler, renderer, or complete napari integration. Its types are intentionally not exported by `zarr_indexing`. ## Running the example ```bash cd packages/zarr-indexing uv run --with-editable . examples/system_memory_chunk_cache/system_memory_chunk_cache.py ``` system_memory_chunk_cache.py000066400000000000000000000347421527014137600406250ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/examples/system_memory_chunk_cache# /// script # requires-python = ">=3.12" # dependencies = [ # "zarr-indexing>=0.1", # "numpy==2.4.3", # ] # /// # from __future__ import annotations from contextlib import contextmanager from dataclasses import dataclass from enum import StrEnum from typing import TYPE_CHECKING, Any import numpy as np from zarr_indexing import ( IndexDomain, LazyArray, ReadContext, ) if TYPE_CHECKING: from collections.abc import Callable, Iterator type ChunkCoords = tuple[int, ...] # --8<-- [start:chunk-cache-types] class ChunkState(StrEnum): NEW = "new" QUEUED = "queued" LOADING = "loading" READY = "ready" FAILED = "failed" EVICTED = "evicted" @dataclass(slots=True) class ChunkRecord: state: ChunkState = ChunkState.NEW buffer: np.ndarray[Any, Any] | None = None error: Exception | None = None last_access: int = -1 @dataclass(frozen=True, slots=True) class ChunkEvent: chunk_coords: ChunkCoords previous: ChunkState current: ChunkState reason: str class ChunkLoadError(RuntimeError): pass # --8<-- [end:chunk-cache-types] # --8<-- [start:chunk-cache-source] class RecordingChunkSource: def __init__(self, data: np.ndarray[Any, Any], chunks: tuple[int, ...]) -> None: self._data = data self.chunks = chunks self.reads: list[ChunkCoords] = [] self.failures: set[ChunkCoords] = set() @property def shape(self) -> tuple[int, ...]: return self._data.shape @property def dtype(self) -> np.dtype[Any]: return self._data.dtype def __getitem__(self, key: Any) -> np.ndarray[Any, Any]: raise AssertionError("the cache must read complete chunks through read_chunk") def read_chunk(self, chunk_coords: ChunkCoords) -> np.ndarray[Any, Any]: self.reads.append(chunk_coords) if chunk_coords in self.failures: raise OSError(f"source read failed for chunk {chunk_coords}") key = tuple( slice(coord * size, min((coord + 1) * size, extent)) for coord, size, extent in zip(chunk_coords, self.chunks, self.shape, strict=True) ) return self._data[key].copy() def _domain_points(domain: IndexDomain) -> np.ndarray[Any, np.dtype[np.intp]]: """Enumerate a rectangular domain with a trailing coordinate axis.""" if domain.ndim == 0: return np.empty((1, 0), dtype=np.intp) points = np.moveaxis(np.indices(domain.shape, dtype=np.intp), 0, -1).reshape(-1, domain.ndim) points += np.asarray(domain.inclusive_min, dtype=np.intp) return points def _gather_and_scatter( destination: np.ndarray[Any, Any], source: np.ndarray[Any, Any], source_points: np.ndarray[Any, np.dtype[np.intp]], destination_points: np.ndarray[Any, np.dtype[np.intp]], ) -> np.ndarray[Any, Any]: """Gather and scatter a flattened point batch, including rank zero.""" values = np.asarray(source[tuple(source_points.T)]).reshape(-1) if destination_points.shape[-1] == 0: destination[()] = values.reshape(destination.shape)[()] else: destination[tuple(destination_points.T)] = values return values # --8<-- [end:chunk-cache-source] # --8<-- [start:chunk-cache-wrapper] LEGAL_TRANSITIONS: dict[ChunkState, frozenset[ChunkState]] = { ChunkState.NEW: frozenset({ChunkState.QUEUED}), ChunkState.QUEUED: frozenset({ChunkState.LOADING}), ChunkState.LOADING: frozenset({ChunkState.READY, ChunkState.FAILED}), ChunkState.READY: frozenset({ChunkState.EVICTED}), ChunkState.FAILED: frozenset({ChunkState.QUEUED}), ChunkState.EVICTED: frozenset({ChunkState.QUEUED}), } class _OrthogonalIndexer: """Expose outer-product indexing without changing ``cache[key]`` semantics.""" def __init__(self, getitem: Callable[[Any], np.ndarray[Any, Any]]) -> None: self._getitem = getitem def __getitem__(self, key: Any) -> np.ndarray[Any, Any]: return self._getitem(key) class SystemMemoryChunkReader: def __init__(self, *, capacity: int) -> None: self.capacity = capacity self._records: dict[ChunkCoords, ChunkRecord] = {} self._queue: list[ChunkCoords] = [] self._clock = 0 self._requests = 0 self.events: list[ChunkEvent] = [] self.projection_uses: list[tuple[str, str]] = [] def state(self, chunk_coords: ChunkCoords) -> ChunkState: return self._record(chunk_coords).state def resident(self) -> tuple[ChunkCoords, ...]: return tuple( sorted( coords for coords, record in self._records.items() if record.state is ChunkState.READY ) ) def _record(self, chunk_coords: ChunkCoords) -> ChunkRecord: return self._records.setdefault(chunk_coords, ChunkRecord()) def _transition(self, chunk_coords: ChunkCoords, current: ChunkState, reason: str) -> None: record = self._record(chunk_coords) if current not in LEGAL_TRANSITIONS[record.state]: raise ValueError(f"illegal chunk transition {record.state} -> {current}") previous = record.state record.state = current self.events.append(ChunkEvent(chunk_coords, previous, current, reason)) def retry(self, chunk_coords: ChunkCoords) -> None: record = self._record(chunk_coords) if record.state is not ChunkState.FAILED: raise ValueError(f"retry requires failed chunk {chunk_coords}, got {record.state}") record.error = None self._transition(chunk_coords, ChunkState.QUEUED, "explicit retry") self._queue.append(chunk_coords) @contextmanager def request(self, required: tuple[ChunkCoords, ...]) -> Iterator[None]: """Prepare parts; evict after the outermost request succeeds, not on failure.""" self._prepare(required) self._requests += 1 try: yield except Exception: self._requests -= 1 raise else: self._requests -= 1 if self._requests == 0: self._evict(pinned=frozenset()) def _touch(self, record: ChunkRecord) -> None: self._clock += 1 record.last_access = self._clock def _queue_once(self, chunk_coords: ChunkCoords) -> None: record = self._record(chunk_coords) if record.state in {ChunkState.QUEUED, ChunkState.LOADING, ChunkState.READY}: return if record.state is ChunkState.FAILED: raise ValueError(f"failed chunk {chunk_coords} requires explicit retry") self._transition(chunk_coords, ChunkState.QUEUED, "requested") self._queue.append(chunk_coords) def _prepare(self, required: tuple[ChunkCoords, ...]) -> None: for chunk_coords in required: record = self._record(chunk_coords) if record.state is ChunkState.FAILED: assert record.error is not None raise ChunkLoadError( f"chunk {chunk_coords} is failed; call retry first" ) from record.error for chunk_coords in required: record = self._record(chunk_coords) if record.state is ChunkState.READY: self._touch(record) else: self._queue_once(chunk_coords) def _ensure_ready( self, source: RecordingChunkSource, required: tuple[ChunkCoords, ...], ) -> None: if self._requests == 0: self._prepare(required) self._drain(source, frozenset(required)) def _drain(self, source: RecordingChunkSource, required: frozenset[ChunkCoords]) -> None: pending = self._queue self._queue = [] for index, chunk_coords in enumerate(pending): if chunk_coords not in required: self._queue.append(chunk_coords) continue record = self._record(chunk_coords) self._transition(chunk_coords, ChunkState.LOADING, "queue drained") try: record.buffer = source.read_chunk(chunk_coords) except OSError as error: record.buffer = None record.error = error self._transition(chunk_coords, ChunkState.FAILED, "source read failed") self._queue.extend(pending[index + 1 :]) raise ChunkLoadError(f"could not load chunk {chunk_coords}") from error record.error = None self._transition(chunk_coords, ChunkState.READY, "source read completed") self._touch(record) def _evict(self, *, pinned: frozenset[ChunkCoords]) -> None: while len(self.resident()) > self.capacity: candidates = ( (record.last_access, chunk_coords) for chunk_coords, record in self._records.items() if record.state is ChunkState.READY and chunk_coords not in pinned ) _, chunk_coords = min(candidates) record = self._record(chunk_coords) record.buffer = None self._transition(chunk_coords, ChunkState.EVICTED, "LRU capacity") def read_into( self, source: RecordingChunkSource, context: ReadContext, out: np.ndarray[Any, Any], /, ) -> None: projection = context.projection if projection is None: raise ValueError("SystemMemoryChunkReader requires context.projection") required = (projection.chunk_coords,) self._ensure_ready(source, required) record = self._record(projection.chunk_coords) assert record.buffer is not None cell_points = _domain_points(projection.chunk_transform.domain) chunk_points = projection.chunk_transform.apply_many(cell_points) destination_points = _domain_points(context.transform.domain) _gather_and_scatter(out, record.buffer, chunk_points, destination_points) self.projection_uses.append(("chunk_transform", "context.transform")) if self._requests == 0: self._evict(pinned=frozenset()) class SystemMemoryChunkCache: def __init__(self, source: RecordingChunkSource, *, capacity: int) -> None: self.source = source self.reader = SystemMemoryChunkReader(capacity=capacity) self._lazy = LazyArray(source).with_reader(self.reader) @property def shape(self) -> tuple[int, ...]: return self.source.shape @property def dtype(self) -> np.dtype[Any]: return self.source.dtype @property def oindex(self) -> _OrthogonalIndexer: return _OrthogonalIndexer(lambda key: self._read(key, orthogonal=True)) @property def events(self) -> list[ChunkEvent]: return self.reader.events @property def projection_uses(self) -> tuple[tuple[str, str], ...]: return tuple(self.reader.projection_uses) def state(self, chunk_coords: ChunkCoords) -> ChunkState: return self.reader.state(chunk_coords) def resident(self) -> tuple[ChunkCoords, ...]: return self.reader.resident() def retry(self, chunk_coords: ChunkCoords) -> None: self.reader.retry(chunk_coords) def __getitem__(self, key: Any) -> np.ndarray[Any, Any]: return self._read(key, orthogonal=False) def _read(self, key: Any, *, orthogonal: bool) -> np.ndarray[Any, Any]: self.reader.projection_uses.clear() lazy = self._lazy view = lazy.oindex[key] if orthogonal else lazy[key] # One prepared tuple is the request plan: queue its chunks and defer # eviction during the request, then hand the # same owned parts back to LazyArray for assembly without replanning. parts = tuple(view.parts()) required = tuple(dict.fromkeys(part.base_coords for part in parts)) with self.reader.request(required): return np.asarray(view.result(parts=parts)) # --8<-- [end:chunk-cache-wrapper] # --8<-- [start:chunk-cache-worked-example] image = np.arange(48).reshape(6, 8) source = RecordingChunkSource(image, chunks=(3, 4)) cache = SystemMemoryChunkCache(source, capacity=2) READS_BEFORE_SELECTION = tuple(source.reads) INITIAL_RESULT = cache[1:5, 2] INITIAL_READS = tuple(source.reads) before_overlap = len(source.reads) OVERLAP_RESULT = cache[3:5, 2] OVERLAP_NEW_READS = tuple(source.reads[before_overlap:]) before_eviction = len(source.reads) EVICTION_RESULT = cache[0:2, 5] EVICTION_NEW_READS = tuple(source.reads[before_eviction:]) AFTER_EVICTION_RESIDENT = cache.resident() before_reload = len(source.reads) RELOAD_RESULT = cache[1:5, 2] RELOAD_NEW_READS = tuple(source.reads[before_reload:]) AFTER_RELOAD_RESIDENT = cache.resident() source.failures.add((1, 1)) failed_once = False try: cache[3:5, 4:6] except ChunkLoadError: failed_once = True assert failed_once FAILED_READ_COUNT = source.reads.count((1, 1)) failed_twice = False try: cache[3:5, 4:6] except ChunkLoadError: failed_twice = True assert failed_twice FAILED_REPEAT_READ_COUNT = source.reads.count((1, 1)) FAILURE_READ_COUNTS = (FAILED_READ_COUNT, FAILED_REPEAT_READ_COUNT) source.failures.remove((1, 1)) cache.retry((1, 1)) before_retry = len(source.reads) RETRY_RESULT = cache[3:5, 4:6] RETRY_NEW_READS = tuple(source.reads[before_retry:]) RETRY_STATE = cache.state((1, 1)).value WORKED_EVENTS = tuple(cache.events) FAILED_TRANSITIONS = tuple( event.current.value for event in WORKED_EVENTS if event.chunk_coords == (1, 1) ) FAILED_EVENT_ROWS = tuple( (event.previous.value, event.current.value, event.reason) for event in WORKED_EVENTS if event.chunk_coords == (1, 1) ) # --8<-- [end:chunk-cache-worked-example] assert READS_BEFORE_SELECTION == () assert INITIAL_RESULT.tolist() == [10, 18, 26, 34] assert INITIAL_READS == ((0, 0), (1, 0)) assert OVERLAP_RESULT.tolist() == [26, 34] assert OVERLAP_NEW_READS == () assert EVICTION_RESULT.tolist() == [5, 13] assert EVICTION_NEW_READS == ((0, 1),) assert AFTER_EVICTION_RESIDENT == ((0, 1), (1, 0)) assert RELOAD_RESULT.tolist() == [10, 18, 26, 34] assert RELOAD_NEW_READS == ((0, 0),) assert AFTER_RELOAD_RESIDENT == ((0, 0), (1, 0)) assert FAILURE_READ_COUNTS == (1, 1) assert RETRY_RESULT.tolist() == [[28, 29], [36, 37]] assert RETRY_NEW_READS == ((1, 1),) assert RETRY_STATE == "ready" assert FAILED_TRANSITIONS == ( "queued", "loading", "failed", "queued", "loading", "ready", ) assert FAILED_EVENT_ROWS == ( ("new", "queued", "requested"), ("queued", "loading", "queue drained"), ("loading", "failed", "source read failed"), ("failed", "queued", "explicit retry"), ("queued", "loading", "queue drained"), ("loading", "ready", "source read completed"), ) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/justfile000066400000000000000000000056201527014137600255770ustar00rootroot00000000000000# Development verbs for the zarr-indexing package. Recipes run with this # directory as the working directory regardless of where `just` is invoked. # Quoted arguments must survive delegation from the root Justfile. set positional-arguments # In the repository, recipes overlay the in-repo zarr (editable) so they run # against the code beside them. pyproject.toml stays standalone, so from an # unpacked sdist, where ../.. is not the repository, there is no overlay and # the suites that need zarr skip. repo_zarr := if path_exists("../../src/zarr/__init__.py") == "true" { "--with-editable ../.." } else { "" } # List available recipes default: @just --list # zarr is not a dependency: repo_zarr supplies it, so the suites that read # through zarr run here instead of skipping. # Run the test suite; extra args are passed to pytest test *args: uv run {{ repo_zarr }} --group test python -m pytest tests src/zarr_indexing "$@" # TensorStore is the oracle for the parity suites, which skip without it. It # ships binary wheels only, so it rides in as a run-time overlay rather than # joining a dependency group; if a future Python lacks a tensorstore wheel, # gate the CI job that calls this on the matrix version. # Run the tensorstore parity suites; extra args are passed to pytest test-tensorstore *args: uv run {{ repo_zarr }} --group test --with 'tensorstore>=0.1.84' python -m pytest tests/test_ndsel_tensorstore.py tests/test_tensorstore_parity.py "$@" # Lint with the same invocation CI uses. Ruff is pinned to the repo-wide # version in the root .pre-commit-config.yaml; bump together. lint: uvx ruff@0.16.0 check . # Type-check the package sources, documentation Python, and their contract tests typecheck: uv run --group test --with pyright pyright # Run these checks with the locally selected interpreter and dependencies; # CI also exercises its configured Python-version matrix check: lint typecheck test test-tensorstore docs-check # Preview the changelog that the next release would generate changelog-draft: uvx towncrier build --draft --version Unreleased # Build this package's documentation site, warnings as errors docs-check: env DISABLE_MKDOCS_2_WARNING=true uv run --group docs mkdocs build --strict # With no argument, uses port 8000 if free, otherwise an ephemeral free port; # an explicitly requested port is used as-is so a conflict fails loudly. # Serve this package's documentation site docs-serve port="": #!/usr/bin/env bash set -euo pipefail port="{{ port }}" if [ -z "$port" ]; then port=$(uv run --group docs python -c ' import socket s = socket.socket() try: s.bind(("127.0.0.1", 8000)) except OSError: s.close() s = socket.socket() s.bind(("127.0.0.1", 0)) print(s.getsockname()[1]) s.close() ') fi exec env DISABLE_MKDOCS_2_WARNING=true uv run --group docs mkdocs serve -a "localhost:$port" zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/mkdocs-base.yml000077700000000000000000000000001527014137600320602../mkdocs-base.ymlustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/mkdocs.yml000066400000000000000000000101371527014137600260310ustar00rootroot00000000000000INHERIT: mkdocs-base.yml site_name: zarr-indexing # The package lives in the zarr-python monorepo; point the header source # widget at the package directory rather than the repository root. repo_name: zarr-python/packages/zarr-indexing repo_url: https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-indexing # Absolute because mkdocs would otherwise append this to repo_url's subpath. edit_uri: https://github.com/zarr-developers/zarr-python/edit/main/packages/zarr-indexing/docs/ site_description: Composable, lazy coordinate transforms for Zarr array indexing. site_url: !ENV [READTHEDOCS_CANONICAL_URL, 'https://zarr-indexing.readthedocs.io/'] exclude_docs: | snippets/*.py # --strict promotes warnings to errors, but broken link anchors and pages # missing from nav are only INFO by default — a strict build passed with # both breakages. Warn so strict actually fails on them. validation: links: anchors: warn nav: omitted_files: warn # Top-level rank is consistent: collections (Guide, Examples, API Reference) # and standalone artifacts (landing, ndsel spec, design notes, release notes). # Guide mirrors the docs/guide/ directory — its three pages are one # collection (learn / look up / integrate), pydantic's Concepts pattern at # small scale; navigation.indexes makes the Guide entry itself land on the # visual guide. No tabs: at eight content pages, hiding sections costs more # than it organizes. nav: - index.md - Guide: - guide/index.md - Indexing patterns: guide/patterns.md - From a selection to chunk operations: guide/selection-flow.md - Integration boundaries: guide/integrations.md - Examples: - Lazy indexing a NumPy array: examples/lazy_indexing_numpy.md - Lazy indexing with Dask: examples/lazy_indexing_dask.md - System-memory chunk cache: examples/system_memory_chunk_cache.md - The ndsel wire format: ndsel.md - Design notes: design-notes.md - API Reference: - api/index.md - ' zarr_indexing.transform': api/transform.md - ' zarr_indexing.domain': api/domain.md - ' zarr_indexing.output_map': api/output_map.md - ' zarr_indexing.chunk_resolution': api/chunk_resolution.md - ' zarr_indexing.grid': api/grid.md - ' zarr_indexing.lazy_array': api/lazy_array.md - ' zarr_indexing.reader': api/reader.md - ' zarr_indexing.writer': api/writer.md - ' zarr_indexing.eager': api/eager.md - ' zarr_indexing.boundary': api/boundary.md - ' zarr_indexing.json': api/json.md - ' zarr_indexing.messages': api/messages.md - ' zarr_indexing.errors': api/errors.md - ' zarr_indexing.testing.stateful': api/testing_stateful.md - ' zarr_indexing.testing.strategies': api/testing_strategies.md - Release notes: release-notes.md # This site is a Read the Docs subproject of zarr-python; give readers a way # back to the parent docs, which list every companion package. - 'zarr-python ↪': https://zarr.readthedocs.io/ zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/pyproject.toml000066400000000000000000000165241527014137600267500ustar00rootroot00000000000000[build-system] requires = ["hatchling>=1.29.0", "hatch-vcs", "setuptools-scm>=9"] build-backend = "hatchling.build" [project] name = "zarr-indexing" dynamic = ["version"] description = "Composable, lazy coordinate transforms for Zarr array indexing." readme = "README.md" requires-python = ">=3.12" license = "MIT" license-files = ["LICENSE.txt"] authors = [ { name = "Davis Bennett", email = "davis.v.bennett@gmail.com" }, ] classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Intended Audience :: Information Technology", "Intended Audience :: Science/Research", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Topic :: Scientific/Engineering", "Topic :: Software Development :: Libraries :: Python Modules", "Typing :: Typed", ] keywords = ["zarr"] dependencies = [ "numpy>=2", ] [project.optional-dependencies] # `zarr_indexing.testing` — a Hypothesis state machine and selection strategies # for projects checking their own array against this package. Nothing else in # the package imports hypothesis. testing = ["hypothesis>=6.160.0"] [project.urls] Homepage = "https://github.com/zarr-developers/zarr-python" Source = "https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-indexing" Issues = "https://github.com/zarr-developers/zarr-python/issues" Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-indexing/CHANGELOG.md" Documentation = "https://zarr-indexing.readthedocs.io/" [dependency-groups] # Package tests use the local grid protocol and grid implementations. The # repository recipes layer this package into the root test environment. # This test group and the optional testing extra both declare Hypothesis; # keep their supported minimum versions aligned. test = ["pytest", "hypothesis>=6.160.0"] docs = [ # Pins match the zarr-python docs environment in the repo-root # pyproject.toml so every site renders with the same toolchain, and the # shared packages/mkdocs-base.yml sees one set of plugin versions. # Bump them together with the root. "mkdocs-material==9.7.7", "mkdocs==1.6.1", "mkdocstrings==1.0.6", # pymdown-extensions 12.3 changed the Highlight extension API; mkdocstrings 1.0.6 # cannot drive it (Highlight.__init__() missing argument "md"). Match the # version in the repo-root uv.lock until mkdocstrings is bumped everywhere. "pymdown-extensions==11.0.1", "mkdocstrings-python==2.0.8", "griffe-inherited-docstrings==1.1.3", # mkdocstrings uses ruff to format rendered signatures "ruff==0.16.8", ] [tool.hatch.version] source = "vcs" tag-pattern = '^zarr_indexing-v(?P.+)$' # `git_describe_command` ensures we get the zarr_indexing tags instead of latest. # `local_scheme` strips the git commit info so the appending info is just a counter from latest tag. # test-pypi doesn't accept git commit info in tags, and the count should be enough to distinguish unique runs. raw-options = { root = "../..", scm = { git = { describe_command = "git describe --dirty --tags --long --match zarr_indexing-v*" } }, local_scheme = "no-local-version" } [tool.hatch.build.targets.wheel] packages = ["src/zarr_indexing"] # An allowlist, so nothing that merely happens to sit in the package directory # — a scratch script, a stray notebook — can ride along in a release. The list # includes test assets in the sdist: `tests/` carries the vendored ndsel conformance # corpus, and `tests/test_doc_examples.py` executes `docs/snippets/*.py` and # `examples/*/*.py`, so those are part of the suite rather than decoration. # `pyproject.toml`, `README.md` and `LICENSE.txt` are added by hatchling itself. [tool.hatch.build.targets.sdist] include = [ "/src", "/tests", "/docs", "/examples", "/mkdocs.yml", "/justfile", "/CHANGELOG.md", "/CONTRIBUTING.md", ] # The shared docs config and header script are symlinks into packages/ in the # repository; ship the files they point at, since a link out of the sdist # dangles and pip refuses to unpack it. exclude = ["/mkdocs-base.yml", "/docs/_static/source-version.js"] [tool.hatch.build.targets.sdist.force-include] "../mkdocs-base.yml" = "mkdocs-base.yml" "../source-version.js" = "docs/_static/source-version.js" [tool.ruff] extend = "../../pyproject.toml" target-version = "py312" [tool.ruff.lint.per-file-ignores] # Chunk discovery deliberately catches Exception: a foreign source's # attributes may fail, so this path provides fallback metadata. Configured here (not as # noqa comments) because different ruff versions # have differed on whether these rules fire; RUF100 can remove unused noqa comments. "src/zarr_indexing/lazy_array.py" = ["BLE001", "S110"] [tool.pytest.ini_options] minversion = "7" # src is collected for its doctests: every public object's Examples section # is eligible for --doctest-modules. Passing examples do not verify all prose # contracts or optional-dependency paths. testpaths = ["tests", "src/zarr_indexing"] pythonpath = ["."] xfail_strict = true addopts = ["-ra", "--strict-config", "--strict-markers", "--doctest-modules"] doctest_optionflags = [ "NORMALIZE_WHITESPACE", "ELLIPSIS", "IGNORE_EXCEPTION_DETAIL", ] filterwarnings = [ "error", ] [tool.pyright] include = [ "src", "docs/snippets", "tests/test_doc_examples.py", ] enableExperimentalFeatures = true typeCheckingMode = "strict" pythonVersion = "3.12" # Partially unknown types from NumPy stubs are reported as warnings. CI's # pyright invocation fails on errors, while retaining these warnings in its output. reportUnknownVariableType = "warning" reportUnknownArgumentType = "warning" reportUnknownMemberType = "warning" reportUnknownParameterType = "warning" [tool.numpydoc_validation] checks = [ "GL10", "SS04", "PR02", "PR03", "PR05", "PR06", ] [tool.towncrier] # Fragments for this package live alongside the package source, separate # from the parent zarr-python `changes/` directory, so a PR touching only # `packages/zarr-indexing/` produces a release note for this package only. directory = "changes" filename = "CHANGELOG.md" package = "zarr_indexing" underlines = ["", "", ""] title_format = "## {version} ({project_date})" issue_format = "[#{issue}](https://github.com/zarr-developers/zarr-python/pull/{issue})" start_string = "\n" # Declaring any type replaces towncrier's built-in set, so all five the # `changes/README.md` menu offers are restated here. They are the defaults # verbatim except for `misc`, whose `showcontent` towncrier defaults to false: # a `misc` entry would render as a bare PR link, which tells a reader nothing. # A change worth a release note is worth a sentence, whatever its category. [[tool.towncrier.type]] directory = "feature" name = "Features" showcontent = true [[tool.towncrier.type]] directory = "bugfix" name = "Bugfixes" showcontent = true [[tool.towncrier.type]] directory = "doc" name = "Improved Documentation" showcontent = true [[tool.towncrier.type]] directory = "removal" name = "Deprecations and Removals" showcontent = true [[tool.towncrier.type]] directory = "misc" name = "Misc" showcontent = true zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/000077500000000000000000000000001527014137600246135ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/000077500000000000000000000000001527014137600274565ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/__init__.py000066400000000000000000000067571527014137600316060ustar00rootroot00000000000000"""Composable, lazy coordinate transforms for zarr array indexing. This package implements TensorStore-inspired index transforms. The core idea: every indexing operation (slicing, fancy indexing, etc.) produces a coordinate mapping from user space to storage space. These mappings compose lazily — no I/O until you explicitly read or write. Key types: - `IndexDomain` — a rectangular region of integer coordinates - `IndexTransform` — maps input coordinates to storage coordinates - `ConstantMap`, `DimensionMap`, `ArrayMap` — the three ways a single output dimension can depend on the input (see `output_map.py`) - `IndexTransform.compose` — chain two transforms into one `LazyArray` wraps a system-memory/basic-indexing source and gives it deferred indexing through `view[...]`, yielding its reads as `Partition`s. Other backends use an explicit `Reader` adapter. `plan_chunks` projects a transform through a caller-selected chunk grid without coupling the result to a storage backend or scheduler; `ChunkPlan.partition` exposes the plan's factored, columnar form as a `GridPartition` of `StridedSet`, `IndexedSet` and `JointSet` tables. Consumers starting from a NumPy-style selection translate it with `zarr_indexing.boundary`. The `DimensionGridLike` Protocol describes the narrow grid surface chunk resolution consumes without importing zarr. """ from importlib.metadata import version from zarr_indexing.chunk_resolution import ( ChunkCoverage, ChunkPlan, ChunkProjection, GridPartition, IndexedSet, JointSet, StridedSet, plan_chunks, ) from zarr_indexing.domain import IndexDomain from zarr_indexing.eager import EagerArrayAdapter from zarr_indexing.errors import BoundsCheckError, VindexInvalidSelectionError from zarr_indexing.grid import ( ChunkGrid, ChunkSpec, DimensionGrid, DimensionGridLike, EdgeDimensionGrid, FixedDimension, VaryingDimension, dimension_grids_from_chunks, ) from zarr_indexing.json import ( IndexDomainJSON, IndexTransformJSON, OutputIndexMapJSON, ) from zarr_indexing.lazy_array import LazyArray, Partition from zarr_indexing.messages import NdselError, normalize_ndsel, parse_ndsel from zarr_indexing.output_map import ( ArrayMap, ConstantMap, DimensionMap, OutputIndexMap, output_index_map_from_json, ) from zarr_indexing.reader import ( BasicReader, NumPyReader, ReadContext, Reader, UnitStepReader, basic_reader, numpy_reader, unit_step_reader, ) from zarr_indexing.transform import ( IndexTransform, ) __version__ = version("zarr-indexing") __all__ = [ "ArrayMap", "BasicReader", "BoundsCheckError", "ChunkCoverage", "ChunkGrid", "ChunkPlan", "ChunkProjection", "ChunkSpec", "ConstantMap", "DimensionGrid", "DimensionGridLike", "DimensionMap", "EagerArrayAdapter", "EdgeDimensionGrid", "FixedDimension", "GridPartition", "IndexDomain", "IndexDomainJSON", "IndexTransform", "IndexTransformJSON", "IndexedSet", "JointSet", "LazyArray", "NdselError", "NumPyReader", "OutputIndexMap", "OutputIndexMapJSON", "Partition", "ReadContext", "Reader", "StridedSet", "UnitStepReader", "VaryingDimension", "VindexInvalidSelectionError", "__version__", "basic_reader", "dimension_grids_from_chunks", "normalize_ndsel", "numpy_reader", "output_index_map_from_json", "parse_ndsel", "plan_chunks", "unit_step_reader", ] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/_affine.py000066400000000000000000000055221527014137600314230ustar00rootroot00000000000000"""Checked affine coordinate arithmetic.""" from __future__ import annotations from typing import Any, overload import numpy as np import numpy.typing as npt _INTP_INFO = np.iinfo(np.intp) def _fits_intp(value: int) -> bool: return _INTP_INFO.min <= value <= _INTP_INFO.max @overload def checked_affine(offset: int, stride: int, coordinates: int) -> int: ... @overload def checked_affine( offset: int, stride: int, coordinates: npt.NDArray[np.integer[Any]], ) -> npt.NDArray[np.intp]: ... def checked_affine( offset: int, stride: int, coordinates: int | npt.NDArray[np.integer[Any]], ) -> int | npt.NDArray[np.intp]: """Evaluate ``offset + stride * coordinates`` without integer overflow. Bounds are established with Python integers before coordinates are cast or NumPy performs fixed-width arithmetic. The common representable case then uses an ``np.intp`` fast path whose multiplication and addition were proven safe; cancellation cases use exact object arithmetic. The identity affine (``offset == 0 and stride == 1``) of an array whose dtype fits ``np.intp`` cannot overflow and is returned re-typed without a scan; `ArrayMap.__post_init__` normalizes every index array through it. """ offset = int(offset) stride = int(stride) if not isinstance(coordinates, np.ndarray): mapped = offset + stride * int(coordinates) if not _fits_intp(mapped): raise OverflowError(f"output coordinate {mapped} is outside np.intp range") return mapped if coordinates.size == 0: return np.empty(coordinates.shape, dtype=np.intp) if offset == 0 and stride == 1 and np.can_cast(coordinates.dtype, np.intp): return np.asarray(coordinates, dtype=np.intp) coordinate_min = int(coordinates.min()) coordinate_max = int(coordinates.max()) product_at_min = stride * coordinate_min product_at_max = stride * coordinate_max mapped_at_min = offset + product_at_min mapped_at_max = offset + product_at_max mapped_min = min(mapped_at_min, mapped_at_max) mapped_max = max(mapped_at_min, mapped_at_max) if not _fits_intp(mapped_min) or not _fits_intp(mapped_max): invalid = mapped_min if not _fits_intp(mapped_min) else mapped_max raise OverflowError(f"output coordinate {invalid} is outside np.intp range") safe_fixed_width = ( _fits_intp(coordinate_min) and _fits_intp(coordinate_max) and _fits_intp(offset) and _fits_intp(stride) and _fits_intp(product_at_min) and _fits_intp(product_at_max) ) if safe_fixed_width: intp_coordinates = coordinates.astype(np.intp, copy=False) return np.asarray(offset + stride * intp_coordinates, dtype=np.intp) exact = offset + stride * coordinates.astype(object) return np.asarray(exact, dtype=np.intp) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/_composition.py000066400000000000000000000236411527014137600325400ustar00rootroot00000000000000"""Composition — chaining two transforms into one. `compose(outer, inner)` is the operation that makes views stack. `outer` maps user coordinates to intermediate coordinates, `inner` maps those intermediate coordinates to output coordinates, and the result maps user coordinates straight through — so a view of a view of an array is still a single `IndexTransform`, and indexing never accumulates layers to walk at read time. Composition works one output map at a time, and each case reduces to substituting the outer map into the inner one: - A `ConstantMap` inner map ignores its input, so it survives unchanged. - A `DimensionMap` inner map is affine, so composing it with an outer `ConstantMap` or `DimensionMap` folds into new `offset`/`stride` values; composing it with an outer `ArrayMap` leaves the index array alone and rescales around it. - An `ArrayMap` inner map must be *evaluated* at the coordinates the outer transform produces. This case gathers from the inner lookup table; other cases can still inspect or copy index arrays during validation/construction. """ from __future__ import annotations from typing import Any import numpy as np from zarr_indexing._affine import checked_affine from zarr_indexing.errors import BoundsCheckError from zarr_indexing.output_map import ( ArrayMap, ConstantMap, DimensionMap, OutputIndexMap, array_map_or_constant, ) from zarr_indexing.transform import IndexTransform def compose(outer: IndexTransform, inner: IndexTransform) -> IndexTransform: """Compose two IndexTransforms. `outer` maps user coords (rank m) to intermediate coords (rank n). `inner` maps intermediate coords (rank n) to output coords (rank p). The result maps user coords (rank m) to output coords (rank p). The output rank of `outer` must equal the input rank of `inner`, and every coordinate produced by a nonempty `outer` must lie in `inner.domain`. A rank mismatch raises `ValueError`; out-of-domain coordinates raise `BoundsCheckError`. Examples -------- Chained indexing — `source[2:5]`, then `[::-1]` on the result — collapses to a single transform (a reversed axis keeps literal coordinates, so the composed domain is `[-4, -1)`): >>> inner = IndexTransform.from_shape((10,))[2:5] >>> outer = IndexTransform.identity(inner.domain)[::-1] >>> chained = compose(outer, inner) >>> chained == inner[::-1] True >>> [chained.apply((i,)) for i in (-4, -3, -2)] [(4,), (3,), (2,)] >>> np.arange(10)[2:5][::-1].tolist() [4, 3, 2] """ if outer.output_rank != inner.domain.ndim: raise ValueError( f"outer output rank ({outer.output_rank}) must match inner input rank " f"({inner.domain.ndim})" ) _validate_outer_outputs(outer, inner) result_output = [ _compose_single(outer, inner_map, inner.domain.inclusive_min) for inner_map in inner.output ] return IndexTransform(domain=outer.domain, output=tuple(result_output)) def _validate_outer_outputs(outer: IndexTransform, inner: IndexTransform) -> None: """Prove that every intermediate coordinate is in the inner domain. An empty outer domain has no points, so containment is vacuously true. For a nonempty domain, each map form has an exact, constant-space range proof: constants are singletons, dimension maps are affine intervals, and array maps need only their extrema. """ if any(extent == 0 for extent in outer.domain.shape): return for axis, (outer_map, inner_lo, inner_hi) in enumerate( zip( outer.output, inner.domain.inclusive_min, inner.domain.exclusive_max, strict=True, ) ): output_lo, output_hi = _output_bounds(outer, outer_map) if output_lo < inner_lo or output_hi >= inner_hi: raise BoundsCheckError( f"outer output dimension {axis} produces coordinates " f"[{output_lo}, {output_hi}] outside the inner input domain " f"[{inner_lo}, {inner_hi})" ) def _output_bounds(outer: IndexTransform, output_map: OutputIndexMap) -> tuple[int, int]: """Return the exact inclusive bounds of one output map on a nonempty domain.""" if isinstance(output_map, ConstantMap): return output_map.offset, output_map.offset if isinstance(output_map, DimensionMap): input_lo = outer.domain.inclusive_min[output_map.input_dimension] input_hi = outer.domain.exclusive_max[output_map.input_dimension] first = output_map.offset + output_map.stride * input_lo last = output_map.offset + output_map.stride * (input_hi - 1) return min(first, last), max(first, last) index_lo = int(output_map.index_array.min()) index_hi = int(output_map.index_array.max()) first = output_map.offset + output_map.stride * index_lo last = output_map.offset + output_map.stride * index_hi return min(first, last), max(first, last) def _compose_single( outer: IndexTransform, inner_map: OutputIndexMap, inner_origin: tuple[int, ...] ) -> OutputIndexMap: """Compose a single inner output map with the full outer transform.""" if isinstance(inner_map, ConstantMap): return ConstantMap(offset=inner_map.offset) if isinstance(inner_map, DimensionMap): return _compose_dimension(outer, inner_map) # inner_map: ArrayMap (OutputIndexMap = ConstantMap | DimensionMap | ArrayMap) return _compose_array(outer, inner_map, inner_origin) def _compose_dimension(outer: IndexTransform, inner_map: DimensionMap) -> OutputIndexMap: """Compose when inner is a DimensionMap. storage = offset_i + stride_i * intermediate[dim_i] where intermediate[dim_i] = outer.output[dim_i](user_input) """ dim_i = inner_map.input_dimension offset_i = inner_map.offset stride_i = inner_map.stride outer_map = outer.output[dim_i] if isinstance(outer_map, ConstantMap): return ConstantMap(offset=checked_affine(offset_i, stride_i, outer_map.offset)) if isinstance(outer_map, DimensionMap): return DimensionMap( input_dimension=outer_map.input_dimension, offset=offset_i + stride_i * outer_map.offset, stride=stride_i * outer_map.stride, ) # outer_map: ArrayMap (OutputIndexMap = ConstantMap | DimensionMap | ArrayMap) # Affine post-composition leaves the index array (and hence its full # input rank and dependency axes) untouched. return ArrayMap( index_array=outer_map.index_array, offset=offset_i + stride_i * outer_map.offset, stride=stride_i * outer_map.stride, ) def _dimension_positions( outer: IndexTransform, outer_map: DimensionMap, inner_origin: int ) -> np.ndarray[Any, np.dtype[np.intp]]: """Build exact positional indices for an affine outer map.""" dimension = outer_map.input_dimension extent = outer.domain.shape[dimension] start = checked_affine( outer_map.offset - inner_origin, outer_map.stride, outer.domain.inclusive_min[dimension], ) shape = (1,) * dimension + (extent,) + (1,) * (outer.input_rank - dimension - 1) if extent == 0: return np.empty(shape, dtype=np.intp) if extent == 1: return np.full(shape, start, dtype=np.intp) steps = np.arange(extent, dtype=np.intp) return checked_affine(start, outer_map.stride, steps).reshape(shape) def _array_positions(outer_map: ArrayMap, inner_origin: int) -> np.ndarray[Any, np.dtype[np.intp]]: """Build exact positional indices without fixed-width affine overflow.""" return checked_affine(outer_map.offset - inner_origin, outer_map.stride, outer_map.index_array) def _positions_for_axis( outer: IndexTransform, outer_map: OutputIndexMap, inner_origin: int ) -> int | np.ndarray[Any, np.dtype[np.intp]]: """Convert one intermediate coordinate map to inner-array positions.""" if isinstance(outer_map, ConstantMap): return outer_map.offset - inner_origin if isinstance(outer_map, DimensionMap): return _dimension_positions(outer, outer_map, inner_origin) return _array_positions(outer_map, inner_origin) def _compose_array( outer: IndexTransform, inner_map: ArrayMap, inner_origin: tuple[int, ...] ) -> OutputIndexMap: """Compose when inner is an ArrayMap. storage = offset_i + stride_i * arr_i[intermediate] We need to evaluate arr_i at the intermediate coordinates produced by outer. Both domains carry their own origin, and neither is necessarily 0 — a step-1 slice keeps its literal bounds and a negative step produces a negative origin, so non-zero origins are the ordinary case here rather than the exotic one. The intermediate coordinates are read over the *outer* domain's own range, and the inner array is addressed positionally from the *inner* domain's origin. """ arr_i = inner_map.index_array if any(extent == 0 for extent in outer.domain.shape): # The empty map is singleton on every non-empty axis: it varies over no # axis at all, and the emptiness lives in the domain emitted alongside. empty_shape = tuple(0 if extent == 0 else 1 for extent in outer.domain.shape) return ArrayMap( index_array=np.empty(empty_shape, dtype=arr_i.dtype), offset=inner_map.offset, stride=inner_map.stride, ) positions = tuple( 0 if size == 1 else _positions_for_axis(outer, outer_map, origin) for outer_map, origin, size in zip(outer.output, inner_origin, arr_i.shape, strict=True) ) # A gather narrowed to one coordinate — scalar or all-singleton — is the # ConstantMap it equals; `array_map_or_constant` normalizes both. gathered = np.asarray(arr_i[positions]) if gathered.ndim == 0: return ConstantMap(offset=checked_affine(inner_map.offset, inner_map.stride, int(gathered))) return array_map_or_constant(gathered, offset=inner_map.offset, stride=inner_map.stride) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/_selector.py000066400000000000000000000030231527014137600320050ustar00rootroot00000000000000"""Internal scalar-selector coercion shared by indexing dialects.""" from __future__ import annotations import operator from typing import Any, SupportsIndex, cast import numpy as np def is_bool_scalar(value: Any) -> bool: """Return whether ``value`` is a Python or NumPy boolean scalar.""" return isinstance(value, (bool, np.bool_)) def as_scalar_index(value: Any) -> int | None: """Return a non-boolean scalar selector as an exact Python integer. Selector coercion follows Python's ``__index__`` protocol, rather than accepting only the concrete integer classes we happen to know about. In particular, ``operator.index`` rejects lossy ``__int__``-only objects and validates that ``__index__`` really returned an integer. """ if is_bool_scalar(value): return None # ndarray defines ``__index__`` for its scalar-integer case, but the # attribute also makes non-scalar and non-integer arrays look like # ``SupportsIndex`` at runtime. Those are array selectors, not malformed # scalar selectors, and must continue through array dtype validation. if isinstance(value, np.ndarray): array = cast("np.ndarray[Any, np.dtype[Any]]", value) if array.ndim != 0 or array.dtype.kind not in "iu": return None if not isinstance(value, SupportsIndex): return None return operator.index(cast(SupportsIndex, value)) def require_index(value: Any) -> int: """Coerce a required slice component through ``__index__``.""" return operator.index(value) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/_wire.py000066400000000000000000000154171527014137600311450ustar00rootroot00000000000000"""Shared lowering rules between the canonical ndsel wire form and the engine. Package-private: the types that serialize themselves (`IndexDomain`, `IndexTransform`, the output map kinds) all need these, so they cannot live in any one of them, and they are not API. Domain bounds, index-array bounds, implicit bounds lowering by value, and integer `index_array` content are handled here, as described in [`zarr_indexing.json`][zarr_indexing.json]. """ from __future__ import annotations from typing import TYPE_CHECKING, Any import numpy as np from zarr_indexing.messages import NdselError, validate_index_array_bounds if TYPE_CHECKING: from zarr_indexing.domain import IndexDomain from zarr_indexing.json import BoundJSON def check_index_array_bounds(array: np.ndarray[Any, Any], bounds: Any, where: str) -> None: """Validate every raw index value against an inclusive interval. Validate interval syntax even for empty arrays. Once checked, immutable index coordinates need no retained constraint. Use Python integer extrema to avoid overflow or floating-point rounding at integer limits. """ lo, hi = validate_index_array_bounds(bounds, where) if array.size == 0 or (lo == "-inf" and hi == "+inf"): return minimum, maximum = int(array.min()), int(array.max()) if ( lo == "+inf" or hi == "-inf" or (isinstance(lo, int) and minimum < lo) or (isinstance(hi, int) and maximum > hi) ): raise NdselError( "invalid_json", f"{where}.index_array values [{minimum}, {maximum}] are outside " f"index_array_bounds {bounds!r}", ) def lower_bound(bound: BoundJSON, where: str) -> int: """Lower a canonical bound to a finite integer, rejecting infinities. Reached only with a bound the message layer has already validated as an `index-value`, so the one thing left to rule out is a sentinel: an `IndexDomain` addresses a finite array. """ value = bound[0] if isinstance(bound, list) else bound if value == "-inf" or value == "+inf": raise NdselError( "invalid_json", f"{where} is infinite ({value!r}); an IndexDomain addresses a finite " f"array and cannot lower an infinite bound", ) return int(value) def lower_index_array(raw: Any, where: str) -> np.ndarray[Any, np.dtype[np.intp]]: """Lower a canonical `index_array` to `intp`, rejecting non-integer content. The message layer carries `index_array` verbatim — the spec defers its shape and type to the engine — so this is where the content is checked. An index array names output coordinates, and nothing but an integer names one: converting `[0.9, 1.9]` would silently read cells 0 and 1, and `[true, false]` cells 1 and 0. Strings raise here rather than leaking NumPy's own conversion error. """ if not isinstance(raw, list): # The wire representation requires a nested array, not a scalar. raise NdselError( "invalid_json", f"{where} must be an array of integers, got {raw!r}", ) # Validate before NumPy inference can coerce mixed booleans to integers or # conversion to intp can wrap unsigned coordinates. limits = np.iinfo(np.intp) pending = [raw] while pending: for value in pending.pop(): if isinstance(value, list): pending.append(value) elif isinstance(value, (bool, np.bool_)) or not isinstance(value, (int, np.integer)): dtype = np.asarray(value).dtype.name raise NdselError( "invalid_json", f"{where} must hold integers, got {dtype}: {value!r}" ) elif not limits.min <= value <= limits.max: raise NdselError( "invalid_json", f"{where} coordinate {value} is outside intp range" ) try: arr = np.asarray(raw) except (TypeError, ValueError) as exc: raise NdselError("invalid_json", f"{where} is not an array: {exc}") from exc if arr.size == 0 and arr.dtype.kind == "f": # An empty JSON list carries no element type and NumPy defaults it to # float64. An empty selection is legal, so take it as an empty index array. return np.zeros(arr.shape, dtype=np.intp) if arr.dtype.kind not in "iu": raise NdselError( "invalid_json", f"{where} must hold integers, got an array of {arr.dtype.name}; an " f"index array names output coordinates, which floats, booleans and " f"strings do not", ) return np.asarray(arr, dtype=np.intp) def lower_labels(labels: list[str]) -> tuple[str, ...] | None: """All-empty labels collapse to `None` so a label-free domain round-trips.""" return None if all(label == "" for label in labels) else tuple(labels) def emit_labels(labels: tuple[str, ...] | None, rank: int) -> list[str]: """Emit canonical labels: `[""]*rank` when the domain is unlabeled.""" return [""] * rank if labels is None else list(labels) def full_rank_index_array( arr: np.ndarray[Any, np.dtype[np.intp]], domain: IndexDomain, where: str, ) -> np.ndarray[Any, np.dtype[np.intp]]: """Give an incoming `index_array` the input rank the engine requires. ndsel intends index arrays to have the input rank, but defers validating that constraint. This engine also accepts lower-rank arrays, aligning them to the *trailing* input dimensions as in NumPy broadcasting. This extension does not imply other ndsel consumers accept the same document. An empty array is a different matter: `[]` is the only spelling of every empty shape once the leading axis is the zero-length one, so the axis it varies over cannot be read off it. When ranks differ, this engine recovers a shape with the domain's single empty axis and singleton axes elsewhere; it rejects recovery if the domain has zero or multiple empty axes. This package never emits such a document (an empty map is degenerate and collapses to a constant, as TensorStore's does), so this path exists for external producers alone. """ if arr.size == 0 and arr.ndim != domain.ndim: empty_axes = [k for k, extent in enumerate(domain.shape) if extent == 0] if len(empty_axes) != 1: raise NdselError( "invalid_json", f"{where}.index_array is empty, but the input domain has " f"{len(empty_axes)} zero-length dimensions, so the axis it varies " f"over cannot be recovered", ) shape = [1] * domain.ndim shape[empty_axes[0]] = 0 return arr.reshape(tuple(shape)) if arr.ndim < domain.ndim: return arr.reshape((1,) * (domain.ndim - arr.ndim) + arr.shape) return arr zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/boundary.py000066400000000000000000000361471527014137600316660ustar00rootroot00000000000000"""The positional (NumPy) selection dialect, lowered onto the transform algebra. The transform algebra uses **literal domain coordinates**: an index is a point in the view's own coordinate system, which after `view = arr[10:50]` runs from 10 to 49, and a negative index is a negative coordinate rather than an offset from the end (TensorStore's convention — see `zarr_indexing.transform`). NumPy uses **positions**: index 0 always means the first element of the object being indexed, and `-1` means the last. This module translates between the two. It validates a selection against the view's shape with NumPy semantics, then shifts every coordinate by the domain's origin so the transform layer sees literal coordinates. Note ---- This module provides the positional boundary used by `LazyArray`, independently of Zarr's own array indexers. Direct `IndexTransform` indexing continues to use literal coordinates. """ from __future__ import annotations from typing import TYPE_CHECKING, Any, Literal import numpy as np from zarr_indexing._selector import as_scalar_index, is_bool_scalar, require_index if TYPE_CHECKING: from zarr_indexing.domain import IndexDomain SelectionMode = Literal["basic", "orthogonal", "vectorized"] def _as_index_array(sel: Any) -> np.ndarray[Any, np.dtype[Any]] | None: """Return `sel` as an ndarray if it is array-like, else None.""" if isinstance(sel, np.ndarray): return sel if isinstance(sel, (list, tuple)): arr = np.asarray(sel) if arr.size == 0 and arr.dtype.kind == "f": # An empty Python list carries no element type and NumPy defaults it # to float64. Selecting nothing is legal — NumPy takes `a[np.ix_([])]` # — so read it as the empty integer selection it spells, rather than # rejecting it for a dtype it never had a chance to have. return np.zeros(arr.shape, dtype=np.intp) if arr.dtype.kind in "biu": return arr raise IndexError( f"arrays used as indices must be of integer or boolean type; got dtype {arr.dtype}" ) return None def _axes_consumed(sel: Any, mode: SelectionMode) -> int: """How many axes of the view a single selection entry consumes.""" if sel is None: return 0 arr = _as_index_array(sel) # A multidimensional boolean mask consumes one axis per mask dimension. # Orthogonal indexing is per-axis by construction, so a mask there is 1-D # and consumes exactly one axis. if mode == "vectorized" and arr is not None and arr.dtype == np.bool_: return arr.ndim return 1 def _normalize_int(value: int, size: int, axis: int) -> int: """Bounds-check a positional integer index, wrapping negatives NumPy-style.""" idx = value if idx < 0: idx += size if idx < 0 or idx >= size: raise IndexError(f"index {value} is out of bounds for axis {axis} with size {size}") return idx def _normalize_slice(sel: slice, size: int, axis: int) -> tuple[int, int, int]: """Resolve a positional slice to `(start, stop, step)`, either direction. `slice.indices` already applies NumPy's rules — negative bounds count from the end, out-of-range bounds clamp, and a reversed slice runs downward with `stop` one *below* the last selected position. The one thing it does not do is canonicalize an empty result: it can hand back a stop on the far side of the start (`5:2` going up, `2:5` going down), which the transform layer reads as a direction error rather than an empty selection. Collapsing it to `stop == start` keeps NumPy's "empty, not an error" answer. """ start_bound = None if sel.start is None else require_index(sel.start) stop_bound = None if sel.stop is None else require_index(sel.stop) step = 1 if sel.step is None else require_index(sel.step) if step == 0: raise ValueError(f"slice step cannot be zero (axis {axis})") # ValueError: NumPy parity start, stop, step = slice(start_bound, stop_bound, step).indices(size) stop = max(stop, start) if step > 0 else min(stop, start) return start, stop, step def _normalize_int_array( arr: np.ndarray[Any, np.dtype[Any]], size: int, axis: int ) -> np.ndarray[Any, np.dtype[np.intp]]: """Bounds-check a positional integer index array, wrapping negatives.""" if arr.dtype.kind not in "iu": raise IndexError( f"arrays used as indices must be of integer or boolean type; got dtype {arr.dtype}" ) if arr.dtype.kind == "u": # Unsigned values are never negative, so check them as they are: a # value beyond the `intp` range would wrap negative on narrowing and # pass as a wrapped index. if arr.size > 0 and int(arr.max()) >= size: raise IndexError( f"index {int(arr.max())} is out of bounds for axis {axis} with size {size}" ) return arr.astype(np.intp) # Cast before wrapping: `intp` covers every index NumPy can address. out = arr.astype(np.intp, copy=True) if out.size > 0: negative = out < 0 if bool(negative.any()): out = np.where(negative, out + size, out) lo, hi = int(out.min()), int(out.max()) if lo < 0 or hi >= size: bad = lo if lo < 0 else hi raise IndexError(f"index {bad} is out of bounds for axis {axis} with size {size}") return out def _expanded_axis_walk(entries: tuple[Any, ...], ndim: int, mode: SelectionMode) -> list[int]: """The starting axis each entry addresses, with an ellipsis expanded. The returned list has one entry per element of `entries`; the value for an `Ellipsis` (or a `newaxis`) is the axis it starts at. The following entry advances past the ellipsis's skipped axes; a `newaxis` consumes no axis. """ for sel in entries: if is_bool_scalar(sel): raise IndexError( "boolean scalars are not valid indices; use a boolean array " "matching the shape of the axes it selects" ) if sum(1 for sel in entries if sel is Ellipsis) > 1: raise IndexError("an index can only have a single ellipsis ('...')") consumed = sum(_axes_consumed(sel, mode) for sel in entries if sel is not Ellipsis) if consumed > ndim: raise IndexError( f"too many indices for array: array has {ndim} dimensions, but {consumed} were indexed" ) axes: list[int] = [] axis = 0 for sel in entries: axes.append(axis) axis += (ndim - consumed) if sel is Ellipsis else _axes_consumed(sel, mode) return axes def validate_advanced_selection( selection: Any, domain: IndexDomain, mode: Literal["orthogonal", "vectorized"], ) -> None: """Validate advanced-index selector dtypes and boolean mask extents. This is the validation shared by positional callers such as `LazyArray` and direct `IndexTransform.oindex` / `.vindex` callers. It deliberately does not normalize coordinates: direct transforms use literal coordinates, whereas positional callers shift and wrap them separately. """ entries: tuple[Any, ...] = selection if isinstance(selection, tuple) else (selection,) axes = _expanded_axis_walk(entries, domain.ndim, mode) for sel, axis in zip(entries, axes, strict=True): arr = _as_index_array(sel) if arr is None: continue if arr.dtype == np.bool_: n_axes = _axes_consumed(sel, mode) expected = domain.shape[axis : axis + n_axes] if arr.shape != tuple(expected): extent = ( f"dimension {expected[0]}" if len(expected) == 1 else f"dimensions {tuple(expected)}" ) raise IndexError( f"boolean index has shape {arr.shape} but {extent} has shape {tuple(expected)}" ) elif arr.dtype.kind not in "iu": raise IndexError( f"arrays used as indices must be of integer or boolean type; got dtype {arr.dtype}" ) def split_scalar_axes( selection: Any, domain: IndexDomain, mode: SelectionMode, ) -> tuple[tuple[Any, ...] | None, Any]: """Peel scalar integer indices out of a fancy selection. A scalar integer drops its axis, and neither the orthogonal nor the vectorized path of the transform algebra models that — both widen a scalar into a length-1 index array, which keeps the axis — so the scalars are split off here and applied as a separate basic step first. Applying them *first* is this package's rule, not NumPy's. NumPy groups a scalar with the advanced indices for the purpose of placing the broadcast result, so the two disagree when a scalar and an index array are separated: `a[0, ..., [1, 2]]` has shape `(2, 3)` for a `(2, 3, 4)` array, where `a[0][..., [1, 2]]` has shape `(3, 2)`. Scalar-first processing is the wrapper's indexing dialect; it does not implement NumPy's full advanced-axis placement rules. Parameters ---------- selection A positional orthogonal or vectorized selection. domain The domain of the view being indexed. mode `"orthogonal"` or `"vectorized"`; controls how many axes each entry covers, which decides where the scalars sit. Returns ------- tuple[tuple[Any, ...] | None, Any] `(basic_selection, remaining_selection)`. `basic_selection` is a full-rank basic selection in **literal** domain coordinates that drops the scalar axes, or `None` when the selection has no scalar entries (in which case `remaining_selection` is `selection` unchanged). Raises ------ IndexError If a boolean scalar is used as an index, an index is out of bounds, or too many indices are supplied. """ entries = selection if isinstance(selection, tuple) else (selection,) axes = _expanded_axis_walk(entries, domain.ndim, mode) scalar_axes: dict[int, int] = {} remaining: list[Any] = [] for sel, axis in zip(entries, axes, strict=True): scalar = as_scalar_index(sel) if scalar is not None: scalar_axes[axis] = _normalize_int(scalar, domain.shape[axis], axis) else: remaining.append(sel) if len(scalar_axes) == 0: return None, selection basic: list[Any] = [] for axis in range(domain.ndim): lo = domain.inclusive_min[axis] if axis in scalar_axes: basic.append(lo + scalar_axes[axis]) else: basic.append(slice(lo, domain.exclusive_max[axis])) return tuple(basic), tuple(remaining) def normalize_positional_selection( selection: Any, domain: IndexDomain, mode: SelectionMode, ) -> Any: """Translate a positional (NumPy-dialect) selection into literal coordinates. Positions are zero-based offsets into the current view; negatives wrap from the end. The returned selection addresses the same cells in the literal coordinate system `domain` uses, ready for `zarr_indexing.transform.selection_to_transform`. Parameters ---------- selection A NumPy-style selection: integers, slices, `Ellipsis`, integer arrays or lists, or boolean arrays. domain The domain of the view being indexed. Its shape defines the positional bounds and its origin the coordinate shift. mode Which selection dialect the entries follow: `"basic"` (integers and slices), `"orthogonal"` (per-axis arrays, outer product), or `"vectorized"` (correlated coordinate arrays or a single mask). Returns ------- tuple[Any, ...] The selection with every coordinate expressed in literal domain coordinates. Raises ------ IndexError If a boolean scalar is used as an index, a boolean mask does not match the shape of the axes it covers, an index is out of bounds, or too many indices are supplied. """ entries = selection if isinstance(selection, tuple) else (selection,) shape = domain.shape origin = domain.inclusive_min ndim = domain.ndim if mode in ("orthogonal", "vectorized"): validate_advanced_selection(selection, domain, mode) else: for sel in entries: if is_bool_scalar(sel): raise IndexError( "boolean scalars are not valid indices; use a boolean array " "matching the shape of the axes it selects" ) n_ellipsis = sum(1 for sel in entries if sel is Ellipsis) if n_ellipsis > 1: raise IndexError("an index can only have a single ellipsis ('...')") consumed = sum(_axes_consumed(sel, mode) for sel in entries if sel is not Ellipsis) if consumed > ndim: raise IndexError( f"too many indices for array: array has {ndim} dimensions, but {consumed} were indexed" ) result: list[Any] = [] axis = 0 for sel in entries: if sel is Ellipsis: # Passed through rather than expanded: every mode of # `selection_to_transform` expands an ellipsis (and pads short # selections) to whole-axis slices itself, and `vectorized` mode # rejects an explicit slice, so expanding here would turn a legal # partial coordinate selection into an error. result.append(Ellipsis) axis += ndim - consumed continue if sel is None: # newaxis: no axis of the view is consumed, and there is no # coordinate to shift. The transform layer decides whether the mode # accepts it. result.append(None) continue arr = _as_index_array(sel) if arr is not None and arr.dtype == np.bool_: n_axes = _axes_consumed(sel, mode) expected = shape[axis : axis + n_axes] if arr.shape != tuple(expected): raise IndexError( f"boolean index has shape {arr.shape} but the axes it " f"covers have shape {tuple(expected)}" ) for offset, positions in enumerate(np.nonzero(arr)): result.append(positions.astype(np.intp) + origin[axis + offset]) axis += n_axes continue if axis >= ndim: raise IndexError( f"too many indices for array: array has {ndim} dimensions, " f"but {consumed} were indexed" ) size = shape[axis] if arr is not None: result.append(_normalize_int_array(arr, size, axis) + origin[axis]) elif isinstance(sel, slice): start, stop, step = _normalize_slice(sel, size, axis) result.append(slice(start + origin[axis], stop + origin[axis], step)) elif (scalar := as_scalar_index(sel)) is not None: result.append(_normalize_int(scalar, size, axis) + origin[axis]) else: raise IndexError(f"unsupported selection type: {type(sel)!r}") axis += 1 # Axes the selection did not mention are left to `selection_to_transform`, # which pads them with whole-axis slices in every mode. return tuple(result) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/chunk_resolution.py000066400000000000000000001367711527014137600334420ustar00rootroot00000000000000"""Chunk resolution — mapping transforms to chunk-level I/O. Given an `IndexTransform` (which coordinates a request reads) and one grid per storage dimension (how storage is divided into chunks), chunk resolution answers: For each chunk, which storage coordinates does this transform touch, and where do those values land in the request? The public result is a lazy, reusable `ChunkPlan` whose rows are `ChunkProjection`s. Each identifies a chunk and pairs a chunk-local transform with a transform back to the request's cells, over one shared zero-origin cell domain, without assuming NumPy selectors, a codec pipeline, or a scheduler. The plan is computed in factored form, the `GridPartition`. Restricting a transform to a chunk box distributes over output dimensions whenever each output map reads its own input axis — every basic and orthogonal selection — so each axis is resolved once against its grid into a table: - `StridedSet` — a `ConstantMap` or `DimensionMap` axis: one row per touched chunk, holding the chunk-local start, the extent, the request position of the first cell, and whether the row covers its chunk exactly once. - `IndexedSet` — an orthogonal `ArrayMap` axis: its coordinates grouped by chunk in CSR form, with the request positions they fill. - `JointSet` — one connected index-array component. Arrays sharing input axes are grouped together; independent components remain separate tables. A projection is one row of each table combined. Independent strided tables resolve each axis separately rather than walking their Cartesian product. Index-array tables also scan and group lookup points, sorting them when needed. Projection rows are materialized only on request, and a consumer may read the tables directly instead. This planner does not support affine maps sharing input axes: two affine maps sharing an axis raise `ValueError`, while an index array varying over an axis also read by an affine map raises `NotImplementedError`. """ from __future__ import annotations import itertools import math from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Literal, cast import numpy as np from zarr_indexing._affine import checked_affine from zarr_indexing.domain import IndexDomain from zarr_indexing.output_map import ArrayMap, ConstantMap, DimensionMap, OutputIndexMap from zarr_indexing.transform import ( IndexTransform, _intersect_dimension_map, # pyright: ignore[reportPrivateUsage] _prepare_correlated, # pyright: ignore[reportPrivateUsage] ) if TYPE_CHECKING: from collections.abc import Iterator, Sequence from zarr_indexing.grid import DimensionGridLike type ChunkCoverage = Literal["full", "partial", "unknown"] def _data_size(dim_grid: DimensionGridLike, chunk_ix: int) -> int: """Return a chunk's data extent, falling back for narrow-protocol grids.""" data_size = getattr(dim_grid, "data_size", None) if data_size is None: return dim_grid.chunk_size(chunk_ix) return int(data_size(chunk_ix)) @dataclass(frozen=True, slots=True) class ChunkProjection: """One source-independent projection of a request through a chunk. Both transforms share a synthetic input domain. ``chunk_transform`` maps that domain to chunk-local storage coordinates; ``cell_transform`` maps it to the original request domain. Attributes ---------- chunk_coords Coordinates of the selected cell in the caller's grid. chunk_domain Bounds of that grid cell in global storage coordinates. chunk_transform Mapping from the shared synthetic domain to chunk-local storage. cell_transform Mapping from the shared synthetic domain to request coordinates. coverage Whether the request is proven to cover the whole grid cell exactly once. Projections retaining ArrayMaps or survivor arrays are conservatively ``"unknown"``. A singleton fancy selection can simplify to a ConstantMap and receive a proven ``"full"`` or ``"partial"`` result. Examples -------- Row 1 of a `(3, 4)` array with `(2, 2)` chunks touches only part of the first chunk, whose domain spans rows `[0, 2)` and columns `[0, 2)`: >>> from zarr_indexing import IndexTransform >>> from zarr_indexing.grid import dimension_grids_from_chunks >>> grids = dimension_grids_from_chunks((2, 2), shape=(3, 4)) >>> plan = plan_chunks(IndexTransform.from_shape((3, 4))[1, :], grids) >>> first = next(iter(plan)) >>> first.chunk_coords (0, 0) >>> first.chunk_domain.shape (2, 2) >>> first.coverage 'partial' """ chunk_coords: tuple[int, ...] chunk_domain: IndexDomain chunk_transform: IndexTransform cell_transform: IndexTransform coverage: ChunkCoverage def __post_init__(self) -> None: if self.chunk_transform.domain != self.cell_transform.domain: raise ValueError( "chunk_transform and cell_transform must share an input domain; " f"got {self.chunk_transform.domain!r} and {self.cell_transform.domain!r}" ) @dataclass(frozen=True, slots=True) class ChunkPlan: """A reusable, lazy partition of an index transform over a chunk grid. Construct plans with `plan_chunks`. The plan's factored form, a `GridPartition`, is built on first use and memoized; iterating the plan or `projections()` materializes fresh `ChunkProjection` rows from it. Examples -------- Row 1 of a `(3, 4)` array with `(2, 2)` chunks crosses two chunks, and the plan can be walked again after it is exhausted: >>> from zarr_indexing import IndexTransform >>> from zarr_indexing.grid import dimension_grids_from_chunks >>> grids = dimension_grids_from_chunks((2, 2), shape=(3, 4)) >>> plan = plan_chunks(IndexTransform.from_shape((3, 4))[1, :], grids) >>> [p.chunk_coords for p in plan] [(0, 0), (0, 1)] >>> [p.chunk_coords for p in plan.projections()] [(0, 0), (0, 1)] """ transform: IndexTransform """The composed request this plan partitions.""" dimension_grids: tuple[DimensionGridLike, ...] """One grid per storage dimension, defining the chunk layout the plan walks.""" # Memoized factored form; derived state, excluded from identity (see # `IndexDomain._shape` for the same pattern). _partition: GridPartition | None = field(default=None, init=False, repr=False, compare=False) def partition(self) -> GridPartition: """The plan in factored, columnar form: one table per axis plus connected index-array tables. Built once per plan and memoized. Raises `ValueError` if two output maps read one input axis through a `DimensionMap` (a diagonal, which no selection produces): its tables are per output dimension, and a diagonal needs a strided set spanning several. """ cached = self._partition if cached is None: cached = _partition_transform(self.transform, self.dimension_grids) object.__setattr__(self, "_partition", cached) return cached def projections(self) -> Iterator[ChunkProjection]: """Return a fresh iterator over the chunks touched by this plan.""" return iter(self.partition()) def __iter__(self) -> Iterator[ChunkProjection]: """Equivalent to `projections()`.""" return self.projections() def plan_chunks( transform: IndexTransform, dimension_grids: Sequence[DimensionGridLike], ) -> ChunkPlan: """Plan a transform against a caller-selected chunk grid. Parameters ---------- transform Mapping from the request domain to storage coordinates. dimension_grids One storage grid per transform output dimension. Returns ------- ChunkPlan A reusable plan whose projections are computed lazily; its `partition()` is the factored form they are derived from. Examples -------- Row 1 of a `(3, 4)` array with `(2, 2)` chunks touches the two chunks in the top grid row, each contributing a `(2, 2)` chunk domain: >>> from zarr_indexing import IndexTransform >>> from zarr_indexing.grid import dimension_grids_from_chunks >>> grids = dimension_grids_from_chunks((2, 2), shape=(3, 4)) >>> plan = plan_chunks(IndexTransform.from_shape((3, 4))[1, :], grids) >>> [p.chunk_coords for p in plan] [(0, 0), (0, 1)] >>> [p.chunk_domain.shape for p in plan] [(2, 2), (2, 2)] """ grids = tuple(dimension_grids) if len(grids) != transform.output_rank: raise ValueError( "dimension_grids must have one entry per transform output dimension; " f"got {len(grids)} grids for output rank {transform.output_rank}" ) return ChunkPlan(transform=transform, dimension_grids=grids) # --------------------------------------------------------------------------- # # Grid partition: the factored form of a plan # --------------------------------------------------------------------------- # def _dimension_map_candidates( m: DimensionMap, dim_lo: int, dim_hi: int, dg: DimensionGridLike ) -> Sequence[tuple[int, ...]]: """The chunks a nonempty `DimensionMap` over input `[dim_lo, dim_hi)` can touch, ascending.""" first_storage = checked_affine(m.offset, m.stride, dim_lo) if m.stride > 0: s_min = first_storage s_max = checked_affine(m.offset, m.stride, dim_hi - 1) elif m.stride < 0: s_min = checked_affine(m.offset, m.stride, dim_hi - 1) s_max = first_storage else: s_min = s_max = first_storage first = dg.index_to_chunk(s_min) last = dg.index_to_chunk(s_max) point_count = dim_hi - dim_lo chunk_count = last - first + 1 if point_count < chunk_count: steps = np.arange(point_count, dtype=np.intp) storage = checked_affine(first_storage, m.stride, steps) chunk_ids = dg.indices_to_chunks(storage) return [(int(c),) for c in np.unique(chunk_ids)] return [(c,) for c in range(first, last + 1)] def _own(column: np.ndarray[Any, Any]) -> np.ndarray[Any, Any]: """A read-only column that cannot be made writable again. A memoized partition must not drift under a consumer. Clearing the writeable flag is advisory (`setflags(write=True)` undoes it), so a column is re-homed over immutable bytes, the way `ArrayMap` owns its index array; a view of such bytes is kept as is. Object columns (exact ints beyond ``np.intp``) hold references and cannot live in bytes, so they keep the flag only. """ if column.dtype == object: column.setflags(write=False) return column owner: Any = column.base while isinstance(owner, np.ndarray): owner = owner.base if isinstance(owner, bytes) and not column.flags.writeable: return column contiguous = np.ascontiguousarray(column) return np.frombuffer(contiguous.tobytes(), dtype=contiguous.dtype).reshape(contiguous.shape) def _own_columns(table: Any, *names: str) -> None: for name in names: object.__setattr__(table, name, _own(getattr(table, name))) def _wide_column(values: Sequence[int]) -> np.ndarray[Any, np.dtype[np.intp]]: """An ``intp`` column, or an object column of exact ints when a value does not fit. Request-domain extents are unbounded Python ints (a zero-stride map over a `(2**63,)` domain is valid and touches one storage cell), so the two columns measured along the request axis keep exact integers rather than imposing a domain-size limit the algebra does not have. """ try: return np.array(values, dtype=np.intp) except OverflowError: return cast("np.ndarray[Any, np.dtype[np.intp]]", np.array(values, dtype=object)) def _chunk_bounds( dg: DimensionGridLike, chunks: np.ndarray[Any, np.dtype[np.intp]] ) -> tuple[np.ndarray[Any, np.dtype[np.intp]], np.ndarray[Any, np.dtype[np.intp]]]: """Storage origin and data extent of each chunk, one grid call per touched chunk.""" n = int(chunks.size) starts = np.fromiter((dg.chunk_offset(int(c)) for c in chunks), dtype=np.intp, count=n) extents = np.fromiter((_data_size(dg, int(c)) for c in chunks), dtype=np.intp, count=n) return starts, extents @dataclass(frozen=True, slots=True) class StridedSet: """One output dimension read through a `ConstantMap` or `DimensionMap`, one row per chunk. Row ``i`` is the map restricted to chunk ``chunk[i]`` and re-based to chunk-local, zero-origin coordinates: the chunk-local map is `DimensionMap(input_dimension, offset=local_start[i], stride=stride)` over ``[0, extent[i])`` (a `ConstantMap(local_start[i])` for a constant), and its cells are request positions ``[origin[i], origin[i] + extent[i])`` along the input axis, counted from the request domain's lower bound. Columns are read-only NumPy arrays. `extent` and `origin` are measured along the request axis, whose bounds are arbitrary Python ints; they are ``intp`` unless a value does not fit, in which case they hold exact ints (``dtype=object``). Examples -------- >>> from zarr_indexing import IndexTransform, plan_chunks >>> from zarr_indexing.grid import dimension_grids_from_chunks >>> grids = dimension_grids_from_chunks((4,), shape=(10,)) >>> (axis,) = plan_chunks(IndexTransform.from_shape((10,))[1:9:2], grids).partition().sets >>> axis.chunk.tolist(), axis.local_start.tolist(), axis.extent.tolist(), axis.origin.tolist() ([0, 1], [1, 1], [2, 2], [0, 2]) """ output_dimension: int """The storage axis this table describes.""" input_dimension: int | None """The request axis the map reads, or `None` for a constant.""" stride: int """Storage step per request cell; ``0`` for a constant.""" chunk: np.ndarray[Any, np.dtype[np.intp]] """Chunk index along the axis, one per row, ascending.""" chunk_start: np.ndarray[Any, np.dtype[np.intp]] """Storage origin of each chunk.""" chunk_extent: np.ndarray[Any, np.dtype[np.intp]] """Data extent of each chunk (clipped at the array boundary).""" local_start: np.ndarray[Any, np.dtype[np.intp]] """Chunk-local storage coordinate of the row's first cell.""" extent: np.ndarray[Any, np.dtype[np.intp]] """Cells the row selects along the request axis (``1`` for a constant).""" origin: np.ndarray[Any, np.dtype[np.intp]] """Position of the row's first cell along the request axis (``0`` for a constant).""" full: np.ndarray[Any, np.dtype[np.bool_]] """Whether the row covers its chunk's data extent exactly once (in either direction).""" def __post_init__(self) -> None: _own_columns( self, "chunk", "chunk_start", "chunk_extent", "local_start", "extent", "origin", "full" ) def __len__(self) -> int: return int(self.chunk.size) @dataclass(frozen=True, slots=True) class IndexedSet: """One output dimension read through an orthogonal `ArrayMap`, one row per chunk. The map's coordinates are grouped by chunk in CSR form: row ``i`` owns ``index[pointer[i]:pointer[i + 1]]`` (the index-array values, in request order) and ``positions[pointer[i]:pointer[i + 1]]`` (their positions along the request axis, ascending). `local` gives the same values as chunk-local storage coordinates. Columns are read-only NumPy arrays. Examples -------- >>> import numpy as np >>> from zarr_indexing import IndexTransform, plan_chunks >>> from zarr_indexing.grid import dimension_grids_from_chunks >>> grids = dimension_grids_from_chunks((4,), shape=(10,)) >>> transform = IndexTransform.from_shape((10,)).oindex[np.array([9, 1, 2, 8])] >>> (axis,) = plan_chunks(transform, grids).partition().sets >>> axis.chunk.tolist(), axis.pointer.tolist() ([0, 2], [0, 2, 4]) >>> axis.local.tolist(), axis.positions.tolist() ([1, 2, 1, 0], [1, 2, 0, 3]) """ output_dimension: int """The storage axis this table describes.""" input_dimension: int """The request axis the index array varies over.""" offset: int """The map's affine offset: storage is ``offset + stride * index``.""" stride: int """The map's affine stride.""" chunk: np.ndarray[Any, np.dtype[np.intp]] """Chunk index along the axis, one per row, ascending.""" chunk_start: np.ndarray[Any, np.dtype[np.intp]] """Storage origin of each chunk.""" chunk_extent: np.ndarray[Any, np.dtype[np.intp]] """Data extent of each chunk.""" pointer: np.ndarray[Any, np.dtype[np.intp]] """CSR row pointer: row ``i`` owns entries ``pointer[i]`` to ``pointer[i + 1]``.""" index: np.ndarray[Any, np.dtype[np.intp]] """Index-array values grouped by chunk.""" positions: np.ndarray[Any, np.dtype[np.intp]] """Positions along the request axis, grouped by chunk, ascending within a row.""" def __post_init__(self) -> None: _own_columns(self, "chunk", "chunk_start", "chunk_extent", "pointer", "index", "positions") def __len__(self) -> int: return int(self.chunk.size) @property def counts(self) -> np.ndarray[Any, np.dtype[np.intp]]: """Entries per row.""" return np.diff(self.pointer) _local: np.ndarray[Any, np.dtype[np.intp]] | None = field( default=None, init=False, repr=False, compare=False ) @property def local(self) -> np.ndarray[Any, np.dtype[np.intp]]: """Memoized, read-only chunk-local coordinates, grouped like `index`.""" cached = self._local if cached is None: storage = checked_affine(self.offset, self.stride, self.index) cached = storage - np.repeat(self.chunk_start, self.counts) cached = _own(cached) object.__setattr__(self, "_local", cached) return cached def run(self, row: int) -> slice: """The slice of `index` / `positions` a row owns.""" return slice(int(self.pointer[row]), int(self.pointer[row + 1])) @dataclass(frozen=True, slots=True) class JointSet: """One connected component of index arrays, grouped by the chunk each point lands in. Arrays connected by shared input axes, possibly transitively, constrain each other and are sorted into chunks together. Row ``i`` is one touched chunk, `chunk[i]` its coordinates on the `output_dimensions`, and CSR range ``pointer[i]:pointer[i + 1]`` its points: `index` holds their index-array values per output dimension, `positions` their flat positions in this component's input block, and `block_coordinates` those positions unravelled over the block. Columns are read-only NumPy arrays. Examples -------- >>> import numpy as np >>> from zarr_indexing import IndexTransform, plan_chunks >>> from zarr_indexing.grid import dimension_grids_from_chunks >>> grids = dimension_grids_from_chunks((3, 4), shape=(7, 9)) >>> transform = IndexTransform.from_shape((7, 9)).vindex[ ... np.array([0, 6, 6, 1]), np.array([8, 0, 1, 8]) ... ] >>> (joint,) = plan_chunks(transform, grids).partition().joint_sets >>> joint.chunk.tolist(), joint.pointer.tolist(), joint.positions.tolist() ([[0, 2], [2, 0]], [0, 2, 4], [0, 3, 1, 2]) """ output_dimensions: tuple[int, ...] """The storage axes read by correlated index arrays.""" offsets: tuple[int, ...] """Affine offset of each array's map, aligned with `output_dimensions`.""" strides: tuple[int, ...] """Affine stride of each array's map.""" broadcast_axes: tuple[int, ...] """The request axes the arrays broadcast over.""" broadcast_shape: tuple[int, ...] """The extent of those axes.""" chunk: np.ndarray[Any, np.dtype[np.intp]] """Chunk coordinates on `output_dimensions`, shape ``(rows, k)``, lexicographic.""" chunk_start: np.ndarray[Any, np.dtype[np.intp]] """Storage origin of each chunk on `output_dimensions`, shape ``(rows, k)``.""" chunk_extent: np.ndarray[Any, np.dtype[np.intp]] """Data extent of each chunk on `output_dimensions`, shape ``(rows, k)``.""" pointer: np.ndarray[Any, np.dtype[np.intp]] """CSR row pointer into `index`, `positions` and `block_coordinates`.""" index: np.ndarray[Any, np.dtype[np.intp]] """Index-array values per point and output dimension, shape ``(points, k)``.""" positions: np.ndarray[Any, np.dtype[np.intp]] """Flat block position of each point, ascending within a row.""" _block_coordinates: np.ndarray[Any, np.dtype[np.intp]] | None = field( default=None, init=False, repr=False, compare=False ) def __post_init__(self) -> None: _own_columns(self, "chunk", "chunk_start", "chunk_extent", "pointer", "index", "positions") def __len__(self) -> int: return int(self.chunk.shape[0]) @property def counts(self) -> np.ndarray[Any, np.dtype[np.intp]]: """Points per row.""" return np.diff(self.pointer) _local: np.ndarray[Any, np.dtype[np.intp]] | None = field( default=None, init=False, repr=False, compare=False ) @property def local(self) -> np.ndarray[Any, np.dtype[np.intp]]: """Memoized, read-only chunk-local coordinates, shape ``(points, k)``.""" cached = self._local if cached is None: storage = np.stack( [ checked_affine(offset, stride, self.index[:, column]) for column, (offset, stride) in enumerate( zip(self.offsets, self.strides, strict=True) ) ], axis=1, ) cached = storage - np.repeat(self.chunk_start, self.counts, axis=0) cached = _own(cached) object.__setattr__(self, "_local", cached) return cached @property def block_coordinates(self) -> np.ndarray[Any, np.dtype[np.intp]]: """Memoized `positions` unravelled over `broadcast_shape`, shape ``(points, len(broadcast_axes))``.""" cached = self._block_coordinates if cached is None: if self.broadcast_shape: cached = np.stack(np.unravel_index(self.positions, self.broadcast_shape), axis=1) else: cached = np.empty((self.positions.size, 0), dtype=np.intp) cached = _own(np.asarray(cached, dtype=np.intp)) object.__setattr__(self, "_block_coordinates", cached) return cached def run(self, row: int) -> slice: """The slice of the point arrays a row owns.""" return slice(int(self.pointer[row]), int(self.pointer[row + 1])) def _strided_set( transform: IndexTransform, out_dim: int, m: ConstantMap | DimensionMap, dg: DimensionGridLike ) -> StridedSet: domain = transform.domain if isinstance(m, ConstantMap): c = dg.index_to_chunk(checked_affine(m.offset, 0, 0)) chunks = np.array([c], dtype=np.intp) starts, extents = _chunk_bounds(dg, chunks) local = np.array([m.offset - int(starts[0])], dtype=np.intp) return StridedSet( output_dimension=out_dim, input_dimension=None, stride=0, chunk=chunks, chunk_start=starts, chunk_extent=extents, local_start=local, extent=np.ones(1, dtype=np.intp), origin=np.zeros(1, dtype=np.intp), full=(extents == 1) & (local == 0), ) k = m.input_dimension lo = domain.inclusive_min[k] hi = domain.exclusive_max[k] stride = m.stride unit = abs(stride) == 1 rows: list[tuple[int, int, int, int, int, int, bool]] = [] # Exact Python-int arithmetic per touched chunk: the request's literal # coordinates can exceed np.intp before cancellation, and the number of # touched chunks along one axis is a sum, not a product. Chunk-local # columns fit np.intp; `extent` and `origin` scale with the request # domain, which has arbitrary Python-int bounds, so they keep exact ints # when they must (`_wide_column`). for (c,) in _dimension_map_candidates(m, lo, hi, dg): c_start = dg.chunk_offset(c) c_extent = _data_size(dg, c) narrowed = _intersect_dimension_map(m, lo, hi, c_start, c_start + c_extent) if narrowed is None: continue nlo, nhi = narrowed extent = nhi - nlo # Chunk-local, then re-based to a zero-origin input axis. local_start = m.offset - c_start + stride * nlo if unit or (extent == 1 and c_extent == 1): last = local_start + stride * (extent - 1) full = min(local_start, last) == 0 and max(local_start, last) == c_extent - 1 else: full = False rows.append((c, c_start, c_extent, local_start, extent, nlo - lo, full)) columns = list(zip(*rows, strict=True)) if rows else [()] * 7 return StridedSet( output_dimension=out_dim, input_dimension=k, stride=stride, chunk=np.array(columns[0], dtype=np.intp), chunk_start=np.array(columns[1], dtype=np.intp), chunk_extent=np.array(columns[2], dtype=np.intp), local_start=np.array(columns[3], dtype=np.intp), extent=_wide_column(columns[4]), origin=_wide_column(columns[5]), full=np.array(columns[6], dtype=np.bool_), ) def _indexed_set(out_dim: int, m: ArrayMap, dg: DimensionGridLike) -> IndexedSet: dependent = m.dependent_axis if dependent is None: # `_partition_transform` sends every map that varies over no axis down # the joint (general) path; an orthogonal map always has its one axis. raise RuntimeError( f"output dimension {out_dim} has an index array of shape " f"{m.index_array.shape} that varies over no input axis; it cannot be " "partitioned as an orthogonal map" ) flat = m.index_array.reshape(-1) n = int(flat.size) storage = checked_affine(m.offset, m.stride, flat) # Probe the extreme coordinates with the scalar lookup first: a grid's # scalar error names the offending coordinate, where the vectorized one # reports a range. dg.index_to_chunk(int(storage.min())) dg.index_to_chunk(int(storage.max())) chunk_ids = dg.indices_to_chunks(storage) # Already-sorted coordinates (the common case) need no sort. if bool((chunk_ids[1:] >= chunk_ids[:-1]).all()): positions = np.arange(n, dtype=np.intp) sorted_ids = chunk_ids index = flat else: positions = np.argsort(chunk_ids, kind="stable") sorted_ids = chunk_ids[positions] index = flat[positions] boundaries = np.flatnonzero(sorted_ids[1:] != sorted_ids[:-1]) + 1 pointer = np.concatenate([[0], boundaries, [n]]).astype(np.intp) chunks = np.asarray(sorted_ids[pointer[:-1]], dtype=np.intp) starts, extents = _chunk_bounds(dg, chunks) return IndexedSet( output_dimension=out_dim, input_dimension=dependent, offset=m.offset, stride=m.stride, chunk=chunks, chunk_start=starts, chunk_extent=extents, pointer=pointer, index=np.asarray(index, dtype=np.intp), positions=np.asarray(positions, dtype=np.intp), ) def _chunk_keys( chunk_ids: Sequence[np.ndarray[Any, np.dtype[np.intp]]], ) -> np.ndarray[Any, np.dtype[np.intp]]: """One sortable key per point whose order is the lexicographic chunk order.""" n = int(chunk_ids[0].size) if len(chunk_ids) == 1: return np.asarray(chunk_ids[0], dtype=np.intp) # Mixed-radix key with the first dimension most significant. Negative # chunk ids do not fit these zero-based digits; group their tuples directly. keys = np.zeros(n, dtype=np.intp) multiplier = 1 for ids in reversed(chunk_ids): radix = int(ids.max()) + 1 if int(ids.min()) < 0 or multiplier * radix >= 2**62: stacked = np.stack([np.asarray(i, dtype=np.intp).ravel() for i in chunk_ids], axis=1) _, inverse = np.unique(stacked, axis=0, return_inverse=True) return np.asarray(inverse, dtype=np.intp).reshape(-1) keys += np.asarray(ids, dtype=np.intp) * multiplier multiplier *= radix return keys def _joint_set( transform: IndexTransform, dim_grids: Sequence[DimensionGridLike], *, output_dimensions: tuple[int, ...] | None = None, input_dimensions: tuple[int, ...] | None = None, ) -> JointSet: block = _prepare_correlated( transform, output_dimensions=output_dimensions, input_dimensions=input_dimensions ) dims = block.correlated_dims # Probe the extreme coordinates with the scalar lookup first: a grid's # vectorized lookup need not check bounds (zarr's does not), and its # scalar error names the offending coordinate. for d in dims: storage = block.flat_storage[d] dim_grids[d].index_to_chunk(int(storage.min())) dim_grids[d].index_to_chunk(int(storage.max())) chunk_ids = [dim_grids[d].indices_to_chunks(block.flat_storage[d]) for d in dims] n_points = int(chunk_ids[0].size) keys = _chunk_keys(chunk_ids) positions = np.argsort(keys, kind="stable") sorted_keys = keys[positions] boundaries = np.flatnonzero(sorted_keys[1:] != sorted_keys[:-1]) + 1 pointer = np.concatenate([[0], boundaries, [n_points]]).astype(np.intp) first = positions[pointer[:-1]] chunk = np.stack([np.asarray(ids, dtype=np.intp)[first] for ids in chunk_ids], axis=1) index = np.stack([block.flat_index[d][positions] for d in dims], axis=1) starts = np.empty_like(chunk) extents = np.empty_like(chunk) for column, d in enumerate(dims): starts[:, column], extents[:, column] = _chunk_bounds(dim_grids[d], chunk[:, column]) maps = [cast("ArrayMap", transform.output[d]) for d in dims] return JointSet( output_dimensions=dims, offsets=tuple(m.offset for m in maps), strides=tuple(m.stride for m in maps), broadcast_axes=block.broadcast_axes, broadcast_shape=block.broadcast_shape, chunk=chunk, chunk_start=starts, chunk_extent=extents, pointer=pointer, index=index, positions=np.asarray(positions, dtype=np.intp), ) # One table row's contribution to a projection: (chunk index, chunk start, # chunk data extent, bound input axis or None, restricted extent of that axis, # the rebased chunk-local map, the cell map for the axis, whole-chunk cover). _AxisPiece = tuple[int, int, int, int | None, int, OutputIndexMap, OutputIndexMap | None, bool] def _strided_piece( axis: StridedSet, row: int, origin: int, input_dimension: int | None = None ) -> _AxisPiece: """One row of a strided table as maps; ``origin`` is the request axis's lower bound. ``input_dimension`` renumbers the chunk-local map's axis for the correlated walk, whose restricted domain collapses the broadcast axes into one leading axis. """ offset = int(axis.local_start[row]) k = axis.input_dimension chunk_map: OutputIndexMap cell_map: OutputIndexMap | None if k is None: chunk_map = ConstantMap(offset=offset) cell_map = None else: chunk_map = DimensionMap( input_dimension=k if input_dimension is None else input_dimension, offset=offset, stride=axis.stride, ) cell_map = DimensionMap(input_dimension=k, offset=origin + int(axis.origin[row])) return ( int(axis.chunk[row]), int(axis.chunk_start[row]), int(axis.chunk_extent[row]), k, int(axis.extent[row]), chunk_map, cell_map, bool(axis.full[row]), ) def _indexed_piece(axis: IndexedSet, row: int, rank: int, origin: int) -> _AxisPiece: run = axis.run(row) k = axis.input_dimension count = run.stop - run.start shape = (1,) * k + (count,) + (1,) * (rank - k - 1) c_start = int(axis.chunk_start[row]) return ( int(axis.chunk[row]), c_start, int(axis.chunk_extent[row]), k, count, ArrayMap( index_array=axis.index[run].reshape(shape), offset=axis.offset - c_start, stride=axis.stride, ), ArrayMap(index_array=axis.positions[run].reshape(shape), offset=origin), False, ) @dataclass(frozen=True, slots=True) class GridPartition: """A plan in factored form: per-axis tables whose product is the chunk walk. `sets` holds one `StridedSet` or `IndexedSet` per output dimension the transform reads independently, in output-dimension order; `joint_sets` holds the connected index-array components. A projection is one row of each table, so the partition has ``prod(row_shape)`` rows, walked in row-major order over `row_shape` (the component tables after `sets`). Rows are materialized into `ChunkProjection` objects only on request; a vectorized consumer can read the tables directly. Take one from `ChunkPlan.partition`. Examples -------- `arr[1:6:2, 5:]` on a `(7, 9)` array with `(3, 4)` chunks touches two chunks along each axis, so the partition has four rows: >>> from zarr_indexing import IndexTransform, plan_chunks >>> from zarr_indexing.grid import dimension_grids_from_chunks >>> grids = dimension_grids_from_chunks((3, 4), shape=(7, 9)) >>> partition = plan_chunks(IndexTransform.from_shape((7, 9))[1:6:2, 5:], grids).partition() >>> partition.row_shape, len(partition) ((2, 2), 4) >>> partition.chunk_coords().tolist() [[0, 1], [0, 2], [1, 1], [1, 2]] >>> [projection.chunk_transform.selection_repr for projection in partition][3] '{ [0, 4) step 2, [0, 1) }' """ transform: IndexTransform """The transform this partition factors.""" dimension_grids: tuple[DimensionGridLike, ...] """One grid per storage dimension.""" sets: tuple[StridedSet | IndexedSet, ...] """Independent per-axis tables, in output-dimension order.""" joint_sets: tuple[JointSet, ...] """Connected index-array components, ordered by their first output dimension.""" row_shape: tuple[int, ...] """Rows per table: `sets` followed by `joint_sets`, in row-major iteration order.""" def __len__(self) -> int: return math.prod(self.row_shape) def chunk_coords(self) -> np.ndarray[Any, np.dtype[np.intp]]: """Chunk coordinates of every row, shape ``(len(self), output rank)``, without materializing rows.""" count = len(self) out = np.empty((count, self.transform.output_rank), dtype=np.intp) if count == 0 or not self.row_shape: return out indices = np.unravel_index(np.arange(count, dtype=np.intp), self.row_shape) for axis, rows in zip(self.sets, indices, strict=False): out[:, axis.output_dimension] = axis.chunk[rows] for joint, rows in zip(self.joint_sets, indices[len(self.sets) :], strict=True): out[:, list(joint.output_dimensions)] = joint.chunk[rows] return out def __iter__(self) -> Iterator[ChunkProjection]: """Materialize every row in order.""" if math.prod(self.row_shape) == 0: return if not self.joint_sets: yield from self._iter_factorized() else: yield from self._iter_correlated() # -- factorized assembly ------------------------------------------------ def _pieces(self, axis: StridedSet | IndexedSet, row: int) -> _AxisPiece: domain = self.transform.domain if isinstance(axis, StridedSet): origin = ( 0 if axis.input_dimension is None else domain.inclusive_min[axis.input_dimension] ) return _strided_piece(axis, row, origin) return _indexed_piece(axis, row, domain.ndim, domain.inclusive_min[axis.input_dimension]) def _unbound(self) -> tuple[list[OutputIndexMap | None], bool]: """Cell maps for request axes no output reads, and whether they permit full coverage. A whole-chunk cover must biject onto the chunk, so an unread axis can only be a singleton. """ domain = self.transform.domain bound = {axis.input_dimension for axis in self.sets if axis.input_dimension is not None} cell_maps: list[OutputIndexMap | None] = [None] * domain.ndim unbound_ok = True for axis in range(domain.ndim): if axis not in bound: cell_maps[axis] = DimensionMap( input_dimension=axis, offset=domain.inclusive_min[axis] ) unbound_ok = unbound_ok and domain.shape[axis] <= 1 return cell_maps, unbound_ok def _iter_factorized(self) -> Iterator[ChunkProjection]: pieces_per_set = [ [self._pieces(axis, row) for row in range(len(axis))] for axis in self.sets ] base_cell_maps, unbound_ok = self._unbound() for combo in itertools.product(*pieces_per_set): yield self._factorized_projection(list(combo), base_cell_maps, unbound_ok) def _factorized_projection( self, pieces: list[_AxisPiece], base_cell_maps: list[OutputIndexMap | None], unbound_ok: bool, ) -> ChunkProjection: domain = self.transform.domain shape = list(domain.shape) cell_maps = list(base_cell_maps) chunk_maps: list[OutputIndexMap] = [] chunk_coords: list[int] = [] chunk_min: list[int] = [] chunk_max: list[int] = [] has_array = False full = unbound_ok for c, c_start, c_extent, k, extent, chunk_map, cell_map, piece_full in pieces: chunk_coords.append(c) chunk_min.append(c_start) chunk_max.append(c_start + c_extent) chunk_maps.append(chunk_map) if isinstance(chunk_map, ArrayMap): has_array = True if k is not None: shape[k] = extent cell_maps[k] = cell_map full = full and piece_full synthetic = IndexDomain._unchecked((0,) * domain.ndim, tuple(shape)) # pyright: ignore[reportPrivateUsage] if has_array: coverage: ChunkCoverage = "unknown" elif full: coverage = "full" else: coverage = "partial" return ChunkProjection( chunk_coords=tuple(chunk_coords), chunk_domain=IndexDomain._unchecked(tuple(chunk_min), tuple(chunk_max)), # pyright: ignore[reportPrivateUsage] chunk_transform=IndexTransform._unchecked(synthetic, tuple(chunk_maps)), # pyright: ignore[reportPrivateUsage] cell_transform=IndexTransform._unchecked( # pyright: ignore[reportPrivateUsage] synthetic, tuple(cast("list[OutputIndexMap]", cell_maps)) ), coverage=coverage, ) # -- correlated assembly ------------------------------------------------ def _slots(self) -> tuple[tuple[int | None, ...], dict[int, int]]: """One synthetic axis per nonconstant component, followed by residual axes.""" component_slots: list[int | None] = [] bound: set[int] = set() n_lead = 0 for joint in self.joint_sets: bound.update(joint.broadcast_axes) component_slots.append(n_lead if joint.broadcast_axes else None) n_lead += bool(joint.broadcast_axes) residual_axes = [axis for axis in range(self.transform.input_rank) if axis not in bound] return tuple(component_slots), { axis: n_lead + slot for slot, axis in enumerate(residual_axes) } def _iter_correlated(self) -> Iterator[ChunkProjection]: component_slots, slot_of = self._slots() lo_all = self.transform.domain.inclusive_min pieces_per_set = [ [ _strided_piece( cast("StridedSet", axis), row, 0 if axis.input_dimension is None else lo_all[axis.input_dimension], None if axis.input_dimension is None else slot_of[axis.input_dimension], ) for row in range(len(axis)) ] for axis in self.sets ] for residual in itertools.product(*pieces_per_set): for rows in itertools.product(*(range(len(joint)) for joint in self.joint_sets)): yield self._correlated_projection(list(residual), rows, component_slots, slot_of) def _correlated_projection( self, residual: list[_AxisPiece], rows: tuple[int, ...], component_slots: tuple[int | None, ...], slot_of: dict[int, int], ) -> ChunkProjection: transform = self.transform domain = transform.domain lo_all = domain.inclusive_min output_rank = transform.output_rank runs = tuple(joint.run(row) for joint, row in zip(self.joint_sets, rows, strict=True)) points_shape = tuple( run.stop - run.start for run, slot in zip(runs, component_slots, strict=True) if slot is not None ) n_lead = len(points_shape) rank = n_lead + len(slot_of) chunk_coords = [0] * output_rank chunk_min = [0] * output_rank chunk_max = [0] * output_rank chunk_maps: list[OutputIndexMap | None] = [None] * output_rank cell_maps: list[OutputIndexMap | None] = [None] * domain.ndim # Unread input axes are independent repetitions, not part of an index # array component. Retain their extent without expanding any arrays. extents = [domain.shape[axis] for axis in slot_of] for axis, slot in slot_of.items(): cell_maps[axis] = DimensionMap(input_dimension=slot, offset=lo_all[axis]) for out_dim, (c, c_start, c_extent, k, extent, chunk_map, cell_map, _full) in zip( (axis.output_dimension for axis in self.sets), residual, strict=True ): chunk_coords[out_dim] = c chunk_min[out_dim] = c_start chunk_max[out_dim] = c_start + c_extent chunk_maps[out_dim] = chunk_map if k is not None: slot = slot_of[k] extents[slot - n_lead] = extent cell_maps[k] = DimensionMap( input_dimension=slot, offset=cast("DimensionMap", cell_map).offset ) for joint, row, run, component_slot in zip( self.joint_sets, rows, runs, component_slots, strict=True ): component_shape = [1] * rank if component_slot is not None: component_shape[component_slot] = run.stop - run.start for column, out_dim in enumerate(joint.output_dimensions): c_start = int(joint.chunk_start[row, column]) chunk_coords[out_dim] = int(joint.chunk[row, column]) chunk_min[out_dim] = c_start chunk_max[out_dim] = c_start + int(joint.chunk_extent[row, column]) chunk_maps[out_dim] = ArrayMap( index_array=joint.index[run, column].reshape(component_shape), offset=joint.offsets[column] - c_start, stride=joint.strides[column], ) for column, axis in enumerate(joint.broadcast_axes): cell_maps[axis] = ArrayMap( index_array=joint.block_coordinates[run, column].reshape(component_shape), offset=lo_all[axis], ) synthetic = IndexDomain._unchecked((0,) * rank, points_shape + tuple(extents)) # pyright: ignore[reportPrivateUsage] return ChunkProjection( chunk_coords=tuple(chunk_coords), chunk_domain=IndexDomain._unchecked(tuple(chunk_min), tuple(chunk_max)), # pyright: ignore[reportPrivateUsage] chunk_transform=IndexTransform._unchecked( # pyright: ignore[reportPrivateUsage] synthetic, tuple(cast("list[OutputIndexMap]", chunk_maps)) ), cell_transform=IndexTransform._unchecked( # pyright: ignore[reportPrivateUsage] synthetic, tuple(cast("list[OutputIndexMap]", cell_maps)) ), coverage="unknown", ) def _component_joint_sets( transform: IndexTransform, grids: tuple[DimensionGridLike, ...] ) -> tuple[JointSet, ...]: """Partition the input/index-array dependency graph into connected components.""" affine_axes = {m.input_dimension for m in transform.output if isinstance(m, DimensionMap)} # Accumulated components have disjoint input axes. A new map can bridge # several of them; merging every overlap maintains that invariant. components: list[tuple[set[int], list[int]]] = [] for dimension, m in enumerate(transform.output): if not isinstance(m, ArrayMap): continue axes = set(m.dependency_axes) if axes.intersection(affine_axes): raise NotImplementedError( "index array varies over an input dimension also bound by a slice map" ) dimensions = [dimension] separate = [] for other_axes, other_dimensions in components: if axes.intersection(other_axes): axes.update(other_axes) dimensions.extend(other_dimensions) else: separate.append((other_axes, other_dimensions)) separate.append((axes, sorted(dimensions))) components = separate return tuple( _joint_set( transform, grids, output_dimensions=tuple(dimensions), input_dimensions=tuple(sorted(axes)), ) for axes, dimensions in sorted(components, key=lambda component: component[1][0]) ) def _shared_input_axis(transform: IndexTransform) -> int | None: """An input axis read by two output maps in a way that does not factor, if any. Two `DimensionMap`s on one axis, or a `DimensionMap` and an orthogonal `ArrayMap` on one axis, make a diagonal: neither restricts independently of the other. Correlated `ArrayMap`s sharing axes are the `JointSet` case and factor fine; an `ArrayMap` varying over a `DimensionMap`'s axis is rejected separately by `_prepare_correlated`. """ seen: set[int] = set() correlated = transform.index_array_structure == "general" for m in transform.output: if isinstance(m, DimensionMap): axis: int | None = m.input_dimension elif isinstance(m, ArrayMap) and not correlated: axis = m.dependent_axis else: continue if axis is None: continue if axis in seen: return axis seen.add(axis) return None def _partition_transform( transform: IndexTransform, grids: tuple[DimensionGridLike, ...] ) -> GridPartition: """Factor a transform over its grids; `ChunkPlan.partition` is the public entry.""" shared = _shared_input_axis(transform) if shared is not None: raise ValueError( f"two output maps read input axis {shared} (a diagonal, which no selection " "produces); it has no per-output-axis factored form" ) if any(size == 0 for size in transform.domain.shape): return GridPartition( transform=transform, dimension_grids=grids, sets=(), joint_sets=(), row_shape=(0,) ) correlated = transform.index_array_structure == "general" sets: list[StridedSet | IndexedSet] = [] for out_dim, m in enumerate(transform.output): if isinstance(m, ArrayMap): if correlated: continue sets.append(_indexed_set(out_dim, m, grids[out_dim])) else: sets.append(_strided_set(transform, out_dim, m, grids[out_dim])) joint_sets = _component_joint_sets(transform, grids) if correlated else () row_shape = tuple(len(axis) for axis in sets) + tuple(len(joint) for joint in joint_sets) return GridPartition( transform=transform, dimension_grids=grids, sets=tuple(sets), joint_sets=joint_sets, row_shape=row_shape, ) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/domain.py000066400000000000000000000334021527014137600313010ustar00rootroot00000000000000"""Index domains — rectangular regions in N-dimensional integer space. An `IndexDomain` represents the set of valid coordinates for an array or array view. It is the cartesian product of per-dimension integer ranges:: IndexDomain(inclusive_min=(2, 5), exclusive_max=(10, 20)) # represents {(i, j) : 2 <= i < 10, 5 <= j < 20} Unlike NumPy, domains can have **non-zero origins**. After slicing `arr[5:10]`, the result has origin 5 and shape 5 — coordinates 5 through 9 are valid. This follows the TensorStore convention. """ from __future__ import annotations from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any from zarr_indexing.errors import BoundsCheckError if TYPE_CHECKING: from zarr_indexing.json import IndexDomainJSON @dataclass(frozen=True, slots=True) class IndexDomain: """A rectangular region in N-dimensional index space. The valid coordinates are the integers in `[inclusive_min[d], exclusive_max[d])` for each dimension `d`. Examples -------- >>> domain = IndexDomain(inclusive_min=(2, 5), exclusive_max=(10, 20)) >>> domain.shape (8, 15) Unlike a NumPy shape, a domain keeps literal coordinates: narrowing to `[5, 10)` gives a region whose valid coordinates are 5 through 9, not re-zeroed: >>> view = IndexDomain.from_shape((10,)).narrow(slice(5, 10)) >>> view.origin, view.shape ((5,), (5,)) >>> view.contains((5,)), view.contains((0,)) (True, False) """ inclusive_min: tuple[int, ...] """The lower corner: each dimension's smallest literal coordinate. May be negative.""" exclusive_max: tuple[int, ...] """Each dimension's upper bound, excluded: valid coordinates end at `exclusive_max - 1`.""" labels: tuple[str, ...] | None = None """Optional per-dimension names; carried through the wire format, never consulted by indexing.""" # Lazily-memoized shape. Excluded from init/repr/eq/hash: it is derived # state, not part of the domain's identity. The domain is frozen, so the # value is computed at most once (see `shape`). `None` is the unset # sentinel; an empty shape caches as `()`. _shape: tuple[int, ...] | None = field(default=None, init=False, repr=False, compare=False) def __post_init__(self) -> None: if len(self.inclusive_min) != len(self.exclusive_max): raise ValueError( f"inclusive_min and exclusive_max must have the same length. " f"Got {len(self.inclusive_min)} and {len(self.exclusive_max)}." ) for i, (lo, hi) in enumerate(zip(self.inclusive_min, self.exclusive_max, strict=True)): if lo > hi: raise ValueError( f"inclusive_min must be <= exclusive_max for all dimensions. " f"Dimension {i}: {lo} > {hi}" ) if self.labels is not None and len(self.labels) != len(self.inclusive_min): raise ValueError( f"labels must have the same length as dimensions. " f"Got {len(self.labels)} labels for {len(self.inclusive_min)} dimensions." ) @classmethod def _unchecked( cls, inclusive_min: tuple[int, ...], exclusive_max: tuple[int, ...] ) -> IndexDomain: """Build an unlabeled domain from validated bounds. Skip __post_init__; internal callers must ensure equal bound lengths and inclusive_min <= exclusive_max in every dimension.""" domain = object.__new__(cls) object.__setattr__(domain, "inclusive_min", inclusive_min) object.__setattr__(domain, "exclusive_max", exclusive_max) object.__setattr__(domain, "labels", None) object.__setattr__(domain, "_shape", None) return domain @classmethod def from_shape(cls, shape: tuple[int, ...]) -> IndexDomain: """Create a domain with origin at zero.""" return cls( inclusive_min=(0,) * len(shape), exclusive_max=shape, ) @property def ndim(self) -> int: """Number of dimensions.""" return len(self.inclusive_min) @property def origin(self) -> tuple[int, ...]: """The lower corner of the domain — an alias for `inclusive_min`, and may be negative.""" return self.inclusive_min @property def shape(self) -> tuple[int, ...]: """Per-dimension extents: `exclusive_max - inclusive_min` for each dimension.""" cached = self._shape if cached is None: cached = tuple( hi - lo for lo, hi in zip(self.inclusive_min, self.exclusive_max, strict=True) ) object.__setattr__(self, "_shape", cached) return cached def contains(self, index: tuple[int, ...]) -> bool: """Whether the literal coordinate `index` lies inside this domain. Coordinates are literal, not NumPy-style offsets: a negative value is the coordinate itself, valid only if the domain's bounds include it. A tuple of the wrong length is simply not contained (returns `False`). """ if len(index) != self.ndim: return False return all( lo <= idx < hi for lo, hi, idx in zip(self.inclusive_min, self.exclusive_max, index, strict=True) ) def contains_domain(self, other: IndexDomain) -> bool: """Whether `other` has the same rank and bounds enclosed by this domain. Empty domains are still checked by their bounds: an empty `other` located outside this domain returns `False`, unlike empty-set containment. A rank mismatch returns `False` rather than raising. """ if other.ndim != self.ndim: return False return all( self_lo <= other_lo and other_hi <= self_hi for self_lo, self_hi, other_lo, other_hi in zip( self.inclusive_min, self.exclusive_max, other.inclusive_min, other.exclusive_max, strict=True, ) ) def intersect(self, other: IndexDomain) -> IndexDomain | None: """Return the overlap of this domain with `other`, or `None` if they are disjoint. Raises ------ ValueError If the two domains have different ranks. """ if other.ndim != self.ndim: raise ValueError( f"Cannot intersect domains with different ranks: {self.ndim} vs {other.ndim}" ) new_min = tuple( max(a, b) for a, b in zip(self.inclusive_min, other.inclusive_min, strict=True) ) new_max = tuple( min(a, b) for a, b in zip(self.exclusive_max, other.exclusive_max, strict=True) ) if any(lo >= hi for lo, hi in zip(new_min, new_max, strict=True)): return None return IndexDomain(inclusive_min=new_min, exclusive_max=new_max) def translate(self, offset: tuple[int, ...]) -> IndexDomain: """Return this domain shifted by `offset` per dimension; the shape is unchanged. Offsets may be negative, and the result may have a negative origin. Raises ------ ValueError If `offset` does not have one entry per dimension. """ if len(offset) != self.ndim: raise ValueError( f"Offset must have same length as domain dimensions. " f"Domain has {self.ndim} dimensions, offset has {len(offset)}." ) new_min = tuple(lo + off for lo, off in zip(self.inclusive_min, offset, strict=True)) new_max = tuple(hi + off for hi, off in zip(self.exclusive_max, offset, strict=True)) return IndexDomain(inclusive_min=new_min, exclusive_max=new_max) def narrow(self, selection: Any) -> IndexDomain: """Apply a basic selection and return a narrowed domain. Indices are absolute coordinates, not NumPy-style offsets: `-3` names the coordinate `-3`, and is out of bounds unless the domain contains it. Integer indices produce a length-1 extent. Strided slices are not supported — use `IndexTransform` for strides. Raises ------ BoundsCheckError If an integer index or explicit slice bound lies outside this domain. Bounds are checked as literal coordinates, without clamping or negative-index wrapping. """ normalized = _normalize_selection(selection, self.ndim) new_inclusive_min: list[int] = [] new_exclusive_max: list[int] = [] for dim_idx, (sel, dim_lo, dim_hi) in enumerate( zip(normalized, self.inclusive_min, self.exclusive_max, strict=True) ): if isinstance(sel, int): if sel < dim_lo or sel >= dim_hi: raise BoundsCheckError( f"index {sel} is out of bounds for dimension {dim_idx} " f"with domain [{dim_lo}, {dim_hi})" ) new_inclusive_min.append(sel) new_exclusive_max.append(sel + 1) else: start, stop, step = sel.start, sel.stop, sel.step if step is not None and step != 1: raise ValueError( "IndexDomain.narrow only supports step=1 slices. " f"Got step={step}. Use IndexTransform for strided access." ) abs_start = dim_lo if start is None else start abs_stop = dim_hi if stop is None else stop for bound, name in ((abs_start, "start"), (abs_stop, "stop")): if bound < dim_lo or bound > dim_hi: raise BoundsCheckError( f"slice {name} {bound} is out of bounds for dimension " f"{dim_idx} with domain [{dim_lo}, {dim_hi}); indices " f"here are absolute coordinates, so they are neither " f"clamped to the domain nor counted from its end" ) # An empty interval is legal; a reversed one is the same request # spelled backwards, and reads as empty rather than as an error. new_inclusive_min.append(abs_start) new_exclusive_max.append(max(abs_stop, abs_start)) return IndexDomain( inclusive_min=tuple(new_inclusive_min), exclusive_max=tuple(new_exclusive_max), ) # -- serialization ------------------------------------------------------ def to_json(self) -> IndexDomainJSON: """Convert to the canonical ndsel JSON representation. Examples -------- >>> IndexDomain(inclusive_min=(0,), exclusive_max=(3,)).to_json() {'input_inclusive_min': [0], 'input_exclusive_max': [3], 'input_labels': ['']} """ from zarr_indexing._wire import emit_labels return { "input_inclusive_min": list(self.inclusive_min), "input_exclusive_max": list(self.exclusive_max), "input_labels": emit_labels(self.labels, self.ndim), } @classmethod def from_json(cls, data: IndexDomainJSON) -> IndexDomain: """Construct from the canonical ndsel JSON representation. The document is validated by the message layer first, exactly as a transform body is. Reading the keys directly would be a second, undefended way into the same objects: `int(value)` alone accepts `3.9`, `"3"` and `True`, and each of those builds a domain that is not the document's. Examples -------- >>> domain = IndexDomain.from_json( ... {"input_inclusive_min": [1], "input_exclusive_max": [4], "input_labels": [""]} ... ) >>> (domain.inclusive_min, domain.exclusive_max, domain.shape) ((1,), (4,), (3,)) >>> IndexDomain.from_json(domain.to_json()) == domain True """ from zarr_indexing._wire import lower_bound, lower_labels from zarr_indexing.messages import NdselError, normalize_ndsel # The annotation says what a well-formed caller passes; this is a parser # of documents that arrive from elsewhere, so the shape is checked # rather than assumed. if not isinstance(data, dict): # pyright: ignore[reportUnnecessaryIsInstance] raise NdselError("invalid_json", f"an index domain must be a JSON object, got {data!r}") body = normalize_ndsel({**data, "kind": "transform"}) return cls( inclusive_min=tuple( lower_bound(b, f"input_inclusive_min[{i}]") for i, b in enumerate(body["input_inclusive_min"]) ), exclusive_max=tuple( lower_bound(b, f"input_exclusive_max[{i}]") for i, b in enumerate(body["input_exclusive_max"]) ), labels=lower_labels(body["input_labels"]), ) def _normalize_selection(selection: Any, ndim: int) -> tuple[int | slice, ...]: """Normalize a basic selection to a tuple of ints/slices with length ndim.""" if not isinstance(selection, tuple): selection = (selection,) result: list[int | slice] = [] ellipsis_seen = False for sel in selection: if sel is Ellipsis: if ellipsis_seen: raise IndexError("an index can only have a single ellipsis ('...')") ellipsis_seen = True num_missing = ndim - (len(selection) - 1) result.extend([slice(None)] * num_missing) else: result.append(sel) while len(result) < ndim: result.append(slice(None)) if len(result) > ndim: raise IndexError( f"too many indices for array: array has {ndim} dimensions, " f"but {len(result)} were indexed" ) return tuple(result) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/eager.py000066400000000000000000000034261527014137600311200ustar00rootroot00000000000000"""Explicit eager indexing for consumers such as Dask's ``from_array``.""" from __future__ import annotations from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from zarr_indexing.lazy_array import LazyArray class EagerArrayAdapter: """Expose an eager array interface over a lazy view. Constructing the adapter does not read source values. Indexing materializes the selected view in fresh memory, using its existing reader and partitioning. Tokenization delegates to the view's source-token contract and may read data. Examples -------- >>> import numpy as np >>> from zarr_indexing import LazyArray >>> view = LazyArray(np.arange(8))[1::2] >>> EagerArrayAdapter(view)[1:3] array([3, 5]) """ __slots__ = ("_view",) def __init__(self, view: LazyArray) -> None: self._view = view @property def shape(self) -> tuple[int, ...]: """Shape of the selected view.""" return self._view.shape @property def ndim(self) -> int: """Number of dimensions of the selected view.""" return self._view.ndim @property def dtype(self) -> Any: """Data type reported by the source.""" return self._view.dtype def __getitem__(self, selection: Any) -> Any: """Read a basic selection into fresh memory.""" return self._view[selection].result() def __array__(self, dtype: Any = None, copy: bool | None = None) -> Any: """Materialize using the view's NumPy conversion contract.""" return self._view.__array__(dtype=dtype, copy=copy) def __dask_tokenize__(self) -> tuple[Any, ...]: """Distinguish eager tasks while preserving the source identity contract.""" return (type(self).__qualname__, self._view.__dask_tokenize__()) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/errors.py000066400000000000000000000045161527014137600313520ustar00rootroot00000000000000"""Canonical index-error types raised by the transform algebra. Both subclass the built-in `IndexError`, so an `except IndexError` catch site keeps working unchanged whichever library raised. `zarr.errors` defines classes of the same names, and they are *not* these objects: `zarr.errors.BoundsCheckError is BoundsCheckError` is false. Catching zarr's class does not catch this package's errors. Import these from here, or catch their shared built-in `IndexError` base to handle both libraries. """ from __future__ import annotations __all__ = [ "BoundsCheckError", "VindexInvalidSelectionError", ] class VindexInvalidSelectionError(IndexError): """A wrapper `vindex` selection contained a slice. Raised by `LazyArray`'s selection validation: the wrapper's vectorized dialect accepts coordinate selections (integer arrays, with scalars and an ellipsis) or a single boolean mask, and rejects slices with this error. Other invalid entries raise plain `IndexError`, and the engine-level `IndexTransform.vindex` is wider — it accepts residual slice dimensions without raising. Examples -------- Raised by the wrapper, not the engine — a slice inside `vindex`: >>> import numpy as np >>> from zarr_indexing import LazyArray >>> view = LazyArray.from_numpy(np.arange(12).reshape(3, 4)) >>> view.vindex[np.array([0, 2]), :] Traceback (most recent call last): ... zarr_indexing.errors.VindexInvalidSelectionError: ... """ class BoundsCheckError(IndexError): """A selection addressed coordinates outside the domain being indexed. Raised for out-of-domain integer indices, slice bounds, index-array values, and points passed to `IndexTransform.apply`. Coordinates in this algebra are literal: they are never clamped, and a negative value below the domain's `inclusive_min` is out of bounds rather than counted from the end. Examples -------- >>> from zarr_indexing import IndexTransform >>> IndexTransform.from_shape((8,)).apply((9,)) Traceback (most recent call last): ... zarr_indexing.errors.BoundsCheckError: ... A negative index is a literal coordinate, not "from the end": >>> IndexTransform.from_shape((8,))[-1] Traceback (most recent call last): ... zarr_indexing.errors.BoundsCheckError: ... """ zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/grid.py000066400000000000000000000761601527014137600307670ustar00rootroot00000000000000"""Compact chunk grids and the narrow planner protocol. ``DimensionGridLike`` describes only the per-axis operations required by ``plan_chunks``. The concrete compact grids below also retain enough metadata to describe chunk data regions and codec buffer regions without importing Zarr's array implementation. """ from __future__ import annotations import bisect import itertools import operator from dataclasses import dataclass, field from functools import reduce from typing import TYPE_CHECKING, Any, Protocol, cast, runtime_checkable import numpy as np if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Sequence import numpy.typing as npt class DimensionGridLike(Protocol): """The per-dimension chunk-mapping surface consumed by chunk resolution. Examples -------- `EdgeDimensionGrid` provides this surface. Chunk sizes `(2, 3)` tile source coordinates `[0, 5)`, so index 4 lands in the second chunk: >>> grid = EdgeDimensionGrid([2, 3]) >>> grid.index_to_chunk(4) 1 >>> grid.chunk_offset(1), grid.chunk_size(1) (2, 3) """ def index_to_chunk(self, idx: int) -> int: """Map a global source index to the index of the chunk that contains it. Raise `IndexError` outside the coordinates supported by the grid. The built-in bounded grids use `[0, extent)`; custom grids may support negative coordinates or an unbounded region. """ ... def chunk_offset(self, chunk_ix: int) -> int: """The global source coordinate at which chunk `chunk_ix` begins.""" ... def chunk_size(self, chunk_ix: int) -> int: """The declared length of chunk `chunk_ix`, i.e. its codec buffer size along this axis.""" ... def indices_to_chunks(self, indices: npt.NDArray[np.intp]) -> npt.NDArray[np.intp]: """Vectorized `index_to_chunk`: map global source indices to chunk indices. Raise `IndexError` if any index is outside the grid's supported region, consistently with the scalar method. """ ... def _bounded_indices(indices: npt.NDArray[np.intp], extent: int) -> npt.NDArray[np.intp]: """Normalize a vector lookup and enforce the scalar grid bounds.""" arr = np.asarray(indices, dtype=np.intp) if arr.size > 0 and (int(arr.min()) < 0 or int(arr.max()) >= extent): raise IndexError( f"indices must lie in [0, {extent}); got [{int(arr.min())}, {int(arr.max())}]" ) return arr @dataclass(frozen=True) class FixedDimension: """Uniform chunk size with a boundary chunk clipped to the axis extent. Examples -------- Chunks of size 3 on an axis of extent 10 give 4 chunks. The last chunk still declares a codec buffer of 3 but holds only 1 valid element: >>> dim = FixedDimension(size=3, extent=10) >>> dim.nchunks 4 >>> dim.index_to_chunk(7) 2 >>> dim.chunk_size(3), dim.data_size(3) (3, 1) """ size: int """The declared chunk length along this axis; every chunk's codec buffer size.""" extent: int """The axis length in global source coordinates.""" nchunks: int = field(init=False, repr=False) """Derived: the number of chunks holding data within `extent`.""" ngridcells: int = field(init=False, repr=False) """Derived: the number of declared grid cells; equals `nchunks` for a fixed dimension.""" def __post_init__(self) -> None: if self.size < 0: raise ValueError(f"FixedDimension size must be >= 0, got {self.size}") if self.extent < 0: raise ValueError(f"FixedDimension extent must be >= 0, got {self.extent}") if self.size == 0 and self.extent > 0: raise ValueError( "FixedDimension size must be > 0 when extent is nonzero; " f"got size {self.size} and extent {self.extent}" ) nchunks = 0 if self.size == 0 else (self.extent + self.size - 1) // self.size object.__setattr__(self, "nchunks", nchunks) object.__setattr__(self, "ngridcells", nchunks) def index_to_chunk(self, idx: int) -> int: """Map a global source index to its chunk index (`idx // size`). Raises `IndexError` when `idx` lies outside `[0, extent)`. """ if idx < 0 or idx >= self.extent: raise IndexError(f"index {idx} is out of bounds for extent {self.extent}") return 0 if self.size == 0 else idx // self.size def chunk_offset(self, chunk_ix: int) -> int: """The global source coordinate where chunk `chunk_ix` begins (`chunk_ix * size`). Not bounds-checked: chunk indices past the last chunk extrapolate linearly. """ return chunk_ix * self.size def chunk_size(self, chunk_ix: int) -> int: """The declared chunk length, `size` for every chunk. The boundary chunk is not clipped here; use `data_size` for the valid data length. """ return self.size def data_size(self, chunk_ix: int) -> int: """The number of valid data elements in chunk `chunk_ix`, clipped to `extent`. Interior chunks report `size`; the boundary chunk reports the remainder, and chunk indices at or past `nchunks` report 0. """ if self.size == 0: return 0 return max(0, min(self.size, self.extent - chunk_ix * self.size)) def indices_to_chunks(self, indices: npt.NDArray[np.intp]) -> npt.NDArray[np.intp]: """Vectorized `index_to_chunk` over an array of global source indices. Raises `IndexError` if any index lies outside `[0, extent)`. """ arr = _bounded_indices(indices, self.extent) if self.size == 0: return np.zeros_like(arr) return arr // self.size def with_extent(self, new_extent: int) -> FixedDimension: """Return a copy with the same chunk size and the axis extent set to `new_extent`.""" return FixedDimension(size=self.size, extent=new_extent) def resize(self, new_extent: int) -> FixedDimension: """Return a copy resized to `new_extent`, retaining the chunk size. A zero chunk size is supported only for an empty extent and cannot grow. """ return FixedDimension(size=self.size, extent=new_extent) @property def size_repr(self) -> str: """The chunk size rendered as a scalar for `ChunkGrid.__repr__`.""" return str(self.size) @dataclass(frozen=True, init=False) class VaryingDimension: """Explicit chunk edge lengths, with trailing data clipped to ``extent``. Examples -------- Edges `(2, 3, 5)` clipped to extent 9: the last chunk declares 5 but holds only 4 valid elements, and index 4 lands in the second chunk: >>> dim = VaryingDimension(edges=(2, 3, 5), extent=9) >>> dim.nchunks 3 >>> dim.index_to_chunk(4) 1 >>> dim.chunk_offset(2) 5 >>> dim.chunk_size(2), dim.data_size(2) (5, 4) """ edges: tuple[int, ...] """The declared per-chunk edge lengths, in order; codec buffer sizes, unclipped.""" cumulative: tuple[int, ...] """Prefix sums of `edges`; derived, and what index lookups binary-search.""" extent: int """The axis length in global source coordinates; at most the sum of `edges`.""" nchunks: int = field(init=False, repr=False) """Derived: the number of chunks holding data within `extent`.""" ngridcells: int = field(init=False, repr=False) """Derived: the number of declared edges; exceeds `nchunks` when trailing cells are empty.""" def __init__(self, edges: Sequence[int], extent: int) -> None: edges_tuple = tuple(edges) if not edges_tuple: raise ValueError("VaryingDimension edges must not be empty") if any(edge <= 0 for edge in edges_tuple): raise ValueError(f"All edge lengths must be > 0, got {edges_tuple}") cumulative = tuple(itertools.accumulate(edges_tuple)) if extent < 0: raise ValueError(f"VaryingDimension extent must be >= 0, got {extent}") if extent > cumulative[-1]: raise ValueError( f"VaryingDimension extent {extent} exceeds sum of edges {cumulative[-1]}" ) object.__setattr__(self, "edges", edges_tuple) object.__setattr__(self, "cumulative", cumulative) object.__setattr__(self, "extent", extent) nchunks = 0 if extent == 0 else bisect.bisect_left(cumulative, extent) + 1 object.__setattr__(self, "nchunks", nchunks) object.__setattr__(self, "ngridcells", len(edges_tuple)) def index_to_chunk(self, idx: int) -> int: """Map a global source index to the chunk whose edge interval contains it. Raises `IndexError` when `idx` lies outside `[0, extent)`. """ if idx < 0 or idx >= self.extent: raise IndexError(f"index {idx} is out of bounds for extent {self.extent}") return bisect.bisect_right(self.cumulative, idx) def chunk_offset(self, chunk_ix: int) -> int: """The global source coordinate where chunk `chunk_ix` begins (sum of prior edges).""" return self.cumulative[chunk_ix - 1] if chunk_ix > 0 else 0 def chunk_size(self, chunk_ix: int) -> int: """The declared edge length of chunk `chunk_ix`. Trailing chunks are not clipped to `extent` here; use `data_size` for that. """ return self.edges[chunk_ix] def data_size(self, chunk_ix: int) -> int: """The number of valid data elements in chunk `chunk_ix`, clipped to `extent`. Grid cells that lie entirely at or past `extent` report 0. """ offset = self.chunk_offset(chunk_ix) return max(0, min(self.edges[chunk_ix], self.extent - offset)) def indices_to_chunks(self, indices: npt.NDArray[np.intp]) -> npt.NDArray[np.intp]: """Vectorized `index_to_chunk` over an array of global source indices. Raises `IndexError` if any index lies outside `[0, extent)`. """ arr = _bounded_indices(indices, self.extent) return np.searchsorted(self.cumulative, arr, side="right") def with_extent(self, new_extent: int) -> VaryingDimension: """Return a copy with the same edges re-clipped to `new_extent`. The existing edges must already cover the new extent; raises `ValueError` when `new_extent` exceeds the sum of edges. Use `resize` to grow past the edges. """ if self.cumulative[-1] < new_extent: raise ValueError( f"VaryingDimension edge sum {self.cumulative[-1]} is less than new extent " f"{new_extent}" ) return VaryingDimension(self.edges, extent=new_extent) def resize(self, new_extent: int) -> VaryingDimension: """Return a copy resized to `new_extent`. Shrinking (or growing within the existing edges) keeps the edges and re-clips them; growing past the sum of edges appends one new trailing edge covering the remainder. """ if new_extent == self.extent: return self if new_extent > self.cumulative[-1]: return VaryingDimension((*self.edges, new_extent - self.cumulative[-1]), new_extent) return VaryingDimension(self.edges, extent=new_extent) @property def size_repr(self) -> str: """The edge lengths rendered as a tuple for `ChunkGrid.__repr__`.""" return repr(self.edges) @runtime_checkable class DimensionGrid(Protocol): """Structural interface shared by the compact dimension grids. Examples -------- `FixedDimension` satisfies the protocol structurally: >>> dim = FixedDimension(size=2, extent=5) >>> isinstance(dim, DimensionGrid) True >>> dim.nchunks, dim.extent (3, 5) >>> dim.with_extent(4).nchunks 2 """ @property def nchunks(self) -> int: """The number of chunks holding data within `extent`.""" ... @property def ngridcells(self) -> int: """The number of declared grid cells; may exceed `nchunks` when trailing cells are empty.""" ... @property def extent(self) -> int: """The axis length in global source coordinates.""" ... def index_to_chunk(self, idx: int) -> int: """Map a global source index to the chunk index that contains it. Implementers must raise `IndexError` when `idx` lies outside `[0, extent)`. """ ... def chunk_offset(self, chunk_ix: int) -> int: """The global source coordinate at which chunk `chunk_ix` begins.""" ... def chunk_size(self, chunk_ix: int) -> int: """The declared (codec buffer) length of chunk `chunk_ix`, never clipped to `extent`.""" ... def data_size(self, chunk_ix: int) -> int: """The valid data length of chunk `chunk_ix`, clipped to `extent` at the boundary.""" ... def indices_to_chunks(self, indices: npt.NDArray[np.intp]) -> npt.NDArray[np.intp]: """Vectorized `index_to_chunk`; must raise `IndexError` for indices outside `[0, extent)`.""" ... def with_extent(self, new_extent: int) -> DimensionGrid: """Return a grid with the existing chunk layout re-clipped to `new_extent`. Preserve the existing layout rule: a fixed positive chunk size can cover a larger extent without changing its size; explicit edge lists raise `ValueError` if their sum cannot cover `new_extent`. """ ... def resize(self, new_extent: int) -> DimensionGrid: """Return a grid covering `new_extent`, extending the chunk layout when it must grow.""" ... @property def size_repr(self) -> str: """A compact rendering of the chunk sizes, used by `ChunkGrid.__repr__`.""" ... @dataclass(frozen=True) class ChunkSpec: """A chunk's valid data region and its full codec buffer shape. Examples -------- The last chunk of a size-10 axis chunked by 3 holds one valid element (`slices`), while its codec buffer still spans 3: >>> spec = ChunkGrid.from_sizes((10,), (3,))[3] >>> spec.slices (slice(9, 10, 1),) >>> spec.shape, spec.codec_shape ((1,), (3,)) >>> spec.is_boundary True """ slices: tuple[slice, ...] """Per-dimension bounds of the valid data region, in global source coordinates.""" codec_shape: tuple[int, ...] """The declared (codec buffer) chunk shape, unclipped by the array extent.""" @property def shape(self) -> tuple[int, ...]: """The shape of the valid data region described by `slices`. Smaller than `codec_shape` on boundary chunks, where the array extent clips the chunk. """ return tuple(chunk_slice.stop - chunk_slice.start for chunk_slice in self.slices) @property def is_boundary(self) -> bool: """Whether the valid data region is smaller than the full codec buffer on any axis.""" return self.shape != self.codec_shape @dataclass(frozen=True) class ChunkGrid: """A concrete regular or rectilinear arrangement of chunks for one array. Examples -------- A `(3, 4)` array with `(2, 2)` chunks has a `(2, 2)` grid whose bottom row of chunks is clipped to one valid row of data: >>> grid = ChunkGrid.from_sizes((3, 4), (2, 2)) >>> grid.grid_shape (2, 2) >>> grid.chunk_sizes ((2, 1), (2, 2)) >>> spec = grid[1, 0] >>> spec.shape, spec.codec_shape, spec.is_boundary ((1, 2), (2, 2), True) """ dimensions: tuple[DimensionGrid, ...] """One per-axis grid, each mapping that axis's source indices to chunks.""" _is_regular: bool = field(init=False, repr=False) def __post_init__(self) -> None: object.__setattr__( self, "_is_regular", all(isinstance(dimension, FixedDimension) for dimension in self.dimensions), ) def __repr__(self) -> str: sizes = ", ".join(dimension.size_repr for dimension in self.dimensions) shape = tuple(dimension.extent for dimension in self.dimensions) return f"ChunkGrid(chunk_sizes=({sizes}), array_shape={shape})" @classmethod def from_sizes( cls, array_shape: Sequence[int], chunk_sizes: Sequence[int | Sequence[int]] ) -> ChunkGrid: """Build a grid from an array shape and one chunk-size spec per dimension. An `int` entry gives a fixed chunk size along that axis; a sequence of ints gives explicit per-chunk edge lengths. A uniform sequence consistent with the axis extent collapses to a fixed dimension, so the result may report `is_regular`. Parameters ---------- array_shape : Sequence[int] The array extent along each dimension, in global source coordinates. chunk_sizes : Sequence[int | Sequence[int]] Per-dimension chunk layout: a single size or explicit edge lengths. """ extents = _shape_tuple(array_shape) if len(extents) != len(chunk_sizes): raise ValueError( f"array_shape has {len(extents)} dimensions but chunk_sizes has " f"{len(chunk_sizes)} dimensions" ) dimensions: list[DimensionGrid] = [] for dimension_spec, extent in zip(chunk_sizes, extents, strict=True): if isinstance(dimension_spec, int): dimensions.append(FixedDimension(size=dimension_spec, extent=extent)) else: edges = tuple(dimension_spec) if not edges: raise ValueError("Each dimension must have at least one chunk") if ( edges[0] > 0 and all(edge == edges[0] for edge in edges) and (extent == sum(edges) or len(edges) == (extent + edges[0] - 1) // edges[0]) ): dimensions.append(FixedDimension(size=edges[0], extent=extent)) else: dimensions.append(VaryingDimension(edges, extent=extent)) return cls(dimensions=tuple(dimensions)) @property def ndim(self) -> int: """The number of dimensions.""" return len(self.dimensions) @property def is_regular(self) -> bool: """Whether every dimension uses a single fixed chunk size. False when any axis carries explicit (rectilinear) per-chunk edge lengths. """ return self._is_regular @property def grid_shape(self) -> tuple[int, ...]: """The number of data-bearing chunks along each dimension.""" return tuple(dimension.nchunks for dimension in self.dimensions) @property def chunk_shape(self) -> tuple[int, ...]: """The uniform declared chunk shape of a regular grid. Raises `ValueError` for rectilinear grids, which have no single chunk shape; use `grid[coords]` for per-chunk sizes instead. """ if not self.is_regular: raise ValueError( "chunk_shape is only available for regular chunk grids. " "Use grid[coords] for per-chunk sizes." ) return tuple( dimension.size for dimension in self.dimensions if isinstance(dimension, FixedDimension) ) @property def chunk_sizes(self) -> tuple[tuple[int, ...], ...]: """Per-dimension tuples of each chunk's valid data length. Boundary chunks report their clipped extent, not the declared codec size. """ return tuple( tuple(dimension.data_size(index) for index in range(dimension.nchunks)) for dimension in self.dimensions ) def __getitem__(self, coords: int | tuple[int, ...]) -> ChunkSpec | None: """Look up the `ChunkSpec` at the given chunk coordinates (grid cells, not indices). Returns `None` when any coordinate falls outside the grid; raises `ValueError` when the number of coordinates does not match `ndim`. The spec's slices are in global source coordinates. """ if isinstance(coords, int): coords = (coords,) if len(coords) != self.ndim: raise ValueError( f"Expected {self.ndim} coordinate(s) for a {self.ndim}-d chunk grid, " f"got {len(coords)}." ) slices: list[slice] = [] codec_shape: list[int] = [] for dimension, index in zip(self.dimensions, coords, strict=True): if index < 0 or index >= dimension.nchunks: return None offset = dimension.chunk_offset(index) slices.append(slice(offset, offset + dimension.data_size(index), 1)) codec_shape.append(dimension.chunk_size(index)) return ChunkSpec(tuple(slices), tuple(codec_shape)) def __iter__(self) -> Iterator[ChunkSpec]: """Yield a `ChunkSpec` for every data-bearing chunk in row-major (C) order.""" for coords in itertools.product( *(range(dimension.nchunks) for dimension in self.dimensions) ): spec = self[coords] if spec is not None: yield spec def all_chunk_coords( self, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None, ) -> Iterator[tuple[int, ...]]: """Iterate chunk coordinates over a rectangular grid region in row-major (C) order. `origin` defaults to the grid origin and `selection_shape` to the rest of the grid. The region is not bounds-checked: an oversized region yields coordinates outside the grid, which `__getitem__` resolves to `None`. """ origin_parsed = (0,) * self.ndim if origin is None else tuple(origin) selection_shape_parsed = ( tuple( grid_size - coordinate for coordinate, grid_size in zip(origin_parsed, self.grid_shape, strict=True) ) if selection_shape is None else tuple(selection_shape) ) return itertools.product( *( range(coordinate, coordinate + size) for coordinate, size in zip(origin_parsed, selection_shape_parsed, strict=True) ) ) def iter_chunk_regions( self, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None, ) -> Iterator[tuple[slice, ...]]: """Yield each chunk's valid-data slices, in global source coordinates. Covers the same region as `all_chunk_coords`, silently skipping coordinates that fall outside the grid. """ for coords in self.all_chunk_coords(origin=origin, selection_shape=selection_shape): spec = self[coords] if spec is not None: yield spec.slices def get_nchunks(self) -> int: """The total number of data-bearing chunks: the product of `grid_shape` (1 if 0-d).""" return reduce(operator.mul, (dimension.nchunks for dimension in self.dimensions), 1) def update_shape(self, new_shape: tuple[int, ...]) -> ChunkGrid: """Return a grid resized to `new_shape` by resizing each dimension. Fixed axes keep their chunk size; rectilinear axes gain one trailing edge when grown past their declared edges. Raises `ValueError` when `new_shape` does not have `ndim` entries. """ if len(new_shape) != self.ndim: raise ValueError( f"new_shape has {len(new_shape)} dimensions but chunk grid has {self.ndim} dimensions" ) return ChunkGrid( dimensions=tuple( dimension.resize(new_extent) for dimension, new_extent in zip(self.dimensions, new_shape, strict=True) ) ) class EdgeDimensionGrid: """An explicitly edge-based grid for coordinate-origin examples and planners. Examples -------- Chunk sizes `(2, 3)` tile source coordinates `[0, 5)`; lookups outside that range raise: >>> grid = EdgeDimensionGrid([2, 3]) >>> grid.num_chunks, grid.extent (2, 5) >>> grid.index_to_chunk(2) 1 >>> grid.index_to_chunk(5) Traceback (most recent call last): ... IndexError: index 5 is out of bounds for an axis of extent 5 """ __slots__ = ("_offsets", "sizes") sizes: tuple[int, ...] """The length of each chunk along the axis, in order; every entry is positive.""" def __init__(self, sizes: Sequence[int]) -> None: """Build a one-axis grid from explicit per-chunk sizes. Every size must be positive; raises `ValueError` otherwise. A zero-length axis is spelled as an empty sequence (no chunks), not as a zero size. Parameters ---------- sizes : Sequence[int] The length of each chunk along the axis, in order. """ normalized = tuple(int(size) for size in sizes) for index, size in enumerate(normalized): if size <= 0: raise ValueError( f"chunk sizes must be positive; got {size} at position {index} of {normalized}. " "A zero-length axis is spelled as no chunks at all: EdgeDimensionGrid(())" ) self.sizes = normalized offsets: np.ndarray[Any, np.dtype[np.intp]] = np.zeros(len(normalized) + 1, dtype=np.intp) if normalized: np.cumsum(np.asarray(normalized, dtype=np.intp), out=offsets[1:]) self._offsets = offsets @property def num_chunks(self) -> int: """The number of chunks along the axis.""" return len(self.sizes) @property def extent(self) -> int: """The axis length in global source coordinates: the sum of all chunk sizes.""" return int(self._offsets[-1]) def __repr__(self) -> str: return f"EdgeDimensionGrid(sizes={self.sizes})" def __eq__(self, other: object) -> bool: if not isinstance(other, EdgeDimensionGrid): return NotImplemented return self.sizes == other.sizes def __hash__(self) -> int: return hash((type(self).__name__, self.sizes)) def index_to_chunk(self, idx: int) -> int: """Map a global source index to the chunk whose interval contains it. Raises `IndexError` when `idx` lies outside `[0, extent)`. """ if idx < 0 or idx >= self.extent: raise IndexError(f"index {idx} is out of bounds for an axis of extent {self.extent}") return int(np.searchsorted(self._offsets, idx, side="right")) - 1 def chunk_offset(self, chunk_ix: int) -> int: """The global source coordinate where chunk `chunk_ix` begins. Raises `IndexError` when `chunk_ix` lies outside `[0, num_chunks)`. """ if chunk_ix < 0 or chunk_ix >= len(self.sizes): raise IndexError( f"chunk index {chunk_ix} is out of bounds for {len(self.sizes)} chunks" ) return int(self._offsets[chunk_ix]) def chunk_size(self, chunk_ix: int) -> int: """The length of chunk `chunk_ix`; every chunk holds data, so no boundary clipping applies. Raises `IndexError` when `chunk_ix` lies outside `[0, num_chunks)`. """ if chunk_ix < 0 or chunk_ix >= len(self.sizes): raise IndexError( f"chunk index {chunk_ix} is out of bounds for {len(self.sizes)} chunks" ) return self.sizes[chunk_ix] def indices_to_chunks(self, indices: npt.NDArray[np.intp]) -> npt.NDArray[np.intp]: """Vectorized `index_to_chunk` over an array of global source indices. Raises `IndexError` if any index lies outside `[0, extent)`. """ arr = _bounded_indices(indices, self.extent) return (np.searchsorted(self._offsets, arr, side="right") - 1).astype(np.intp) def _shape_tuple(shape: Sequence[int]) -> tuple[int, ...]: result = tuple(int(size) for size in shape) if any(size < 0 for size in result): raise ValueError(f"shape entries must be non-negative; got {result}") return result def _entry_kind(entry: Any) -> str: if isinstance(entry, (int, np.integer)) and not isinstance(entry, bool): return "int" if isinstance(entry, (str, bytes)): return "neither" try: iter(cast("Iterable[Any]", entry)) except TypeError: return "neither" return "sequence" def dimension_grids_from_chunks( chunks: Sequence[int] | Sequence[Sequence[int]], shape: Sequence[int] ) -> tuple[DimensionGrid, ...]: """Build compact dimensions from regular sizes or explicit per-axis edges. Examples -------- One integer per dimension builds fixed grids, ready for `plan_chunks`: >>> from zarr_indexing import IndexTransform, plan_chunks >>> grids = dimension_grids_from_chunks((2, 2), shape=(3, 4)) >>> [type(grid).__name__ for grid in grids] ['FixedDimension', 'FixedDimension'] >>> plan = plan_chunks(IndexTransform.from_shape((3, 4))[1, :], grids) >>> [p.chunk_coords for p in plan] [(0, 0), (0, 1)] """ shape_t = _shape_tuple(shape) entries: tuple[Any, ...] = tuple(chunks) if len(entries) != len(shape_t): raise ValueError( f"chunks must have one entry per dimension; got {len(entries)} entries for shape {shape_t}" ) conventions = ( "chunks must be either a uniform chunk shape (one integer per dimension) " "or per-axis chunk sizes (one sequence of integers per dimension)" ) kinds = [_entry_kind(entry) for entry in entries] neither = [(axis, entries[axis]) for axis, kind in enumerate(kinds) if kind == "neither"] if neither: described = ", ".join(f"{entry!r} at dimension {axis}" for axis, entry in neither) verb = "is" if len(neither) == 1 else "are" raise ValueError(f"{conventions}; {described} {verb} neither") integer_count = sum(kind == "int" for kind in kinds) if entries and integer_count == len(entries): dimensions: list[DimensionGrid] = [] for entry, extent in zip(entries, shape_t, strict=True): size = int(entry) if size <= 0: raise ValueError(f"chunk shape entries must be positive; got {size}") dimensions.append(FixedDimension(size=size, extent=extent)) return tuple(dimensions) if integer_count: raise ValueError(f"{conventions}, not a mixture; got {entries!r}") dimensions = [] for axis, (entry, extent) in enumerate(zip(entries, shape_t, strict=True)): elements: tuple[Any, ...] = tuple(cast("Iterable[Any]", entry)) if any( not isinstance(element, (int, np.integer)) or isinstance(element, bool) for element in elements ): raise ValueError( f"per-axis chunk sizes must be integers; dimension {axis} has {entry!r}" ) edges = tuple(int(element) for element in elements) total = sum(edges) if total != extent: raise ValueError( f"per-axis chunk sizes for dimension {axis} sum to {total}, but the array extent is {extent}" ) if extent == 0 and all(edge == 0 for edge in edges): dimensions.append(FixedDimension(size=0, extent=0)) continue if any(edge <= 0 for edge in edges): raise ValueError(f"chunk sizes must be positive; got {edges}") dimensions.append(VaryingDimension(edges=edges, extent=extent)) return tuple(dimensions) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/json.py000066400000000000000000000166161527014137600310130ustar00rootroot00000000000000"""The canonical ndsel wire vocabulary, and the rules for lowering it. This is the **engine layer**. Where `messages.py` is pure JSON→JSON, this module holds the JSON shapes a canonical ndsel body takes (spec section 4.3, as produced by `zarr_indexing.messages.normalize_ndsel`) together with the lowering rules that turn one into the numpy-backed engine representation. The conversions themselves are **methods on the types**, since each type owns its one serialization: `IndexTransform.to_json` / `from_json`, `IndexDomain.to_json` / `from_json`, and `to_json` on each output map kind, with `output_index_map_from_json` in `zarr_indexing.output_map` dispatching the wire's structurally discriminated union back to the right kind. This module is what they share. The engine lowering rules include: - **Validated index arrays.** Raw index values must satisfy the inclusive `index_array_bounds` before offset and stride are applied. Lowering checks all supplied values eagerly; validated immutable maps need not retain bounds. - **Finite bounds.** An `IndexDomain` addresses a finite array, so a canonical body carrying a `"-inf"`/`"+inf"` bound cannot be lowered; `from_json` raises. - **Implicit bounds lower by value.** The `[n]`-bracket implicit/explicit flag is a message-layer concern; the engine keeps only the integer value. - **Integer `index_array` content.** The message layer carries `index_array` verbatim (the spec defers its shape and type), so lowering is where a float, boolean or string array is rejected — as an `NdselError`, rather than truncating `[0.9, 1.9]` to cells 0 and 1 or leaking a raw NumPy error. ## The `index_array` wire format (and the degenerate-collapse it documents) ndsel and TensorStore both **reject** an output map that carries *both* `input_dimension` and `index_array`; `input_dimension` belongs to affine (`single_input_dimension`) maps. The in-memory `ArrayMap` matches: what a map depends on is read from its full-rank array's shape (its non-singleton axes), so there is nothing to reconstruct on load. On serialize (`to_json`): 1. An all-singleton `index_array` (size 1) selects a single coordinate regardless of input, so it is **collapsed to a `constant` map** `{offset: offset + stride*value}`. The size-1 input dimension stays in the domain, unconsumed — a valid transform. The selection layer already builds such maps as `ConstantMap` (`output_map.array_map_or_constant`); this covers hand-built transforms. 2. An **empty** `index_array` (size 0) collapses the same way, to `{offset: 0}`. It names no cell, and it can only be empty because an input dimension is — the full-rank invariant makes every axis either 1 or the domain's extent — so nothing is ever read through it and the emptiness is carried by the domain, which is emitted separately. TensorStore does the same: `t[ts.d[0][[]]]` is `out[0] = 0`, emitted as `{}`. This avoids losing trailing shape information: `ndarray.tolist()` renders every empty array as `[]` once the leading axis is zero-length, and nested lists cannot spell the shape back — `[[]]` is `(1, 0)` and nothing spells `(0, 1)`. 3. Non-degenerate `index_array` maps are emitted with their array and bounds only. """ from __future__ import annotations from typing import Any, Required, TypedDict # --------------------------------------------------------------------------- # TypedDict definitions (canonical JSON shapes) # --------------------------------------------------------------------------- # An `index_array` serializes via `ndarray.tolist()`, so it is a nested list of # ints whose nesting depth equals the array rank. NestedIntList = list[Any] # A canonical *lowered* body carries only finite integer bounds, but the JSON # shape admits the full ndsel `bound` grammar: an explicit int / sentinel, or a # one-element implicit `[value]` array. IndexValueJSON = int | str BoundJSON = int | str | list[IndexValueJSON] class IndexDomainJSON(TypedDict, total=False): """Canonical JSON representation of an IndexDomain. Examples -------- >>> doc: IndexDomainJSON = { ... "input_inclusive_min": [0], ... "input_exclusive_max": [4], ... "input_labels": ["x"], ... } >>> from zarr_indexing import IndexDomain >>> IndexDomain.from_json(doc).shape (4,) """ input_inclusive_min: Required[list[BoundJSON]] """Per-dimension lower bounds; `"-inf"` is legal on the wire but cannot be lowered.""" input_exclusive_max: Required[list[BoundJSON]] """Per-dimension exclusive upper bounds; `"+inf"` is legal on the wire but cannot be lowered.""" input_labels: Required[list[str]] """Per-dimension names; the empty string marks an unlabeled dimension.""" class OutputIndexMapJSON(TypedDict, total=False): """Canonical JSON representation of a single output index map. Exactly one of three forms (distinguished by which fields are present): - `{"offset": 5}` — constant - `{"offset": 0, "stride": 1, "input_dimension": 0}` — single_input_dimension - `{"offset": 0, "stride": 1, "index_array": [...], "index_array_bounds": ["-inf", "+inf"]}` — index_array Examples -------- >>> from zarr_indexing import output_index_map_from_json >>> constant: OutputIndexMapJSON = {"offset": 5} >>> output_index_map_from_json(constant) ConstantMap(offset=5) >>> affine: OutputIndexMapJSON = {"offset": 0, "stride": 2, "input_dimension": 1} >>> output_index_map_from_json(affine) DimensionMap(input_dimension=1, offset=0, stride=2) """ offset: int """Constant term; alone it is the whole constant form.""" stride: int """Multiplier applied to the input coordinate or to each `index_array` value.""" input_dimension: int """The input dimension the single_input_dimension form reads.""" index_array: NestedIntList """Nested lists of output coordinates, one nesting level per input dimension.""" index_array_bounds: list[IndexValueJSON] """Wire bounds on index-array values; `["-inf", "+inf"]` if unconstrained. The message layer preserves these inclusive constraints on raw index values. Engine lowering validates all values eagerly before offset and stride. Serialization emits unconstrained bounds for validated non-degenerate maps. """ class IndexTransformJSON(TypedDict, total=False): """Canonical JSON representation of an IndexTransform (spec section 4.3). Examples -------- >>> doc: IndexTransformJSON = { ... "input_rank": 1, ... "input_inclusive_min": [0], ... "input_exclusive_max": [2], ... "input_labels": [""], ... "output": [{"offset": 1, "stride": 2, "input_dimension": 0}], ... } >>> from zarr_indexing import IndexTransform >>> IndexTransform.from_json(doc).domain.shape (2,) """ input_rank: Required[int] """The number of input dimensions; the bounds and labels lists match it in length.""" input_inclusive_min: Required[list[BoundJSON]] """Per-dimension lower bounds; `"-inf"` is legal on the wire but cannot be lowered.""" input_exclusive_max: Required[list[BoundJSON]] """Per-dimension exclusive upper bounds; `"+inf"` is legal on the wire but cannot be lowered.""" input_labels: Required[list[str]] """Per-dimension names; the empty string marks an unlabeled dimension.""" output: Required[list[OutputIndexMapJSON]] """One output map per output dimension.""" zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/lazy_array.py000066400000000000000000001674741527014137600322300ustar00rootroot00000000000000"""`LazyArray` — TensorStore-style lazy indexing over array-like sources. `LazyArray` wraps a source with `shape`, `dtype`, and basic integer/slice `__getitem__`, whose reads can be lowered through NumPy system memory. It adds indexing operations that build up an [`IndexTransform`](transform.md) instead of reading data: ```python view = LazyArray(source)[10:50, ::2].oindex[[3, 1, 1], :] view.shape # known without touching the data values = view.result() ``` Selection construction does not read source values: `result()`, `__array__`, and scalar conversions perform reads. Source tokenization is delegated to Dask and may inspect source values. Indexing operations inspect selection metadata and may copy or process supplied index arrays. Composition does not accumulate wrapper layers: a view of a view is still a single transform and retains its reader. Parts ----- A `LazyArray` carries a **partitioning** of the array it wraps: a grid of boxes that a read is broken into. `parts()` walks those boxes as they fall through the view, yielding a [`Partition`](#zarr_indexing.lazy_array.Partition) per box. Its paired projection describes the chunk-local read and where its cells land in the request; `view` carries that partition's transform. `result()` allocates one fresh output buffer, then reads each partition once through the selected reader into the final buffer or an owned temporary for fancy placement. The part view's transform directly addresses its raw wrapped array. The paired projection deliberately retains the chunk-local frame. Parent materialization passes both in `ReadContext`. Every view retains the source's partitioning and plans its own reads, including views obtained from partitions. Calling `part.view.result()` therefore supplies the reader with the selected chunk's projection as well. Output placement is relative to the view being executed. The partitioning is discovered from the wrapped array at construction — first `read_chunk_sizes` (zarr's clipped per-axis sizes, sharding-aware), then `chunks`, read as per-axis sizes if its entries are sequences and as a uniform box shape if they are integers. Those attribute names belong to the wrapped array; this API refers only to parts. An array that advertises neither gets a single whole-array part, and resolving it reads the whole view through its selected reader in one pass. `with_parts` replaces the partitioning without touching the data or the view: ```python view.with_parts((64, 64)) # uniform boxes, tail clipped view.with_parts_per_axis(((3, 3, 1),)) # explicit per-axis sizes view.unpartitioned() # one whole-array part; resolve in one shot ``` Repartitioning changes how the read is divided, not what `result()` returns. Parts that do not align with the source's own boxes are permitted and can be useful for controlling per-read sizes or batching small reads. They can change I/O costs; the intended selected values remain the same for an unchanged source and a conforming reader. The full result buffer is still allocated. Readers ------- Every wrapper carries a reader that owns the backend-specific request. The transform answers **which values?** and is independent of the backend; the reader answers **how does this backend obtain them?** and must preserve the complete transform exactly. Readers do not define indexing semantics, partitioning, scheduling, or result ownership. The conservative `LazyArray(source)` uses `basic_reader`, which needs only basic slicing. `LazyArray.from_numpy(array)` explicitly opts into `numpy_reader` for direct NumPy indexing. `with_reader()` replaces the reader without reading or changing the view metadata. The reader object is shared by all derived views and their parts. Consumers may materialize part views concurrently; `LazyArray` does not serialize calls, so a stateful reader must synchronize its own mutable state. The built-in readers lower through NumPy system memory using NumPy conversion of source slices. Device arrays that refuse implicit conversion need a custom reader that explicitly transfers values into the output buffer. Boxes and queries ----------------- `is_box` reports whether the current transform contains only constant and affine output maps. An index-array gather commonly produces a query, but singleton indices and later selections can remove its index-array maps and make it a box. `bounding_box()` gives a coordinate hull, and `strides()` gives stride magnitudes for a box. These describe storage coordinates, not the full result layout or traversal order. A query can fill its hull, and a singleton box can fill its hull even when its recorded stride exceeds one. Relative keys, absolute domain ------------------------------ **The type of a key decides whether it is relative or absolute.** NumPy keys (slices, integers, `...`, `None`, index arrays, masks) are positions relative to the current view. An `IndexDomain` key names absolute coordinates of the view's domain, and an `IndexTransform` key composes onto the view. No key type has two readings, and whichever key produced a view, `view.transform.domain` is absolute. See the guide section "The key's type picks the frame". Selections with NumPy keys are **positional, NumPy-style**: index 0 is the first element of the current view, `-1` is the last, boolean masks must match the view's shape, integer coordinates are bounds-checked, and slices are clipped to the view's extent. The dialect governs how a *key* is read, not what a view remembers. A derived view keeps its **literal domain**, as a TensorStore view does: `a[10:20]` has domain `[10, 20)` and `a[10:20][2:5]` has domain `[12, 15)`, while `a[10:20][0]` still names the first element because positional keys are normalized against the domain's origin. The literal frame is reachable through `view.transform`, and directly through the two literal keys: an `IndexDomain` restricts the view to those coordinates, and an `IndexTransform` composes onto it. A reversed view carries a negative origin, as in TensorStore, because a reversing map traverses the source frame backwards. The current main Zarr `Array` does not expose this wrapper as an `Array.lazy` attribute; use `LazyArray(array)` explicitly. Scalar integers drop axes. Non-boolean objects implementing `SupportsIndex` are accepted as scalar indices and in slice bounds; `__int__` alone is not enough. For orthogonal and vectorized modes this wrapper applies scalar indices first, then the remaining advanced selection. Orthogonal indices form an outer product. Vectorized indexing accepts coordinate arrays or a shape-matching boolean mask. An ellipsis can retain unindexed axes (`[..., i, j]`), but explicit slice entries in vectorized selections are currently rejected. Scalar-first processing can also differ from NumPy's advanced-axis placement: for shape `(2, 3, 4)`, `lazy.vindex[0, ..., [1, 2]]` has shape `(3, 2)`, whereas NumPy's same selection has shape `(2, 3)`. These modes do not implement every NumPy indexing form. Materializing on fallback ------------------------- `LazyArray` implements `__array__` but deliberately implements neither `__array_ufunc__` nor `__array_function__`. Many NumPy operations therefore materialize through `__array__` and work on the resulting array: `numpy.sum(view)`, `numpy.add(view, 1)` and `numpy.stack([view, view])` all do, and so does `numpy.ones(view.shape) + view`, where the ndarray on the left dispatches. Metadata queries such as `numpy.shape(view)` and `numpy.ndim(view)` can use the exposed attributes without materializing; other unsupported operations may fail. Python's arithmetic *operators* do not: `view + 1` raises `TypeError`, because the wrapper defines no arithmetic dunders and an `int` has nothing to dispatch to. Both facts follow from the same intent — laziness here applies to indexing, not to building a deferred compute graph — and a `LazyArray` is not a drop-in for arithmetic on a large array either way. Use `[...]` to narrow the view first, or pass `EagerArrayAdapter(view)` to `dask.array.from_array` so that Dask owns the compute graph. Ownership --------- `result()` always allocates fresh system memory before reading through the selected reader. A `numpy.ma` source keeps its mask by receiving a masked output buffer; other source-specific array types do not survive materializing. """ from __future__ import annotations import hashlib import json import math import operator from collections.abc import Sequence from dataclasses import dataclass, field, replace from typing import TYPE_CHECKING, Any, Protocol, cast import numpy as np from zarr_indexing.boundary import ( SelectionMode, normalize_positional_selection, split_scalar_axes, ) from zarr_indexing.chunk_resolution import ( ChunkProjection, plan_chunks, ) from zarr_indexing.domain import IndexDomain from zarr_indexing.errors import BoundsCheckError from zarr_indexing.grid import DimensionGrid, FixedDimension, dimension_grids_from_chunks from zarr_indexing.output_map import ArrayMap, ConstantMap, DimensionMap from zarr_indexing.reader import ( ReadContext, Reader, basic_reader, numpy_reader, ) from zarr_indexing.transform import ( IndexTransform, ) if TYPE_CHECKING: from collections.abc import Callable, Iterator SelectFn = Callable[[Any, SelectionMode], "LazyArray"] __all__ = ["LazyArray", "Partition"] def _invoke_reader( reader: Reader, source: Any, context: ReadContext, out: np.ndarray[Any, Any], ) -> None: """Invoke a reader and enforce its in-place return contract.""" returned = reader.read_into(source, context, out) if returned is not None: raise TypeError(f"reader.read_into must return None, got {type(returned).__name__}") def _is_correlated(transform: IndexTransform) -> bool: """True when the transform gathers a list of points rather than an outer product.""" return transform.index_array_structure == "general" class ArrayLike(Protocol): """The surface `LazyArray` needs from the array it wraps.""" @property def shape(self) -> tuple[int, ...]: ... @property def dtype(self) -> Any: ... def __getitem__(self, key: Any) -> Any: ... # NumPy's dtype-specialized ``__getitem__`` overloads do not structurally match # the deliberately broad protocol above under strict type checking. Accept an # ndarray explicitly so users do not have to erase its type with ``cast(Any, …)``. _WrappedArray = ArrayLike | np.ndarray[Any, Any] # --------------------------------------------------------------------------- # # Partition discovery # --------------------------------------------------------------------------- # def _read_source_attribute(array: Any, name: str) -> Any: """Read a partition-describing attribute, treating any failure as "absent". Discovery inspects an object we did not write. A missing attribute is the common case, but zarr raises an `AttributeError` subclass from `read_chunk_sizes` on a lazy view, and other backends compute the attribute lazily and may fail for their own reasons. Any failure here means "this array does not advertise a partitioning", never a hard error. """ try: return getattr(array, name, None) # Guarded properties (e.g. zarr's LazyViewError) and broken foreign # attributes may raise anything; discovery must degrade to None. except Exception: return None def _discover_parts(array: Any, shape: tuple[int, ...]) -> tuple[DimensionGrid, ...] | None: """Resolve the partitioning advertised by `array`, or None for one whole part. Discovery parses external input: an attribute that does not describe a partitioning of `shape` means "this object does not advertise one I understand", and the array is treated as unpartitioned rather than rejected. With a compatible reader, resolving the view without partitioning preserves its values, though it may require larger reads. `with_parts` is a public API and validates strictly. """ declared = _read_source_attribute(array, "read_chunk_sizes") if declared is None: declared = _read_source_attribute(array, "chunks") if declared is None: return None try: return dimension_grids_from_chunks(declared, shape) except (ValueError, TypeError): return None def _discover_write_grid(array: Any, shape: tuple[int, ...]) -> tuple[DimensionGrid, ...] | None: """Resolve the grid `array` commits writes in, or None for one whole cell. The write grid can be coarser than the read grid: a sharded zarr array decodes inner chunks but replaces whole shards, so `write_chunk_sizes` is consulted before `chunks`. As with read discovery, anything unusable means "no advertised grid" rather than an error. """ declared = _read_source_attribute(array, "write_chunk_sizes") if declared is None: declared = _read_source_attribute(array, "chunks") if declared is None: return None try: return dimension_grids_from_chunks(declared, shape) except (ValueError, TypeError): return None def _whole_array_grids(shape: tuple[int, ...]) -> tuple[DimensionGrid, ...]: """A partitioning with a single part covering the whole array.""" return tuple(FixedDimension(size=extent, extent=extent) for extent in shape) # --------------------------------------------------------------------------- # # The lowering engine # --------------------------------------------------------------------------- # def _pure_translation(transform: IndexTransform) -> tuple[int, ...] | None: """The per-axis shift when `transform` only translates its zero-origin domain, else None.""" if transform.output_rank != transform.input_rank or any( transform.domain.inclusive_min[i] != 0 for i in range(transform.input_rank) ): return None shift: list[int] = [] for axis, m in enumerate(transform.output): if not (isinstance(m, DimensionMap) and m.input_dimension == axis and m.stride == 1): return None shift.append(m.offset) return tuple(shift) def _is_identity_transform(transform: IndexTransform, shape: tuple[int, ...]) -> bool: """True when `transform` maps every coordinate of `shape` to itself. Structural rather than an `==` against `IndexTransform.from_shape`: an `ArrayMap` holds an ndarray, so equality on two transforms that both carry one would try to take the truth value of an array. """ domain = transform.domain if domain.inclusive_min != (0,) * len(shape) or domain.exclusive_max != shape: return False if len(transform.output) != len(shape): return False return all( isinstance(m, DimensionMap) and m.input_dimension == i and m.offset == 0 and m.stride == 1 for i, m in enumerate(transform.output) ) # --------------------------------------------------------------------------- # # Partitions # --------------------------------------------------------------------------- # @dataclass(frozen=True, slots=True, eq=False) class _PartOwner: """Opaque identity shared only by one view and the parts it prepared.""" def _partition_out_selection( cell_transform: IndexTransform, ) -> tuple[Any, ...]: """Lower ``cell_transform`` to NumPy selectors on the request buffer.""" domain = cell_transform.domain # A compact cell transform may permute a slice axis and a gather axis. # NumPy's mixed basic/advanced selectors would place the gathered block # differently from the synthetic domain; broadcast coordinate selectors # retain that domain's axis order for scatter. dependencies = [ m.input_dimension if isinstance(m, DimensionMap) else m.dependent_axis for m in cell_transform.output if not isinstance(m, ConstantMap) ] permuted = all(d is not None for d in dependencies) and dependencies != sorted( d for d in dependencies if d is not None ) if _is_correlated(cell_transform) or permuted: correlated_selectors: list[np.ndarray[Any, np.dtype[np.intp]]] = [] for output_map in cell_transform.output: if isinstance(output_map, ConstantMap): coordinates = np.full((1,) * domain.ndim, output_map.offset, dtype=np.intp) elif isinstance(output_map, DimensionMap): input_dimension = output_map.input_dimension axis = np.arange( domain.inclusive_min[input_dimension], domain.exclusive_max[input_dimension], dtype=np.intp, ) shape = ( (1,) * input_dimension + (axis.size,) + ((1,) * (domain.ndim - input_dimension - 1)) ) coordinates = output_map.offset + output_map.stride * axis.reshape(shape) else: coordinates = output_map.offset + output_map.stride * output_map.index_array correlated_selectors.append(np.asarray(coordinates, dtype=np.intp)) return tuple(correlated_selectors) selectors: list[int | slice | np.ndarray[Any, np.dtype[np.intp]]] = [] n_array_maps = sum(isinstance(output_map, ArrayMap) for output_map in cell_transform.output) for output_map in cell_transform.output: if isinstance(output_map, ConstantMap): selectors.append(output_map.offset) elif isinstance(output_map, DimensionMap): input_dimension = output_map.input_dimension lo = domain.inclusive_min[input_dimension] hi = domain.exclusive_max[input_dimension] selectors.append( slice( output_map.offset + output_map.stride * lo, output_map.offset + output_map.stride * hi, output_map.stride, ) ) else: selectors.append( (output_map.offset + output_map.stride * output_map.index_array.ravel()).astype( np.intp ) ) if n_array_maps > 1: axes = [ np.asarray([selector], dtype=np.intp) if isinstance(selector, int) else ( np.arange(selector.start, selector.stop, selector.step, dtype=np.intp) if isinstance(selector, slice) else selector ) for selector in selectors ] return np.ix_(*axes) return tuple(selectors) def _out_selection_cell_count(selection: tuple[Any, ...], out_shape: tuple[int, ...]) -> int: """How many cells of an array of shape `out_shape` a `Partition.out_selection` writes. Counted from the selectors' own shapes, so nothing is read and no index array is materialized. The selectors are slices and integer arrays: the slices contribute their lengths, and the arrays broadcast against each other exactly as NumPy's advanced indexing broadcasts them, whether they arrive as an open mesh (`numpy.ix_`) or as parallel coordinates (`unravel_index`). """ total = 1 array_shapes: list[tuple[int, ...]] = [] if len(selection) != len(out_shape): raise AssertionError( f"a partition addressed {len(selection)} of the view's {len(out_shape)} " "dimensions; this is a bug in zarr-indexing's partition walk" ) for selector, extent in zip(selection, out_shape, strict=True): if isinstance(selector, slice): start: Any = selector.start stop: Any = selector.stop step: Any = selector.step if step is None and start is not None and stop is not None and 0 <= start <= stop: # The shape a partition walk actually produces: a concrete, # forward, in-bounds interval. Sized directly, so the common # path allocates neither a tuple nor a range. total *= int(stop) - int(start) else: total *= len(range(*selector.indices(extent))) else: array_shapes.append(tuple(int(s) for s in np.shape(selector))) if len(array_shapes) > 0: total *= math.prod(np.broadcast_shapes(*array_shapes)) return total @dataclass(frozen=True, kw_only=True) class Partition: """One box of a `LazyArray`'s partitioning, as it falls through the view. Yielded by [`LazyArray.parts`][zarr_indexing.lazy_array.LazyArray.parts]. The parts of a view tile it exactly and disjointly: assembling every `part.view.result()` at its `out_selection` reproduces the whole view's `result()`. Each view plans its reads using the retained source grid. Parts can be resolved concurrently when the source and reader permit it. Derived parts retain the same reader object; a shared stateful reader owns synchronization for concurrent calls. A consumer that needs the plan before materialization can prepare it once and reuse the same part records for both scheduling and assembly. The frozen records do not snapshot the mutable source or reader: ```python parts = tuple(view.parts()) schedule(part.base_coords for part in parts) values = view.result(parts=parts) ``` Prepared parts are owned by the exact view that created them and must tile it completely. Passing parts from another view, even an equivalent one, is rejected without reading; omitting a part is likewise rejected rather than returning a partly initialized result. Attributes ---------- projection The source-independent description of this part. Its paired `chunk_transform` and `cell_transform` share one compact synthetic domain, mapping each selected cell to chunk-local storage coordinates and to positions in this view's zero-origin result buffer respectively. (The planner's own projections place cells in the request's literal domain; `parts()` re-bases them.) This is the authoritative placement model; `base_coords` and `is_complete` are conveniences derived from it. base_coords Which box of the base partitioning this is, one coordinate per dimension of the wrapped array. box The box itself, in the global storage coordinates of the wrapped array: one `[inclusive_min, exclusive_max)` interval per dimension. It describes the whole partition cell, while `view.bounding_box()` is the global hull of only the selected values in that cell. For a nested or repartitioned view this box may be narrower than `projection.chunk_domain`. view A `LazyArray` covering exactly the cells of the view that live in this box. Its transform directly addresses its raw wrapped `array`; only the projection's `chunk_transform` is chunk-local. A box part keeps the request's literal coordinates, so its domain is a sub-domain of the parent view's; a part placed by index arrays has a fresh zero-origin domain, and `out_selection` is the placement in both cases. The view retains the source grid and reader and can be read or indexed like any other view. Named `view` rather than `array` because `LazyArray.array` is the opposite thing — the raw wrapped source — and the two sat next to each other meaning inverses. out_selection Where `view.result()` belongs in an array of the whole view's shape — a NumPy index tuple with one entry per dimension of the view, usable directly as `out[part.out_selection] = ...`. is_complete Whether the view covers the whole box. Useful to a writer deciding whether coverage is complete; it does not by itself establish buffer order, storage-unit alignment, or concurrent-write safety. Fancy projections report `False` because their coverage is deliberately `unknown` until duplicate-aware proof is added. Examples -------- Assembling every part's result at its `out_selection` reproduces the view: >>> import numpy as np >>> source = np.arange(12).reshape(3, 4) >>> view = LazyArray.from_numpy(source).with_parts((2, 2)) >>> out = np.empty(view.shape, dtype=view.dtype) >>> for part in view.parts(): ... out[part.out_selection] = part.view.result() >>> bool((out == source).all()) True """ projection: ChunkProjection box: tuple[tuple[int, int], ...] view: LazyArray out_selection: tuple[Any, ...] _owner: _PartOwner | None = field(default=None, repr=False, compare=False) @property def base_coords(self) -> tuple[int, ...]: """Coordinates of this partition in the selected base grid.""" return self.projection.chunk_coords @property def is_complete(self) -> bool: """Whether the projection proves it covers the entire selected cell.""" return self.projection.coverage == "full" def _validate_prepared_parts(parts: Sequence[Partition], out_shape: tuple[int, ...]) -> None: """Require `parts` to address every output cell exactly once. Prepared parts are caller-supplied input, so a plan that does not tile the view is a `ValueError`, not an assertion about this library's own walk. """ coverage = np.zeros(out_shape, dtype=np.bool_) addressed = 0 try: for part in parts: # Name a rank mismatch explicitly instead of letting NumPy treat # omitted selectors as implicit full slices. addressed += _out_selection_cell_count(part.out_selection, out_shape) if all(isinstance(selector, slice) for selector in part.out_selection): # The common box part: plain assignment, no pointwise walk. coverage[part.out_selection] = True else: # `logical_or.at` applies duplicate advanced coordinates one by # one instead of buffering them as ordinary advanced indexing # would. np.logical_or.at(coverage, part.out_selection, True) except (AssertionError, IndexError, TypeError, ValueError) as error: raise ValueError("prepared parts do not tile the view exactly") from error if addressed != math.prod(out_shape) or not np.all(coverage): raise ValueError("prepared parts do not tile the view exactly") # --------------------------------------------------------------------------- # # The wrapper # --------------------------------------------------------------------------- # class LazyArray: """A lazily-indexable view over a system-memory/basic-indexing source. Wrapping neither copies nor reads the wrapped array at construction time. Indexing composes an `IndexTransform` and returns another `LazyArray`; `result()` materializes. Selections use the **positional NumPy dialect** and reads are broken up along a **partitioning** discovered from the wrapped array. Every derived view retains its reader; that reader receives the complete projected transform once per part. See the module docstring, which also covers how the dialect differs from low-level literal transforms and which NumPy operations materialize the view. Selection is lazy; `result()` reads and `write(values)` writes synchronously. Assignment is shorthand for selecting a view and calling its `write` method. The source owns storage errors and concurrency; writes are not transactional. Parameters ---------- array The array to wrap. It must expose `shape`, `dtype`, and `__getitem__` with basic (integer/slice) indexing; `__setitem__` is not required, so a read-only source wraps as well as a writable one. Its partitioning, if it advertises one, is discovered here; use `with_parts` to choose a different one. This conservative constructor selects `basic_reader`; use `from_numpy` for a NumPy array or `with_reader` to select another backend adapter. Examples -------- >>> import numpy as np >>> source = np.arange(12).reshape(3, 4) >>> view = LazyArray.from_numpy(source).with_parts((2, 2))[1:, ::2] >>> view.shape (2, 2) >>> view.result() array([[ 4, 6], [ 8, 10]]) """ __slots__ = ("_array", "_part_owner", "_parts", "_reader", "_transform") def __init__(self, array: _WrappedArray) -> None: """Wrap `array` without reading it; parameters are documented on the class. Reject `numpy.matrix`, normalize the shape, and inspect partition metadata. Shape conversion and transform construction can also reject invalid input. """ if isinstance(array, np.matrix): # `np.matrix` keeps every result two-dimensional, so `m[1]` has shape # `(1, n)` where every other array-like gives `(n,)`. A view's shape # comes from the transform, which follows NumPy's rule, so the two # disagree on every rank-reducing selection. Refused at the door # rather than resolved into a shape the view did not promise. raise TypeError( "numpy.matrix cannot be wrapped: it never reduces rank, so a " "view's shape and its result would disagree. Convert it first, " "with numpy.asarray(m)." ) shape = tuple(int(s) for s in array.shape) self._array = array self._transform = IndexTransform.from_shape(shape) self._parts = _discover_parts(array, shape) self._reader = basic_reader self._part_owner = _PartOwner() @classmethod def from_numpy(cls, array: np.ndarray[Any, Any]) -> LazyArray: """Wrap a NumPy array with its explicitly selected NumPy reader.""" if not isinstance(cast(object, array), np.ndarray): raise TypeError( f"LazyArray.from_numpy requires a numpy.ndarray, got {type(array).__name__}" ) return cls(array).with_reader(numpy_reader) @classmethod def _derive( cls, array: _WrappedArray, transform: IndexTransform, parts: tuple[DimensionGrid, ...] | None, reader: Reader, ) -> LazyArray: """Build a wrapper sharing `array` but carrying a new transform or partitioning.""" view = cls.__new__(cls) view._array = array # The domain keeps its literal coordinates, as TensorStore's does: a # slice of `[10, 20)` remembers that it is `[10, 20)`. Positional keys # are normalized against that origin at selection time instead. view._transform = transform view._parts = parts view._reader = reader view._part_owner = _PartOwner() return view @property def _base_shape(self) -> tuple[int, ...]: """The shape of what this wrapper treats as its base array.""" return tuple(int(s) for s in self._array.shape) # -- array-like surface ------------------------------------------------- @property def array(self) -> _WrappedArray: """The wrapped array.""" return self._array @property def base_shape(self) -> tuple[int, ...]: """The shape the partitioning is expressed in — not this view's shape. `with_parts` and `with_parts_per_axis` describe boxes of the array being read, not of the view reading it. All derived views, including partition views, retain the full source shape as their partitioning frame. """ return self._base_shape @property def transform(self) -> IndexTransform: """The composed transform from this view's coordinates to storage.""" return self._transform @property def shape(self) -> tuple[int, ...]: """The shape of this view — the transform's input domain, not the source's.""" return self._transform.domain.shape @property def ndim(self) -> int: """Number of dimensions of this view — the transform's input rank.""" return self._transform.input_rank @property def size(self) -> int: """Total number of elements in this view (the product of `shape`).""" return math.prod(self.shape) @property def dtype(self) -> Any: """The wrapped array's dtype; views never change it.""" return self._array.dtype @property def reader(self) -> Reader: """The backend adapter used when this view materializes.""" return self._reader def with_reader(self, reader: Reader) -> LazyArray: """Return the same metadata view resolved through `reader`.""" if not callable(getattr(reader, "read_into", None)): raise TypeError(f"reader.read_into must be callable, got {type(reader).__name__}") return LazyArray._derive( self._array, self._transform, self._parts, reader, ) # -- shape of the selection --------------------------------------------- @property def is_box(self) -> bool: """Whether this view selects a rectangular region rather than a point list. True exactly when the composed transform has no `ArrayMap` output maps. Basic indexing of a box stays a box. Advanced indexing can introduce index arrays, but singleton gathers and later selections may remove them. `bounding_box()` and `strides()` describe the touched coordinate region and stride magnitudes; use the transform for order, result axes, and repetitions. Neither this flag nor the hull alone proves dense coverage. Examples -------- >>> import numpy as np >>> array = LazyArray.from_numpy(np.arange(12).reshape(3, 4)) >>> (array[1:, ::2].is_box, array.oindex[[2, 0], :].is_box) (True, False) """ return not any(isinstance(m, ArrayMap) for m in self._transform.output) def bounding_box(self) -> tuple[tuple[int, int], ...] | None: """The storage region this view touches, one interval per storage dimension. Defined for any selection, box or not, as the hull: the smallest `[inclusive_min, exclusive_max)` interval per dimension of the array this view reads from that contains every coordinate the selection reaches. A strided box can leave gaps, and a query hull can be arbitrarily loose: `oindex[[0, 999]]` spans a 1000-wide hull over two selected rows. Conversely, a query may cover every coordinate in its hull. Inspect the transform when coverage or traversal order matters. Returns ------- tuple of (int, int), or None One interval per storage dimension, or `None` when the view is empty (`size == 0`) and so touches no coordinate at all, leaving no interval to report. Notes ----- The coordinates directly address the raw `array` this view exposes. Consequently, partition views report source-global hulls; [`Partition.box`][zarr_indexing.lazy_array.Partition] separately gives the whole global partition cell rather than only the selected hull. Examples -------- >>> import numpy as np >>> array = LazyArray.from_numpy(np.arange(12).reshape(3, 4)) >>> array[1:, ::2].bounding_box() ((1, 3), (0, 3)) >>> array.oindex[[2, 0], :].bounding_box() ((0, 3), (0, 4)) >>> array[1:1].bounding_box() is None True """ if self.size == 0: return None domain = self._transform.domain bounds: list[tuple[int, int]] = [] for m in self._transform.output: if isinstance(m, ConstantMap): bounds.append((m.offset, m.offset + 1)) elif isinstance(m, DimensionMap): d = m.input_dimension first = m.offset + m.stride * domain.inclusive_min[d] last = m.offset + m.stride * (domain.exclusive_max[d] - 1) bounds.append((min(first, last), max(first, last) + 1)) else: coords = m.offset + m.stride * m.index_array bounds.append((int(coords.min()), int(coords.max()) + 1)) return tuple(bounds) def strides(self) -> tuple[int, ...] | None: """The step between selected coordinates, one per storage dimension. Together with `bounding_box()`, this describes a box's per-axis coordinates: `bounding_box()` gives the interval per dimension, `strides()` gives the step per dimension. A stride of 1 means every cell of the hull along that dimension is selected; `k` means every `k`-th. Dimensions fixed by an integer index report 1 — they span a single coordinate. Returns ------- tuple of int, or None One positive stride per storage dimension, or `None` when [`is_box`][zarr_indexing.lazy_array.LazyArray.is_box] is false: a query's coordinates are a lookup table and have no step. An empty box still reports its strides even though [`bounding_box`][zarr_indexing.lazy_array.LazyArray.bounding_box] returns `None`, because the step is a property of the selection's shape, not of the (empty) region it touches. Notes ----- Magnitudes only. A reversing view (`lazy[::-1]`) selects the same set of coordinates as the equivalent forward view, so it reports the same bounding box and the same strides. The traversal direction is recorded in the transform, not in this description of the region touched. A consumer that needs the order reads the transform, or reverses the block it gets back. Examples -------- >>> import numpy as np >>> array = LazyArray.from_numpy(np.arange(24).reshape(4, 6)) >>> (array[1:, ::2].bounding_box(), array[1:, ::2].strides()) (((1, 4), (0, 5)), (1, 2)) >>> array[2, ::3].strides() (1, 3) >>> array.oindex[[2, 0], :].strides() is None True """ if not self.is_box: return None return tuple( 1 if isinstance(m, ConstantMap) else abs(m.stride) for m in self._transform.output ) # -- partitioning ------------------------------------------------------- def with_parts(self, parts: Sequence[int]) -> LazyArray: """Return the same view, read in uniform boxes of shape `parts`. One integer per dimension of `base_shape`, with the trailing box in each dimension clipped to the extent. The transform, the wrapped array, and therefore `result()` are all unchanged; only the boxes the read is broken into differ. Nothing is copied and nothing is read. For per-axis sizes see [`with_parts_per_axis`][zarr_indexing.lazy_array.LazyArray.with_parts_per_axis], and to read in one pass see [`unpartitioned`][zarr_indexing.lazy_array.LazyArray.unpartitioned]. Parameters ---------- parts The box shape, one integer per dimension of `base_shape`. Returns ------- LazyArray The same view with a new partitioning. Raises ------ ValueError If `parts` has the wrong length or contains a non-positive extent. Uniform part sizes must remain positive even for a zero-length axis; use `with_parts_per_axis` for the accepted explicit zero-axis spellings. Examples -------- >>> import numpy as np >>> view = LazyArray.from_numpy(np.arange(12).reshape(3, 4)) >>> [part.base_coords for part in view.with_parts((2, 3)).parts()] [(0, 0), (0, 1), (1, 0), (1, 1)] """ entries = self._part_entries(parts, "with_parts") if any(isinstance(entry, Sequence) for entry in entries): raise ValueError( "with_parts takes one integer per dimension; for per-axis box " "sizes use with_parts_per_axis" ) return self._with_grids(dimension_grids_from_chunks(entries, self._base_shape)) def with_parts_per_axis(self, sizes: Sequence[Sequence[int]]) -> LazyArray: """Return the same view, read in boxes of explicitly listed sizes. The dask convention: one sequence of box extents per dimension of `base_shape`, each summing to that dimension's extent. Use it when the boxes are not uniform — a partitioning discovered from a store, or one whose last box differs by more than clipping. Parameters ---------- sizes One sequence of box extents per dimension of `base_shape`. Returns ------- LazyArray The same view with a new partitioning. Raises ------ ValueError If `sizes` has the wrong length, contains a negative extent, uses a zero extent on a nonempty axis, or declares sizes that do not sum to `base_shape`. On a zero-length axis, `()`, `(0,)`, and repeated zeros all describe no chunks. Examples -------- >>> import numpy as np >>> view = LazyArray.from_numpy(np.arange(12).reshape(3, 4)) >>> [part.box for part in view.with_parts_per_axis(((1, 2), (4,))).parts()] [((0, 1), (0, 4)), ((1, 3), (0, 4))] """ entries = self._part_entries(sizes, "with_parts_per_axis") return self._with_grids(dimension_grids_from_chunks(entries, self._base_shape)) @staticmethod def _part_entries(parts: Sequence[Any], method: str) -> tuple[Any, ...]: """Materialize a partitioning argument, naming a non-iterable a ValueError. Both partitioning methods document ValueError for malformed input; a bare integer would otherwise surface as a TypeError from iteration. """ try: return tuple(parts) except TypeError as error: raise ValueError( f"{method} takes one entry per dimension of base_shape; got {parts!r}" ) from error def unpartitioned(self) -> LazyArray: """Return the same view, read in one pass. The source is treated as a single grid cell. Nonempty reads still pass its projection to the reader; empty reads make no reader calls. Returns ------- LazyArray The same view with no partitioning. """ return self._with_grids(None) def _with_grids(self, grids: tuple[DimensionGrid, ...] | None) -> LazyArray: return LazyArray._derive(self._array, self._transform, grids, self._reader) def parts(self) -> Iterator[Partition]: """Iterate the base partitioning, projected through this view. Single-use: this is a generator, so it is consumed by the first walk and a second `for` over the same object yields nothing. Call `parts()` again for a fresh walk, or keep a `list` of it if you need to revisit. Yields one [`Partition`][zarr_indexing.lazy_array.Partition] per box the view actually touches. The parts tile the view exactly and disjointly, and each view can be resolved with `part.view.result()`: in another thread, in another order, or not at all. Those views share this view's reader, and `LazyArray` does not serialize calls, so a stateful reader must synchronize its own mutable state. A wrapper with no partitioning (see `with_parts`) yields a single part covering the whole array. Yields ------ Partition One per touched box, in the resolver's own order. Examples -------- >>> import numpy as np >>> view = LazyArray.from_numpy(np.arange(12).reshape(3, 4)).with_parts((2, 2)) >>> part = next(view[:, 1:].parts()) >>> (part.base_coords, part.view.shape, part.is_complete) ((0, 0), (2, 1), False) """ grids = self._parts if self._parts is not None else _whole_array_grids(self._base_shape) to_buffer = tuple(-o for o in self._transform.domain.inclusive_min) for planned in plan_chunks(self._transform, grids): domain = planned.chunk_domain part_transform = planned.chunk_transform.translate(domain.inclusive_min) placement = _pure_translation(planned.cell_transform) if placement is not None: # A box part sits at a fixed offset in the request, so its view # keeps the request's literal coordinates: its domain is a # sub-domain of this view's, and placement is readable from it. part_transform = part_transform.translate_domain_by(placement) # The planner places cells in the request's literal domain; the # partition places them in this view's zero-origin result buffer. projection = replace( planned, cell_transform=planned.cell_transform.translate(to_buffer) ) yield Partition( projection=projection, box=tuple(zip(domain.inclusive_min, domain.exclusive_max, strict=True)), view=LazyArray._derive(self._array, part_transform, self._parts, self._reader), out_selection=_partition_out_selection(projection.cell_transform), _owner=self._part_owner, ) # -- indexing ----------------------------------------------------------- @property def oindex(self) -> _LazyOIndex: """Build a view using orthogonal (outer-product) indexing.""" return _LazyOIndex(self._select) @property def vindex(self) -> _LazyVIndex: """Build a view using vectorized coordinate or mask indexing.""" return _LazyVIndex(self._select) def _select(self, selection: Any, mode: SelectionMode) -> LazyArray: transform = self._transform if mode != "basic": # This frontend applies scalar integers first, dropping their axes, # then normalizes its supported advanced-indexing forms. scalar_selection, selection = split_scalar_axes(selection, transform.domain, mode) if scalar_selection is not None: transform = transform.select(scalar_selection, "basic") literal = normalize_positional_selection(selection, transform.domain, mode) if mode == "basic": # IndexTransform's basic path includes NumPy's `None`/newaxis. # `selection_to_transform` intentionally exposes a narrower basic # selection contract and rejects it. composed = transform[literal] else: composed = transform.select(literal, mode) return LazyArray._derive(self._array, composed, self._parts, self._reader) def __getitem__(self, selection: Any) -> LazyArray: """Build a view without reading source values. A NumPy key (integers, slices, ellipsis, `None`) is **positional**: `view[0]` is the first element and `view[-1]` the last, whatever literal coordinates the view's domain carries. An `IndexDomain` key is **literal**, restricting the view to those coordinates of its domain, and an `IndexTransform` key composes onto the view, mapping the key's domain into the view's domain. These are the two ways TensorStore's `__getitem__` addresses a view, offered here without changing what a NumPy key means. """ if isinstance(selection, IndexDomain): return self._select_domain(selection) if isinstance(selection, IndexTransform): return self._select_transform(selection) return self._select(selection, "basic") def _select_domain(self, domain: IndexDomain) -> LazyArray: """Restrict the view to a literal sub-domain of its own domain.""" if domain.ndim != self._transform.input_rank: raise ValueError( f"domain rank {domain.ndim} does not match view rank {self._transform.input_rank}" ) own = self._transform.domain for axis, (lo, hi) in enumerate( zip(domain.inclusive_min, domain.exclusive_max, strict=True) ): # Stricter than a literal slice, which admits an empty interval # anywhere: a domain key names coordinates of this view's domain. if lo < own.inclusive_min[axis] or hi > own.exclusive_max[axis]: raise BoundsCheckError( f"domain [{lo}, {hi}) on dimension {axis} is outside the view's domain " f"[{own.inclusive_min[axis]}, {own.exclusive_max[axis]})" ) literal = tuple( slice(lo, hi) for lo, hi in zip(domain.inclusive_min, domain.exclusive_max, strict=True) ) return LazyArray._derive(self._array, self._transform[literal], self._parts, self._reader) def _select_transform(self, transform: IndexTransform) -> LazyArray: """Compose a literal transform onto the view, as TensorStore's `store[transform]`.""" return LazyArray._derive( self._array, transform.compose(self._transform), self._parts, self._reader ) def __setitem__(self, selection: Any, values: Any) -> None: """Select a basic view and synchronously write its values.""" self[selection].write(values) def write(self, values: Any) -> None: """Synchronously write broadcastable values through this view. The source must support integer/slice assignment. Values are copied before mutation, including when they alias the source. Writes follow C-order view coordinates; the last occurrence wins for repeated source coordinates. Affine selections use one basic assignment. Other selections are scattered against the source's **write grid**, discovered from `write_chunk_sizes` or `chunks`: each touched cell is read once, updated in memory, and written back, so the number of storage round trips is bounded by the number of touched cells rather than selected elements. A NumPy source receives one fancy assignment instead, and a source with no advertised grid is written one element at a time without reading. The read-side partitioning (`with_parts`) does not affect writes. Writes go to the source directly and bypass the reader. A reader that caches source data is not invalidated, so reading after writing through such a reader may return stale values. Backend errors propagate and may leave a partially written source. This method does not provide transactions, concurrency control, or asynchronous execution. """ from zarr_indexing.writer import write_into write_into( self._array, self._transform, values, write_grid=_discover_write_grid(self._array, self._base_shape), ) def result(self, *, parts: Sequence[Partition] | None = None) -> Any: """Materialize this view. Every result starts as a fresh system-memory buffer. Each touched partition is read through the selected reader directly into its rectangular destination, or into an owned dense temporary before fancy placement. Empty views allocate without reading the source. Every reader call includes a chunk projection, even when the source is treated as a single grid cell by `unpartitioned()`. Parameters ---------- parts A reusable sequence previously returned by this exact view's `parts()` method. Supplying it reuses that partition plan instead of constructing another one. The parts must tile the view exactly. Returns ------- numpy.ndarray An array of shape `self.shape`, identical whatever partitioning is in force, always in fresh system memory. A view with a zero-rank domain returns a zero-dimensional array, not a scalar. Raises ------ ValueError If supplied parts were prepared by another view, or do not tile this view exactly. The output buffer is uninitialized where nothing was written, so a bad plan is reported rather than returned. AssertionError If this library's own partition walk fails to cover the view — a bug in zarr-indexing, never a consequence of the caller's input. """ prepared_parts = None if parts is None else tuple(parts) if prepared_parts is not None and any( # Module-private provenance deliberately crosses the two public # wrapper types without becoming part of either public surface. part._owner is not self._part_owner # pyright: ignore[reportPrivateUsage] for part in prepared_parts ): raise ValueError("prepared parts do not belong to this view") out_shape = self.shape if prepared_parts is not None: _validate_prepared_parts(prepared_parts, out_shape) out = self._output_buffer(out_shape) size = math.prod(out_shape) if size == 0: return out written = 0 selected_parts = self.parts() if prepared_parts is None else prepared_parts for part in selected_parts: # Counted before the scatter, so a part addressing the wrong # number of axes is named rather than reported as a broadcast # failure against the buffer. written += _out_selection_cell_count(part.out_selection, out_shape) direct = all(isinstance(selector, slice) for selector in part.out_selection) if direct: destination = out if len(part.out_selection) == 0 else out[part.out_selection] else: destination = part.view._output_buffer(part.view.shape) _invoke_reader( self._reader, self._array, ReadContext(part.view.transform, part.projection), destination, ) if not direct: out[part.out_selection] = destination if written != size: # The buffer is uninitialized where no part wrote, so a partition # walk that does not tile the view exactly would otherwise hand # back process memory dressed as data. The parts are disjoint by # contract, so counting the cells each addresses is enough: # a gap alone undercounts and an overlap alone overcounts. This # count cannot detect a compensating gap and overlap; absence of overlap # still relies on the planner contract. if prepared_parts is not None: raise ValueError( "prepared parts do not tile the view exactly: " f"they addressed {written} of the view's {size} cells" ) raise AssertionError( f"the partition walk addressed {written} of the view's {size} " "cells; this is a bug in zarr-indexing's partition walk" ) return out def _output_buffer(self, out_shape: tuple[int, ...]) -> Any: """The buffer `result()` scatters parts into. Deliberately uninitialized: every cell is written by exactly one part, as required by the planner contract. `result()` checks the total count; caller-supplied parts additionally undergo coverage validation. A masked source gets a masked buffer so that reader writes preserve the mask; other source- specific array types do not survive materializing. """ dtype = np.dtype(self.dtype) if isinstance(self._array, np.ma.MaskedArray): return np.ma.masked_all(out_shape, dtype=dtype) return np.empty(out_shape, dtype=dtype) # -- protocols ---------------------------------------------------------- def __array__(self, dtype: Any = None, copy: bool | None = None) -> Any: """Materialize the view as a NumPy array. The result never shares memory with the wrapped array, whatever `copy` asks for: `result()` already allocates, so `copy=True` gets an array the caller owns. `copy=None` need not copy again when the dtype already matches; requesting another dtype can allocate a conversion buffer. `copy=False` is refused, because materializing means reading — this API always materializes into its own result buffer. """ if copy is False: raise ValueError( "a LazyArray cannot be converted to a NumPy array without a " "copy: a view is a description of a read, and the values only " "exist once the read is made" ) return np.asarray(self.result(), dtype=dtype) def __dask_tokenize__(self) -> Any: """Tokenize the source and serialized transform. The transform contributes a digest of JSON produced by `to_json()`, so its JSON is not embedded in the returned token, but is allocated while computing the digest. Equal serialized transforms and equal source tokens produce equal tokens; arbitrary semantically equivalent mappings are not guaranteed to serialize identically. Dask tokenizes the wrapped source using its normal dispatch and determinism policy. This may read or hash source values. Dask is imported only when this method is called and is otherwise optional. The reader and partitioning are omitted because they must preserve values. Mutating a source does not update keys in existing Dask graphs. """ from dask.base import tokenize # pyright: ignore[reportMissingImports] canonical = json.dumps(self._transform.to_json(), sort_keys=True) return ( type(self).__qualname__, tokenize(self._array), hashlib.sha256(canonical.encode()).hexdigest(), ) def __len__(self) -> int: """The length of the first axis, as for a NumPy array; `TypeError` on a 0-d view.""" if self.ndim == 0: raise TypeError("len() of unsized object") return self.shape[0] def __iter__(self) -> Iterator[LazyArray]: """Iterate over lazy first-axis views without reading source values. Each element is a `LazyArray`, not a value: call `result()` or convert with NumPy before doing arithmetic on it. The rank check happens in `__iter__` itself rather than in the generator, so `iter(view)` on a zero-rank view raises immediately as NumPy's does, instead of waiting for the first `next`. """ if self.ndim == 0: raise TypeError("iteration over a 0-d array") return (self[position] for position in range(self.shape[0])) # NumPy's own conversions decide what a size-1 (or wrong-sized) view means, # including which exception it raises, so these delegate rather than # reimplement. Each materializes the view first. def __bool__(self) -> bool: return bool(self.result()) def __int__(self) -> int: return int(self.result()) def __float__(self) -> float: return float(self.result()) def __index__(self) -> int: return operator.index(self.result()) def __repr__(self) -> str: wrapped = type(self._array).__name__ described = [f"{wrapped} shape={self.shape} dtype={self.dtype}"] if not _is_identity_transform(self._transform, self._base_shape): described.append(f"view={self._transform.selection_repr}") return f"" class _LazyOIndex: """`view.oindex[...]` — one selection per axis, combined as an outer product.""" __slots__ = ("_select",) def __init__(self, select: SelectFn) -> None: self._select = select def __getitem__(self, selection: Any) -> LazyArray: return self._select(selection, "orthogonal") def __setitem__(self, selection: Any, values: Any) -> None: self[selection].write(values) class _LazyVIndex: """`view.vindex[...]` — correlated coordinate arrays, or a single mask.""" __slots__ = ("_select",) def __init__(self, select: SelectFn) -> None: self._select = select def __getitem__(self, selection: Any) -> LazyArray: return self._select(selection, "vectorized") def __setitem__(self, selection: Any, values: Any) -> None: self[selection].write(values) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/messages.py000066400000000000000000000721421527014137600316450ustar00rootroot00000000000000"""The ndsel message layer — pure JSON in, canonical JSON out. This module implements the [ndsel](https://github.com/zarr-developers/ndsel) draft wire format: a JSON-serializable representation of NumPy-style n-dimensional selections that adapts TensorStore's `IndexTransform` model. It is a **pure JSON→JSON** layer depending only on the standard library. It validates and desugars messages, removing redundant fields on constant maps. It limits input rank to 32 and checks affine references against that rank. Finite bounds and in-memory array construction are enforced by the engine lowering layer. Two entry points: - `parse_ndsel(obj)` — structurally validate an ndsel message of any of the five kinds (`point`/`box`/`slice`/`points`/`transform`), returning it unchanged. Raises `NdselError` (carrying a spec reason code) on any defect. - `normalize_ndsel(obj)` — desugar and canonicalize a message to the single deterministic **canonical transform body** of the spec (section 4.3): a bare `IndexTransform` JSON body, without the `kind` discriminator. `normalize` is idempotent when its output is re-tagged with `kind: "transform"`. The canonical body uses TensorStore's `IndexTransform` field vocabulary. Normalization alone does not guarantee TensorStore acceptance: index-array content is deferred, and TensorStore has additional coordinate and label limits. Value rules for validated fields (excluding the verbatim `index_array` payload and discarded constant-map fields): integers are 64-bit signed values; JSON booleans are **not** integers (Python's `isinstance(True, int)` is guarded against explicitly); the `"-inf"`/`"+inf"` sentinels are legal only in bound positions; an implicit bound is the one-element `[n]`-bracket form, and its implicit/explicit flag is preserved through normalization. """ from __future__ import annotations from typing import Any __all__ = [ "NdselError", "normalize_ndsel", "parse_ndsel", ] # --------------------------------------------------------------------------- # Error taxonomy # --------------------------------------------------------------------------- #: Current ndsel reason codes plus the recognized retired negative-step code. REASON_CODES = frozenset( { "invalid_json", "unknown_kind", "unknown_field", "multiple_upper_bounds", "bounds_out_of_order", "output_map_conflict", "rank_mismatch", "step_zero", # Retired in 1.0-draft.2, when negative `step` became specified. Kept in # the set so a message carrying the code is still recognized, but no # condition in this implementation emits it. "negative_step_unsupported", } ) class NdselError(ValueError): """An ndsel message failed validation. Carries the spec `reason` code (one of `REASON_CODES`) so callers and the conformance harness can assert on it directly, plus a human-readable `detail`. Examples -------- >>> try: ... normalize_ndsel({"kind": "bogus"}) ... except NdselError as error: ... (error.reason, str(error)) ('unknown_kind', "unknown_kind: unknown kind 'bogus'") """ def __init__(self, reason: str, detail: str = "") -> None: """Store `reason` and `detail` and compose the message as `"reason: detail"`. `reason` is a spec reason code (one of `REASON_CODES`); `detail` is optional human-readable context, and when empty the message is the bare `reason`. """ self.reason = reason self.detail = detail super().__init__(f"{reason}: {detail}" if detail else reason) # --------------------------------------------------------------------------- # 64-bit signed integer range (spec section 3.5) # --------------------------------------------------------------------------- _I64_MIN = -(2**63) _I64_MAX = 2**63 - 1 _KNOWN_KINDS = frozenset({"point", "box", "slice", "points", "transform"}) # The two upper-bound spellings, keyed by message prefix. Only one of the three # per group may appear (spec section 4.1 / 5.2). _BOX_UPPER = ("exclusive_max", "inclusive_max", "shape") _TRANSFORM_UPPER = ("input_exclusive_max", "input_inclusive_max", "input_shape") _OUTPUT_MAP_FIELDS = frozenset( { "offset", "stride", "input_dimension", "index_array", "index_array_bounds", } ) # An upper bound on `input_rank`, because normalization allocates proportionally # to it — an identity `output`, a bound per dimension, a label per dimension — # from a document that carries no data behind the number. Matches the rank # TensorStore accepts. This is an implementation limit, not an ndsel limit. _MAX_RANK = 32 # --------------------------------------------------------------------------- # Leaf value validators # --------------------------------------------------------------------------- def _is_int(value: Any) -> bool: """True iff `value` is a JSON integer — an `int` that is not a `bool`. JSON has no boolean-as-integer: `True`/`False` are rejected even though Python makes `bool` a subclass of `int` (spec section 3.6). """ return isinstance(value, int) and not isinstance(value, bool) def _check_int(value: Any, where: str) -> int: """Validate a plain-integer position: an in-range i64, never a sentinel.""" if not _is_int(value): raise NdselError("invalid_json", f"{where} must be an integer, got {value!r}") if value < _I64_MIN or value > _I64_MAX: raise NdselError("invalid_json", f"{where} is outside the 64-bit signed range: {value}") return int(value) def _is_sentinel(value: Any) -> bool: return value in ("-inf", "+inf") def _check_index_value(value: Any, where: str) -> int | str: """Validate an `index-value`: an in-range i64 or a `"-inf"`/`"+inf"` sentinel.""" if _is_sentinel(value): return str(value) return _check_int(value, where) def _check_bound(value: Any, where: str) -> int | str | list[int | str]: """Validate a `bound`: an explicit `index-value`, or a one-element implicit `[index-value]`.""" if isinstance(value, list): if len(value) != 1: raise NdselError( "invalid_json", f"{where} implicit bound must be a one-element array, got {value!r}", ) return [_check_index_value(value[0], where)] return _check_index_value(value, where) def _check_int_list(value: Any, where: str) -> list[int]: if not isinstance(value, list): raise NdselError("invalid_json", f"{where} must be an array, got {value!r}") return [_check_int(v, f"{where}[{i}]") for i, v in enumerate(value)] def _check_bound_list(value: Any, where: str) -> list[Any]: if not isinstance(value, list): raise NdselError("invalid_json", f"{where} must be an array, got {value!r}") return [_check_bound(v, f"{where}[{i}]") for i, v in enumerate(value)] def _check_label_list(value: Any, where: str) -> list[str]: if not isinstance(value, list): raise NdselError("invalid_json", f"{where} must be an array, got {value!r}") for i, v in enumerate(value): if not isinstance(v, str): raise NdselError("invalid_json", f"{where}[{i}] must be a string, got {v!r}") return list(value) # --------------------------------------------------------------------------- # Extended-integer order for bounds (spec section 4.1) # --------------------------------------------------------------------------- def _bound_value(bound: int | str | list[int | str]) -> int | str: """The underlying `index-value` of a bound, dropping the implicit bracket.""" return bound[0] if isinstance(bound, list) else bound def _bound_is_implicit(bound: int | str | list[int | str]) -> bool: return isinstance(bound, list) def _finite(value: int | str, where: str) -> int: """The integer behind a validated `index-value` that is not a sentinel. Every value reaching the desugaring helpers has passed `_check_index_value`, so a string here is one of the two sentinels, which each helper handles before calling this. Anything else is an internal error, not bad input. """ if isinstance(value, int): return value raise RuntimeError(f"{where} is {value!r}; expected an integer or an infinity sentinel") def _ext_key(value: int | str) -> tuple[int, int]: """A sort key giving the extended-integer order `-inf < n < +inf` exactly. Uses an integer tier plus the value, so no float rounding of near-`2**63` integers can misorder the `inclusive_min <= exclusive_max` check. """ if value == "-inf": return (0, 0) if value == "+inf": return (2, 0) return (1, _finite(value, "bound")) def _rewrap(value: int | str, *, implicit: bool) -> int | str | list[int | str]: return [value] if implicit else value # --------------------------------------------------------------------------- # Message-level helpers # --------------------------------------------------------------------------- def _require_object(obj: Any) -> dict[str, Any]: if not isinstance(obj, dict): raise NdselError("invalid_json", f"message must be a JSON object, got {type(obj).__name__}") return obj def _message_kind(obj: dict[str, Any]) -> str: kind = obj.get("kind") if not isinstance(kind, str): raise NdselError("invalid_json", "message must have a string 'kind' field") if kind not in _KNOWN_KINDS: raise NdselError("unknown_kind", f"unknown kind {kind!r}") return kind def _check_membership(obj: dict[str, Any], allowed: frozenset[str], what: str) -> None: """Strict membership (spec section 3.7): reject any undefined member.""" for key in obj: if key not in allowed: raise NdselError("unknown_field", f"{what} has undefined member {key!r}") def _single_upper_bound(obj: dict[str, Any], fields: tuple[str, str, str]) -> str | None: present = [f for f in fields if f in obj] if len(present) > 1: raise NdselError( "multiple_upper_bounds", f"at most one of {fields} may be present; got {present}", ) return present[0] if present else None def _resolve_upper_bound( upper_field: str | None, upper_raw: list[Any] | None, inclusive_min: list[Any], rank: int, *, kind_of: str, ) -> list[int | str | list[int | str]]: """Produce `exclusive_max` from whichever upper-bound spelling was supplied. - `exclusive_max`/`input_exclusive_max` → used directly. - `inclusive_max`/`input_inclusive_max` → each element `+1`. - `shape`/`input_shape` → `inclusive_min + shape` per element. - none → an **implicit `+inf`** in every dimension. The implicit/explicit bracket travels with the extent-bearing field (the upper bound, or `shape`), matching the spec's `[n]`-bracket convention. """ if upper_raw is None: # No upper-bound field was supplied (`upper_field` is None as well). return [["+inf"] for _ in range(rank)] if kind_of == "exclusive": return list(upper_raw) result: list[int | str | list[int | str]] = [] for k in range(rank): raw = upper_raw[k] implicit = _bound_is_implicit(raw) value = _bound_value(raw) if kind_of == "inclusive": new = _inclusive_to_exclusive(value, f"{upper_field}[{k}]") else: # shape new = _shape_to_exclusive(_bound_value(inclusive_min[k]), value, f"{upper_field}[{k}]") result.append(_rewrap(new, implicit=implicit)) return result def _checked_i64(value: int, where: str) -> int: """An arithmetic result that must still be a 64-bit signed integer. Normalization is idempotent (spec section 4.3): whatever it emits must pass the same validation on the way back in. Desugaring adds — `inclusive_max + 1`, `inclusive_min + shape` — so a bound at the top of the range would otherwise be emitted one past it and rejected by the next call on our own output. """ if value < _I64_MIN or value > _I64_MAX: raise NdselError( "invalid_json", f"{where} is {value}, which is outside the 64-bit signed range; the " f"normalized form cannot represent it", ) return value def _inclusive_to_exclusive(value: int | str, where: str) -> int | str: if value == "+inf" or value == "-inf": return value return _checked_i64(_finite(value, where) + 1, f"{where} converted to an exclusive bound") def _shape_to_exclusive(min_value: int | str, shape_value: int | str, where: str) -> int | str: if shape_value == "-inf": raise NdselError( "invalid_json", f"{where} is '-inf'; a shape counts cells and cannot be negatively infinite", ) if shape_value == "+inf" or min_value == "+inf": return "+inf" if min_value == "-inf": return "-inf" return _checked_i64( _finite(min_value, f"inclusive_min for {where}") + _finite(shape_value, where), f"{where} added to its inclusive_min", ) def _validate_domain(inclusive_min: list[Any], exclusive_max: list[Any], *, prefix: str) -> None: """Every dimension must satisfy `inclusive_min <= exclusive_max` (empty is valid).""" for k, (lo, hi) in enumerate(zip(inclusive_min, exclusive_max, strict=True)): if _ext_key(_bound_value(lo)) > _ext_key(_bound_value(hi)): raise NdselError( "bounds_out_of_order", f"{prefix}[{k}]: inclusive_min {_bound_value(lo)!r} > " f"exclusive_max {_bound_value(hi)!r}", ) def _identity_output(rank: int) -> list[dict[str, Any]]: return [{"offset": 0, "stride": 1, "input_dimension": k} for k in range(rank)] # --------------------------------------------------------------------------- # Per-kind desugaring # --------------------------------------------------------------------------- def _normalize_point(obj: dict[str, Any]) -> dict[str, Any]: _check_membership(obj, frozenset({"kind", "coords"}), "point") if "coords" not in obj: raise NdselError("invalid_json", "point requires 'coords'") coords = _check_int_list(obj["coords"], "coords") return { "input_rank": 0, "input_inclusive_min": [], "input_exclusive_max": [], "input_labels": [], "output": [{"offset": c} for c in coords], } def _infer_rank( obj: dict[str, Any], named_lengths: list[tuple[str, int]], *, declared: int | None, ) -> int: """Reconcile a declared rank (if any) with every present array's length.""" rank = declared for name, length in named_lengths: if rank is None: rank = length elif rank != length: raise NdselError( "rank_mismatch", f"{name} has length {length}, inconsistent with rank {rank}", ) return rank if rank is not None else 0 def _normalize_box(obj: dict[str, Any]) -> dict[str, Any]: allowed = frozenset( {"kind", "inclusive_min", "exclusive_max", "inclusive_max", "shape", "labels"} ) _check_membership(obj, allowed, "box") inclusive_min_raw = ( _check_bound_list(obj["inclusive_min"], "inclusive_min") if "inclusive_min" in obj else None ) upper_field = _single_upper_bound(obj, _BOX_UPPER) upper_raw = _check_bound_list(obj[upper_field], upper_field) if upper_field else None labels_raw = _check_label_list(obj["labels"], "labels") if "labels" in obj else None named_lengths: list[tuple[str, int]] = [] if inclusive_min_raw is not None: named_lengths.append(("inclusive_min", len(inclusive_min_raw))) if upper_raw is not None: named_lengths.append((upper_field or "", len(upper_raw))) if labels_raw is not None: named_lengths.append(("labels", len(labels_raw))) rank = _infer_rank(obj, named_lengths, declared=None) inclusive_min = inclusive_min_raw if inclusive_min_raw is not None else [0] * rank exclusive_max = _resolve_upper_bound( upper_field, upper_raw, inclusive_min, rank, kind_of=_upper_kind(upper_field, _BOX_UPPER) ) labels = labels_raw if labels_raw is not None else [""] * rank _validate_domain(inclusive_min, exclusive_max, prefix="box") return { "input_rank": rank, "input_inclusive_min": inclusive_min, "input_exclusive_max": exclusive_max, "input_labels": labels, "output": _identity_output(rank), } def _upper_kind(upper_field: str | None, fields: tuple[str, str, str]) -> str: if upper_field is None or upper_field == fields[0]: return "exclusive" if upper_field == fields[1]: return "inclusive" return "shape" def _normalize_slice(obj: dict[str, Any]) -> dict[str, Any]: allowed = frozenset({"kind", "start", "stop", "step", "labels"}) _check_membership(obj, allowed, "slice") if "start" not in obj: raise NdselError("invalid_json", "slice requires 'start'") if "stop" not in obj: raise NdselError("invalid_json", "slice requires 'stop'") start = _check_int_list(obj["start"], "start") stop = _check_int_list(obj["stop"], "stop") step = _check_int_list(obj["step"], "step") if "step" in obj else [1] * len(start) labels_raw = _check_label_list(obj["labels"], "labels") if "labels" in obj else None n = len(start) for name, arr in (("stop", stop), ("step", step)): if len(arr) != n: raise NdselError( "rank_mismatch", f"{name} has length {len(arr)}, expected {n} (from start)" ) if labels_raw is not None and len(labels_raw) != n: raise NdselError( "rank_mismatch", f"labels has length {len(labels_raw)}, expected {n} (from start)" ) for k, s in enumerate(step): if s == 0: raise NdselError("step_zero", f"step[{k}] is zero") inclusive_min: list[Any] = [] exclusive_max: list[Any] = [] output: list[dict[str, Any]] = [] for k in range(n): a, b, s = start[k], stop[k], step[k] # One rule for both signs (spec 5.3): the traversal runs from `a` # toward `b`, so the source interval's length is `b - a` going up and # `a - b` going down. length = (b - a) if s > 0 else (a - b) if length < 0: # A reversed interval is a mistake about the direction of travel, # not an empty selection. `b == a` is the way to select nothing. raise NdselError( "bounds_out_of_order", f"start[{k}]={a} and stop[{k}]={b} with step {s} run the wrong " "way; an empty selection is spelled stop == start", ) m = -(-length // abs(s)) # ceil(length / |s|) o = _trunc_div(a, s) # trunc(a / s), toward zero, both signs offset = a - s * o # lattice phase, |offset| < |s| inclusive_min.append(_checked_i64(o, f"input_inclusive_min[{k}]")) exclusive_max.append(_checked_i64(o + m, f"input_exclusive_max[{k}]")) output.append({"offset": offset, "stride": s, "input_dimension": k}) labels = labels_raw if labels_raw is not None else [""] * n return { "input_rank": n, "input_inclusive_min": inclusive_min, "input_exclusive_max": exclusive_max, "input_labels": labels, "output": output, } def _normalize_points(obj: dict[str, Any]) -> dict[str, Any]: _check_membership(obj, frozenset({"kind", "coords"}), "points") if "coords" not in obj: raise NdselError("invalid_json", "points requires 'coords'") coords = obj["coords"] if not isinstance(coords, list): raise NdselError("invalid_json", f"points coords must be an array, got {coords!r}") rows: list[list[int]] = [] n: int | None = None for i, row in enumerate(coords): if not isinstance(row, list): raise NdselError("invalid_json", f"points coords[{i}] must be an array, got {row!r}") row_ints = [_check_int(v, f"coords[{i}][{j}]") for j, v in enumerate(row)] if n is None: n = len(row_ints) elif len(row_ints) != n: raise NdselError( "rank_mismatch", f"points coords[{i}] has length {len(row_ints)}, expected {n} (ragged)", ) rows.append(row_ints) m = len(rows) n = n if n is not None else 0 output = [ { "offset": 0, "stride": 1, "index_array": [rows[i][k] for i in range(m)], "index_array_bounds": ["-inf", "+inf"], } for k in range(n) ] return { "input_rank": 1, "input_inclusive_min": [0], "input_exclusive_max": [m], "input_labels": [""], "output": output, } def _normalize_output_map(raw: Any, where: str) -> dict[str, Any]: if not isinstance(raw, dict): raise NdselError("invalid_json", f"{where} must be a JSON object, got {raw!r}") _check_membership(raw, _OUTPUT_MAP_FIELDS, where) has_index_array = "index_array" in raw has_input_dim = "input_dimension" in raw if has_index_array and has_input_dim: raise NdselError( "output_map_conflict", f"{where} carries both 'input_dimension' and 'index_array'", ) offset = _check_int(raw["offset"], f"{where}.offset") if "offset" in raw else 0 if has_index_array: stride = _check_int(raw["stride"], f"{where}.stride") if "stride" in raw else 1 bounds = ( validate_index_array_bounds(raw["index_array_bounds"], where) if "index_array_bounds" in raw else ["-inf", "+inf"] ) # index_array is carried verbatim (spec section 7 defers shape validation). normalized: dict[str, Any] = { "offset": offset, "stride": stride, "index_array": raw["index_array"], "index_array_bounds": bounds, } return normalized if has_input_dim: input_dim = _check_int(raw["input_dimension"], f"{where}.input_dimension") if input_dim < 0: raise NdselError( "invalid_json", f"{where}.input_dimension must be >= 0, got {input_dim}" ) stride = _check_int(raw["stride"], f"{where}.stride") if "stride" in raw else 1 return {"offset": offset, "stride": stride, "input_dimension": input_dim} # Constant map: only offset survives. A stray `stride`/`index_array_bounds` # is schema-valid (the output-map schema permits those members on any map), # so it is silently dropped rather than rejected — a constant carries only # `offset` in canonical form (spec section 4.3). return {"offset": offset} def validate_index_array_bounds(value: Any, where: str) -> list[int | str]: """Validate the syntax and ordering of an inclusive index-array interval.""" if not isinstance(value, list) or len(value) != 2: raise NdselError( "invalid_json", f"{where}.index_array_bounds must be a two-element array, got {value!r}", ) lo = _check_index_value(value[0], f"{where}.index_array_bounds[0]") hi = _check_index_value(value[1], f"{where}.index_array_bounds[1]") if _ext_key(lo) > _ext_key(hi): raise NdselError( "bounds_out_of_order", f"{where}.index_array_bounds: lower bound {lo!r} > upper bound {hi!r}", ) return [lo, hi] def _normalize_transform(obj: dict[str, Any]) -> dict[str, Any]: allowed = frozenset( { "kind", "input_rank", "input_inclusive_min", "input_exclusive_max", "input_inclusive_max", "input_shape", "input_labels", "output", } ) _check_membership(obj, allowed, "transform") declared_rank: int | None = None if "input_rank" in obj: declared_rank = _check_int(obj["input_rank"], "input_rank") if declared_rank < 0: raise NdselError("invalid_json", f"input_rank must be >= 0, got {declared_rank}") if declared_rank > _MAX_RANK: # Normalization fills a bound, a label and an identity output map per # dimension, so an unbacked rank is a request to allocate from a # document that carries nothing. raise NdselError( "invalid_json", f"input_rank must be <= {_MAX_RANK}, got {declared_rank}", ) inclusive_min_raw = ( _check_bound_list(obj["input_inclusive_min"], "input_inclusive_min") if "input_inclusive_min" in obj else None ) upper_field = _single_upper_bound(obj, _TRANSFORM_UPPER) upper_raw = _check_bound_list(obj[upper_field], upper_field) if upper_field else None labels_raw = ( _check_label_list(obj["input_labels"], "input_labels") if "input_labels" in obj else None ) named_lengths: list[tuple[str, int]] = [] if inclusive_min_raw is not None: named_lengths.append(("input_inclusive_min", len(inclusive_min_raw))) if upper_raw is not None: named_lengths.append((upper_field or "", len(upper_raw))) if labels_raw is not None: named_lengths.append(("input_labels", len(labels_raw))) rank = _infer_rank(obj, named_lengths, declared=declared_rank) inclusive_min = inclusive_min_raw if inclusive_min_raw is not None else [0] * rank exclusive_max = _resolve_upper_bound( upper_field, upper_raw, inclusive_min, rank, kind_of=_upper_kind(upper_field, _TRANSFORM_UPPER), ) labels = labels_raw if labels_raw is not None else [""] * rank _validate_domain(inclusive_min, exclusive_max, prefix="input") if "output" in obj: if not isinstance(obj["output"], list): raise NdselError("invalid_json", f"output must be an array, got {obj['output']!r}") output = [_normalize_output_map(m, f"output[{i}]") for i, m in enumerate(obj["output"])] for i, m in enumerate(output): # An `input_dimension` names one of *this* transform's input # dimensions, so the rank is what bounds it. Checked here rather than # in `_normalize_output_map`, which sees one map and not the rank. if "input_dimension" in m and m["input_dimension"] >= rank: raise NdselError( "rank_mismatch", f"output[{i}].input_dimension is {m['input_dimension']}, " f"outside the valid range [0, {rank}) for input_rank {rank}", ) else: output = _identity_output(rank) return { "input_rank": rank, "input_inclusive_min": inclusive_min, "input_exclusive_max": exclusive_max, "input_labels": labels, "output": output, } _NORMALIZERS = { "point": _normalize_point, "box": _normalize_box, "slice": _normalize_slice, "points": _normalize_points, "transform": _normalize_transform, } # --------------------------------------------------------------------------- # trunc division (spec section 5.3 correction, matches _trunc_div in transform.py) # --------------------------------------------------------------------------- def _trunc_div(a: int, b: int) -> int: """Integer division rounded toward zero (C semantics).""" q = a // b if q < 0 and q * b != a: q += 1 return q # --------------------------------------------------------------------------- # Public entry points # --------------------------------------------------------------------------- def normalize_ndsel(obj: Any) -> dict[str, Any]: """Desugar and canonicalize an ndsel message to its canonical transform body. Accepts any of the five message kinds and returns the bare canonical `IndexTransform` body of spec section 4.3 — no `kind` field. Raises `NdselError` (carrying a reason code) for any invalid input. Examples -------- >>> body = normalize_ndsel({"kind": "box", "shape": [2, 3]}) >>> (body["input_rank"], body["input_inclusive_min"], body["input_exclusive_max"]) (2, [0, 0], [2, 3]) >>> body["output"][0] {'offset': 0, 'stride': 1, 'input_dimension': 0} """ message = _require_object(obj) kind = _message_kind(message) canonical = _NORMALIZERS[kind](message) # Apply the same limit to inferred and shorthand ranks as to explicit # transform ranks, so every result can be normalized again. if canonical["input_rank"] > _MAX_RANK: raise NdselError( "invalid_json", f"input_rank must be <= {_MAX_RANK}, got {canonical['input_rank']}" ) return canonical def parse_ndsel(obj: Any) -> dict[str, Any]: """Structurally validate an ndsel message, returning it unchanged. Runs the same validation and desugaring as `normalize_ndsel`, discards the canonical body, and returns the original object. Useful for validating a message you intend to keep in its compact shorthand form. Index-array payload validation remains the engine's responsibility. Examples -------- >>> message = {"kind": "point", "coords": [3, 4]} >>> parse_ndsel(message) is message True >>> normalize_ndsel(message)["output"] [{'offset': 3}, {'offset': 4}] """ message = _require_object(obj) _message_kind(message) # Validation and desugaring share one pass; run it and discard the body. normalize_ndsel(message) return message zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/output_map.py000066400000000000000000000403551527014137600322340ustar00rootroot00000000000000"""Output index maps — three ordered mappings to integer coordinates. An output index map describes how input cells address one dimension of the output space. Its coordinates form an **ordered, duplicate-preserving sequence** aligned with the input domain, never a mathematical set. Three representations cover the cases that arise in practice: - `ConstantMap(offset=5)` — every request cell maps to coordinate `5` - `DimensionMap(input_dimension=0, offset=3, stride=2)` over input `[0, 5)` — the ordered arithmetic progression `[3, 5, 7, 9, 11]` - `ArrayMap(index_array=[5, 1, 1])` — the explicit sequence `[5, 1, 1]`, preserving both order and the repeated coordinate Every output map participates in two operations defined on `IndexTransform`, which provides the input-domain context these maps lack: - **intersect** — retain mapped cells whose coordinates lie within a range (e.g., a chunk), without changing their order or multiplicity. Restricting `[3, 5, 5, 9]` to `[4, 8)` produces `[5, 5]`. - **translate** — shift every coordinate by a constant (e.g., make chunk-local). Translating `[5, 5, 7]` by `-4` produces `[1, 1, 3]`. These two operations are the foundation of chunk resolution: for each chunk, intersect the map with the chunk's range, then translate to chunk-local coordinates. The three types exist because they trade off generality for efficiency: - `ConstantMap`: O(1) storage, O(1) intersection - `DimensionMap`: O(1) storage, O(1) intersection (analytical) - `ArrayMap`: O(n) storage, O(n) intersection (must scan the array) Collapsing everything to `ArrayMap` would be correct but wasteful — a billion-element slice would materialize a billion coordinates just to group them by chunk, when `DimensionMap` does it with three integers. """ from __future__ import annotations from dataclasses import dataclass from typing import TYPE_CHECKING, Any import numpy as np from zarr_indexing._affine import checked_affine if TYPE_CHECKING: import numpy.typing as npt from zarr_indexing.json import OutputIndexMapJSON def _array_map_dependency_axes(index_array: np.ndarray[Any, Any]) -> tuple[int, ...]: """Return the input axes on which a normalized index array varies. Normalized `ArrayMap` index arrays carry the full input rank of their enclosing transform: an axis the array varies over has its full size, while an axis the array is independent of is a singleton (size 1). The dependency axes are therefore exactly the axes of size 2 or more. This is structural: values are not inspected for constant or repeated coordinates. An array with shape `(0, 2)` is empty but still reports axis 1; the zero-size axis itself is not reported. """ return tuple(axis for axis, size in enumerate(index_array.shape) if size > 1) @dataclass(frozen=True, slots=True) class ConstantMap: """A constant output-coordinate mapping. Every input cell maps to `offset`. Arises from integer indexing (e.g., `arr[5]` fixes one dimension to coordinate 5). Examples -------- Every input cell maps to the same output coordinate, like broadcasting coordinate 5 with `np.broadcast_to(5, (3,))`. Repeated fancy indices can also describe these coordinates, using an explicit list: >>> from zarr_indexing.domain import IndexDomain >>> from zarr_indexing.transform import IndexTransform >>> domain = IndexDomain.from_shape((3,)) >>> t = IndexTransform(domain=domain, output=(ConstantMap(offset=5),)) >>> t.apply((0,)), t.apply((1,)), t.apply((2,)) ((5,), (5,), (5,)) """ offset: int = 0 """The fixed output coordinate every input cell maps to.""" def to_json(self) -> OutputIndexMapJSON: """Convert to the canonical wire form: the bare `constant` map. Examples -------- >>> ConstantMap(5).to_json() {'offset': 5} """ return {"offset": self.offset} @dataclass(frozen=True, slots=True) class DimensionMap: """An ordered affine mapping to output coordinates. Maps each input coordinate `i` to `offset + stride * i`, where the input range comes from the enclosing `IndexTransform`'s domain. Arises from slice indexing (e.g., `arr[2:10:3]` gives offset=2, stride=3). Examples -------- The slice `arr[2:11:3]` reads coordinates `2, 5, 8` — the rule `offset + stride * i` with `offset=2`, `stride=3`: >>> m = DimensionMap(input_dimension=0, offset=2, stride=3) >>> [m.offset + m.stride * i for i in range(3)] [2, 5, 8] >>> np.arange(11)[2:11:3].tolist() [2, 5, 8] """ input_dimension: int """The input (domain) dimension whose coordinate this map reads.""" offset: int = 0 """The output coordinate that input coordinate `0` maps to.""" stride: int = 1 """The output-coordinate step per unit input step; negative walks backward, zero repeats `offset`.""" def to_json(self) -> OutputIndexMapJSON: """Convert to the canonical wire form: the `single_input_dimension` map. Examples -------- >>> DimensionMap(input_dimension=1, offset=0, stride=2).to_json() {'offset': 0, 'stride': 2, 'input_dimension': 1} """ return { "offset": self.offset, "stride": self.stride, "input_dimension": self.input_dimension, } @dataclass(frozen=True, slots=True) class ArrayMap: """An explicit ordered, duplicate-preserving coordinate mapping. Maps each input position `i` to `offset + stride * index_array[i]`. Index-array order and repeated entries are semantic and remain present in the result. Arises from fancy indexing (e.g., `arr[[5, 1, 1]]` or boolean masks). A map used in a transform must have its **full input rank**: `index_array` has the enclosing domain's rank, sized fully on the axes it varies over and singleton (size 1) elsewhere. The shape is the single source of truth for what the map depends on — its **dependency axes** are exactly its axes of size greater than one (see `_array_map_dependency_axes`) — and it distinguishes the two flavors of multi-array fancy indexing: - **orthogonal** (`oindex`): each array varies along a single, *distinct* axis (all others singleton); the result is their outer product. - **vectorized** (`vindex`): the arrays are correlated and share the same non-singleton (broadcast) axes; the result is a pointwise scatter. A map holding exactly one coordinate carries no shape to read a dependency from, and none is needed: it is the `ConstantMap` it equals, and the selection layer builds that instead (see `array_map_or_constant`). A hand-built all-singleton `ArrayMap` is still a valid value; resolution classifies it with the correlated maps and reads it pointwise. Examples -------- The fancy selection `arr[[5, 1, 1]]` reads coordinate 5, then 1, then 1 — order and the duplicate preserved, exactly as NumPy fancy indexing: >>> m = ArrayMap(index_array=np.array([5, 1, 1])) >>> [m.offset + m.stride * c for c in m.index_array.tolist()] [5, 1, 1] >>> np.arange(10)[[5, 1, 1]].tolist() [5, 1, 1] """ index_array: npt.NDArray[np.integer[Any]] """Explicit coordinates at the enclosing transform's full input rank; order and duplicates are semantic. Its non-singleton axes are the map's dependency axes.""" offset: int = 0 """Constant term of the affine adjustment: the output coordinate is `offset + stride * index_array[i]`.""" stride: int = 1 """Multiplier applied to each `index_array` value before `offset` is added.""" def __post_init__(self) -> None: """Own an immutable snapshot of the integer index coordinates. The snapshot is backed by immutable bytes, so callers cannot modify it or re-enable its WRITEABLE flag. Changes to the supplied array do not change the map's coordinates or hash. This freezes the coordinate mapping, not the source values read through it.""" # Immutable bytes are the ultimate owner so callers cannot re-enable # the WRITEABLE flag, as they can on a read-only array that owns its # allocation. `asarray` also accepts the NumPy scalars that reach here # after indexing an array down to one element. array = np.asarray(self.index_array) if not np.issubdtype(array.dtype, np.integer): raise TypeError(f"index_array must have an integer dtype, got {array.dtype}") normalized = checked_affine(0, 1, array) frozen = np.frombuffer(normalized.tobytes(), dtype=np.intp).reshape(normalized.shape) object.__setattr__(self, "index_array", frozen) def __reduce__(self) -> tuple[object, tuple[object, int, int]]: """Reconstruct through `__init__`, preserving the ownership invariant.""" return ( type(self), (self.index_array, self.offset, self.stride), ) def _with_affine(self, offset: int, stride: int) -> ArrayMap: """Return a map with a different affine adjustment. Share the immutable index array while replacing the offset and stride. This preserves coordinate ownership without copying the array.""" new = object.__new__(ArrayMap) object.__setattr__(new, "index_array", self.index_array) object.__setattr__(new, "offset", offset) object.__setattr__(new, "stride", stride) return new def __eq__(self, other: object) -> bool: """Compare offset, stride, array shape, and index values. Return a scalar boolean for another ArrayMap and NotImplemented for other types.""" if not isinstance(other, ArrayMap): return NotImplemented return ( self.offset == other.offset and self.stride == other.stride and self.index_array.shape == other.index_array.shape and bool(np.array_equal(self.index_array, other.index_array)) ) def __hash__(self) -> int: """Hash the offset, stride, array shape, and index bytes. The immutable coordinate snapshot keeps the hash stable, and equal maps have equal hashes.""" return hash( ( self.offset, self.stride, self.index_array.shape, self.index_array.tobytes(), ) ) @property def dependency_axes(self) -> tuple[int, ...]: """Structural dependency axes: axes of size greater than one. Axes of size greater than one are reported, regardless of coordinate values or a zero-size axis elsewhere. Whether the whole transform is orthogonal also depends on how other maps use these axes; a single map's shape does not establish independence. Examples -------- >>> ArrayMap(index_array=np.array([[4, 0, 2]])).dependency_axes (1,) >>> ArrayMap(index_array=np.array([[1, 2], [3, 4]])).dependency_axes (0, 1) """ return _array_map_dependency_axes(self.index_array) @property def dependent_axis(self) -> int | None: """Return the single input axis an orthogonal `ArrayMap` varies over. This is the array's one non-singleton axis, read from the shape — the single source of truth for what a map depends on. The selection layer collapses a single-coordinate map to a `ConstantMap` (`array_map_or_constant`), so a non-empty map built by this package always has at least one dependency axis. Returns ------- int or None The axis the map varies over, or `None` when it varies over no input axis of size greater than one, such as an all-singleton map. `None` is a valid result, not an error; such maps resolve through the pointwise (general) path. Raises ------ ValueError If the map varies over more than one axis, which makes it correlated rather than orthogonal. Examples -------- An `oindex` selection on axis 1 of a rank-2 transform stores its coordinates full-sized on axis 1 and singleton on axis 0, so the dependency axis is read straight off the shape: >>> m = ArrayMap(index_array=np.array([[4, 0, 2]])) >>> m.index_array.shape (1, 3) >>> m.dependent_axis 1 """ dep = self.dependency_axes if len(dep) == 1: return dep[0] if len(dep) == 0: return None raise ValueError( f"orthogonal ArrayMap must vary over exactly one axis; got dependency axes {dep}" ) def to_json(self) -> OutputIndexMapJSON: """Convert to the canonical wire form, collapsing a degenerate map. A map holding exactly one coordinate, or none at all, is emitted as a `constant` map — see the module note on the wire format in [`zarr_indexing.json`][zarr_indexing.json]. Both are degenerate: the first selects one coordinate whatever the input, and the second names no cell and can only be empty because an input dimension is, so the emptiness travels in the domain instead. Examples -------- >>> ArrayMap(np.array([[4], [1], [1]])).to_json()["index_array"] [[4], [1], [1]] >>> ArrayMap(np.array([7])).to_json() # degenerate: one coordinate {'offset': 7} """ if self.index_array.size == 1: value = int(self.index_array.reshape(-1)[0]) return {"offset": self.offset + self.stride * value} if self.index_array.size == 0: return {"offset": 0} return { "offset": self.offset, "stride": self.stride, "index_array": self.index_array.tolist(), "index_array_bounds": ["-inf", "+inf"], } def output_index_map_from_json(data: OutputIndexMapJSON) -> OutputIndexMap: """Construct the output map a canonical wire form names. The wire form is structurally discriminated: the presence of `index_array` selects an array map, `input_dimension` selects a dimension map, and neither selects a constant map. Validate every raw index value against the inclusive `index_array_bounds` before applying offset and stride. Out-of-bounds values raise `NdselError` at load time. Validated immutable maps do not retain the bounds. Examples -------- >>> output_index_map_from_json({"offset": 5}) ConstantMap(offset=5) >>> output_index_map_from_json({"offset": 0, "stride": 2, "input_dimension": 1}) DimensionMap(input_dimension=1, offset=0, stride=2) """ from zarr_indexing._wire import check_index_array_bounds, lower_index_array if "index_array" in data: array = lower_index_array(data["index_array"], "index_array") check_index_array_bounds(array, data.get("index_array_bounds", ["-inf", "+inf"]), "output") return ArrayMap( index_array=array, offset=data.get("offset", 0), stride=data.get("stride", 1), ) if "input_dimension" in data: return DimensionMap( input_dimension=data["input_dimension"], offset=data.get("offset", 0), stride=data.get("stride", 1), ) return ConstantMap(offset=data.get("offset", 0)) def array_map_or_constant( index_array: npt.NDArray[np.integer[Any]], offset: int = 0, stride: int = 1, ) -> ArrayMap | ConstantMap: """An `ArrayMap`, collapsed to the `ConstantMap` it equals when it can be. An index array holding exactly one coordinate maps every input cell to the same place; representing it as a lookup table would leave a map whose shape names no dependency axis, the one form the shape-derived classifier cannot read. The selection and composition layers build their array maps through this helper so that a non-empty `ArrayMap` always varies over at least one axis. An empty array stays an `ArrayMap`: it maps no cell at all, and the emptiness lives in the domain that accompanies it. """ arr = np.asarray(index_array) if arr.size == 1: return ConstantMap(offset=checked_affine(offset, stride, int(arr.reshape(-1)[0]))) return ArrayMap(index_array=arr, offset=offset, stride=stride) OutputIndexMap = ConstantMap | DimensionMap | ArrayMap zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/py.typed000066400000000000000000000000001527014137600311430ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/reader.py000066400000000000000000000605621527014137600313030ustar00rootroot00000000000000"""Backend reader protocol and built-in system-memory implementations.""" from __future__ import annotations import math from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final, Protocol if TYPE_CHECKING: from collections.abc import Callable import numpy as np from zarr_indexing._affine import checked_affine from zarr_indexing.chunk_resolution import ChunkProjection # noqa: TC001 (runtime annotation) from zarr_indexing.output_map import ArrayMap, ConstantMap, DimensionMap, OutputIndexMap from zarr_indexing.transform import ( IndexTransform, ) __all__ = [ "BasicReader", "NumPyReader", "ReadContext", "Reader", "UnitStepReader", "basic_reader", "numpy_reader", "unit_step_reader", ] @dataclass(frozen=True, slots=True) class ReadContext: """A source-global transform and optional projection for a partitioned read. Examples -------- >>> transform = IndexTransform.from_shape((6,))[1:5:2] >>> context = ReadContext(transform) >>> context.transform.domain.shape (2,) >>> context.projection is None True """ transform: IndexTransform """Maps zero-origin output-buffer coordinates to global coordinates in the source. A transform carrying a literal (nonzero-origin) domain, such as a view's, is re-based to origin zero on construction: readers address the buffer they fill, so the origin is fixed here rather than by every caller. """ projection: ChunkProjection | None = None """The read plan, always supplied by `LazyArray` execution. Direct reader callers may omit it if their reader supports unplanned reads. """ def __post_init__(self) -> None: origin = self.transform.domain.inclusive_min if any(origin): object.__setattr__( self, "transform", self.transform.translate_domain_to((0,) * len(origin)) ) class Reader(Protocol): """Backend adapter that fills supplied system-memory result buffers. A reader may be shared by every view and part derived from one [`LazyArray`][zarr_indexing.lazy_array.LazyArray]. Part reads may run concurrently, so a stateful implementation must synchronize its own mutable state. `LazyArray` deliberately adds no serialization. Examples -------- The protocol is not `runtime_checkable`; an object satisfies it by exposing a conforming `read_into`, as `basic_reader` does: >>> transform = IndexTransform.from_shape((6,))[1:5:2] >>> source = np.arange(6) >>> out = np.empty(transform.domain.shape, dtype=source.dtype) >>> basic_reader.read_into(source, ReadContext(transform), out) >>> out.tolist() [1, 3] """ def read_into( self, source: Any, context: ReadContext, out: np.ndarray[Any, Any], /, ) -> None: """Fill `out` with the exact source values selected by `context`. `context.transform` maps zero-origin coordinates in the output buffer to global coordinates in `source`, and its domain shape equals `out.shape`. `context.projection`, when present, is the corresponding partition plan: its `chunk_transform` is chunk-local, its `cell_transform` places cells in the zero-origin result buffer of the view that planned the read, and its `chunk_domain` describes the grid cell. Fill every cell in place, preserving the transform's exact values, order, and dtype, then return `None`. Do not replace or retain `out`; it may be a strided writable view rather than an owning array. Backend exceptions propagate unchanged. Because callers may resolve parts concurrently through the same reader object, stateful readers are responsible for synchronizing their own state. """ ... class BasicReader: """Reader for system-memory sources exposing basic integer/slice indexing. Each transform is decomposed into the smallest enclosing positive-slice slab and a residual transform. The slab is read once with basic indexing, so fancy selections may over-read. Reversals read their selected coordinates in ascending order without requiring a negative source slice. The residual is then lowered through NumPy system-memory operations into the supplied buffer. Slice results must permit conversion to NumPy system memory. Device arrays that reject implicit conversion require a custom reader responsible for transferring values into the supplied system-memory output buffer. Examples -------- >>> transform = IndexTransform.from_shape((6,))[1:5:2] >>> source = np.arange(6) >>> out = np.empty(transform.domain.shape, dtype=source.dtype) >>> BasicReader().read_into(source, ReadContext(transform), out) >>> out.tolist() == source[1:5:2].tolist() True """ __slots__ = () def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: """Read one transform through a positive-slice slab and residual lowering.""" transform = context.transform key, residual = _decompose_basic(transform) block = np.asanyarray(source[key]) out[...] = _lower(block, residual) class NumPyReader: """Explicit reader for NumPy system-memory arrays. This is the reader selected by [`LazyArray.from_numpy`][zarr_indexing.lazy_array.LazyArray.from_numpy]. It applies the complete transform with NumPy operations and is applicable to `numpy.ndarray` sources, including `numpy.ma.MaskedArray`. Its current implementation uses the same slab and residual path as `BasicReader`. Examples -------- >>> transform = IndexTransform.from_shape((3, 4))[::2, 1:3] >>> source = np.arange(12).reshape(3, 4) >>> out = np.empty(transform.domain.shape, dtype=source.dtype) >>> NumPyReader().read_into(source, ReadContext(transform), out) >>> out.tolist() [[1, 2], [9, 10]] """ __slots__ = () def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: """Read one transform through a narrowed slab into `out`.""" transform = context.transform key, residual = _decompose_basic(transform) block = np.asanyarray(source[key]) out[...] = _lower(block, residual) class UnitStepReader: """Reader for sources whose basic indexing accepts only step-1 slices. Each transform is decomposed into the smallest enclosing ascending unit-step slab and a residual transform, so the source only ever receives `slice(start, stop, 1)` on every axis — the one form an API without general strided reads (an FFI binding, an HTTP range endpoint) supports. `BasicReader` instead requests positive-stride slices for both forward and reversed affine selections, then reverses in memory when needed. The residual lowering applies strides, reversals, and gathers to the in-memory block. For n selected points at positive spacing k on one axis, the cover contains (n - 1) * k + 1 elements when n is nonzero; its over-read ratio approaches k for long selections. Partitioning the wrapping [`LazyArray`][zarr_indexing.lazy_array.LazyArray] (`with_parts`) bounds each cover by a part. Slice results must permit conversion to NumPy system memory, exactly as for `BasicReader`. Examples -------- The source below is only ever asked for step-1 slices — here the cover `slice(1, 4, 1)` — and the stride is replayed against the block: >>> transform = IndexTransform.from_shape((6,))[1:5:2] >>> source = np.arange(6) >>> out = np.empty(transform.domain.shape, dtype=source.dtype) >>> UnitStepReader().read_into(source, ReadContext(transform), out) >>> out.tolist() [1, 3] """ __slots__ = () def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: """Read one transform through an ascending unit-step slab into `out`.""" transform = context.transform key, residual = _decompose_unit_step(transform) block = np.asanyarray(source[key]) out[...] = _lower(block, residual) basic_reader: Final = BasicReader() numpy_reader: Final = NumPyReader() unit_step_reader: Final = UnitStepReader() def _take(array: Any, indices: np.ndarray[Any, np.dtype[np.intp]], axis: int) -> Any: return np.take(array, indices, axis=axis) def _reshape(array: Any, shape: tuple[int, ...]) -> Any: return np.reshape(array, shape) def _transpose(array: Any, permutation: tuple[int, ...]) -> Any: return np.transpose(array, permutation) def _expand_dims(array: Any, axis: int) -> Any: return np.expand_dims(array, axis) def _dimension_map_coords( m: DimensionMap, transform: IndexTransform ) -> np.ndarray[Any, np.dtype[np.intp]]: """The storage coordinates a DimensionMap enumerates, in view order.""" d = m.input_dimension lo = transform.domain.inclusive_min[d] hi = transform.domain.exclusive_max[d] extent = hi - lo if extent == 0: return np.empty((0,), dtype=np.intp) first = checked_affine(m.offset, m.stride, lo) return checked_affine(first, m.stride, np.arange(extent, dtype=np.intp)) def _array_map_coords(m: ArrayMap) -> np.ndarray[Any, np.dtype[np.intp]]: """The storage coordinates an ArrayMap enumerates, flattened.""" return checked_affine(m.offset, m.stride, m.index_array).reshape(-1) def _correlated_map_coords( m: ArrayMap, broadcast_axes: list[int], broadcast_shape: tuple[int, ...], input_rank: int ) -> np.ndarray[Any, np.dtype[np.intp]]: """One storage coordinate per point of the correlated block, flattened. A correlated `ArrayMap`'s index array carries the transform's full input rank, with a singleton on every axis it does not vary over — including broadcast axes it shares with the *other* correlated maps but is itself constant along. Flattening it directly would then yield fewer coordinates than there are points, so it is reduced to the broadcast block and broadcast up to it explicitly. """ coords = checked_affine(m.offset, m.stride, m.index_array) if math.prod(broadcast_shape) == 0: # A zero-extent broadcast axis makes the correlated block empty — for # example an ArrayMap composed over an empty domain, which the package # promises resolves like any other. The per-axis reshape below cannot # express that block (a 0-size array does not reshape to the non-zero # singleton axes), and there is no coordinate to produce anyway. return np.empty(0, dtype=np.intp) if coords.ndim == input_rank: # Drop the axes bound by a slice, which the map is singleton along. # Removing size-1 axes by reshape preserves element order wherever they # sit, so no transpose is needed. coords = coords.reshape(tuple(coords.shape[axis] for axis in broadcast_axes)) return np.ascontiguousarray(np.broadcast_to(coords, broadcast_shape)).reshape(-1) def _restore_domain_axis_order( result: Any, axis_input_dims: list[int], domain_shape: tuple[int, ...] ) -> Any: """Permute `result`'s axes into input-domain order, restoring dropped axes. `axis_input_dims[k]` is the input (domain) dimension that axis `k` of `result` corresponds to. Axes are permuted so that they appear in increasing domain-dimension order, and any domain dimension no output map depends on is reinserted at **its own extent**. An unreferenced dimension is not always a singleton. A `vindex` coordinate array with a broadcast axis it does not vary over leaves that axis in the domain; a later basic index that consumes the axis the array *does* vary over collapses the map to a `ConstantMap` and leaves the broadcast axis behind, with whatever extent the basic index gave it — including 0. Every position along such an axis holds the same values, so it is restored by repeating the block, and an extent of 0 restores an empty result rather than fabricating a row. This is also where NumPy's advanced-index placement rules are absorbed: whatever order the gather produced, the lowered result always comes back in the view's own axis order. """ if len(set(axis_input_dims)) != len(axis_input_dims): raise NotImplementedError( "resolving a transform whose output maps share an input dimension " "(a diagonal view) is not supported" ) order = sorted(range(len(axis_input_dims)), key=lambda k: axis_input_dims[k]) if order != list(range(len(order))): result = _transpose(result, tuple(order)) covered = set(axis_input_dims) for dim, extent in enumerate(domain_shape): if dim in covered: continue result = _expand_dims(result, dim) if extent != 1: result = _take(result, np.zeros(extent, dtype=np.intp), axis=dim) return result def _lower(array: Any, transform: IndexTransform) -> Any: """Lower a transform to one pass of array operations over `array`. Every read, partitioned or not, goes through this function. The result is always in the transform's own domain axis order and of exactly its domain shape. """ if math.prod(transform.domain.shape) == 0: # An empty domain selects nothing, and its maps may legitimately be # empty along the vanished axes (an ArrayMap composed over an empty # domain, which the package promises resolves like any other). The # resolvers below cannot evaluate such maps — and have no reason to. return np.empty(transform.domain.shape, dtype=np.asanyarray(array).dtype) if transform.index_array_structure == "general": result = _lower_general(array, transform) else: result = _lower_orthogonal(array, transform) if isinstance(result, np.generic): # Basic indexing every axis of a NumPy array yields a scalar; `result()` # documents a zero-dimensional array. return np.asarray(result) return result def _lower_orthogonal(array: Any, transform: IndexTransform) -> Any: """Basic slicing plus one `take` per fancy-indexed axis (an outer product). Orthogonal `ArrayMap`s vary over distinct input axes, so gathering them one axis at a time is exact — a `take` along one storage axis leaves every other axis's coordinates untouched. """ outputs = transform.output gathered: dict[int, np.ndarray[Any, np.dtype[np.intp]]] = {} for out_dim, m in enumerate(outputs): if isinstance(m, ArrayMap): gathered[out_dim] = _array_map_coords(m) elif isinstance(m, DimensionMap) and m.stride <= 0: # Reversing and repeating maps have no positive-step slice; gather them. gathered[out_dim] = _dimension_map_coords(m, transform) result = array for out_dim, coords in gathered.items(): result = _take(result, coords, axis=out_dim) selection: list[Any] = [] axis_input_dims: list[int] = [] for out_dim, m in enumerate(outputs): if isinstance(m, ConstantMap): selection.append(m.offset) continue if isinstance(m, DimensionMap) and out_dim not in gathered: d = m.input_dimension lo = transform.domain.inclusive_min[d] hi = transform.domain.exclusive_max[d] selection.append(slice(m.offset + m.stride * lo, m.offset + m.stride * hi, m.stride)) else: # Every ArrayMap, and every non-positive-stride DimensionMap, was gathered above. selection.append(slice(None)) if isinstance(m, ArrayMap): axis = m.dependent_axis if axis is None: raise NotImplementedError( "resolving an orthogonal ArrayMap that varies over no input " "dimension is not supported; such a map should have been " "collapsed to a ConstantMap" ) axis_input_dims.append(axis) else: axis_input_dims.append(m.input_dimension) result = result[tuple(selection)] return _restore_domain_axis_order(result, axis_input_dims, transform.domain.shape) def _lower_general(array: Any, transform: IndexTransform) -> Any: """Flatten the index-array axes, gather the points once, reshape back. The general path for every index-array structure the orthogonal resolver cannot take: correlated (`vindex`) maps, maps sharing an input axis (a diagonal gather), and mixtures of correlated and orthogonal maps. All index arrays are treated as lookup tables over the joint block of non-slice axes: the corresponding storage axes are moved to the front and flattened, the per-point coordinates are converted to offsets into that flat axis with row-major strides, and a single `take` collects them. """ outputs = transform.output correlated = [(d, m) for d, m in enumerate(outputs) if isinstance(m, ArrayMap)] correlated_dims = [d for d, _ in correlated] slice_input_dims = {m.input_dimension for m in outputs if isinstance(m, DimensionMap)} broadcast_axes = [d for d in range(transform.input_rank) if d not in slice_input_dims] broadcast_shape = tuple(transform.domain.shape[d] for d in broadcast_axes) for _, arr_map in correlated: # The axes the array varies over (its non-singleton axes; see # transform._array_map_dependency_axes) must all live in the block. dependency = (axis for axis, size in enumerate(arr_map.index_array.shape) if size > 1) if any(a not in broadcast_axes for a in dependency): # Reachable only by hand-building a transform: no selection binds # the same input axis to both a slice map and an index array. raise NotImplementedError( "resolving a transform whose index array varies over an input " "dimension also bound by a slice map is not supported" ) # Gather any reversing or repeating slice axis first, then take the basic-slice # cut. The correlated axes keep their full extent: their coordinates are absolute. gathered: dict[int, np.ndarray[Any, np.dtype[np.intp]]] = { d: _dimension_map_coords(m, transform) for d, m in enumerate(outputs) if isinstance(m, DimensionMap) and m.stride <= 0 } result = array for out_dim, coords in gathered.items(): result = _take(result, coords, axis=out_dim) selection: list[Any] = [] residual_axis_dims: list[int] = [] correlated_positions: list[int] = [] residual_positions: list[int] = [] axis = 0 for out_dim, m in enumerate(outputs): if isinstance(m, ConstantMap): selection.append(m.offset) continue if isinstance(m, ArrayMap): selection.append(slice(None)) correlated_positions.append(axis) elif out_dim in gathered: selection.append(slice(None)) residual_positions.append(axis) residual_axis_dims.append(m.input_dimension) else: d = m.input_dimension lo = transform.domain.inclusive_min[d] hi = transform.domain.exclusive_max[d] selection.append(slice(m.offset + m.stride * lo, m.offset + m.stride * hi, m.stride)) residual_positions.append(axis) residual_axis_dims.append(d) axis += 1 result = result[tuple(selection)] # Correlated axes to the front, in output order, so the flattening strides # below match the order the coordinates are combined in. perm = tuple(correlated_positions) + tuple(residual_positions) if perm != tuple(range(len(perm))): result = _transpose(result, perm) n_corr = len(correlated_dims) corr_sizes = tuple(int(s) for s in result.shape[:n_corr]) tail_shape = tuple(int(s) for s in result.shape[n_corr:]) result = _reshape(result, (math.prod(corr_sizes), *tail_shape)) flat_index = np.zeros(math.prod(broadcast_shape), dtype=np.intp) stride = 1 for position in range(n_corr - 1, -1, -1): _, corr_map = correlated[position] flat_index = flat_index + ( _correlated_map_coords(corr_map, broadcast_axes, broadcast_shape, transform.input_rank) * stride ) stride *= corr_sizes[position] result = _take(result, flat_index, axis=0) result = _reshape(result, broadcast_shape + tail_shape) return _restore_domain_axis_order( result, list(broadcast_axes) + residual_axis_dims, transform.domain.shape ) def _push_slice_for_dimension_map( m: DimensionMap, transform: IndexTransform ) -> tuple[slice, DimensionMap]: """The positive-step slice covering a `DimensionMap`, and its block-local map. A negative step is read forwards and reversed by the residual: a source is only asked for positive-stride slices. Supporting those slices is part of this reader's source contract. """ d = m.input_dimension lo = transform.domain.inclusive_min[d] hi = max(transform.domain.exclusive_max[d], lo) if hi == lo: return slice(0, 0, 1), DimensionMap(input_dimension=d, offset=-lo, stride=1) first = checked_affine(m.offset, m.stride, lo) last = checked_affine(m.offset, m.stride, hi - 1) if m.stride > 0: return ( slice(first, last + 1, m.stride), DimensionMap(input_dimension=d, offset=-lo, stride=1), ) if m.stride == 0: return ( slice(first, first + 1, 1), DimensionMap(input_dimension=d, offset=0, stride=0), ) # Descending: the block holds the same coordinates in ascending order, so # the residual walks it backwards from the last block position. return ( slice(last, first + 1, -m.stride), DimensionMap(input_dimension=d, offset=hi - 1, stride=-1), ) def _push_unit_slice_for_dimension_map( m: DimensionMap, transform: IndexTransform ) -> tuple[slice, DimensionMap]: """The unit-step slice covering a `DimensionMap`, and its block-local map. Strides and reversals stay in the residual: the source is only ever asked for a contiguous ascending slice, and the original stride is replayed against the in-memory block. The cover may include unselected elements; for n distinct points at spacing k it has (n - 1) * k + 1 elements. """ d = m.input_dimension lo = transform.domain.inclusive_min[d] hi = max(transform.domain.exclusive_max[d], lo) if hi == lo: return slice(0, 0, 1), DimensionMap(input_dimension=d, offset=-lo, stride=1) first = checked_affine(m.offset, m.stride, lo) if m.stride == 0: return ( slice(first, first + 1, 1), DimensionMap(input_dimension=d, offset=0, stride=0), ) last = checked_affine(m.offset, m.stride, hi - 1) origin = min(first, last) return ( slice(origin, max(first, last) + 1, 1), DimensionMap(input_dimension=d, offset=m.offset - origin, stride=m.stride), ) def _decompose_basic(transform: IndexTransform) -> tuple[tuple[slice, ...], IndexTransform]: return _decompose(transform, _push_slice_for_dimension_map) def _decompose_unit_step(transform: IndexTransform) -> tuple[tuple[slice, ...], IndexTransform]: return _decompose(transform, _push_unit_slice_for_dimension_map) def _decompose( transform: IndexTransform, push_dimension_map: Callable[[DimensionMap, IndexTransform], tuple[slice, DimensionMap]], ) -> tuple[tuple[slice, ...], IndexTransform]: key: list[slice] = [] residual: list[OutputIndexMap] = [] for output_map in transform.output: if isinstance(output_map, ConstantMap): coordinate = checked_affine(output_map.offset, 0, 0) key.append(slice(coordinate, coordinate + 1, 1)) residual.append(ConstantMap(offset=0)) elif isinstance(output_map, DimensionMap): pushed, local = push_dimension_map(output_map, transform) key.append(pushed) residual.append(local) else: coordinates = checked_affine( output_map.offset, output_map.stride, output_map.index_array ) if coordinates.size == 0: key.append(slice(0, 0, 1)) local_index = coordinates else: origin = int(coordinates.min()) key.append(slice(origin, int(coordinates.max()) + 1, 1)) local_index = checked_affine(-origin, 1, coordinates) residual.append(ArrayMap(index_array=local_index)) return tuple(key), IndexTransform(domain=transform.domain, output=tuple(residual)) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/testing/000077500000000000000000000000001527014137600311335ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/testing/__init__.py000066400000000000000000000031601527014137600332440ustar00rootroot00000000000000"""Test support for projects whose arrays are read through `LazyArray`. A Hypothesis state machine that composes indexing steps onto a `LazyArray` wrapping your array and checks every step against NumPy ([`stateful`][zarr_indexing.testing.stateful]), and the selection strategies it draws from, exported on their own for a project that has its own harness ([`strategies`][zarr_indexing.testing.strategies]). ```python from zarr_indexing.testing import ChainedIndexingStateMachine, state_machine_test class MyArrayIndexing(ChainedIndexingStateMachine): def make_source(self, data): array = my_format.create(shape=data.shape, dtype=data.dtype) array[:] = data return array TestMyArrayIndexing = state_machine_test(MyArrayIndexing) ``` This subpackage needs `hypothesis`, which the rest of `zarr_indexing` does not: install it with the `testing` extra (`pip install zarr-indexing[testing]`). """ from zarr_indexing.testing.stateful import ( DEFAULT_DATA, DEFAULT_PARTITIONINGS, DEFAULT_SETTINGS, ChainedIndexingStateMachine, apply_selection, outer_selection, repartition, state_machine_test, ) from zarr_indexing.testing.strategies import ( basic_selections, masks, orthogonal_selections, slice_selections, vectorized_selections, ) __all__ = [ "DEFAULT_DATA", "DEFAULT_PARTITIONINGS", "DEFAULT_SETTINGS", "ChainedIndexingStateMachine", "apply_selection", "basic_selections", "masks", "orthogonal_selections", "outer_selection", "repartition", "slice_selections", "state_machine_test", "vectorized_selections", ] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/testing/stateful.py000066400000000000000000000363241527014137600333440ustar00rootroot00000000000000"""A stateful property test for indexing an array through `LazyArray`. `ChainedIndexingStateMachine` composes indexing steps onto a `LazyArray` wrapping *your* array — `lazy[...]`, `lazy.oindex[...]`, `lazy.vindex[...]`, each step applied to the view the last one produced — while applying the same steps to a NumPy array holding the same values. After every step the view must still agree with that model three ways: its shape, its `result()`, and the assembly of its `parts()`. Point it at an array by subclassing and overriding `make_source`: ```python from zarr_indexing.testing import ChainedIndexingStateMachine, state_machine_test class MyArrayIndexing(ChainedIndexingStateMachine): def make_source(self, data): array = my_format.create(shape=data.shape, dtype=data.dtype) array[:] = data return array TestMyArrayIndexing = state_machine_test(MyArrayIndexing) ``` `data`, `partitionings`, and `readers` are class attributes; override any of them to widen or narrow what is drawn. The base class needs no `make_source` at all — left alone it wraps the NumPy array itself, which is a useful smoke test of this package but says nothing about yours. What it is checking ------------------- The parts invariant is the one with teeth. [`Partition`][zarr_indexing.lazy_array.Partition] documents `out[part.out_selection] = part.view.result()` as the assembly procedure, so this checks that literally: a part's values must arrive at exactly the shape its `out_selection` addresses — not merely a shape that broadcasts into it — land there, and cover the view once. Checking through `result()` alone would prove only that `result()` is self-consistent. The `choose_reader` rule draws a reader and applies it to the view, so the execution strategy becomes part of the chain. Every reader listed by a subclass must preserve the NumPy model for its source. The universal `basic_reader` is always eligible to be drawn, even when a subclass lists only specialized readers; with no declared readers it is the sole strategy drawn. Requires the `testing` extra (`pip install zarr-indexing[testing]`). """ from __future__ import annotations import math from collections.abc import Sequence from typing import TYPE_CHECKING, Any, ClassVar import numpy as np from hypothesis import HealthCheck, settings from hypothesis import strategies as st from hypothesis.stateful import RuleBasedStateMachine, initialize, invariant, precondition, rule from zarr_indexing.lazy_array import LazyArray from zarr_indexing.reader import Reader, basic_reader from zarr_indexing.testing.strategies import ( basic_selections, orthogonal_selections, slice_selections, vectorized_selections, ) if TYPE_CHECKING: from zarr_indexing.boundary import SelectionMode __all__ = [ "DEFAULT_DATA", "DEFAULT_PARTITIONINGS", "DEFAULT_SETTINGS", "ChainedIndexingStateMachine", "apply_selection", "outer_selection", "repartition", "state_machine_test", ] DEFAULT_DATA = np.arange(7 * 5 * 4, dtype=np.int64).reshape(7, 5, 4) """The values the source holds by default: distinct, so a misplaced cell shows.""" DEFAULT_PARTITIONINGS: tuple[Any, ...] = ( None, (2, 2, 2), (7, 5, 4), (3, 2, 3), ((3, 3, 1), (2, 2, 1), (3, 1)), (4, 3, 3), ) """Partitionings to read under: a single whole-array part, uniform boxes of several shapes (some of which do not divide the extent), and explicit per-axis sizes. Boxes that straddle whatever the source declares are deliberate — they cost extra I/O but must not change an answer.""" DEFAULT_SETTINGS = settings( max_examples=250, stateful_step_count=10, deadline=None, suppress_health_check=[ HealthCheck.data_too_large, HealthCheck.filter_too_much, HealthCheck.too_slow, ], ) """A budget of 250 examples with at most 10 state-machine steps per example. Invariants materialize the whole view after steps. Deadlines and selected health checks are disabled to accommodate those potentially expensive reads; the budget does not guarantee discovery of every defect. """ # --------------------------------------------------------------------------- # # The NumPy model # --------------------------------------------------------------------------- # def outer_selection(array: Any, selection: Sequence[Any]) -> Any: """Apply an orthogonal selection to a NumPy array: the outer product of its axes. NumPy has no operator for this, so the model is built from `numpy.ix_`. Scalar integers are basic indices — NumPy applies them first and drops the axis — so they are peeled off before the outer product is formed. """ def is_scalar(sel: Any) -> bool: return isinstance(sel, (int, np.integer)) and not isinstance(sel, bool) scalars = tuple(sel if is_scalar(sel) else slice(None) for sel in selection) reduced = array[scalars] axes = [ np.arange(size)[sel] for size, sel in zip(reduced.shape, [s for s in selection if not is_scalar(s)], strict=True) ] if len(axes) == 0: return reduced return reduced[np.ix_(*axes)] def apply_selection(array: Any, selection: tuple[Any, ...], mode: SelectionMode) -> Any: """Apply a selection to a NumPy array in the given mode — the model a view is checked against. NumPy's own semantics *are* basic and vectorized indexing, so only the orthogonal mode needs building (see `outer_selection`). """ if mode == "orthogonal": return outer_selection(array, selection) return array[selection] def state_machine_test( machine: type[RuleBasedStateMachine], *, config: settings = DEFAULT_SETTINGS ) -> Any: """The pytest-collectable `TestCase` for a machine, with settings applied. Hypothesis builds a fresh `TestCase` per state-machine class, so settings set on a base class do not reach a subclass's; this applies them where they land. Assign the result to a module-level name beginning with `Test`. """ case = machine.TestCase case.settings = config return case def repartition(view: LazyArray, parts: Any) -> LazyArray: """Apply one of `partitionings` to a view. The three partitioning spellings are three named methods, so a list holding a mix of them needs a dispatch somewhere. Choosing among them is what a test harness drawing from that list is doing, so it lives here rather than being pushed back into the public API as a type-inspecting parameter. """ if parts is None: return view.unpartitioned() if any(isinstance(entry, Sequence) for entry in parts): return view.with_parts_per_axis(parts) return view.with_parts(parts) # --------------------------------------------------------------------------- # # The machine # --------------------------------------------------------------------------- # _SOURCE = "_zarr_indexing_cached_source" class ChainedIndexingStateMachine(RuleBasedStateMachine): """Indexing steps composed onto one `LazyArray`, against NumPy as the model. Subclass and override `make_source` to point it at your own array. See the module docstring for the shape of that subclass and for what the invariants check. Attributes ---------- data The values the source holds, and the model every step is checked against. Any shape and dtype NumPy supports; every axis must be non-empty. partitionings Drawn once per run, before any indexing: `with_parts` is a pure setter that carries through composition untouched and is read only when a view resolves, so choosing it up front reaches the same states choosing it mid-chain does, and spends the whole step budget on indexing. readers Execution strategies `choose_reader` may draw. Every listed reader must preserve the model for the source. `basic_reader` is always included; `None` means the reader already carried by the constructed view. """ data: ClassVar[Any] = DEFAULT_DATA partitionings: ClassVar[Sequence[Any]] = DEFAULT_PARTITIONINGS readers: ClassVar[Sequence[Reader] | None] = None def make_source(self, data: Any) -> Any: """Build the array under test, holding `data`. Called once per machine class and cached, not once per example: an example is cheap and a source may not be. The machine only ever reads, so the same object serves every run — but it must therefore not be mutated by anything else while the test runs. The default returns `data` itself, so an unsubclassed machine exercises this package against NumPy. """ return data def __init__(self) -> None: super().__init__() cls = type(self) self.model: Any = np.asarray(cls.data) source = cls.__dict__.get(_SOURCE) if source is None: source = self.make_source(self.model) setattr(cls, _SOURCE, source) self.view = LazyArray(source) self.reader_choices = _reader_set(self.view, cls.readers) self.chain: list[tuple[str, Any]] = [] def _indexable(self) -> bool: """Whether there is anything left to index. This harness stops drawing selections for rank-0 or empty views. Reader changes, repartitioning and invariant checks remain possible. NumPy itself permits some indexing of such arrays, such as `[()]`. """ return self.model.ndim > 0 and self.model.size > 0 def _step(self, mode: SelectionMode, selection: tuple[Any, ...]) -> None: self.chain.append((mode, selection)) self.model = apply_selection(self.model, selection, mode) if mode == "basic": self.view = self.view[selection] elif mode == "orthogonal": self.view = self.view.oindex[selection] else: self.view = self.view.vindex[selection] # -- rules -------------------------------------------------------------- @initialize(data=st.data()) def choose_partitioning(self, data: st.DataObject) -> None: """Fix how the read is broken up, before any indexing.""" parts = data.draw(st.sampled_from(list(type(self).partitionings))) self.view = repartition(self.view, parts) self.chain.append(("parts", parts)) @precondition(lambda self: self._indexable()) @rule(data=st.data()) def basic(self, data: st.DataObject) -> None: self._step("basic", data.draw(basic_selections(self.model.shape))) @precondition(lambda self: self._indexable()) @rule(data=st.data()) def orthogonal(self, data: st.DataObject) -> None: self._step("orthogonal", data.draw(orthogonal_selections(self.model.shape))) @precondition(lambda self: self._indexable()) @rule(data=st.data()) def vectorized(self, data: st.DataObject) -> None: self._step("vectorized", data.draw(vectorized_selections(self.model.shape))) @precondition(lambda self: self._indexable()) @rule(data=st.data()) def slices_only(self, data: st.DataObject) -> None: """An `oindex` step carrying only slices is not a fancy selection. It narrows the view's own axes and composes like basic indexing. Drawn as its own rule so that narrowing an existing index array by slices — a distinct code path from narrowing it with coordinates — stays exercised at full weight. """ self._step("orthogonal", data.draw(slice_selections(self.model.shape))) @rule(data=st.data()) def choose_reader(self, data: st.DataObject) -> None: """Read the rest of the chain through another conforming strategy.""" reader = data.draw(st.sampled_from(list(self.reader_choices))) self.view = self.view.with_reader(reader) self.chain.append(("reader", type(reader).__qualname__)) @precondition(lambda self: not self._indexable()) @rule(data=st.data()) def repartition(self, data: st.DataObject) -> None: """Re-box a chain that has run out of axes to index. Once the harness stops drawing selections, this rule tests another partitioning of the same view while the reader-change rule also remains enabled. The invariants must continue to match the NumPy model. """ parts = data.draw(st.sampled_from(list(type(self).partitionings))) self.view = repartition(self.view, parts) self.chain.append(("parts", parts)) # -- invariants --------------------------------------------------------- @invariant() def the_view_has_the_models_shape(self) -> None: assert self.view.shape == self.model.shape, self.chain @invariant() def result_matches_the_model(self) -> None: np.testing.assert_array_equal( np.asarray(self.view.result()), self.model, err_msg=str(self.chain) ) @invariant() def parts_tile_the_view(self) -> None: """The documented assembly, run literally. Every part's values arrive at exactly the shape its `out_selection` addresses — not merely a shape that broadcasts into it — and together the parts cover the view once. """ assembled = np.zeros(self.view.shape, dtype=self.view.dtype) hits = np.zeros(self.view.shape, dtype=np.int64) for part in self.view.parts(): value = np.asarray(part.view.result()) assert value.shape == assembled[part.out_selection].shape, ( f"part {part.base_coords} carries {value.shape} for an out_selection " f"addressing {assembled[part.out_selection].shape}: {self.chain}" ) # `is_complete` is what a consumer reads to decide it may take a # whole-box read and skip assembling anything, so a wrongly-`True` # one is the silent-corruption case. Asserted one way only: the flag # is documented as conservative, free to say `False` about a part it # does cover (a strided walk over a one-cell box, a fancy axis that # happens to enumerate everything), and only the claim to cover # everything has to be earned. Both quantities are already in hand # here, and nothing else in the suite compares them. if part.is_complete: box_cells = math.prod(stop - start for start, stop in part.box) assert value.size == box_cells, ( f"part {part.base_coords} reports is_complete but carries " f"{value.size} of its box's {box_cells} cells: {self.chain}" ) assembled[part.out_selection] = value np.add.at(hits, part.out_selection, 1) np.testing.assert_array_equal(assembled, self.model, err_msg=str(self.chain)) np.testing.assert_array_equal( hits, np.ones(self.view.shape, dtype=np.int64), err_msg=str(self.chain) ) def _reader_set(view: LazyArray, declared: Sequence[Reader] | None) -> tuple[Reader, ...]: """The readers `choose_reader` draws from, `basic_reader` always among them.""" readers = list(declared) if declared is not None else [view.reader] if all(reader is not basic_reader for reader in readers): readers.insert(0, basic_reader) unique: list[Reader] = [] for reader in readers: if all(reader is not existing for existing in unique): unique.append(reader) return tuple(unique) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/testing/strategies.py000066400000000000000000000171131527014137600336620ustar00rootroot00000000000000"""Hypothesis strategies for the selections `LazyArray` accepts. Each strategy takes the shape of the array being indexed and generates one selection for it, in the spelling its mode expects. Some vectorized selections use a partial coordinate tuple or a mask with an ellipsis. They are the generators behind [`ChainedIndexingStateMachine`][zarr_indexing.testing.stateful.ChainedIndexingStateMachine] and are exported on their own for a project that has its own test harness and wants only the hard part. ```python from hypothesis import given, strategies as st from zarr_indexing.testing.strategies import basic_selections @given(selection=basic_selections((7, 5, 4))) def test_my_array_slices_like_numpy(selection): assert_array_equal(my_array[selection], reference[selection]) ``` Coordinate-drawing strategies require non-empty axes; vectorized selections also require positive rank. The `empty_masks` strategy can generate a mask for an empty shape because it does not draw element coordinates. Requires the `testing` extra (`pip install zarr-indexing[testing]`). """ from __future__ import annotations import operator from typing import TYPE_CHECKING, Any import numpy as np from hypothesis import strategies as st if TYPE_CHECKING: from collections.abc import Callable __all__ = [ "basic_selections", "empty_masks", "masks", "orthogonal_selections", "slice_selections", "vectorized_selections", ] def _entries(shape: tuple[int, ...], entry: Callable[[int], st.SearchStrategy[Any]]) -> Any: """One `entry` strategy per axis, as an index tuple.""" return st.tuples(*[entry(size) for size in shape]) def _basic_entry(size: int) -> st.SearchStrategy[Any]: steps = st.integers(1, 3) return st.one_of( # A scalar integer drops its axis, in every mode, exactly as NumPy does. st.integers(-size, size - 1), st.builds(slice, st.integers(0, size), st.integers(0, size), steps), # Downward. The start is drawn from below `-size` as well, where the walk # begins off the front and selects nothing — a case that reads as an # ordinary negative index but is empty — and a stop that falls off the # front is spelled `None`. st.builds( slice, st.integers(-2 * size - 1, size - 1), st.none() | st.integers(0, size), steps.map(operator.neg), ), st.just(slice(None)), ) def _orthogonal_entry(size: int) -> st.SearchStrategy[Any]: """Generate one axis of an orthogonal selection. Include scalar coordinates, coordinate lists, boolean masks, and slices with positive or negative steps and varying endpoints. Empty coordinate lists and all-False masks exercise selections with zero-length axes.""" coordinate = st.integers(-size, size - 1) return st.one_of( coordinate, st.lists(coordinate, min_size=1, max_size=4), st.just([]), masks((size,)), empty_masks((size,)), st.builds( slice, st.integers(0, size - 1), st.integers(0, size) | st.none(), st.integers(1, 3) | st.integers(-3, -1), ), ) def _slice_entry(size: int) -> st.SearchStrategy[slice]: return st.one_of( st.builds(slice, st.integers(0, size - 1), st.just(size), st.integers(1, 2)), st.just(slice(None, None, -1)), st.just(slice(None)), ) @st.composite def masks(draw: st.DrawFn, shape: tuple[int, ...]) -> np.ndarray[Any, np.dtype[np.bool_]]: """Boolean masks over `shape`, each selecting at least one cell. An all-False mask is legal but is a separate concern — it empties the view, and a chain of selections is more interesting when every step leaves something to index — so one cell is always forced True. """ size = int(np.prod(shape)) flags = np.array(draw(st.lists(st.booleans(), min_size=size, max_size=size))) flags[draw(st.integers(0, size - 1))] = True return flags.reshape(shape) def empty_masks(shape: tuple[int, ...]) -> st.SearchStrategy[np.ndarray[Any, np.dtype[np.bool_]]]: """Generate an all-False mask with the given shape. This selects no elements. Unlike masks(), which includes a True cell, this strategy exercises empty fancy selections.""" return st.just(np.zeros(shape, dtype=np.bool_)) def basic_selections(shape: tuple[int, ...]) -> st.SearchStrategy[tuple[Any, ...]]: """Basic selections: one scalar integer or slice per axis. Slices run in both directions, including the two empty spellings — a forward slice whose stop precedes its start, and a backward one whose start is off the front of the axis. """ return _entries(shape, _basic_entry) def orthogonal_selections(shape: tuple[int, ...]) -> st.SearchStrategy[tuple[Any, ...]]: """Orthogonal (`oindex`) selections: an outer product of per-axis choices. Each axis draws a scalar, a coordinate list (unsorted, with duplicates), a boolean mask, or a slice. """ return _entries(shape, _orthogonal_entry) def slice_selections(shape: tuple[int, ...]) -> st.SearchStrategy[tuple[Any, ...]]: """Selections of slices alone, for the `oindex` spelling that carries no coordinates. Such a step is not a fancy selection — it narrows the view's own axes and composes like basic indexing. The starts reach past the origin, which is what distinguishes a step that walks an existing index array's dependency axes from one that walks its broadcast singletons. """ return _entries(shape, _slice_entry) @st.composite def vectorized_selections(draw: st.DrawFn, shape: tuple[int, ...]) -> tuple[Any, ...]: """Vectorized (`vindex`) selections over a leading or trailing block of axes. `vindex` is coordinate-only — it rejects a slice outright — so a partial selection names its axes by position: a leading block, or a trailing one reached through an ellipsis. Either a single boolean mask spanning the whole covered block, or one entry per axis, each a coordinate array or a scalar (a scalar being a basic index NumPy applies before the coordinates). """ ndim = len(shape) trailing = draw(st.booleans()) count = draw(st.integers(1, ndim)) axes = range(ndim - count, ndim) if trailing else range(count) sizes = [shape[axis] for axis in axes] entries: list[Any] if draw(st.booleans()): entries = [draw(masks(tuple(sizes)))] else: # The coordinate arrays share one shape, which is what makes the # selection correlated. That shape is not always one-dimensional: a # vectorized read of a (2, 3) block of points is an ordinary thing to # ask for and produces a result of that rank. Drawing only 1-D arrays # meant no rank-raising vindex was ever generated — and a length of 0 # covers the empty case the same way `_orthogonal_entry` does. coordinate_shape = draw( st.one_of( st.integers(0, 4).map(lambda length: (length,)), st.tuples(st.integers(1, 2), st.integers(1, 3)), ) ) entries = [ draw( st.one_of( st.integers(-size, size - 1), st.lists( st.integers(-size, size - 1), min_size=int(np.prod(coordinate_shape)), max_size=int(np.prod(coordinate_shape)), ).map(lambda values: np.array(values, dtype=np.intp).reshape(coordinate_shape)), ) ) for size in sizes ] return (Ellipsis, *entries) if trailing else tuple(entries) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/transform.py000066400000000000000000002546671527014137600320670ustar00rootroot00000000000000"""Index transforms — composable, lazy coordinate mappings. An `IndexTransform` pairs an **input domain** (the coordinates a user sees) with a tuple of **output maps** (the output coordinates those inputs map to). One output map per output dimension. See `output_map.py` for the three output map types. Key operations: - **Indexing** (`transform[2:8]`, `.oindex[idx]`, `.vindex[idx]`) — produces a new transform with a new input domain and adjusted output maps. Slices may narrow a domain; fancy indices can repeat cells and enlarge it, and integer/newaxis indexing changes its rank. No I/O occurs. - **intersect(output_domain)** — restrict to output coordinates within a region. This is chunk resolution: "which of my coordinates fall in this chunk?" - **translate(shift)** — shift all output coordinates. This makes coordinates chunk-local: "express my coordinates relative to the chunk origin." - **`transform.compose(inner)`** — chain two transforms into one. The transform is the atomic unit that connects user-facing indexing to chunk-level I/O. A wrapper holds one — `LazyArray` starts from the identity — and `view[...]` composes a new transform lazily rather than reading. Reading resolves the transform against the chunk grid via intersect + translate. """ from __future__ import annotations import math from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal, cast import numpy as np from zarr_indexing._affine import checked_affine from zarr_indexing._selector import as_scalar_index, require_index from zarr_indexing.boundary import validate_advanced_selection from zarr_indexing.domain import IndexDomain from zarr_indexing.errors import BoundsCheckError, VindexInvalidSelectionError from zarr_indexing.output_map import ( ArrayMap, ConstantMap, DimensionMap, OutputIndexMap, array_map_or_constant, ) if TYPE_CHECKING: from collections.abc import Sequence import numpy.typing as npt from zarr_indexing.json import IndexTransformJSON @dataclass(frozen=True, slots=True) class _PointOutOfBounds(Exception): """Internal signal from the shared point kernel: one coordinate left the domain. Never escapes this module. `apply` and `apply_many` format it as the public `BoundsCheckError`, each in its own vocabulary — the kernel knows batches, but a single-point caller must never hear about them. """ dimension: int value: int lower: int upper: int batch_position: tuple[int, ...] @dataclass(frozen=True, slots=True) class IndexTransform: """A composable mapping from input coordinates to output coordinates. An `IndexTransform` has: - `domain`: an `IndexDomain` describing the valid input coordinates (the result's coordinate range, possibly with non-zero origin). - `output`: a tuple of output maps (one per output dimension), each describing which output coordinates the inputs touch. In array-indexing terms: `domain` describes the coordinates of the result array an indexing operation produces, and `output` is the rule relating each result coordinate to a coordinate in the source. Note the direction — the transform's input side is the result, its output side addresses the source; the coordinate mapping runs opposite to the data flow. Indexing an existing transform composes a new one without I/O. Examples -------- The operation "every other element of a 100-element array, starting at index 0" — `array[::2]` — is a 50-cell domain whose cell `i` reads output coordinate `2 * i`: >>> domain = IndexDomain.from_shape((50,)) >>> output = (DimensionMap(input_dimension=0, offset=0, stride=2),) >>> transform = IndexTransform(domain=domain, output=output) >>> transform.apply((0,)), transform.apply((1,)), transform.apply((49,)) ((0,), (2,), (98,)) The selection compiler derives the identical transform from the source's shape and the slice: >>> transform == IndexTransform.from_shape((100,))[::2] True """ domain: IndexDomain """The input domain: the request coordinates this transform accepts.""" output: tuple[OutputIndexMap, ...] """One output map per output dimension, each producing that dimension's coordinate.""" def __post_init__(self) -> None: for i, m in enumerate(self.output): if isinstance(m, DimensionMap): if m.input_dimension < 0 or m.input_dimension >= self.domain.ndim: raise ValueError( f"output[{i}].input_dimension = {m.input_dimension} " f"is out of range for input rank {self.domain.ndim}" ) elif isinstance(m, ArrayMap): # An index array carries the transform's full input rank: the axis # a map varies over is full-sized, every other axis a singleton. # The rank is what makes the dependency axes readable from the # shape, so a mismatch is a bug rather than a spelling. External # JSON may use a lower-rank array that broadcasts against the # domain; `from_json` widens those on the way in, # so the invariant holds for every transform that exists. if m.index_array.ndim != self.domain.ndim: raise ValueError( f"output[{i}].index_array has {m.index_array.ndim} dims " f"but input domain has {self.domain.ndim} dims" ) # Every axis is either the domain's extent or a singleton it # broadcasts over. Any other size addresses input coordinates the # array has no entry for, which reads as a smaller selection # rather than as the error it is. bad = [ (axis, size, extent) for axis, (size, extent) in enumerate( zip(m.index_array.shape, self.domain.shape, strict=True) ) if size not in (1, extent) ] if len(bad) > 0: axis, size, extent = bad[0] raise ValueError( f"output[{i}].index_array has {size} entries on axis {axis}, " f"which is neither 1 nor the domain's extent of {extent} " f"(index_array shape {m.index_array.shape}, " f"domain shape {self.domain.shape})" ) def __eq__(self, other: object) -> bool: """Value equality. `ArrayMap` compares its index array element-wise, so a transform holding one can be compared at all — the generated `__eq__` raised `ValueError: the truth value of an array ... is ambiguous`.""" if not isinstance(other, IndexTransform): return NotImplemented return self.domain == other.domain and self.output == other.output def __hash__(self) -> int: """Hashed by value, so a transform can key a cache or enter a set.""" return hash((self.domain, self.output)) @property def input_rank(self) -> int: """Number of input dimensions — the rank of `domain`.""" return self.domain.ndim @property def output_rank(self) -> int: """Number of output dimensions — one per output map.""" return len(self.output) @classmethod def _unchecked(cls, domain: IndexDomain, output: tuple[OutputIndexMap, ...]) -> IndexTransform: """Build a transform whose maps the caller has already fitted to `domain`. Skips `__post_init__`. For internal producers such as chunk resolution, which derive every map from an already-valid transform and build two transforms per chunk. """ transform = object.__new__(cls) object.__setattr__(transform, "domain", domain) object.__setattr__(transform, "output", output) return transform @classmethod def identity(cls, domain: IndexDomain) -> IndexTransform: """The identity transform over `domain`: every result cell reads the source at its own address.""" output = tuple(DimensionMap(input_dimension=i) for i in range(domain.ndim)) return cls(domain=domain, output=output) @classmethod def from_shape(cls, shape: tuple[int, ...]) -> IndexTransform: """The identity transform over a zero-origin domain of the given `shape`.""" return cls.identity(IndexDomain.from_shape(shape)) def apply(self, point: Sequence[int]) -> tuple[int, ...]: """Map one coordinate of `domain` to the source coordinate that fills it. In array-indexing terms: `point` names a cell of the result array, and the returned tuple — each `output` map evaluated at `point` — names the source-array cell its value is read from: the coordinate arrow, running result to source. Parameters ---------- point : Sequence[int] One literal coordinate for each input dimension. Returns ------- tuple[int, ...] One coordinate for each output map. Raises ------ ValueError If ``point`` does not have exactly one coordinate per input dimension. TypeError If the coordinates do not have an integer dtype. BoundsCheckError If a coordinate lies outside the input domain. OverflowError If a mapped output coordinate cannot be represented by ``np.intp``. Examples -------- The `[::2]` transform reads result cell `i` from source coordinate `2 * i`, so cell 3 of the result holds `source[6]`: >>> transform = IndexTransform.from_shape((100,))[::2] >>> transform.apply((3,)) (6,) """ coordinates = np.asarray(point) expected_shape = (self.input_rank,) if coordinates.shape != expected_shape: raise ValueError(f"point must have shape {expected_shape}, got {coordinates.shape}") # An empty Python sequence has no elements from which NumPy can infer # an integer dtype, but it is the unique point in a rank-zero domain. if self.input_rank == 0 and isinstance(point, (list, tuple)): coordinates = coordinates.astype(np.intp) try: result = self._apply_points(coordinates) except _PointOutOfBounds as error: raise BoundsCheckError( f"coordinate {error.value} on input dimension {error.dimension} " f"is outside the domain [{error.lower}, {error.upper})" ) from None return tuple(int(value) for value in result) def apply_many(self, points: npt.ArrayLike) -> npt.NDArray[np.intp]: """Map a batch of `domain` coordinates to the source coordinates that fill them. The vectorized form of `apply`: each row of `points` names a result cell, and the corresponding output row names the source-array cell its value is read from. Parameters ---------- points : numpy.typing.ArrayLike Integer coordinates with shape ``batch_shape + (input_rank,)``. Returns ------- numpy.typing.NDArray[numpy.intp] An owned array with shape ``batch_shape + (output_rank,)``. Raises ------ ValueError If ``points`` has no trailing coordinate axis or that axis does not contain exactly one coordinate per input dimension. TypeError If the coordinates do not have an integer dtype. BoundsCheckError If a coordinate lies outside the input domain. OverflowError If a mapped output coordinate cannot be represented by ``np.intp``. Examples -------- Three result cells of the `[::2]` transform, located in one call: >>> transform = IndexTransform.from_shape((100,))[::2] >>> transform.apply_many(np.array([[0], [1], [49]])).tolist() [[0], [2], [98]] """ coordinates = np.asarray(points) if coordinates.ndim == 0 or coordinates.shape[-1] != self.input_rank: raise ValueError( "points must have a trailing coordinate axis of size " f"{self.input_rank}, got shape {coordinates.shape}" ) try: return self._apply_points(coordinates) except _PointOutOfBounds as error: raise BoundsCheckError( f"point at batch position {error.batch_position} has input dimension " f"{error.dimension} coordinate {error.value} outside " f"[{error.lower}, {error.upper})" ) from None def _apply_points(self, points: np.ndarray[Any, Any]) -> npt.NDArray[np.intp]: """Vectorized implementation shared by ``apply`` and ``apply_many``. Out-of-domain coordinates raise the internal `_PointOutOfBounds` signal; each public entry point formats it in its own vocabulary — `apply` never mentions a batch, `apply_many` names the batch position.""" if not np.issubdtype(points.dtype, np.integer): raise TypeError(f"points must have an integer dtype, got {points.dtype}") invalid = np.zeros(points.shape, dtype=np.bool_) for dimension, (lower, upper) in enumerate( zip(self.domain.inclusive_min, self.domain.exclusive_max, strict=True) ): invalid[..., dimension] = (points[..., dimension] < lower) | ( points[..., dimension] >= upper ) invalid_positions = np.argwhere(invalid) if invalid_positions.size > 0: first = invalid_positions[0] dimension = int(first[-1]) batch_position = tuple(int(position) for position in first[:-1]) point_index = tuple(int(position) for position in first) value = int(points[point_index]) lower = self.domain.inclusive_min[dimension] upper = self.domain.exclusive_max[dimension] raise _PointOutOfBounds(dimension, value, lower, upper, batch_position) batch_shape = points.shape[:-1] result = np.empty(batch_shape + (self.output_rank,), dtype=np.intp) for output_dimension, output_map in enumerate(self.output): if isinstance(output_map, ConstantMap): if result[..., output_dimension].size == 0: continue result[..., output_dimension] = checked_affine(output_map.offset, 0, 0) elif isinstance(output_map, DimensionMap): result[..., output_dimension] = checked_affine( output_map.offset, output_map.stride, points[..., output_map.input_dimension], ) else: index = tuple( np.zeros(batch_shape, dtype=np.intp) if output_map.index_array.shape[axis] == 1 else _positions_from_origin(points[..., axis], self.domain.inclusive_min[axis]) for axis in range(self.input_rank) ) result[..., output_dimension] = checked_affine( output_map.offset, output_map.stride, np.asarray(output_map.index_array[index]), ) return result def inverted(self) -> IndexTransform: """Return the restricted, exactly representable inverse transform. Inversion is defined for square transforms containing only constants and unique unit-stride dimension maps. Any input dimension not named by a dimension map must have singleton extent, so its coordinate can be recovered as a constant. Returns ------- IndexTransform A new transform mapping output coordinates back to input coordinates. Raises ------ ValueError If this transform does not have a representable inverse, including when input labels cannot be transferred to unlabeled output dimensions. """ if self.domain.labels is not None: raise ValueError( "cannot invert transform: input labels cannot be represented " "because output dimensions do not carry labels" ) if self.input_rank != self.output_rank: raise ValueError( "cannot invert transform: input rank must equal output rank, got " f"{self.input_rank} and {self.output_rank}" ) # Nothing below touches `self`, so validating and building in one pass # leaves no partial state behind when a map is rejected. referenced: set[int] = set() inverse_min: list[int] = [] inverse_max: list[int] = [] inverse_output: dict[int, OutputIndexMap] = {} for output_dimension, output_map in enumerate(self.output): if isinstance(output_map, ArrayMap): raise ValueError( # noqa: TRY004 - valid map, invalid inverse f"cannot invert transform: output[{output_dimension}] is an ArrayMap" ) if isinstance(output_map, ConstantMap): inverse_min.append(output_map.offset) inverse_max.append(output_map.offset + 1) continue if output_map.stride not in (-1, 1): raise ValueError( "cannot invert transform: DimensionMap stride must be +1 or -1, " f"got {output_map.stride} for output[{output_dimension}]" ) input_dimension = output_map.input_dimension if input_dimension in referenced: raise ValueError( "cannot invert transform: input dimension " f"{input_dimension} is referenced more than once" ) referenced.add(input_dimension) lower = self.domain.inclusive_min[input_dimension] upper = self.domain.exclusive_max[input_dimension] if output_map.stride == 1: inverse_min.append(output_map.offset + lower) inverse_max.append(output_map.offset + upper) inverse_output[input_dimension] = DimensionMap( output_dimension, offset=-output_map.offset, ) else: inverse_min.append(output_map.offset - upper + 1) inverse_max.append(output_map.offset - lower + 1) inverse_output[input_dimension] = DimensionMap( output_dimension, offset=output_map.offset, stride=-1, ) for input_dimension, (lower, extent) in enumerate( zip(self.domain.inclusive_min, self.domain.shape, strict=True) ): if input_dimension in referenced: continue if extent != 1: raise ValueError( "cannot invert transform: unreferenced input dimension " f"{input_dimension} has extent {extent}, not 1" ) inverse_output[input_dimension] = ConstantMap(lower) return IndexTransform( domain=IndexDomain(tuple(inverse_min), tuple(inverse_max)), output=tuple(inverse_output[dimension] for dimension in range(self.input_rank)), ) @property def selection_repr(self) -> str: """Compact domain string, e.g. `'{ [2, 8), [0, 10) }'`. Follows TensorStore's IndexDomain notation: each dimension shown as `[inclusive_min, exclusive_max)` with stride annotation if not 1. Constant (integer-indexed) dimensions show as a single value. Array-indexed dimensions show the set of selected coordinates. """ parts: list[str] = [] for m in self.output: if isinstance(m, ConstantMap): parts.append(str(m.offset)) elif isinstance(m, DimensionMap): d = m.input_dimension lo = self.domain.inclusive_min[d] hi = self.domain.exclusive_max[d] start = m.offset + m.stride * lo stop = m.offset + m.stride * hi if m.stride == 1: parts.append(f"[{start}, {stop})") else: parts.append(f"[{start}, {stop}) step {m.stride}") else: # m: ArrayMap (OutputIndexMap = ConstantMap | DimensionMap | ArrayMap) storage = m.offset + m.stride * m.index_array n = int(storage.size) # .size, not len(): index_array may be 0-d if n <= 5: vals = ", ".join(str(int(v)) for v in storage.ravel()) parts.append("{" + vals + "}") else: parts.append("{" + f"array({n})" + "}") return "{ " + ", ".join(parts) + " }" def __repr__(self) -> str: maps: list[str] = [] for i, m in enumerate(self.output): if isinstance(m, ConstantMap): maps.append(f"out[{i}] = {m.offset}") elif isinstance(m, DimensionMap): maps.append(f"out[{i}] = {m.offset} + {m.stride} * in[{m.input_dimension}]") else: # m: ArrayMap (OutputIndexMap = ConstantMap | DimensionMap | ArrayMap) maps.append(f"out[{i}] = {m.offset} + {m.stride} * arr{m.index_array.shape}[in]") maps_str = ", ".join(maps) return f"IndexTransform(domain={self.domain}, {maps_str})" def intersect( self, output_domain: IndexDomain ) -> ( tuple[ IndexTransform, dict[int, np.ndarray[Any, np.dtype[np.intp]]] | np.ndarray[Any, np.dtype[np.intp]] | None, ] | None ): """Keep only the cells whose source coordinates fall inside `output_domain`. Chunk resolution is the canonical caller: intersecting a request with one chunk's box keeps the cells that chunk can serve. Returns `(restricted_transform, out_indices)` or None if empty. `out_indices` is bookkeeping for chunk resolution. `None` means there are no ArrayMaps; the restricted domain itself records surviving input coordinates, which may be fewer than the original domain's. One orthogonal ArrayMap returns its surviving positional indices; multiple orthogonal ArrayMaps return a dict keyed by output dimension. General array maps return a shared scatter array. For paired public coordinate mappings, use `plan_chunks` and its `cell_transform`. Raises ------ NotImplementedError For a nonempty transform whose ArrayMap has a non-singleton axis also referenced by a DimensionMap. These dependencies cannot be filtered independently by the current intersection implementation. """ return _intersect(self, output_domain) def translate(self, shift: tuple[int, ...]) -> IndexTransform: """Shift the source coordinates every cell reads by `shift`, per dimension. The domain is untouched: the result keeps its cells, and each one reads from a shifted source address — for example, making a chunk's global addresses chunk-local by translating by the chunk's negated origin. """ if len(shift) != self.output_rank: raise ValueError(f"shift must have length {self.output_rank}, got {len(shift)}") new_output: list[OutputIndexMap] = [] for m, s in zip(self.output, shift, strict=True): if isinstance(m, ConstantMap): new_output.append(ConstantMap(offset=m.offset + s)) elif isinstance(m, DimensionMap): new_output.append( DimensionMap( input_dimension=m.input_dimension, offset=m.offset + s, stride=m.stride, ) ) else: # m: ArrayMap (OutputIndexMap = ConstantMap | DimensionMap | ArrayMap) new_output.append( m._with_affine(m.offset + s, m.stride) # pyright: ignore[reportPrivateUsage] ) return IndexTransform(domain=self.domain, output=tuple(new_output)) def __getitem__(self, selection: Any) -> IndexTransform: """Compose a basic selection (int, slice, ellipsis, newaxis) into a new transform. No I/O occurs. Integers and slice bounds are literal domain coordinates (TensorStore convention): negative values are not counted from the end, and nonempty out-of-domain slice intervals raise `BoundsCheckError`. Empty slices (`start == stop`) are accepted even outside the domain. Integer indices must lie inside the domain and drop their input dimension; `None` inserts a size-1 dimension. """ return _apply_basic_indexing(self, selection) def translate_domain_by(self, shift: tuple[int, ...]) -> IndexTransform: """Shift the *input* domain by `shift`, preserving which cells are addressed. TensorStore's `translate_by`: the domain moves, and every output map is re-offset so that new coordinate `c` addresses the cell that `c - shift` addressed before. ArrayMaps are indexed positionally over the domain, so their index arrays are unchanged. """ if len(shift) != self.input_rank: raise ValueError(f"shift must have length {self.input_rank}, got {len(shift)}") new_domain = self.domain.translate(shift) new_output: list[OutputIndexMap] = [] for m in self.output: if isinstance(m, DimensionMap): s = shift[m.input_dimension] new_output.append( DimensionMap( input_dimension=m.input_dimension, offset=m.offset - m.stride * s, stride=m.stride, ) ) else: # ConstantMap: no input dependence. ArrayMap: positional over # the domain, invariant under domain translation. new_output.append(m) return IndexTransform(domain=new_domain, output=tuple(new_output)) def translate_domain_to(self, origins: tuple[int, ...]) -> IndexTransform: """Move the input domain so its per-dimension origins equal `origins`. TensorStore's `translate_to`; `translate_domain_to((0,) * rank)` re-zeros a view's coordinate system without changing which cells it addresses. """ if len(origins) != self.input_rank: raise ValueError(f"origins must have length {self.input_rank}, got {len(origins)}") shift = tuple(o - m for o, m in zip(origins, self.domain.inclusive_min, strict=True)) return self.translate_domain_by(shift) @property def oindex(self) -> _OIndexHelper: """Accessor for the orthogonal (outer-product) indexing dialect. `transform.oindex[sel]` applies each index array independently per dimension and returns a new transform. """ return _OIndexHelper(self) @property def vindex(self) -> _VIndexHelper: """Accessor for the vectorized (coordinate/mask) indexing dialect. `transform.vindex[sel]` broadcasts all index arrays together, NumPy fancy-indexing style, and returns a new transform. """ return _VIndexHelper(self) @property def index_array_structure(self) -> Literal["none", "orthogonal", "general"]: """Classify how a transform's index arrays relate to its input axes. Returns ------- `"none"` when no output map is an `ArrayMap`; `"orthogonal"` when every `ArrayMap` has exactly one non-singleton input axis, distinct from all other ArrayMaps' axes and all DimensionMap input dimensions (one independent gather per axis); `"general"` otherwise — correlated (`vindex`) maps sharing their non-singleton axes, maps produced by composing fancy steps, maps sharing an input axis (a diagonal gather), and maps whose shape names no dependency axis. A map can have a non-singleton axis even when another axis has size zero. The orthogonal resolvers narrow one axis at a time and are only sound for `"orthogonal"`; everything else takes the pointwise path that collapses the joint block or rejects unsupported shared affine/array dependencies. Classification uses index-array shapes and DimensionMap input dimensions. Examples -------- >>> t = IndexTransform.from_shape((4, 5)) >>> t.index_array_structure 'none' `oindex` arrays each vary over their own axis (an outer product): >>> t.oindex[[0, 2], [1, 3]].index_array_structure 'orthogonal' `vindex` arrays are correlated — they share the broadcast axis: >>> t.vindex[np.array([0, 2]), np.array([1, 3])].index_array_structure 'general' """ seen = {m.input_dimension for m in self.output if isinstance(m, DimensionMap)} has_array = False for m in self.output: if not isinstance(m, ArrayMap): continue has_array = True dep = m.dependency_axes if len(dep) != 1 or dep[0] in seen: return "general" seen.add(dep[0]) return "orthogonal" if has_array else "none" def select( self, selection: Any, mode: Literal["basic", "orthogonal", "vectorized"] = "basic", ) -> IndexTransform: """Convert a user selection into a composed IndexTransform. Negative indices are treated as literal coordinates (TensorStore convention). The caller (Array layer) is responsible for converting numpy-style negative indices before calling this function. Examples -------- The `mode` picks the dialect; the result is the composed self the corresponding accessor builds: >>> t = IndexTransform.from_shape((10,)) >>> t.select(slice(2, 8)) == t[2:8] True >>> s = t.select(([9, 0, 0],), mode="orthogonal") >>> s.apply((0,)), s.apply((1,)), s.apply((2,)) ((9,), (0,), (0,)) """ if mode == "basic": _validate_basic_selection(selection) return self[selection] elif mode == "orthogonal": _validate_array_selection(selection, self.domain.shape, mode) return self.oindex[selection] elif mode == "vectorized": _validate_array_selection(selection, self.domain.shape, mode) return self.vindex[selection] else: raise ValueError(f"Unknown mode: {mode!r}") def compose(self, inner: IndexTransform) -> IndexTransform: """Chain `inner` onto this transform, yielding one direct transform. This transform maps its own input coordinates to `inner`'s input coordinates, and `inner` maps those onward; the result maps this transform's input coordinates straight to `inner`'s output coordinates. Composition is what keeps a view of a view a single description rather than a stack of layers, and it is exact: index arrays are evaluated at the new coordinates rather than accumulated. The precondition is that this transform's output rank equals `inner`'s input rank; a mismatch, or coordinates leaving `inner`'s domain, raises. Examples -------- Chained indexing — `source[2:5]`, then `[::-1]` on the result — collapses to one transform (a reversed axis keeps literal coordinates, so the composed domain is `[-4, -1)`): >>> inner = IndexTransform.from_shape((10,))[2:5] >>> outer = IndexTransform.identity(inner.domain)[::-1] >>> chained = outer.compose(inner) >>> chained == inner[::-1] True >>> [chained.apply((i,)) for i in (-4, -3, -2)] [(4,), (3,), (2,)] """ from zarr_indexing._composition import compose return compose(self, inner) # -- serialization ------------------------------------------------------ def to_json(self) -> IndexTransformJSON: """Convert to the canonical ndsel transform body (spec section 4.3). The result is fully explicit: `input_rank`, fully written bounds and labels, and an `output` carrying `offset`/`stride` on every affine and array map. It is field-for-field a TensorStore `IndexTransform` minus the `kind` discriminator, so it loads directly into `tensorstore.IndexTransform(json=...)`. Examples -------- >>> body = IndexTransform.from_shape((6,))[1:5:2].to_json() >>> (body["input_inclusive_min"], body["input_exclusive_max"]) ([0], [2]) >>> body["output"] [{'offset': 1, 'stride': 2, 'input_dimension': 0}] """ from zarr_indexing._wire import emit_labels return { "input_rank": self.domain.ndim, "input_inclusive_min": list(self.domain.inclusive_min), "input_exclusive_max": list(self.domain.exclusive_max), "input_labels": emit_labels(self.domain.labels, self.domain.ndim), "output": [m.to_json() for m in self.output], } @classmethod def from_json(cls, data: IndexTransformJSON) -> IndexTransform: """Construct from a canonical (or canonicalizable) ndsel transform body. The body is first run through the message layer (`normalize_ndsel`) so that omitted fields — identity `output`, default bounds and labels — are filled and validated, then lowered to the engine representation. Lower-rank `index_array`s are widened to the full input rank on the way in. Every supplied raw index value is checked against the inclusive `index_array_bounds` before offset, stride, or map simplification. Out-of-bounds values raise `NdselError` immediately, even if a later selection would avoid them. Validated immutable maps do not retain bounds. Examples -------- >>> body = IndexTransform.from_shape((6,))[1:5:2].to_json() >>> transform = IndexTransform.from_json(body) >>> transform.domain.shape (2,) >>> transform.to_json() == body # the round trip is exact True """ from zarr_indexing._wire import ( check_index_array_bounds, full_rank_index_array, lower_bound, lower_index_array, lower_labels, ) from zarr_indexing.messages import NdselError, normalize_ndsel if not isinstance(data, dict): # pyright: ignore[reportUnnecessaryIsInstance] raise NdselError( "invalid_json", f"a transform body must be a JSON object, got {data!r}" ) kind = data.get("kind", "transform") if kind != "transform": # Spelled before normalization so a body carrying its own `kind` # cannot reinterpret the document as some other message and return # a selection this constructor never promised. raise NdselError("invalid_json", f"a transform body cannot carry kind {kind!r}") body = normalize_ndsel({**data, "kind": "transform"}) domain = IndexDomain( inclusive_min=tuple( lower_bound(b, f"input_inclusive_min[{i}]") for i, b in enumerate(body["input_inclusive_min"]) ), exclusive_max=tuple( lower_bound(b, f"input_exclusive_max[{i}]") for i, b in enumerate(body["input_exclusive_max"]) ), labels=lower_labels(body["input_labels"]), ) output: list[OutputIndexMap] = [] for i, om in enumerate(body["output"]): if "index_array" in om: where = f"output[{i}]" arr = lower_index_array(om["index_array"], f"{where}.index_array") check_index_array_bounds(arr, om["index_array_bounds"], where) # ndsel leaves index-array rank unvalidated, so an external # producer may send an array of lower rank that broadcasts # against the domain. Widen it here, on the way in, so every # transform that exists holds the full-rank invariant the # engine reads dependency axes from. output.append( ArrayMap( index_array=full_rank_index_array(arr, domain, where), offset=om.get("offset", 0), stride=om.get("stride", 1), ) ) elif "input_dimension" in om: output.append( DimensionMap( input_dimension=om["input_dimension"], offset=om.get("offset", 0), stride=om.get("stride", 1), ) ) else: output.append(ConstantMap(offset=om.get("offset", 0))) try: return cls(domain=domain, output=tuple(output)) except ValueError as exc: # The engine's invariants are the last gate a document passes, and # they speak in the engine's vocabulary. A document that fails them # is invalid input, so it leaves here as one — with the engine's # account of what was wrong kept, since it names the offending # output map and axis. raise NdselError("rank_mismatch", str(exc)) from exc def _positions_from_origin(coordinates: np.ndarray[Any, Any], origin: int) -> npt.NDArray[np.intp]: """Convert literal coordinates to positional indices without wrapping.""" return checked_affine(-int(origin), 1, coordinates) def _intersect( transform: IndexTransform, output_domain: IndexDomain ) -> ( tuple[ IndexTransform, dict[int, np.ndarray[Any, np.dtype[np.intp]]] | np.ndarray[Any, np.dtype[np.intp]] | None, ] | None ): """Intersect a transform with an output domain (e.g., a chunk's bounds). For each output dimension, restrict to output coordinates within `[output_domain.inclusive_min[d], output_domain.exclusive_max[d])`. Two flavors of fancy indexing require different treatment, distinguished by the ArrayMaps' dependency axes (see `ArrayMap.dependency_axes`): - **orthogonal** (`oindex`): each ArrayMap varies over a single, distinct input axis, forming an outer product. Every output dimension is intersected independently and the input domain narrowed per axis. - **correlated** (`vindex`): the ArrayMaps share their (broadcast) dependency axes and scatter through a single flat index. A point survives only if ALL its output coordinates fall within the output domain; residual slice dimensions are intersected independently, as in the orthogonal case. The routing is `index_array_structure`: only a pure per-axis outer product takes the orthogonal path. Returns `None` if the intersection is empty. """ if output_domain.ndim != transform.output_rank: raise ValueError( f"output_domain rank ({output_domain.ndim}) != " f"transform output rank ({transform.output_rank})" ) if any(size == 0 for size in transform.domain.shape): # An empty input domain addresses no coordinates at all, so it meets no # output domain. Deciding it here keeps the per-flavor intersections from # having to reconcile an empty domain with an index array that is *not* # empty: a genuine extent-1 axis is stored as a broadcast singleton, so # emptying its domain leaves the array at size 1. return None if transform.index_array_structure == "general": return _intersect_general(transform, output_domain) return _intersect_orthogonal(transform, output_domain) def _intersect_dimension_map( m: DimensionMap, input_lo: int, input_hi: int, lo: int, hi: int ) -> tuple[int, int] | None: """Narrow a DimensionMap's input range to output coordinates in `[lo, hi)`. `input_lo`/`input_hi` are the current (possibly already narrowed) input range for the map's axis. Returns the new `(input_lo, input_hi)` or `None` if no input produces an in-bounds output coordinate. """ if input_lo >= input_hi: return None if m.stride > 0: new_input_lo = max(input_lo, _ceil_div(lo - m.offset, m.stride)) new_input_hi = min(input_hi, _ceil_div(hi - m.offset, m.stride)) elif m.stride < 0: new_input_lo = max(input_lo, _ceil_div(hi - 1 - m.offset, m.stride)) new_input_hi = min(input_hi, _ceil_div(lo - 1 - m.offset, m.stride)) else: if lo <= m.offset < hi: new_input_lo, new_input_hi = input_lo, input_hi else: return None if new_input_lo >= new_input_hi: return None return new_input_lo, new_input_hi def _ceil_div(numerator: int, denominator: int) -> int: """Return ``ceil(numerator / denominator)`` using exact integer arithmetic.""" return -((-numerator) // denominator) def _intersect_orthogonal( transform: IndexTransform, output_domain: IndexDomain ) -> ( tuple[ IndexTransform, dict[int, np.ndarray[Any, np.dtype[np.intp]]] | np.ndarray[Any, np.dtype[np.intp]] | None, ] | None ): """Intersect a transform with no correlated ArrayMaps. Every output dimension is intersected independently. Multiple ArrayMaps bound to distinct input dimensions form an outer product, so each array's surviving *output* positions are tracked separately. """ new_min = list(transform.domain.inclusive_min) new_max = list(transform.domain.exclusive_max) new_output: list[OutputIndexMap] = [] out_positions: dict[int, np.ndarray[Any, np.dtype[np.intp]]] = {} for out_dim, m in enumerate(transform.output): lo = output_domain.inclusive_min[out_dim] hi = output_domain.exclusive_max[out_dim] if isinstance(m, ConstantMap): if lo <= m.offset < hi: new_output.append(m) else: return None elif isinstance(m, DimensionMap): d = m.input_dimension narrowed = _intersect_dimension_map(m, new_min[d], new_max[d], lo, hi) if narrowed is None: return None new_min[d], new_max[d] = narrowed new_output.append(m) else: # m: ArrayMap (OutputIndexMap = ConstantMap | DimensionMap | ArrayMap) # Orthogonal: the array varies over a single axis. Filter along that # axis and keep the array at full input rank so the singleton axes # it broadcasts over are preserved. axis = m.dependent_axis if axis is None: raise ValueError( f"output[{out_dim}] is an ArrayMap that varies over no input " "dimension; a map with no dependency axis should have been " "collapsed to a ConstantMap" ) d = axis storage = checked_affine(m.offset, m.stride, m.index_array) mask = (storage >= lo) & (storage < hi) # The array is singleton on every axis but `d`, so its mask reduces # to a 1-D vector along `d`. survivors = np.nonzero(mask.reshape(-1))[0].astype(np.intp) if survivors.size == 0: return None filtered = np.take(m.index_array, survivors, axis=d) new_output.append( ArrayMap( index_array=np.asarray(filtered, dtype=np.intp), offset=m.offset, stride=m.stride, ) ) new_max[d] = new_min[d] + int(survivors.size) out_positions[out_dim] = survivors new_domain = IndexDomain( inclusive_min=tuple(new_min), exclusive_max=tuple(new_max), ) result = IndexTransform(domain=new_domain, output=tuple(new_output)) # Hand back the surviving output positions in the shape the bridge expects: # None (no arrays), a single vector (one array), or a per-output-dim dict # (>= 2 orthogonal arrays → outer product). out_indices: ( dict[int, np.ndarray[Any, np.dtype[np.intp]]] | np.ndarray[Any, np.dtype[np.intp]] | None ) if len(out_positions) == 0: out_indices = None elif len(out_positions) == 1: out_indices = next(iter(out_positions.values())) else: out_indices = out_positions return (result, out_indices) @dataclass(frozen=True, slots=True) class _CorrelatedBlock: """The correlated index arrays of a general transform, flattened over the broadcast block. Built by `_prepare_correlated`: `flat_storage` holds the storage coordinate of every block point per correlated output dimension, `flat_index` the raw index values. Chunk resolution sorts `flat_storage` into chunks once to build a `JointSet`; `_intersect_general` masks it for one output domain. """ correlated_dims: tuple[int, ...] broadcast_axes: tuple[int, ...] broadcast_shape: tuple[int, ...] flat_index: dict[int, np.ndarray[Any, np.dtype[np.intp]]] flat_storage: dict[int, np.ndarray[Any, np.dtype[np.intp]]] def _prepare_correlated( transform: IndexTransform, *, output_dimensions: tuple[int, ...] | None = None, input_dimensions: tuple[int, ...] | None = None, ) -> _CorrelatedBlock: """Flatten a general transform's index arrays over its broadcast block.""" correlated_dims = ( tuple(i for i, m in enumerate(transform.output) if isinstance(m, ArrayMap)) if output_dimensions is None else output_dimensions ) # The broadcast axes are exactly the input axes no `DimensionMap` binds: a # correlated transform's input domain is its residual slice axes plus the # collapsed broadcast block. Deriving them by complement rather than from the # index array's non-singleton axes keeps this correct when a broadcast axis # is itself size 1, and when NumPy's placement rule puts the broadcast block # somewhere other than the front (see `_broadcast_insertion_point`). bound_axes = {m.input_dimension for m in transform.output if isinstance(m, DimensionMap)} broadcast_axes = ( tuple(a for a in range(transform.input_rank) if a not in bound_axes) if input_dimensions is None else input_dimensions ) broadcast_shape = tuple(transform.domain.shape[a] for a in broadcast_axes) flat_index: dict[int, np.ndarray[Any, np.dtype[np.intp]]] = {} flat_storage: dict[int, np.ndarray[Any, np.dtype[np.intp]]] = {} for out_dim in correlated_dims: arr_map = cast("ArrayMap", transform.output[out_dim]) if any(a not in broadcast_axes for a in arr_map.dependency_axes): # Reachable only by hand-building a transform: no selection binds # the same input axis to both a slice map and an index array. raise NotImplementedError( "intersecting a transform whose index array varies over an " "input dimension also bound by a slice map is not supported" ) arr = arr_map.index_array # Index arrays are singleton on every non-broadcast axis, so they # collapse (C-order) to the broadcast block. A map may also be singleton # along a block axis it does not vary over (an orthogonal member, or a # leftover broadcast axis), so the collapsed array is broadcast up to the # full block rather than reshaped. block = arr.reshape(tuple(arr.shape[a] for a in broadcast_axes)) flat = np.ascontiguousarray(np.broadcast_to(block, broadcast_shape)).reshape(-1) flat_index[out_dim] = np.asarray(flat, dtype=np.intp) flat_storage[out_dim] = checked_affine(arr_map.offset, arr_map.stride, flat) return _CorrelatedBlock( correlated_dims=correlated_dims, broadcast_axes=broadcast_axes, broadcast_shape=broadcast_shape, flat_index=flat_index, flat_storage=flat_storage, ) def _intersect_general( transform: IndexTransform, output_domain: IndexDomain, ) -> tuple[IndexTransform, np.ndarray[Any, np.dtype[np.intp]]] | None: """Intersect array maps jointly when they are independent of residual affine axes. Every `ArrayMap` — correlated, orthogonal, or several sharing an axis — is treated as a lookup table over the joint block of non-slice axes: a block point survives only if ALL its output coordinates fall within the output domain. Residual DimensionMap dimensions are intersected independently (as in the orthogonal case) and preserved, so a partial vindex — e.g. two coordinate arrays over a 3-D array, leaving one slice dimension — resolves correctly. Treating an orthogonal map this way forfeits its per-axis independence (the block enumerates the outer product), which is why the pure-orthogonal case keeps its own resolver. The surviving broadcast axes collapse to a single axis; the returned `out_indices` combines positional broadcast offsets with literal residual slice coordinates, of shape `(surviving_points,) + (residual slice sizes)`. Chunk resolution removes residual origins before treating it as a flat offset into the request buffer. A rank-0 broadcast block — every coordinate array a scalar, as after `vindex[...]` narrowed to a single point — has no axis to collapse and stays rank 0: the block either survives whole or the intersection is empty. The result keeps only the residual slice axes and `out_indices` loses its leading points axis, so the sub-transform's rank still matches the view's. """ block = _prepare_correlated(transform) correlated_dims = block.correlated_dims broadcast_axes = block.broadcast_axes broadcast_shape = block.broadcast_shape # Joint bounds mask over the broadcast block. combined = np.ones(math.prod(broadcast_shape), dtype=np.bool_) for out_dim in correlated_dims: storage = block.flat_storage[out_dim] lo = output_domain.inclusive_min[out_dim] hi = output_domain.exclusive_max[out_dim] combined &= (storage >= lo) & (storage < hi) surviving = np.flatnonzero(combined).astype(np.intp) if surviving.size == 0: return None # Intersect residual (slice / constant) dimensions independently. Slice dims # are ordered by input dimension so their flat-buffer strides are row-major. slice_dims: list[tuple[int, int, int, int, DimensionMap]] = [] # (in_dim, lo, hi, full, m) for out_dim, m in enumerate(transform.output): if out_dim in correlated_dims: continue lo = output_domain.inclusive_min[out_dim] hi = output_domain.exclusive_max[out_dim] if isinstance(m, ConstantMap): if not (lo <= m.offset < hi): return None elif isinstance(m, DimensionMap): d = m.input_dimension input_lo = transform.domain.inclusive_min[d] input_hi = transform.domain.exclusive_max[d] narrowed = _intersect_dimension_map(m, input_lo, input_hi, lo, hi) if narrowed is None: return None slice_dims.append((d, narrowed[0], narrowed[1], input_hi - input_lo, m)) slice_dims.sort(key=lambda item: item[0]) n_points = int(surviving.size) n_slice = len(slice_dims) corr_values: dict[int, np.ndarray[Any, np.dtype[np.intp]]] = { out_dim: block.flat_index[out_dim][surviving] for out_dim in correlated_dims } # A rank-0 broadcast block contributes no axis: the leading `(n_points,)` of # the domain, of every index array, and of `out_indices` is present only when # there was a block to collapse. points_shape = (n_points,) if len(broadcast_shape) > 0 else () # New domain: the collapsed broadcast axis if there is one, then one axis per # residual slice. new_min = [0] * len(points_shape) new_max = list(points_shape) new_input_dim_of = {} for new_axis, (d, nlo, nhi, _full, _m) in enumerate(slice_dims, start=len(points_shape)): new_min.append(nlo) new_max.append(nhi) new_input_dim_of[d] = new_axis new_domain = IndexDomain(inclusive_min=tuple(new_min), exclusive_max=tuple(new_max)) corr_shape = points_shape + (1,) * n_slice new_output: list[OutputIndexMap] = [] for out_dim, m in enumerate(transform.output): if isinstance(m, ArrayMap): # `correlated_dims` is exactly the ArrayMap output dimensions. new_output.append( ArrayMap( index_array=corr_values[out_dim].reshape(corr_shape), offset=m.offset, stride=m.stride, ) ) elif isinstance(m, ConstantMap): new_output.append(m) else: new_output.append( DimensionMap( input_dimension=new_input_dim_of[m.input_dimension], offset=m.offset, stride=m.stride, ) ) result = IndexTransform(domain=new_domain, output=tuple(new_output)) # Scatter bookkeeping with row-major input-domain strides. Broadcast # coordinates below are positional; residual coordinates remain literal. # These are not uniformly zero-origin flat indices for nonzero domains. # # Each surviving point is a flat index into the broadcast block; unravel it # to per-axis coordinates so the buffer stride of each broadcast axis is # applied at its real position, wherever NumPy's placement rule put it. domain_shape = transform.domain.shape buffer_strides = [1] * len(domain_shape) for axis in range(len(domain_shape) - 2, -1, -1): buffer_strides[axis] = buffer_strides[axis + 1] * domain_shape[axis + 1] point_offsets = np.zeros(n_points, dtype=np.intp) if len(broadcast_shape) > 0: for axis, coords_along_axis in zip( broadcast_axes, np.unravel_index(surviving, broadcast_shape), strict=True ): point_offsets = point_offsets + coords_along_axis.astype(np.intp) * buffer_strides[axis] n_lead = len(points_shape) out_indices: np.ndarray[Any, np.dtype[np.intp]] = point_offsets.reshape( points_shape + (1,) * n_slice ) for j in range(n_slice): d, nlo, nhi, _full, _m = slice_dims[j] coords = np.arange(nlo, nhi, dtype=np.intp) * buffer_strides[d] shape = [1] * (n_lead + n_slice) shape[n_lead + j] = coords.size out_indices = out_indices + coords.reshape(shape) return (result, out_indices.astype(np.intp)) def _normalize_basic_selection(selection: Any, ndim: int) -> tuple[int | slice | None, ...]: """Normalize a selection to a tuple of int, slice, or None (newaxis), expanding ellipsis and padding with slice(None) as needed. """ if not isinstance(selection, tuple): selection = (selection,) # Count non-newaxis, non-ellipsis entries to determine how many real dims are addressed n_newaxis = sum(1 for s in selection if s is None) has_ellipsis = any(s is Ellipsis for s in selection) n_real = len(selection) - n_newaxis - (1 if has_ellipsis else 0) if n_real > ndim: raise IndexError( f"too many indices for array: array has {ndim} dimensions, but {n_real} were indexed" ) result: list[int | slice | None] = [] ellipsis_seen = False for sel in selection: if sel is Ellipsis: if ellipsis_seen: raise IndexError("an index can only have a single ellipsis ('...')") ellipsis_seen = True num_missing = ndim - n_real result.extend([slice(None)] * num_missing) elif (scalar := as_scalar_index(sel)) is not None: result.append(scalar) elif isinstance(sel, slice) or sel is None: result.append(sel) else: raise IndexError(f"unsupported selection type for basic indexing: {type(sel)!r}") # Pad remaining dimensions with slice(None) while sum(1 for s in result if s is not None) < ndim: result.append(slice(None)) return tuple(result) def _positional_slice(pos: int, size: int, step: int) -> slice: """A NumPy slice selecting `size` elements from `pos`, walking by `step`. The stop is `pos + size*step`, except in two cases. An empty selection is written out explicitly, because the arithmetic form can be a negative stop that NumPy would read as counting from the end. And a downward walk reaching the start of the array must stop at `None`, for the same reason: `slice(6, -1, -1)` selects nothing where `slice(6, None, -1)` selects the first seven elements in reverse. """ if size <= 0: return slice(0, 0, 1) stop = pos + size * step if step < 0 and stop < 0: return slice(pos, None, step) return slice(pos, stop, step) def _reindex_array( m: ArrayMap, normalized: tuple[int | slice | None, ...], domain: IndexDomain, ) -> np.ndarray[Any, np.dtype[np.intp]]: """Apply basic indexing operations to an ArrayMap's index_array. The array's axes correspond to the transform's input dimensions (0-indexed over the domain shape). Each axis is either a **dependency axis** — the array varies with that input dimension — or a **singleton** axis it broadcasts over. Integer indexing, slicing, or newaxis is applied to the array only along its dependency axes; a selection on a singleton axis does not touch the array's values (it just narrows or drops that broadcast axis). """ dependent = set(m.dependency_axes) arr = m.index_array # Build a numpy indexing tuple: one entry per old input dimension idx: list[Any] = [] old_dim = 0 newaxis_positions: list[int] = [] result_axis = 0 for sel in normalized: if sel is None: newaxis_positions.append(result_axis) result_axis += 1 elif isinstance(sel, int): if old_dim < arr.ndim: if old_dim in dependent: # Convert absolute domain coordinate to 0-based array index idx.append(sel - domain.inclusive_min[old_dim]) else: # Broadcast axis: keep the single element and drop the axis. idx.append(0) old_dim += 1 else: # sel: slice (normalized: tuple[int | slice | None, ...]) if old_dim < arr.ndim: if old_dim in dependent: lo = domain.inclusive_min[old_dim] hi = domain.exclusive_max[old_dim] # Bounds are literal domain coordinates; the stored array is # indexed positionally, so shift by the domain origin. start, step, _origin, size = _resolve_slice_ts(sel, old_dim, lo, hi) pos = start - lo idx.append(_positional_slice(pos, size, step)) else: # Broadcast axis: preserve the singleton (it still broadcasts # over the narrowed domain), regardless of the slice bounds. idx.append(slice(None)) old_dim += 1 result_axis += 1 result = arr[tuple(idx)] if idx else arr for pos in newaxis_positions: result = np.expand_dims(result, axis=pos) return np.asarray(result, dtype=np.intp) def _compose_selection( transform: IndexTransform, selection: Any, mode: Literal["orthogonal", "vectorized"], ) -> IndexTransform: """Apply an advanced selection to an array-carrying transform by composition. The selection is applied to an identity transform over the current domain — the same code path a fresh transform takes, so the dialect (placement, bounds, domains) is identical by construction — and the result is chained onto `transform` with `compose`, which evaluates the existing index arrays at the new coordinates. This is how a second fancy step lands on *any* axis of an already-fancy view: axes an existing array varies over, axes it merely broadcasts along, or a mixture. """ # Deferred import: `composition` imports this module at import time. identity = IndexTransform.identity(transform.domain) if mode == "orthogonal": outer = _apply_oindex(identity, selection) else: outer = _apply_vindex(identity, selection) return outer.compose(transform) def _apply_basic_indexing(transform: IndexTransform, selection: Any) -> IndexTransform: """Apply basic indexing (int, slice, ellipsis, newaxis) to an IndexTransform.""" normalized = _normalize_basic_selection(selection, transform.domain.ndim) new_inclusive_min: list[int] = [] new_exclusive_max: list[int] = [] old_dim = 0 new_dim_idx = 0 old_to_new_dim: dict[int, int] = {} dropped_dims: set[int] = set() # Per old-dim: the slice parameters (for computing new output maps) dim_slice_params: dict[int, tuple[int, int, int]] = {} # old_dim -> (start, stop, step) dim_int_val: dict[int, int] = {} # old_dim -> integer index value for sel in normalized: if sel is None: # newaxis: add a size-1 dimension new_inclusive_min.append(0) new_exclusive_max.append(1) new_dim_idx += 1 elif isinstance(sel, int): # Integer index: drop this input dimension. # Negative indices are literal coordinates (TensorStore convention), # NOT "from the end" like NumPy. The Array layer handles conversion. lo = transform.domain.inclusive_min[old_dim] hi = transform.domain.exclusive_max[old_dim] idx = sel if idx < lo or idx >= hi: hint = _LITERAL_HINT if sel < 0 else "" raise BoundsCheckError( f"index {sel} is out of bounds for dimension {old_dim} " f"(valid indices [{lo}, {hi})){hint}" ) dropped_dims.add(old_dim) dim_int_val[old_dim] = idx old_dim += 1 else: # sel: slice (normalized: tuple[int | slice | None, ...]) lo = transform.domain.inclusive_min[old_dim] hi = transform.domain.exclusive_max[old_dim] # TensorStore semantics: bounds are literal coordinates; a step-1 # slice keeps them as the new domain, a strided slice's domain is # [trunc(start/step), trunc(start/step) + size). start, step, origin, size = _resolve_slice_ts(sel, old_dim, lo, hi) new_inclusive_min.append(origin) new_exclusive_max.append(origin + size) dim_slice_params[old_dim] = (start, step, origin) old_to_new_dim[old_dim] = new_dim_idx new_dim_idx += 1 old_dim += 1 new_domain = IndexDomain( inclusive_min=tuple(new_inclusive_min), exclusive_max=tuple(new_exclusive_max), ) # Now update output maps new_output: list[OutputIndexMap] = [] for m in transform.output: if isinstance(m, ConstantMap): new_output.append(m) elif isinstance(m, DimensionMap): d = m.input_dimension if d in dropped_dims: # Integer index: this output becomes constant new_offset = m.offset + m.stride * dim_int_val[d] new_output.append(ConstantMap(offset=new_offset)) elif d in old_to_new_dim: # Slice: new coordinate `origin + k` maps to old coordinate # `start + k*step`, i.e. old = start - step*origin + step*new. start, step, origin = dim_slice_params[d] new_offset = m.offset + m.stride * (start - step * origin) new_stride = m.stride * step new_input_dim = old_to_new_dim[d] new_output.append( DimensionMap( input_dimension=new_input_dim, offset=new_offset, stride=new_stride ) ) else: raise RuntimeError(f"unexpected: dimension {d} not handled") else: # m: ArrayMap (OutputIndexMap = ConstantMap | DimensionMap | ArrayMap). # A result narrowed to a single coordinate collapses to the # ConstantMap it equals — whether an integer consumed the dependency # axis or a slice narrowed it to one entry — so a non-empty ArrayMap # always varies over at least one axis. Nothing here renumbers: the # array's axes are the new domain's axes by construction. new_arr = _reindex_array(m, normalized, transform.domain) new_output.append(array_map_or_constant(new_arr, offset=m.offset, stride=m.stride)) return IndexTransform(domain=new_domain, output=tuple(new_output)) def _reshape_to_axis( values: np.ndarray[Any, np.dtype[np.intp]], axis: int, ndim: int ) -> np.ndarray[Any, np.dtype[np.intp]]: """Reshape a 1-D selection to full rank `ndim` varying only along `axis`. The result has `values` laid out along `axis` and singleton (size-1) axes everywhere else, so its dependency axis is derivable from its shape. """ flat = np.asarray(values, dtype=np.intp).ravel() shape = [1] * ndim shape[axis] = flat.shape[0] return flat.reshape(shape) class _OIndexHelper: """Helper that provides orthogonal (outer) indexing via `transform.oindex[...]`.""" def __init__(self, transform: IndexTransform) -> None: self._transform = transform def __getitem__(self, selection: Any) -> IndexTransform: return _apply_oindex(self._transform, selection) def _normalize_oindex_selection( selection: Any, ndim: int ) -> tuple[np.ndarray[Any, np.dtype[np.intp]] | slice, ...]: """Normalize an oindex selection: arrays, slices, booleans, integers.""" if not isinstance(selection, tuple): selection = (selection,) # Expand ellipsis has_ellipsis = any(s is Ellipsis for s in selection) n_ellipsis = 1 if has_ellipsis else 0 n_real = len(selection) - n_ellipsis result: list[np.ndarray[Any, np.dtype[np.intp]] | slice] = [] for sel in selection: if sel is Ellipsis: num_missing = ndim - n_real result.extend([slice(None)] * num_missing) elif isinstance(sel, np.ndarray) and sel.dtype == np.bool_: # Boolean array -> integer indices (indices,) = np.nonzero(sel) result.append(indices.astype(np.intp)) elif isinstance(sel, np.ndarray): result.append(checked_affine(0, 1, sel)) elif isinstance(sel, slice): result.append(sel) elif (scalar := as_scalar_index(sel)) is not None: # Convert integer scalars to 1-element arrays for orthogonal indexing result.append(np.array([scalar], dtype=np.intp)) elif isinstance(sel, (list, tuple)): array = np.asarray(sel) if array.dtype == np.bool_: (indices,) = np.nonzero(array) result.append(indices.astype(np.intp)) else: # Advanced selection validation has already checked the element types. integer_array = cast("npt.NDArray[np.integer[Any]]", array) result.append(checked_affine(0, 1, integer_array)) else: result.append(sel) # Pad with slice(None) while len(result) < ndim: result.append(slice(None)) return tuple(result) def _apply_oindex(transform: IndexTransform, selection: Any) -> IndexTransform: """Apply orthogonal indexing to an IndexTransform. Each index array is applied independently per dimension (outer product). A transform that already carries index arrays takes the composition path (`_compose_selection`) instead of being rewritten in place, so the new selection may land on any axis — including axes an existing array merely broadcasts along. """ validate_advanced_selection(selection, transform.domain, "orthogonal") if any(isinstance(m, ArrayMap) for m in transform.output): return _compose_selection(transform, selection, "orthogonal") normalized = _normalize_oindex_selection(selection, transform.domain.ndim) new_inclusive_min: list[int] = [] new_exclusive_max: list[int] = [] new_dim_idx = 0 old_to_new_dim: dict[int, int] = {} # Info per old dim dim_array: dict[int, np.ndarray[Any, np.dtype[np.intp]]] = {} dim_slice_params: dict[int, tuple[int, int, int]] = {} for old_dim, sel in enumerate(normalized): if isinstance(sel, np.ndarray): lo = transform.domain.inclusive_min[old_dim] hi = transform.domain.exclusive_max[old_dim] # Index-array values are literal domain coordinates; the fancy dim # they create gets a fresh zero-origin [0, n) domain (TensorStore). _check_array_in_bounds(sel, lo, hi) dim_array[old_dim] = sel new_inclusive_min.append(0) new_exclusive_max.append(len(sel)) old_to_new_dim[old_dim] = new_dim_idx new_dim_idx += 1 else: # sel: slice (_normalize_oindex_selection returns # tuple[np.ndarray | slice, ...]) lo = transform.domain.inclusive_min[old_dim] hi = transform.domain.exclusive_max[old_dim] start, step, origin, size = _resolve_slice_ts(sel, old_dim, lo, hi) new_inclusive_min.append(origin) new_exclusive_max.append(origin + size) dim_slice_params[old_dim] = (start, step, origin) old_to_new_dim[old_dim] = new_dim_idx new_dim_idx += 1 new_domain = IndexDomain( inclusive_min=tuple(new_inclusive_min), exclusive_max=tuple(new_exclusive_max), ) new_output: list[OutputIndexMap] = [] for m in transform.output: if isinstance(m, ConstantMap): new_output.append(m) elif isinstance(m, DimensionMap): d = m.input_dimension if d in dim_array: new_axis = old_to_new_dim[d] # Normalize to full input rank: the selection varies along its # own new axis and is singleton on every other axis, so the # dependency axis is readable from the shape. A single-entry # selection holds one coordinate and collapses to the # ConstantMap it equals; its length-1 axis stays in the domain. full_arr = _reshape_to_axis(dim_array[d], new_axis, new_dim_idx) new_output.append(array_map_or_constant(full_arr, offset=m.offset, stride=m.stride)) elif d in dim_slice_params: start, step, origin = dim_slice_params[d] new_offset = m.offset + m.stride * (start - step * origin) new_stride = m.stride * step new_input_dim = old_to_new_dim[d] new_output.append( DimensionMap( input_dimension=new_input_dim, offset=new_offset, stride=new_stride ) ) else: raise RuntimeError(f"unexpected: dimension {d} not handled") else: # m: ArrayMap — unreachable: array-carrying transforms took the # composition path at the top of this function. raise AssertionError( # noqa: TRY004 - unreachable, not a dispatch "unreachable: ArrayMap transforms are composed" ) return IndexTransform(domain=new_domain, output=tuple(new_output)) class _VIndexHelper: """Helper that provides vectorized (fancy) indexing via `transform.vindex[...]`.""" def __init__(self, transform: IndexTransform) -> None: self._transform = transform def __getitem__(self, selection: Any) -> IndexTransform: return _apply_vindex(self._transform, selection) def _broadcast_insertion_point(array_dims: Sequence[int], slice_dims: Sequence[int]) -> int: """Where the broadcast dimensions land, as a count of leading slice dimensions. NumPy's advanced-indexing placement rule: when the advanced indices are all next to each other in the index tuple, the broadcast dimensions are inserted at the spot they occupied; when a slice separates them, they lead. So `a[:, i, j]` has shape `(len(a), *broadcast)` while `a[i, :, j]` has shape `(*broadcast, a.shape[1])`. Returns the number of slice dimensions that precede the broadcast block; `0` means the broadcast dimensions lead. """ if len(array_dims) == 0: return 0 first, last = array_dims[0], array_dims[-1] separated = any(first < d < last for d in slice_dims) if separated: return 0 return sum(1 for d in slice_dims if d < first) def _as_boolean_index_array(selection: Any) -> np.ndarray[Any, np.dtype[np.bool_]] | None: """Return an array-like boolean index as an ndarray, else None.""" if not isinstance(selection, (np.ndarray, list, tuple)): return None array = np.asarray(selection) if array.dtype != np.bool_: return None return array def _selection_axis_count(selection: Any) -> int: """Return how many input axes one vectorized selection entry consumes.""" boolean_array = _as_boolean_index_array(selection) return boolean_array.ndim if boolean_array is not None else 1 def _apply_vindex(transform: IndexTransform, selection: Any) -> IndexTransform: """Apply vectorized indexing to an IndexTransform. All array indices are broadcast together. Their broadcast dimensions occupy the advanced indices' position when adjacent, and lead when a slice axis separates the advanced indices (see `_broadcast_insertion_point`). A transform that already carries index arrays takes the composition path (`_compose_selection`) instead of being rewritten in place; see `_apply_oindex`. """ validate_advanced_selection(selection, transform.domain, "vectorized") if any(isinstance(m, ArrayMap) for m in transform.output): return _compose_selection(transform, selection, "vectorized") if not isinstance(selection, tuple): selection = (selection,) # Expand ellipsis and count consumed dimensions. Boolean masks consume one # input axis per mask dimension, whether spelled as an ndarray or a list. n_consumed = sum(_selection_axis_count(s) for s in selection if s is not Ellipsis) ndim = transform.domain.ndim expanded: list[Any] = [] for sel in selection: if sel is Ellipsis: num_missing = ndim - n_consumed expanded.extend([slice(None)] * num_missing) else: expanded.append(sel) # Count dimensions already consumed by expanded entries n_expanded_dims = sum(_selection_axis_count(sel) for sel in expanded) while n_expanded_dims < ndim: expanded.append(slice(None)) n_expanded_dims += 1 # Convert booleans, lists, ints to integer arrays processed: list[np.ndarray[Any, np.dtype[np.intp]] | slice] = [] for sel in expanded: boolean_array = _as_boolean_index_array(sel) if boolean_array is not None: indices_tuple = np.nonzero(boolean_array) processed.extend(indices.astype(np.intp) for indices in indices_tuple) elif isinstance(sel, np.ndarray): processed.append(checked_affine(0, 1, sel)) elif isinstance(sel, (list, tuple)): # Advanced selection validation has already checked the element types. integer_array = cast("npt.NDArray[np.integer[Any]]", np.asarray(sel)) processed.append(checked_affine(0, 1, integer_array)) elif (scalar := as_scalar_index(sel)) is not None: processed.append(np.array([scalar], dtype=np.intp)) else: processed.append(sel) # Separate array dims and slice dims array_dims: list[int] = [] slices: dict[int, slice] = {} arrays: list[np.ndarray[Any, np.dtype[np.intp]]] = [] for i, sel in enumerate(processed): if isinstance(sel, np.ndarray): lo = transform.domain.inclusive_min[i] hi = transform.domain.exclusive_max[i] _check_array_in_bounds(sel, lo, hi) array_dims.append(i) arrays.append(sel) else: slices[i] = sel slice_dims = list(slices) # Determine the shared domain without expanding the index arrays: their # singleton axes describe independent dependencies during grid partitioning. broadcast_shape = np.broadcast_shapes(*(array.shape for array in arrays)) if arrays else () # Slice dimensions (preserved-domain literal semantics, like basic indexing) slice_dim_params: dict[int, tuple[int, int, int]] = {} slice_bounds: list[tuple[int, int]] = [] for old_dim, sl in slices.items(): lo = transform.domain.inclusive_min[old_dim] hi = transform.domain.exclusive_max[old_dim] start, step, origin, size = _resolve_slice_ts(sl, old_dim, lo, hi) slice_bounds.append((origin, origin + size)) slice_dim_params[old_dim] = (start, step, origin) n_before = _broadcast_insertion_point(array_dims, slice_dims) # Build the new domain with NumPy's placement rule: the broadcast # (correlated) dimensions sit where the advanced indices sat when those are # adjacent, and lead when a slice separates them. new_inclusive_min = [lo for lo, _ in slice_bounds[:n_before]] new_exclusive_max = [hi for _, hi in slice_bounds[:n_before]] new_inclusive_min.extend([0] * len(broadcast_shape)) new_exclusive_max.extend(broadcast_shape) new_inclusive_min.extend(lo for lo, _ in slice_bounds[n_before:]) new_exclusive_max.extend(hi for _, hi in slice_bounds[n_before:]) new_domain = IndexDomain( inclusive_min=tuple(new_inclusive_min), exclusive_max=tuple(new_exclusive_max), ) # Build output maps array_dim_to_broadcast: dict[int, np.ndarray[Any, np.dtype[np.intp]]] = {} for i, d in enumerate(array_dims): array_dim_to_broadcast[d] = arrays[i] # New dim index for slice dims starts after broadcast dims n_broadcast_dims = len(broadcast_shape) new_output: list[OutputIndexMap] = [] for m in transform.output: if isinstance(m, ConstantMap): new_output.append(m) elif isinstance(m, DimensionMap): d = m.input_dimension if d in array_dim_to_broadcast: # Pad to full input rank while retaining broadcast singletons. broadcast_arr = array_dim_to_broadcast[d] padded_shape = (1,) * (n_broadcast_dims - broadcast_arr.ndim) + broadcast_arr.shape full_arr = broadcast_arr.reshape( (1,) * n_before + padded_shape + (1,) * (len(slice_dims) - n_before) ) new_output.append(array_map_or_constant(full_arr, offset=m.offset, stride=m.stride)) else: # Slice dim: new coord `origin + k` maps to old `start + k*step` start, step, origin = slice_dim_params[d] new_offset = m.offset + m.stride * (start - step * origin) new_stride = m.stride * step position = slice_dims.index(d) new_input_dim = position if position < n_before else position + n_broadcast_dims new_output.append( DimensionMap( input_dimension=new_input_dim, offset=new_offset, stride=new_stride ) ) else: # m: ArrayMap — unreachable: array-carrying transforms took the # composition path at the top of this function. raise AssertionError( # noqa: TRY004 - unreachable, not a dispatch "unreachable: ArrayMap transforms are composed" ) return IndexTransform(domain=new_domain, output=tuple(new_output)) _LITERAL_HINT = ( "; within this transform layer, indices are literal domain coordinates (the " "public Array boundary wraps NumPy-style negatives before they reach here)" ) def _trunc_div(a: int, b: int) -> int: """Integer division rounded toward zero (C semantics), as TensorStore uses for strided-slice domain origins — distinct from Python's floor division for negative operands (`trunc(-9/2) == -4` where `-9 // 2 == -5`).""" q = a // b if q < 0 and q * b != a: q += 1 return q def _resolve_slice_ts(sel: slice, dim: int, lo: int, hi: int) -> tuple[int, int, int, int]: """Resolve a slice against domain `[lo, hi)` with TensorStore semantics. Slice bounds are **literal domain coordinates** — never from-the-end, never clamped. One rule covers both signs of the step (each part verified against tensorstore 0.1.84, and matching ndsel 1.0-draft.2 section 5.3): - defaults follow the direction of travel: `start = lo`, `stop = hi` going up; `start = hi - 1`, `stop = lo - 1` going down; - the traversal runs from `start` toward `stop`, which is excluded, so the source interval is `[start, stop)` going up and `[stop + 1, start + 1)` going down; - a non-empty interval must be contained in the domain (no clamping — a NumPy-style out-of-range or negative bound is an error, not a shorter or wrapped result); - an empty interval is valid anywhere, for either sign; - an interval running the wrong way (`stop` on the far side of `start` from the direction of travel) is an error, not an empty result; - the result's domain origin is `trunc(start/step)` — toward zero, for both signs — and coordinate `origin + k` maps to input `start + k*step`. A negative step normally produces a negative origin: reversing a zero-origin axis of length 20 gives the domain `[-19, 1)`. The coordinate frame stays anchored to the source; a caller that needs non-negative coordinates re-bases explicitly with `translate_domain_to`. Returns `(start, step, origin, size)` in domain coordinates. """ start_bound = None if sel.start is None else require_index(sel.start) stop_bound = None if sel.stop is None else require_index(sel.stop) step = 1 if sel.step is None else require_index(sel.step) if step == 0: raise IndexError("slice step must not be zero") if step > 0: start = lo if start_bound is None else start_bound stop = hi if stop_bound is None else stop_bound interval_lo, interval_hi = start, stop else: start = hi - 1 if start_bound is None else start_bound stop = lo - 1 if stop_bound is None else stop_bound interval_lo, interval_hi = stop + 1, start + 1 length = interval_hi - interval_lo if length < 0: raise IndexError( f"slice from {start} to {stop} with step {step} does not specify a " f"valid interval for dimension {dim}: the derived interval " f"[{interval_lo}, {interval_hi}) runs the wrong way. An empty " "selection is spelled stop == start." ) if length > 0 and (interval_lo < lo or interval_hi > hi): hint = _LITERAL_HINT if (start < 0 or stop < 0) and lo >= 0 else "" raise BoundsCheckError( f"slice interval [{interval_lo}, {interval_hi}) is not contained " f"within domain [{lo}, {hi}) for dimension {dim}{hint}" ) size = -(-length // abs(step)) # ceil(length / |step|) origin = _trunc_div(start, step) return start, step, origin, size def _check_array_in_bounds(arr: np.ndarray[Any, np.dtype[np.intp]], lo: int, hi: int) -> None: """Reject index-array values outside the domain `[lo, hi)`. Index-array values are literal domain coordinates (TensorStore semantics): a value below `inclusive_min` is out of bounds rather than counting from the end. Out-of-range values raise instead of silently wrapping. """ if arr.size == 0: return lo_val, hi_val = int(arr.min()), int(arr.max()) if lo_val < lo: hint = _LITERAL_HINT if lo_val < 0 and lo >= 0 else "" raise BoundsCheckError( f"index {lo_val} is out of bounds (valid indices [{lo}, {hi})){hint}" ) if hi_val >= hi: raise BoundsCheckError(f"index {hi_val} is out of bounds (valid indices [{lo}, {hi}))") def _validate_array_selection(selection: Any, shape: tuple[int, ...], mode: str) -> None: """Validate array-based selections (orthogonal, vectorized). Rejects types that are not valid for coordinate/vectorized indexing. Does not check bounds — the transform operations handle that. """ items = selection if isinstance(selection, tuple) else (selection,) for sel in items: if isinstance(sel, slice): # vindex is coordinate-only (matches eager zarr): every axis needs an # integer/boolean array, never a slice. Orthogonal (oindex) allows slices. if mode == "vectorized": raise VindexInvalidSelectionError( "unsupported selection type for vectorized indexing; only " "coordinate selection (tuple of integer arrays) and mask selection " f"(single Boolean array) are supported; got {selection!r}" ) continue if sel is Ellipsis or as_scalar_index(sel) is not None: continue if isinstance(sel, (list, np.ndarray)): if mode == "orthogonal": array = np.asarray(sel) # An orthogonal selection is per-axis, so an integer array names # coordinates along one axis and can only be one-dimensional. # Left to the engine, this surfaced much later as a rank # complaint about an `index_array` the caller never wrote. if array.dtype.kind in "iu" and array.ndim > 1: raise IndexError( f"integer arrays in an orthogonal selection must be " f"1-dimensional only; got one with {array.ndim} dimensions" ) continue raise IndexError(f"unsupported selection type for {mode} indexing: {type(sel)!r}") def _validate_basic_selection(selection: Any) -> None: """Validate that a selection only contains basic indexing types (int, slice, Ellipsis). Rejects None (newaxis), arrays, lists, floats, strings, etc. """ items = selection if isinstance(selection, tuple) else (selection,) for s in items: if s is Ellipsis or isinstance(s, slice) or as_scalar_index(s) is not None: continue raise IndexError(f"unsupported selection type for basic indexing: {type(s)!r}") zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/src/zarr_indexing/writer.py000066400000000000000000000237621527014137600313560ustar00rootroot00000000000000"""Synchronous writes through coordinate transforms using basic source assignment.""" from __future__ import annotations from typing import TYPE_CHECKING, Any import numpy as np from zarr_indexing.chunk_resolution import plan_chunks from zarr_indexing.output_map import ArrayMap, ConstantMap, DimensionMap if TYPE_CHECKING: from collections.abc import Sequence from zarr_indexing.chunk_resolution import ChunkProjection from zarr_indexing.grid import DimensionGridLike from zarr_indexing.transform import IndexTransform def write_into( source: Any, transform: IndexTransform, values: Any, write_grid: Sequence[DimensionGridLike] | None = None, ) -> None: """Write broadcast values to precisely the cells addressed by ``transform``. The source must expose ``shape``, a NumPy-compatible ``dtype``, and basic integer/slice assignment. Values are snapshotted and converted before the first mutation, so source aliases and conversion failures are safe. Extra leading singleton value axes are accepted, as in NumPy assignment. Repeated source coordinates receive the last value in C order of the view. Independent affine maps use one basic assignment. Other selections are scattered in bulk: a NumPy source receives one fancy assignment, and a readable source with a ``write_grid`` is written one grid cell at a time — the cell's touched hull is read once, updated in memory, and written back with one basic assignment, so storage round trips are bounded by touched cells. Without a grid, or for the few transforms the planner cannot factor (such as one input axis feeding two output maps), or for a source that cannot be read, the fallback is one integer assignment per element, which never reads. Backend assignment may itself read storage units as part of updating them. Source assignment failures propagate and may leave preceding writes applied; this operation is not transactional. """ from zarr_indexing.lazy_array import LazyArray shape = transform.domain.shape source_shape = tuple(int(extent) for extent in source.shape) if transform.output_rank != len(source_shape): raise ValueError("transform output rank must match the source rank") if isinstance(values, LazyArray): # Its NumPy conversion protocol may discard MaskedArray metadata. values = values.result() if isinstance(source, np.ma.MaskedArray): prepared = np.ma.array(values, dtype=source.dtype, copy=True) else: prepared = np.array(values, dtype=source.dtype, copy=True) while prepared.ndim > len(shape) and prepared.shape[0] == 1: prepared = prepared[0] if isinstance(prepared, np.ma.MaskedArray): broadcast = np.ma.array( np.broadcast_to(prepared.data, shape), mask=np.broadcast_to(np.ma.getmaskarray(prepared), shape), copy=False, ) else: broadcast = np.broadcast_to(prepared, shape) if any(extent == 0 for extent in shape): # Values were still converted and broadcast, so a mis-sized RHS is # reported even when the selection happens to be empty. return # Check every map's extremal outputs before any assignment. Python integer # arithmetic avoids overflow while computing bounds of affine maps. for extent, output in zip(source_shape, transform.output, strict=True): if isinstance(output, ConstantMap): low = high = output.offset else: if isinstance(output, DimensionMap): axis = output.input_dimension first = transform.domain.inclusive_min[axis] last = transform.domain.exclusive_max[axis] - 1 else: first = int(output.index_array.min()) last = int(output.index_array.max()) endpoints = ( output.offset + output.stride * first, output.offset + output.stride * last, ) low, high = min(endpoints), max(endpoints) if low < 0 or high >= extent: raise IndexError("transform output coordinates are outside the source shape") if _write_affine(source, transform, broadcast): return if len(source_shape) == 0: # A zero-rank source has one cell; no grid or scatter applies. source[()] = broadcast[(0,) * len(shape)] return if isinstance(source, np.ndarray): _scatter_numpy(source, transform, broadcast) return if write_grid is not None and hasattr(source, "__getitem__"): try: # Materialized before any write: an unfactorable transform is # reported by planning, never after some cells were rewritten. projections = list(plan_chunks(transform, tuple(write_grid))) except (ValueError, NotImplementedError): projections = None if projections is not None: _scatter_planned(source, transform, broadcast, projections) return _scatter_elementwise(source, transform, broadcast) def _write_affine(source: Any, transform: IndexTransform, broadcast: Any) -> bool: """Write with one basic assignment when every map is an independent affine map.""" shape = transform.domain.shape axes = [ output.input_dimension for output in transform.output if isinstance(output, DimensionMap) ] if ( len(set(axes)) != len(axes) or any(shape[axis] != 1 for axis in range(len(shape)) if axis not in axes) or any( isinstance(output, DimensionMap) and output.stride == 0 for output in transform.output ) ): return False selection: list[int | slice] = [] reverse_axes: list[int] = [] for output in transform.output: if isinstance(output, ArrayMap): # An index array is a gather, not one basic assignment. return False if isinstance(output, ConstantMap): selection.append(output.offset) continue axis = output.input_dimension start = output.offset + output.stride * transform.domain.inclusive_min[axis] if output.stride < 0: reverse_axes.append(axes.index(axis)) selection.append( slice(start + output.stride * (shape[axis] - 1), start + 1, -output.stride) ) else: selection.append(slice(start, start + output.stride * shape[axis], output.stride)) unused = tuple(axis for axis in range(len(shape)) if axis not in axes) squeezed = np.squeeze(broadcast, axis=unused) remaining = sorted(axes) ordered = np.transpose(squeezed, tuple(remaining.index(axis) for axis in axes)) if reverse_axes: # `np.flip` with no axes would hand a zero-rank masked value back as # the shared `masked` singleton, discarding its payload. ordered = np.flip(ordered, axis=tuple(reverse_axes)) source[tuple(selection)] = ordered return True def _domain_cells(shape: tuple[int, ...]) -> np.ndarray[Any, np.dtype[np.intp]]: """Every coordinate of a zero-origin domain, one row per cell in C order.""" if len(shape) == 0: return np.zeros((1, 0), dtype=np.intp) return np.indices(shape, dtype=np.intp).reshape(len(shape), -1).T def _last_occurrences(flat: np.ndarray[Any, Any]) -> np.ndarray[Any, np.dtype[np.intp]]: """Positions to keep so that repeated flat indices take their last value.""" _, first_in_reversed = np.unique(flat[::-1], return_index=True) return np.asarray(len(flat) - 1 - first_in_reversed, dtype=np.intp) def _scatter_numpy(source: np.ndarray[Any, Any], transform: IndexTransform, broadcast: Any) -> None: """One fancy assignment: NumPy sources need no read-modify-write.""" positions = _domain_cells(transform.domain.shape) coords = transform.apply_many(positions + np.asarray(transform.domain.inclusive_min)) flat = np.asarray(np.ravel_multi_index(tuple(coords.T), source.shape), dtype=np.intp) keep = _last_occurrences(flat) source[tuple(coords[keep].T)] = broadcast[tuple(positions[keep].T)] def _scatter_planned( source: Any, transform: IndexTransform, broadcast: Any, projections: Sequence[ChunkProjection], ) -> None: """One read-modify-write per touched grid cell, through basic slices only.""" origin = np.asarray(transform.domain.inclusive_min, dtype=np.intp) for projection in projections: cells = _domain_cells(projection.chunk_transform.domain.shape) local = projection.chunk_transform.apply_many(cells) positions = projection.cell_transform.apply_many(cells) - origin cell_origin = np.asarray(projection.chunk_domain.inclusive_min, dtype=np.intp) hull_min = local.min(axis=0) hull_shape = tuple(int(n) for n in local.max(axis=0) + 1 - hull_min) box = tuple( slice(int(lo), int(lo + n)) for lo, n in zip(cell_origin + hull_min, hull_shape, strict=True) ) block = np.array(source[box], dtype=source.dtype, copy=True) flat = np.asarray( np.ravel_multi_index(tuple((local - hull_min).T), hull_shape), dtype=np.intp ) keep = _last_occurrences(flat) np.put(block, flat[keep], np.asarray(broadcast[tuple(positions[keep].T)])) source[box] = block def _scatter_elementwise(source: Any, transform: IndexTransform, broadcast: Any) -> None: """One integer assignment per element, for sources that cannot be read.""" origin = transform.domain.inclusive_min for position in np.ndindex(transform.domain.shape): point = tuple(start + offset for start, offset in zip(origin, position, strict=True)) if isinstance(broadcast, np.ma.MaskedArray): # Masked scalar indexing returns the shared ``masked`` singleton, # discarding the payload. A zero-dimensional array retains both. value = np.ma.array( broadcast.data[position], mask=np.ma.getmaskarray(broadcast)[position] ) else: value = broadcast[position] source[transform.apply(point)] = value zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/000077500000000000000000000000001527014137600251665ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/conformance/000077500000000000000000000000001527014137600274605ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/conformance/PROVENANCE.md000066400000000000000000000041511527014137600314430ustar00rootroot00000000000000# Provenance of the ndsel conformance corpus The JSON fixtures in this directory (`point.json`, `box.json`, `slice.json`, `points.json`, `transform.json`, `errors.json`) and `README.md` are **vendored, unmodified**, from the ndsel reference repository. - **Source:** - **Branch:** `main` (merge of d-v-b/ndsel#3, empty `index_array` serialization) - **Commit:** `49b9e1db1ca93c55f320b025a666367de87a9014` (previously vendored: `92d6a32df0cd1ac47d548f14f42909a95997cf19`, before that `c59bc556c`, itself byte-identical to `c132b4c1caa3205830ce35a42502363171f650a7`) - **Path in source:** `conformance/` **Do not edit these files.** They are vendored as-is so that `zarr_indexing`' ndsel message layer can be checked against the same language-agnostic corpus every other ndsel implementation runs. To update the corpus, re-vendor from a newer ndsel commit and update the commit SHA above. ndsel PR #1 (merged) corrected the `slice` desugaring origin from `floor(a/s)` to `trunc(a/s)` (rounding toward zero), which matches `zarr_indexing`' existing `_trunc_div` semantics. ndsel PR #2 (merged) specified negative `step`, which changed two fixtures: - `slice.json` gained the negative-step cases (full reverse, `|s| > 1` non-divisible, negative-coordinate intervals, empty-at-any-coordinate). - `errors.json` retired `error/negative-step` — the reason code `negative_step_unsupported` is retired with it — and replaced it with three `bounds_out_of_order` fixtures pinning that a reversed interval is an error for either sign of the step, rather than being clamped to empty. Re-vendoring those two files and teaching `zarr_indexing.messages` the new desugaring are one change: the corpus is the definition of correct here, so it lands in the same commit as the code that satisfies it. ndsel PR #3 (merged) specified empty `index_array` serialization, adding two fixtures to `transform.json`: `normalize` carries an empty `index_array` verbatim (it is not rewritten to a constant map), while a producer SHOULD collapse it to a constant output map — which `zarr_indexing.json` already does. zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/conformance/README.md000066400000000000000000000012061527014137600307360ustar00rootroot00000000000000# ndsel conformance corpus Language-agnostic fixtures. Each file is a JSON array of cases. A **success** case: { "name": "...", "input": , "normalized": } An **error** case: { "name": "...", "input": , "error": "" } An implementation is conformant iff, for every success case, `normalize(input)` equals `normalized` by structural JSON equality, and for every error case, `normalize(input)` is rejected with the given reason code. The `normalized` value is a canonical `transform` body (the `kind` field is omitted; implementations compare the transform structure). zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/conformance/box.json000066400000000000000000000030151527014137600311420ustar00rootroot00000000000000[ { "name": "box/2d-min-max", "input": { "kind": "box", "inclusive_min": [0, 0], "exclusive_max": [3, 4] }, "normalized": { "input_rank": 2, "input_inclusive_min": [0, 0], "input_exclusive_max": [3, 4], "input_labels": ["", ""], "output": [ { "offset": 0, "stride": 1, "input_dimension": 0 }, { "offset": 0, "stride": 1, "input_dimension": 1 } ] } }, { "name": "box/shape-only-origin-zero", "input": { "kind": "box", "shape": [5] }, "normalized": { "input_rank": 1, "input_inclusive_min": [0], "input_exclusive_max": [5], "input_labels": [""], "output": [ { "offset": 0, "stride": 1, "input_dimension": 0 } ] } }, { "name": "box/inclusive-max", "input": { "kind": "box", "inclusive_min": [2], "inclusive_max": [9] }, "normalized": { "input_rank": 1, "input_inclusive_min": [2], "input_exclusive_max": [10], "input_labels": [""], "output": [ { "offset": 0, "stride": 1, "input_dimension": 0 } ] } }, { "name": "box/implicit-and-infinite-bounds", "input": { "kind": "box", "inclusive_min": [["-inf"], 0], "exclusive_max": [["+inf"], 4], "labels": ["t", ""] }, "normalized": { "input_rank": 2, "input_inclusive_min": [["-inf"], 0], "input_exclusive_max": [["+inf"], 4], "input_labels": ["t", ""], "output": [ { "offset": 0, "stride": 1, "input_dimension": 0 }, { "offset": 0, "stride": 1, "input_dimension": 1 } ] } } ] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/conformance/errors.json000066400000000000000000000057631527014137600317020ustar00rootroot00000000000000[ { "name": "error/step-zero", "input": { "kind": "slice", "start": [0], "stop": [4], "step": [0] }, "error": "step_zero" }, { "name": "error/slice-reversed-interval-unit-step", "input": { "kind": "slice", "start": [9], "stop": [0] }, "error": "bounds_out_of_order" }, { "name": "error/slice-reversed-interval-positive-step", "input": { "kind": "slice", "start": [9], "stop": [0], "step": [2] }, "error": "bounds_out_of_order" }, { "name": "error/slice-reversed-interval-negative-step", "input": { "kind": "slice", "start": [5], "stop": [6], "step": [-1] }, "error": "bounds_out_of_order" }, { "name": "error/multiple-upper-bounds", "input": { "kind": "box", "shape": [3], "exclusive_max": [3] }, "error": "multiple_upper_bounds" }, { "name": "error/rank-mismatch", "input": { "kind": "slice", "start": [0, 0], "stop": [4] }, "error": "rank_mismatch" }, { "name": "error/unknown-kind", "input": { "kind": "bogus" }, "error": "unknown_kind" }, { "name": "error/transform-multiple-upper-bounds", "input": { "kind": "transform", "input_shape": [3], "input_exclusive_max": [3] }, "error": "multiple_upper_bounds" }, { "name": "error/transform-rank-mismatch", "input": { "kind": "transform", "input_rank": 2, "input_inclusive_min": [0] }, "error": "rank_mismatch" }, { "name": "error/missing-kind", "input": { "coords": [1, 2] }, "error": "invalid_json" }, { "name": "error/point-missing-coords", "input": { "kind": "point" }, "error": "invalid_json" }, { "name": "error/point-bool-coord", "input": { "kind": "point", "coords": [true] }, "error": "invalid_json" }, { "name": "error/slice-missing-stop", "input": { "kind": "slice", "start": [0] }, "error": "invalid_json" }, { "name": "error/box-non-list-bound", "input": { "kind": "box", "inclusive_min": 5 }, "error": "invalid_json" }, { "name": "error/points-bool-coord", "input": { "kind": "points", "coords": [[true]] }, "error": "invalid_json" }, { "name": "error/integer-out-of-i64-range", "input": { "kind": "point", "coords": [99999999999999999999] }, "error": "invalid_json" }, { "name": "error/box-inverted-bounds", "input": { "kind": "box", "inclusive_min": [5], "exclusive_max": [3] }, "error": "bounds_out_of_order" }, { "name": "error/box-negative-shape", "input": { "kind": "box", "shape": [-3] }, "error": "bounds_out_of_order" }, { "name": "error/transform-inverted-bounds", "input": { "kind": "transform", "input_inclusive_min": [0], "input_exclusive_max": [-1] }, "error": "bounds_out_of_order" }, { "name": "error/output-map-conflict", "input": { "kind": "transform", "output": [{ "input_dimension": 0, "index_array": [1, 2] }] }, "error": "output_map_conflict" }, { "name": "error/box-unknown-field", "input": { "kind": "box", "shapee": [3] }, "error": "unknown_field" }, { "name": "error/point-unknown-field", "input": { "kind": "point", "coords": [1], "extra": true }, "error": "unknown_field" }, { "name": "error/output-map-unknown-field", "input": { "kind": "transform", "output": [{ "offset": 0, "bogus": 1 }] }, "error": "unknown_field" } ] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/conformance/point.json000066400000000000000000000014421527014137600315050ustar00rootroot00000000000000[ { "name": "point/2d", "input": { "kind": "point", "coords": [4, 7] }, "normalized": { "input_rank": 0, "input_inclusive_min": [], "input_exclusive_max": [], "input_labels": [], "output": [ { "offset": 4 }, { "offset": 7 } ] } }, { "name": "point/scalar-0d", "input": { "kind": "point", "coords": [] }, "normalized": { "input_rank": 0, "input_inclusive_min": [], "input_exclusive_max": [], "input_labels": [], "output": [] } }, { "name": "point/large-i64", "input": { "kind": "point", "coords": [1152921504606846976] }, "normalized": { "input_rank": 0, "input_inclusive_min": [], "input_exclusive_max": [], "input_labels": [], "output": [ { "offset": 1152921504606846976 } ] } } ] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/conformance/points.json000066400000000000000000000020721527014137600316700ustar00rootroot00000000000000[ { "name": "points/three-2d", "input": { "kind": "points", "coords": [[1, 10], [2, 20], [3, 30]] }, "normalized": { "input_rank": 1, "input_inclusive_min": [0], "input_exclusive_max": [3], "input_labels": [""], "output": [ { "offset": 0, "stride": 1, "index_array": [1, 2, 3], "index_array_bounds": ["-inf", "+inf"] }, { "offset": 0, "stride": 1, "index_array": [10, 20, 30], "index_array_bounds": ["-inf", "+inf"] } ] } }, { "name": "points/1d", "input": { "kind": "points", "coords": [[5], [9], [2]] }, "normalized": { "input_rank": 1, "input_inclusive_min": [0], "input_exclusive_max": [3], "input_labels": [""], "output": [ { "offset": 0, "stride": 1, "index_array": [5, 9, 2], "index_array_bounds": ["-inf", "+inf"] } ] } }, { "name": "points/empty", "input": { "kind": "points", "coords": [] }, "normalized": { "input_rank": 1, "input_inclusive_min": [0], "input_exclusive_max": [0], "input_labels": [""], "output": [] } } ] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/conformance/slice.json000066400000000000000000000130051527014137600314510ustar00rootroot00000000000000[ { "name": "slice/unit-step-preserves-frame", "input": { "kind": "slice", "start": [5], "stop": [10] }, "normalized": { "input_rank": 1, "input_inclusive_min": [5], "input_exclusive_max": [10], "input_labels": [""], "output": [ { "offset": 0, "stride": 1, "input_dimension": 0 } ] } }, { "name": "slice/divisible-stride", "input": { "kind": "slice", "start": [4], "stop": [10], "step": [2] }, "normalized": { "input_rank": 1, "input_inclusive_min": [2], "input_exclusive_max": [5], "input_labels": [""], "output": [ { "offset": 0, "stride": 2, "input_dimension": 0 } ] } }, { "name": "slice/nondivisible-stride-phase-offset", "input": { "kind": "slice", "start": [5], "stop": [10], "step": [2] }, "normalized": { "input_rank": 1, "input_inclusive_min": [2], "input_exclusive_max": [5], "input_labels": [""], "output": [ { "offset": 1, "stride": 2, "input_dimension": 0 } ] } }, { "name": "slice/2d-mixed-step", "input": { "kind": "slice", "start": [0, 5], "stop": [10, 10], "step": [2, 1] }, "normalized": { "input_rank": 2, "input_inclusive_min": [0, 5], "input_exclusive_max": [5, 10], "input_labels": ["", ""], "output": [ { "offset": 0, "stride": 2, "input_dimension": 0 }, { "offset": 0, "stride": 1, "input_dimension": 1 } ] } }, { "name": "slice/negative-start-trunc-origin", "input": { "kind": "slice", "start": [-9], "stop": [5], "step": [2] }, "normalized": { "input_rank": 1, "input_inclusive_min": [-4], "input_exclusive_max": [3], "input_labels": [""], "output": [ { "offset": -1, "stride": 2, "input_dimension": 0 } ] } }, { "name": "slice/negative-start-trunc-origin-step3", "input": { "kind": "slice", "start": [-8], "stop": [6], "step": [3] }, "normalized": { "input_rank": 1, "input_inclusive_min": [-2], "input_exclusive_max": [3], "input_labels": [""], "output": [ { "offset": -2, "stride": 3, "input_dimension": 0 } ] } }, { "name": "slice/negative-step-full-reverse", "input": { "kind": "slice", "start": [19], "stop": [-1], "step": [-1] }, "normalized": { "input_rank": 1, "input_inclusive_min": [-19], "input_exclusive_max": [1], "input_labels": [""], "output": [ { "offset": 0, "stride": -1, "input_dimension": 0 } ] } }, { "name": "slice/negative-step-divisible-span", "input": { "kind": "slice", "start": [15], "stop": [5], "step": [-2] }, "normalized": { "input_rank": 1, "input_inclusive_min": [-7], "input_exclusive_max": [-2], "input_labels": [""], "output": [ { "offset": 1, "stride": -2, "input_dimension": 0 } ] } }, { "name": "slice/negative-step-nondivisible-span", "input": { "kind": "slice", "start": [15], "stop": [5], "step": [-4] }, "normalized": { "input_rank": 1, "input_inclusive_min": [-3], "input_exclusive_max": [0], "input_labels": [""], "output": [ { "offset": 3, "stride": -4, "input_dimension": 0 } ] } }, { "name": "slice/negative-step-down-to-zero", "input": { "kind": "slice", "start": [9], "stop": [0], "step": [-2] }, "normalized": { "input_rank": 1, "input_inclusive_min": [-4], "input_exclusive_max": [1], "input_labels": [""], "output": [ { "offset": 1, "stride": -2, "input_dimension": 0 } ] } }, { "name": "slice/negative-step-negative-interval", "input": { "kind": "slice", "start": [-1], "stop": [-6], "step": [-2] }, "normalized": { "input_rank": 1, "input_inclusive_min": [0], "input_exclusive_max": [3], "input_labels": [""], "output": [ { "offset": -1, "stride": -2, "input_dimension": 0 } ] } }, { "name": "slice/negative-step-negative-interval-step3", "input": { "kind": "slice", "start": [-2], "stop": [-9], "step": [-3] }, "normalized": { "input_rank": 1, "input_inclusive_min": [0], "input_exclusive_max": [3], "input_labels": [""], "output": [ { "offset": -2, "stride": -3, "input_dimension": 0 } ] } }, { "name": "slice/negative-step-single-point", "input": { "kind": "slice", "start": [5], "stop": [4], "step": [-3] }, "normalized": { "input_rank": 1, "input_inclusive_min": [-1], "input_exclusive_max": [0], "input_labels": [""], "output": [ { "offset": 2, "stride": -3, "input_dimension": 0 } ] } }, { "name": "slice/negative-step-empty", "input": { "kind": "slice", "start": [5], "stop": [5], "step": [-1] }, "normalized": { "input_rank": 1, "input_inclusive_min": [-5], "input_exclusive_max": [-5], "input_labels": [""], "output": [ { "offset": 0, "stride": -1, "input_dimension": 0 } ] } }, { "name": "slice/negative-step-keeps-labels", "input": { "kind": "slice", "start": [19], "stop": [-1], "step": [-1], "labels": ["x"] }, "normalized": { "input_rank": 1, "input_inclusive_min": [-19], "input_exclusive_max": [1], "input_labels": ["x"], "output": [ { "offset": 0, "stride": -1, "input_dimension": 0 } ] } }, { "name": "slice/2d-mixed-sign-step", "input": { "kind": "slice", "start": [19, 0], "stop": [-1, 10], "step": [-1, 2] }, "normalized": { "input_rank": 2, "input_inclusive_min": [-19, 0], "input_exclusive_max": [1, 5], "input_labels": ["", ""], "output": [ { "offset": 0, "stride": -1, "input_dimension": 0 }, { "offset": 0, "stride": 2, "input_dimension": 1 } ] } } ] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/conformance/transform.json000066400000000000000000000056711527014137600323770ustar00rootroot00000000000000[ { "name": "transform/omitted-output-identity", "input": { "kind": "transform", "input_inclusive_min": [0, 0], "input_exclusive_max": [3, 4] }, "normalized": { "input_rank": 2, "input_inclusive_min": [0, 0], "input_exclusive_max": [3, 4], "input_labels": ["", ""], "output": [ { "offset": 0, "stride": 1, "input_dimension": 0 }, { "offset": 0, "stride": 1, "input_dimension": 1 } ] } }, { "name": "transform/implicit-bounds-and-labels", "input": { "kind": "transform", "input_inclusive_min": [["-inf"], 7], "input_exclusive_max": [["+inf"], 11], "input_labels": ["x", "y"] }, "normalized": { "input_rank": 2, "input_inclusive_min": [["-inf"], 7], "input_exclusive_max": [["+inf"], 11], "input_labels": ["x", "y"], "output": [ { "offset": 0, "stride": 1, "input_dimension": 0 }, { "offset": 0, "stride": 1, "input_dimension": 1 } ] } }, { "name": "transform/explicit-output-all-three-map-kinds", "input": { "kind": "transform", "input_inclusive_min": [0], "input_exclusive_max": [3], "output": [ { "offset": 7 }, { "input_dimension": 0, "stride": 2 }, { "index_array": [1, 2, 3] } ] }, "normalized": { "input_rank": 1, "input_inclusive_min": [0], "input_exclusive_max": [3], "input_labels": [""], "output": [ { "offset": 7 }, { "offset": 0, "stride": 2, "input_dimension": 0 }, { "offset": 0, "stride": 1, "index_array": [1, 2, 3], "index_array_bounds": ["-inf", "+inf"] } ] } }, { "name": "transform/empty-index-array-carried-verbatim", "input": { "kind": "transform", "input_inclusive_min": [0, 0], "input_exclusive_max": [0, 3], "output": [{ "index_array": [] }, { "input_dimension": 1 }] }, "normalized": { "input_rank": 2, "input_inclusive_min": [0, 0], "input_exclusive_max": [0, 3], "input_labels": ["", ""], "output": [ { "offset": 0, "stride": 1, "index_array": [], "index_array_bounds": ["-inf", "+inf"] }, { "offset": 0, "stride": 1, "input_dimension": 1 } ] } }, { "name": "transform/empty-index-array-is-idempotent", "input": { "input_rank": 2, "input_inclusive_min": [0, 0], "input_exclusive_max": [0, 3], "input_labels": ["", ""], "kind": "transform", "output": [ { "offset": 0, "stride": 1, "index_array": [], "index_array_bounds": ["-inf", "+inf"] }, { "offset": 0, "stride": 1, "input_dimension": 1 } ] }, "normalized": { "input_rank": 2, "input_inclusive_min": [0, 0], "input_exclusive_max": [0, 3], "input_labels": ["", ""], "output": [ { "offset": 0, "stride": 1, "index_array": [], "index_array_bounds": ["-inf", "+inf"] }, { "offset": 0, "stride": 1, "input_dimension": 1 } ] } } ] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_boundary.py000066400000000000000000000034451527014137600304300ustar00rootroot00000000000000"""Positional array selectors retain their values through bounds validation.""" from typing import Literal import numpy as np import pytest from zarr_indexing.boundary import normalize_positional_selection from zarr_indexing.domain import IndexDomain @pytest.mark.parametrize("dtype", ["i1", "i2", "i4", "i8", "u1", "u2", "u4", "u8"]) @pytest.mark.parametrize("mode", ["orthogonal", "vectorized"]) @pytest.mark.parametrize("origin", [-3, 0, 10]) def test_positional_integer_arrays( dtype: str, mode: Literal["orthogonal", "vectorized"], origin: int ) -> None: domain = IndexDomain((origin,), (origin + 4,)) values = [-4, -1, 0, 3] if np.dtype(dtype).kind == "i" else [0, 3, 0, 3] selector = np.array(values, dtype=dtype) snapshot = selector.copy() normalized = normalize_positional_selection(selector, domain, mode) expected = np.array([value % 4 + origin for value in values], dtype=np.intp) np.testing.assert_array_equal(normalized[0], expected) np.testing.assert_array_equal(selector, snapshot) assert normalized[0].dtype == np.dtype(np.intp) @pytest.mark.parametrize( ("dtype", "value"), [ (dtype, value) for dtype in ("i1", "i2", "i4", "i8", "u1", "u2", "u4", "u8") for value in ( (int(np.iinfo(dtype).min), -5, 4, int(np.iinfo(dtype).max)) if np.dtype(dtype).kind == "i" else (4, int(np.iinfo(dtype).max)) ) ], ) @pytest.mark.parametrize("mode", ["orthogonal", "vectorized"]) def test_positional_integer_array_out_of_bounds( dtype: str, value: int, mode: Literal["orthogonal", "vectorized"] ) -> None: selector = np.array([value], dtype=dtype) with pytest.raises(IndexError, match="out of bounds"): normalize_positional_selection(selector, IndexDomain.from_shape((4,)), mode) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_chunk_resolution.py000066400000000000000000001276551527014137600322120ustar00rootroot00000000000000from __future__ import annotations from typing import Any import numpy as np import pytest from hypothesis import assume, given, settings from hypothesis import strategies as st import zarr_indexing from zarr_indexing import ( ChunkGrid, ChunkPlan, ChunkProjection, EdgeDimensionGrid, FixedDimension, VaryingDimension, chunk_resolution, plan_chunks, ) from zarr_indexing.domain import IndexDomain from zarr_indexing.grid import dimension_grids_from_chunks from zarr_indexing.output_map import ArrayMap, ConstantMap, DimensionMap from zarr_indexing.transform import IndexTransform def _storage_of(transform: IndexTransform, point: tuple[int, ...]) -> tuple[int, ...]: """Evaluate the three map forms at one point, independently of planning.""" result: list[int] = [] for output_map in transform.output: if isinstance(output_map, ConstantMap): result.append(output_map.offset) elif isinstance(output_map, DimensionMap): result.append(output_map.offset + output_map.stride * point[output_map.input_dimension]) else: index = tuple( 0 if output_map.index_array.shape[axis] == 1 else point[axis] - transform.domain.inclusive_min[axis] for axis in range(output_map.index_array.ndim) ) result.append( output_map.offset + output_map.stride * int(output_map.index_array[index]) ) return tuple(result) def _points(domain: IndexDomain) -> list[tuple[int, ...]]: """Enumerate a small finite domain in its own coordinates.""" return [ tuple( coordinate + origin for coordinate, origin in zip(position, domain.inclusive_min, strict=True) ) for position in np.ndindex(*domain.shape) ] @pytest.mark.parametrize("shape", [(4,), (2, 2)]) @pytest.mark.parametrize( "coordinates", [ [(-1, 0), (0, -1), (-1, 0), (0, 0)], [(-2, -1), (-1, -2), (-2, -2), (-1, -1)], [(0, 1), (1, 0), (0, 1), (0, 0)], [(-(2**63), 0), (0, -(2**63)), (-1, -1), (0, 0)], ], ) def test_correlated_plan_with_signed_chunk_ids( shape: tuple[int, ...], coordinates: list[tuple[int, int]] ) -> None: """Shared-axis lookups group distinct signed chunk tuples without collisions.""" class SignedUnitGrid: def index_to_chunk(self, index: int) -> int: return index def indices_to_chunks(self, indices: Any) -> Any: return indices def chunk_offset(self, chunk: int) -> int: return chunk def chunk_size(self, chunk: int) -> int: return 1 values = np.array(coordinates, dtype=np.intp) transform = IndexTransform( IndexDomain.from_shape(shape), tuple(ArrayMap(values[:, axis].reshape(shape)) for axis in range(2)), ) plan = plan_chunks(transform, (SignedUnitGrid(), SignedUnitGrid())) expected_chunks = sorted(set(coordinates)) assert [tuple(row) for row in plan.partition().chunk_coords()] == expected_chunks seen = [] for projection in plan: for point in _points(projection.chunk_transform.domain): assert _storage_of(projection.chunk_transform, point) == (0, 0) request_point = _storage_of(projection.cell_transform, point) assert _storage_of(transform, request_point) == projection.chunk_coords seen.append(request_point) assert sorted(seen) == sorted(_points(transform.domain)) def test_basic_plan_is_reiterable_and_projects_both_spaces() -> None: """A plan can be revisited without losing either side of each projection.""" transform = IndexTransform.from_shape((6,))[1:6] grids = dimension_grids_from_chunks((3,), (6,)) plan = plan_chunks(transform, grids) first = list(plan) second = list(plan.projections()) assert isinstance(plan, ChunkPlan) assert all(isinstance(projection, ChunkProjection) for projection in first) assert [projection.chunk_coords for projection in first] == [(0,), (1,)] assert [projection.chunk_domain for projection in first] == [ IndexDomain((0,), (3,)), IndexDomain((3,), (6,)), ] assert [projection.coverage for projection in first] == ["partial", "full"] assert first == second assert all( projection.chunk_transform.domain == projection.cell_transform.domain for projection in first ) assert all(projection.chunk_transform.domain.origin == (0,) for projection in first) def test_projection_requires_one_shared_synthetic_domain() -> None: """Paired transforms with different cell domains are rejected as incoherent.""" with pytest.raises(ValueError, match="must share an input domain"): ChunkProjection( chunk_coords=(0,), chunk_domain=IndexDomain.from_shape((3,)), chunk_transform=IndexTransform.identity(IndexDomain.from_shape((2,))), cell_transform=IndexTransform.identity(IndexDomain.from_shape((1,))), coverage="partial", ) def test_projection_plan_is_the_only_public_chunk_resolution_surface() -> None: """The greenfield API does not retain tuple or NumPy-selector bridges.""" assert {"ChunkCoverage", "ChunkPlan", "ChunkProjection", "plan_chunks"} <= set( zarr_indexing.__all__ ) assert "iter_chunk_transforms" not in zarr_indexing.__all__ assert "sub_transform_to_selections" not in zarr_indexing.__all__ def test_plan_rejects_grid_rank_different_from_transform_output_rank() -> None: """A missing storage grid dimension is rejected before iteration.""" transform = IndexTransform.from_shape((2, 3)) with pytest.raises(ValueError, match="1 grids for output rank 2"): plan_chunks(transform, dimension_grids_from_chunks((2,), (2,))) @pytest.mark.parametrize( ("transform", "expected"), [ (IndexTransform.from_shape((5,)), ["full", "full"]), (IndexTransform.from_shape((5,))[::-1], ["full", "full"]), (IndexTransform.from_shape((5,))[::2], ["partial", "partial"]), (IndexTransform.from_shape((5,))[2], ["partial"]), ( IndexTransform.from_shape((5,)).oindex[np.array([0, 1, 2, 3, 4])], ["unknown", "unknown"], ), ], ids=["clipped-edge", "reverse", "strided", "scalar", "fancy-is-conservative"], ) def test_coverage_classification(transform: IndexTransform, expected: list[str]) -> None: """Coverage is exact for affine requests and conservative for gathers.""" grids = dimension_grids_from_chunks((3,), (5,)) assert [projection.coverage for projection in plan_chunks(transform, grids)] == expected def test_repeated_input_dependency_partition_is_rejected() -> None: """The per-output-axis table API cannot represent a diagonal.""" transform = IndexTransform( domain=IndexDomain.from_shape((2,)), output=(DimensionMap(input_dimension=0), DimensionMap(input_dimension=0)), ) grids = dimension_grids_from_chunks((2, 2), (2, 2)) with pytest.raises(ValueError, match="read input axis 0"): plan_chunks(transform, grids).partition() def test_unused_input_axis_is_not_full_coverage() -> None: transform = IndexTransform( domain=IndexDomain.from_shape((2, 2)), output=(DimensionMap(input_dimension=0),), ) grids = dimension_grids_from_chunks((2,), (2,)) assert [projection.coverage for projection in plan_chunks(transform, grids)] == ["partial"] @pytest.mark.parametrize( ("transform", "grids"), [ ( IndexTransform.from_shape((2, 3)), dimension_grids_from_chunks((2, 3), (2, 3)), ), ( IndexTransform( domain=IndexDomain.from_shape((2, 3)), output=(DimensionMap(input_dimension=1), DimensionMap(input_dimension=0)), ), dimension_grids_from_chunks((3, 2), (3, 2)), ), ( IndexTransform.from_shape((2, 3))[::-1, ::-1], dimension_grids_from_chunks((2, 3), (2, 3)), ), ( IndexTransform( domain=IndexDomain((4, 7), (6, 10)), output=( DimensionMap(input_dimension=0, offset=-4), DimensionMap(input_dimension=1, offset=-7), ), ), dimension_grids_from_chunks((2, 3), (2, 3)), ), ], ids=["identity", "axis-permutation", "reversal", "translated-unit-affine"], ) def test_bijective_unit_affine_transforms_retain_full_coverage( transform: IndexTransform, grids: tuple[Any, ...] ) -> None: assert [projection.coverage for projection in plan_chunks(transform, grids)] == ["full"] def test_rank_zero_transform_has_full_coverage() -> None: transform = IndexTransform.identity(IndexDomain((), ())) assert [projection.coverage for projection in plan_chunks(transform, ())] == ["full"] @pytest.mark.parametrize( ("transform", "grids", "expected_coords"), [ ( IndexTransform.from_shape((30,)), dimension_grids_from_chunks((10,), (30,)), [(0,), (1,), (2,)], ), ( IndexTransform.from_shape((20, 30)), dimension_grids_from_chunks((10, 10), (20, 30)), [(i, j) for i in range(2) for j in range(3)], ), ( IndexTransform.from_shape((100, 100))[25, :], dimension_grids_from_chunks((10, 10), (100, 100)), [(2, j) for j in range(10)], ), ( IndexTransform.from_shape((100,))[8:15], dimension_grids_from_chunks((10,), (100,)), [(0,), (1,)], ), ], ids=["one-dimensional", "two-dimensional", "constant-map", "slice"], ) def test_affine_plans_touch_the_expected_chunks( transform: IndexTransform, grids: tuple[Any, ...], expected_coords: list[tuple[int, ...]], ) -> None: """Identity, constant, and sliced transforms enumerate literal grid cells.""" assert [projection.chunk_coords for projection in plan_chunks(transform, grids)] == ( expected_coords ) @pytest.mark.parametrize( ("transform", "grids"), [ ( IndexTransform.from_shape((6,)).oindex[np.array([4, 0, 4, 2])], dimension_grids_from_chunks((3,), (6,)), ), ( IndexTransform.from_shape((4, 5)).oindex[np.array([3, 0]), np.array([4, 1, 1])], dimension_grids_from_chunks(((1, 3), (2, 3)), (4, 5)), ), ( IndexTransform.from_shape((2, 4, 5)).vindex[ ..., np.array([3, 0, 3]), np.array([4, 1, 1]) ], dimension_grids_from_chunks((1, 2, 3), (2, 4, 5)), ), ], ids=["repeated-oindex", "irregular-oindex", "vindex-with-residual"], ) def test_projection_invariants_for_fancy_selections( transform: IndexTransform, grids: tuple[Any, ...] ) -> None: """Both transforms agree pointwise and cell ranges tile request space once.""" plan = plan_chunks(transform, grids) request_points: list[tuple[int, ...]] = [] for projection in plan: assert projection.coverage == "unknown" assert projection.chunk_transform.domain == projection.cell_transform.domain for cell_point in _points(projection.cell_transform.domain): request_point = _storage_of(projection.cell_transform, cell_point) chunk_point = _storage_of(projection.chunk_transform, cell_point) storage_point = _storage_of(plan.transform, request_point) chunk_origin = projection.chunk_domain.inclusive_min assert chunk_point == tuple( value - origin for value, origin in zip(storage_point, chunk_origin, strict=True) ) assert all( 0 <= value < extent for value, extent in zip(chunk_point, projection.chunk_domain.shape, strict=True) ) request_points.append(request_point) assert sorted(request_points) == sorted(_points(transform.domain)) @pytest.mark.parametrize( "grid", [ pytest.param(FixedDimension(size=2, extent=4), id="fixed"), pytest.param(VaryingDimension(edges=(1, 3), extent=4), id="varying"), ], ) def test_orthogonal_array_map_plan_rejects_coordinate_below_grid(grid: Any) -> None: transform = IndexTransform( domain=IndexDomain.from_shape((2,)), output=(ArrayMap(np.array([-1, 1], dtype=np.intp)),), ) # The sorted 1-D fast path reports the first offending coordinate. with pytest.raises(IndexError, match=r"index -1 is out of bounds"): list(plan_chunks(transform, (grid,))) @pytest.mark.parametrize( "grid", [ pytest.param(FixedDimension(size=2, extent=4), id="fixed"), pytest.param(VaryingDimension(edges=(1, 3), extent=4), id="varying"), ], ) def test_orthogonal_array_map_plan_rejects_coordinate_above_grid(grid: Any) -> None: transform = IndexTransform( domain=IndexDomain.from_shape((2,)), output=(ArrayMap(np.array([1, 4], dtype=np.intp)),), ) # The sorted 1-D fast path reports the first offending coordinate. with pytest.raises(IndexError, match=r"index 4 is out of bounds"): list(plan_chunks(transform, (grid,))) def test_nonempty_identity_plan_rejects_zero_size_fixed_dimension() -> None: transform = IndexTransform.from_shape((4,)) with pytest.raises(ValueError, match="size must be > 0 when extent is nonzero"): list(plan_chunks(transform, (FixedDimension(size=0, extent=4),))) @given( origin=st.integers(min_value=-4, max_value=4), extent=st.integers(min_value=0, max_value=8), stride=st.integers(min_value=-3, max_value=3), ) def test_affine_projection_pairs_reconstruct_independent_source_coordinates( origin: int, extent: int, stride: int ) -> None: """Bounded literal-domain examples preserve every request/storage pair.""" anchor = extent - 1 if stride < 0 else 0 offset = anchor - stride * origin expected_pairs = [ ((coordinate,), (source_coordinate,)) for coordinate in range(origin, origin + extent) if 0 <= (source_coordinate := offset + stride * coordinate) < extent ] assume(expected_pairs) unrestricted = IndexTransform( domain=IndexDomain((origin,), (origin + extent,)), output=(DimensionMap(input_dimension=0, offset=offset, stride=stride),), ) intersection = unrestricted.intersect(IndexDomain.from_shape((extent,))) assume(intersection is not None) transform, _ = intersection grids = dimension_grids_from_chunks((min(3, extent),), (extent,)) reconstructed_pairs = [ ( _storage_of(projection.cell_transform, cell_coordinate), tuple( local_coordinate + chunk_origin for local_coordinate, chunk_origin in zip( _storage_of(projection.chunk_transform, cell_coordinate), projection.chunk_domain.inclusive_min, strict=True, ) ), ) for projection in plan_chunks(transform, grids) for cell_coordinate in _points(projection.cell_transform.domain) ] assert sorted(reconstructed_pairs) == sorted(expected_pairs) def test_correlated_projection_preserves_nonzero_request_coordinates() -> None: base = IndexTransform.identity(IndexDomain((2, 5), (4, 8))) transform = base.vindex[np.array([2, 3], dtype=np.intp), :] grids = dimension_grids_from_chunks((2, 4), (4, 8)) points = [ transform.apply(projection.cell_transform.apply(cell)) for projection in plan_chunks(transform, grids) for cell in _points(projection.cell_transform.domain) ] assert sorted(points) == [(2, 5), (2, 6), (2, 7), (3, 5), (3, 6), (3, 7)] def test_correlated_projection_preserves_translated_advanced_axis_coordinates() -> None: transform = ( IndexTransform.from_shape((4,)) .vindex[np.array([0, 3], dtype=np.intp)] .translate_domain_by((5,)) ) grids = dimension_grids_from_chunks((2,), (4,)) request_points = [ projection.cell_transform.apply(cell) for projection in plan_chunks(transform, grids) for cell in _points(projection.cell_transform.domain) ] assert sorted(request_points) == [(5,), (6,)] def test_empty_request_has_no_projections() -> None: """An empty fancy selection does not fabricate a touched chunk.""" transform = IndexTransform.from_shape((10,)).oindex[np.array([], dtype=np.intp)] grids = dimension_grids_from_chunks((3,), (10,)) assert list(plan_chunks(transform, grids)) == [] class TestSortedOneDimensionalPlan: def test_sorted_coordinates_bypass_intersection(self) -> None: """Sorted coordinates partition directly at touched chunk boundaries.""" transform = IndexTransform.from_shape((12,)).vindex[ np.array([0, 3, 4, 4, 9, 11], dtype=np.intp) ] grid = ChunkGrid(dimensions=(FixedDimension(size=4, extent=12),)) projections = list(plan_chunks(transform, grid.dimensions)) assert [projection.chunk_coords for projection in projections] == [(0,), (1,), (2,)] assert [ [ _storage_of(projection.cell_transform, point)[0] for point in _points(projection.cell_transform.domain) ] for projection in projections ] == [[0, 1], [2, 3], [4, 5]] def test_unsorted_coordinates_use_intersection(self) -> None: """Unsorted coordinates are grouped by chunk, in chunk order.""" transform = IndexTransform.from_shape((12,)).vindex[np.array([9, 0, 4], dtype=np.intp)] grid = ChunkGrid(dimensions=(FixedDimension(size=4, extent=12),)) projections = list(plan_chunks(transform, grid.dimensions)) assert [projection.chunk_coords for projection in projections] == [(0,), (1,), (2,)] class CountingUnitGrid: """A real unit grid that counts every planner-grid operation.""" def __init__(self, extent: int) -> None: self._grid = FixedDimension(size=1, extent=extent) self.calls = 0 def index_to_chunk(self, idx: int) -> int: self.calls += 1 return self._grid.index_to_chunk(idx) def chunk_offset(self, chunk_ix: int) -> int: self.calls += 1 return self._grid.chunk_offset(chunk_ix) def chunk_size(self, chunk_ix: int) -> int: self.calls += 1 return self._grid.chunk_size(chunk_ix) def indices_to_chunks( self, indices: np.ndarray[Any, np.dtype[np.intp]] ) -> np.ndarray[Any, np.dtype[np.intp]]: self.calls += 1 return self._grid.indices_to_chunks(indices) def test_sparse_affine_plan_does_not_visit_intervening_chunks() -> None: grid = CountingUnitGrid(extent=100_001) transform = IndexTransform.from_shape((100_001,))[::100_000] assert [projection.chunk_coords for projection in plan_chunks(transform, (grid,))] == [ (0,), (100_000,), ] assert grid.calls <= 12 def test_sparse_affine_plan_handles_large_origin_cancellation() -> None: origin = int(np.iinfo(np.intp).max) transform = IndexTransform( domain=IndexDomain((origin,), (origin + 2,)), output=(DimensionMap(input_dimension=0, offset=-2 * origin, stride=2),), ) grids = dimension_grids_from_chunks((1,), (3,)) assert [projection.chunk_coords for projection in plan_chunks(transform, grids)] == [ (0,), (2,), ] class TestTouchedOnlyCandidateEnumeration: def test_sparse_one_dimensional_selection_skips_the_dense_span( self, ) -> None: """Two sorted points on a 1000-cell grid touch two chunks.""" transform = IndexTransform.from_shape((4000,)).vindex[np.array([1, 3997], dtype=np.intp)] grid = ChunkGrid(dimensions=(FixedDimension(size=4, extent=4000),)) projections = list(plan_chunks(transform, grid.dimensions)) assert [projection.chunk_coords for projection in projections] == [(0,), (999,)] @pytest.mark.parametrize( ("mode", "expected_coords"), [ ("orthogonal", [(0, 0), (0, 999), (999, 0), (999, 999)]), ("correlated", [(0, 0), (999, 999)]), ], ) def test_sparse_two_dimensional_selection_uses_only_touched_combinations( self, mode: str, expected_coords: list[tuple[int, int]], ) -> None: """Orthogonal points use their outer product; correlated points remain paired.""" base = IndexTransform.from_shape((4000, 4000)) first = np.array([1, 3997], dtype=np.intp) second = np.array([2, 3998], dtype=np.intp) transform = ( base.oindex[first, second] if mode == "orthogonal" else base.vindex[first, second] ) grid = ChunkGrid( dimensions=( FixedDimension(size=4, extent=4000), FixedDimension(size=4, extent=4000), ) ) projections = list(plan_chunks(transform, grid.dimensions)) assert sorted(projection.chunk_coords for projection in projections) == expected_coords def test_correlated_diagonal_scales_with_points_not_their_product( self, ) -> None: """Fifty diagonal points touch fifty chunks, not 2500.""" n_points = 50 coordinates = np.arange(n_points, dtype=np.intp) * 8 transform = IndexTransform.from_shape((4000, 4000)).vindex[coordinates, coordinates] grid = ChunkGrid( dimensions=( FixedDimension(size=4, extent=4000), FixedDimension(size=4, extent=4000), ) ) projections = list(plan_chunks(transform, grid.dimensions)) assert sorted(projection.chunk_coords for projection in projections) == [ (2 * index, 2 * index) for index in range(n_points) ] # --------------------------------------------------------------------------- # GridPartition: the factored form # --------------------------------------------------------------------------- def _partition_cases() -> list[tuple[str, IndexTransform, tuple[Any, ...]]]: base = IndexTransform.from_shape((7, 9, 5)) fixed = dimension_grids_from_chunks((3, 4, 2), shape=(7, 9, 5)) varying = ( VaryingDimension(edges=(2, 5), extent=7), VaryingDimension(edges=(1, 4, 4), extent=9), FixedDimension(size=2, extent=5), ) # Declared edges past the extent: the last chunk's data extent is shorter # than its declared size, which is where `data_size` and `chunk_size` part. clipped = ( VaryingDimension(edges=(5, 10), extent=7), VaryingDimension(edges=(4, 4, 4), extent=9), FixedDimension(size=3, extent=5), ) # The minimal grid protocol: no `data_size`, so resolution falls back to `chunk_size`. edges = (EdgeDimensionGrid([3, 4]), EdgeDimensionGrid([4, 5]), EdgeDimensionGrid([2, 3])) return [ ("identity", base, fixed), ("strided", base[1:6:2, 5:, ::3], fixed), ("strided varying", base[1:6:2, 5:, ::3], varying), ("scalars", base[3, :, 4], fixed), ("reversed", base[::-1, 8:1:-1, :], fixed), ("empty", base[3:3, :, :], fixed), ("oindex arrays", base.oindex[np.array([6, 0, 0, 2]), :, np.array([4, 1])], fixed), ("oindex mixed", base.oindex[np.array([1, 5]), 2, 1:5:2], varying), ("oindex one element", base.oindex[np.array([2]), :, :], fixed), ("vindex", base.vindex[np.array([0, 6, 6, 1]), np.array([8, 0, 1, 8]), :], fixed), ("vindex varying", base.vindex[np.array([0, 6, 6, 1]), np.array([8, 0, 1, 8]), 2], varying), ("strided clipped", base[1:7:2, 5:, ::3], clipped), ("oindex clipped", base.oindex[np.array([6, 0, 6]), 8, 1:5:2], clipped), ("vindex clipped", base.vindex[np.array([6, 6, 1]), np.array([8, 0, 8]), :], clipped), ("strided edge grid", base[1:7:2, 5:, ::3], edges), ("vindex edge grid", base.vindex[np.array([6, 6, 1]), np.array([8, 0, 8]), :], edges), ( "vindex 2-d block", base.vindex[ np.array([[0, 6], [3, 1]]), np.array([[8, 0], [2, 8]]), np.array([[4, 0], [1, 1]]) ], fixed, ), ( "vindex 1-d sorted", IndexTransform.from_shape((20,)).vindex[np.array([1, 5, 9, 17])], dimension_grids_from_chunks((4,), shape=(20,)), ), ("rank 0", IndexTransform.from_shape(()), ()), ] def _check_projections(transform: IndexTransform, projections: list[ChunkProjection]) -> None: """The evaluation oracle: both transforms agree pointwise, cells tile the request exactly once, chunk-local coordinates lie in the chunk, and `coverage` is `full` exactly when a chunk's cells are each read once.""" has_array = any(isinstance(m, ArrayMap) for m in transform.output) request_points: list[tuple[int, ...]] = [] for projection in projections: assert projection.chunk_transform.domain == projection.cell_transform.domain chunk_points: list[tuple[int, ...]] = [] for cell_point in _points(projection.cell_transform.domain): request_point = _storage_of(projection.cell_transform, cell_point) chunk_point = _storage_of(projection.chunk_transform, cell_point) storage_point = _storage_of(transform, request_point) chunk_origin = projection.chunk_domain.inclusive_min assert chunk_point == tuple( value - origin for value, origin in zip(storage_point, chunk_origin, strict=True) ) assert all( 0 <= value < extent for value, extent in zip(chunk_point, projection.chunk_domain.shape, strict=True) ) request_points.append(request_point) chunk_points.append(chunk_point) if has_array: assert projection.coverage == "unknown" else: covers = sorted(chunk_points) == sorted( tuple(int(c) for c in cell) for cell in np.ndindex(*projection.chunk_domain.shape) ) assert (projection.coverage == "full") == covers, projection assert sorted(request_points) == sorted(_points(transform.domain)) @pytest.mark.parametrize("case", _partition_cases(), ids=lambda case: case[0]) def test_partition_rows_are_the_plan_and_satisfy_the_oracle( case: tuple[str, IndexTransform, tuple[Any, ...]], ) -> None: _, transform, grids = case plan = plan_chunks(transform, grids) partition = plan.partition() rows = list(partition) assert rows == list(plan) assert len(partition) == len(rows) _check_projections(transform, rows) @pytest.mark.parametrize("case", _partition_cases(), ids=lambda case: case[0]) def test_partition_chunk_coords_are_vectorized_rows( case: tuple[str, IndexTransform, tuple[Any, ...]], ) -> None: """`chunk_coords` reads the tables without materializing a row per chunk.""" _, transform, grids = case partition = plan_chunks(transform, grids).partition() coords = partition.chunk_coords() assert coords.shape == (len(partition), transform.output_rank) assert coords.tolist() == [list(p.chunk_coords) for p in partition] def test_partition_tables_describe_chunk_local_coordinates() -> None: """The columnar tables carry exactly what each row's chunk transform maps to.""" transform = IndexTransform.from_shape((7, 9)).oindex[np.array([6, 0, 0, 2]), 5:] grids = dimension_grids_from_chunks((3, 4), shape=(7, 9)) partition = plan_chunks(transform, grids).partition() indexed, strided = partition.sets assert isinstance(indexed, chunk_resolution.IndexedSet) assert isinstance(strided, chunk_resolution.StridedSet) # rows are in chunk order; the array [6, 0, 0, 2] lands in chunks 0, 0, 0, 2 assert indexed.chunk.tolist() == [0, 2] assert indexed.pointer.tolist() == [0, 3, 4] assert indexed.local.tolist() == [0, 0, 2, 0] assert indexed.positions.tolist() == [1, 2, 3, 0] assert strided.chunk.tolist() == [1, 2] assert strided.local_start.tolist() == [1, 0] assert strided.extent.tolist() == [3, 1] assert strided.origin.tolist() == [0, 3] # positions along the request axis assert strided.full.tolist() == [False, True] for row, projection in enumerate(partition): table_rows = np.unravel_index(row, partition.row_shape) run = indexed.run(int(table_rows[0])) storage = projection.chunk_transform.apply_many( np.array(list(np.ndindex(*projection.chunk_transform.domain.shape))) ) assert sorted(set(storage[:, 0].tolist())) == sorted(set(indexed.local[run].tolist())) def test_joint_set_groups_points_by_chunk() -> None: transform = IndexTransform.from_shape((7, 9)).vindex[ np.array([0, 6, 6, 1]), np.array([8, 0, 1, 8]) ] grids = dimension_grids_from_chunks((3, 4), shape=(7, 9)) joint = plan_chunks(transform, grids).partition().joint_sets[0] assert joint is not None assert joint.chunk.tolist() == [[0, 2], [2, 0]] assert joint.pointer.tolist() == [0, 2, 4] assert joint.positions.tolist() == [0, 3, 1, 2] assert joint.local.tolist() == [[0, 0], [1, 0], [0, 0], [0, 1]] def test_partition_rejects_correlated_residual_diagonal() -> None: """A diagonal among the residual slice axes of a correlated transform is rejected too.""" transform = IndexTransform( domain=IndexDomain.from_shape((2, 3)), output=( ArrayMap(np.array([[0], [2]])), ArrayMap(np.array([[1], [0]])), DimensionMap(input_dimension=1), DimensionMap(input_dimension=1), ), ) grids = dimension_grids_from_chunks((2, 2, 2, 2), shape=(3, 3, 3, 3)) with pytest.raises(ValueError, match="read input axis 1"): plan_chunks(transform, grids).partition() with pytest.raises(ValueError, match="read input axis 1"): list(plan_chunks(transform, grids)) def test_correlated_plan_checks_storage_bounds_with_any_grid() -> None: """Out-of-range coordinates raise even on grids whose vectorized lookup does not check.""" class UncheckedGrid: # like zarr's FixedDimension: only the scalar lookup validates def __init__(self, size: int, extent: int) -> None: self.size, self.extent = size, extent def index_to_chunk(self, index: int) -> int: if not 0 <= index < self.extent: raise IndexError(f"index {index} is out of bounds for extent {self.extent}") return index // self.size def chunk_offset(self, chunk: int) -> int: return chunk * self.size def chunk_size(self, chunk: int) -> int: return self.size def indices_to_chunks(self, indices: Any) -> Any: return indices // self.size transform = IndexTransform.from_shape((10, 10)).vindex[np.array([1, 9]), np.array([2, 9])] with pytest.raises(IndexError, match="out of bounds"): list(plan_chunks(transform, (UncheckedGrid(2, 5), UncheckedGrid(2, 5)))) def test_wide_request_extent_is_preserved() -> None: """A zero-stride map over a domain wider than np.intp touches one cell and keeps its extent.""" width = 2**63 transform = IndexTransform( domain=IndexDomain((0,), (width,)), output=(DimensionMap(input_dimension=0, offset=3, stride=0),), ) (projection,) = plan_chunks(transform, dimension_grids_from_chunks((2,), shape=(5,))) assert projection.chunk_coords == (1,) assert projection.chunk_transform.domain.shape == (width,) assert projection.coverage == "partial" @pytest.mark.parametrize("width", [3, 10_000]) def test_correlated_projection_keeps_residual_slices_compact(width: int) -> None: indices = np.array([0, 2, 1]) transform = IndexTransform.from_shape((3, 3, width)).vindex[indices, indices, :] grids = dimension_grids_from_chunks((3, 3, width), shape=(3, 3, width)) (projection,) = plan_chunks(transform, grids) assert projection.cell_transform.apply((2, width - 1)) == (2, width - 1) assert projection.chunk_transform.apply((2, width - 1)) == (1, 1, width - 1) coordinate_bytes = sum( m.index_array.nbytes for m in projection.cell_transform.output if isinstance(m, ArrayMap) ) # Request placement needs only the three point positions, regardless of slab width. assert coordinate_bytes <= indices.nbytes def test_mixed_affine_array_dependency_is_rejected() -> None: """A slice map and an index array reading one input axis is a diagonal too.""" transform = IndexTransform( IndexDomain.from_shape((4,)), (DimensionMap(0), ArrayMap(np.array([3, 2, 1, 0]))) ) grids = dimension_grids_from_chunks((2, 2), shape=(4, 4)) with pytest.raises(NotImplementedError, match="also bound by a slice map"): list(plan_chunks(transform, grids)) def test_partition_tables_are_memoized_and_immutable() -> None: """The plan builds its tables once, and nothing a consumer does to a column can change them.""" base = IndexTransform.from_shape((7, 9)) indices = np.array([6, 0, 6, 1]) for transform in (base.oindex[indices, 5:], base.vindex[indices, indices]): plan = plan_chunks(transform, dimension_grids_from_chunks((3, 4), (7, 9))) partition = plan.partition() assert plan.partition() is partition tables: list[Any] = [*partition.sets, *partition.joint_sets] for table in tables: columns = [table.chunk, table.chunk_start, table.chunk_extent] if hasattr(table, "pointer"): assert table.local is table.local columns += [table.local, table.pointer, table.index, table.positions] else: columns += [table.origin, table.extent] for column in columns: assert not column.flags.writeable with pytest.raises(ValueError): column.setflags(write=True) def test_independent_correlated_groups_stay_compact() -> None: n = 1000 indices = np.arange(n) transform = IndexTransform.from_shape((n, n, n)).vindex[ indices[:, None], indices[:, None], indices[None, :] ] assert sum(m.index_array.size for m in transform.output if isinstance(m, ArrayMap)) == 3 * n partition = plan_chunks( transform, dimension_grids_from_chunks((n, n, n), (n, n, n)) ).partition() assert [j.output_dimensions for j in partition.joint_sets] == [(0, 1), (2,)] assert sum(j.index.size for j in partition.joint_sets) == 3 * n assert partition.row_shape == (1, 1) (projection,) = partition assert projection.chunk_transform.domain.shape == (n, n) assert projection.chunk_transform.apply((5, 9)) == (5, 5, 9) assert projection.cell_transform.apply((5, 9)) == (5, 9) @pytest.mark.parametrize("origin", [0, 5]) @pytest.mark.parametrize("transpose", [False, True]) def test_component_projections_preserve_request_cells(origin: int, transpose: bool) -> None: maps = ( ArrayMap(np.array([2, 0, 2]).reshape(3, 1, 1, 1)), ArrayMap(np.array([0, 3, 1]).reshape(3, 1, 1, 1)), ArrayMap(np.array([4, 0, 4, 2]).reshape(1, 4, 1, 1)), DimensionMap(2, offset=-origin), ) transform = IndexTransform( IndexDomain((origin,) * 4, (origin + 3, origin + 4, origin + 2, origin + 1)), maps ) if transpose: transform = IndexTransform(transform.domain, (maps[2], maps[0], maps[3], maps[1])) shape = (5, 3, 2, 4) if transpose else (3, 4, 5, 2) grids = dimension_grids_from_chunks((2, 2, 2, 2), shape) partition = plan_chunks(transform, grids).partition() assert len(partition.joint_sets) == 2 rows = list(partition) _check_projections(transform, rows) assert partition.chunk_coords().tolist() == [list(p.chunk_coords) for p in rows] @pytest.mark.parametrize("origin", [0, 5]) @pytest.mark.parametrize("shape", [(6,), (2, 3)]) @pytest.mark.parametrize("reverse_outputs", [False, True]) def test_single_component_projection_coordinates( origin: int, shape: tuple[int, ...], reverse_outputs: bool ) -> None: """A sole component preserves repeated points, affine maps, and input origins.""" values = np.array([4, 0, 4, 2, 1, 3]).reshape(shape) maps = (ArrayMap(values, offset=1, stride=2), ArrayMap(values, offset=9, stride=-2)) transform = IndexTransform( IndexDomain((origin,) * len(shape), tuple(origin + size for size in shape)), maps[::-1] if reverse_outputs else maps, ) partition = plan_chunks( transform, dimension_grids_from_chunks(((2, 3, 5), (4, 6)), (10, 10)) ).partition() assert len(partition.joint_sets) == 1 assert not partition.sets projections = list(partition) _check_projections(transform, projections) assert partition.chunk_coords().tolist() == [list(p.chunk_coords) for p in projections] def test_index_array_components_merge_transitively() -> None: transform = IndexTransform( IndexDomain.from_shape((2, 3, 4)), ( ArrayMap(np.zeros((2, 1, 1), dtype=np.intp)), ArrayMap(np.zeros((1, 1, 4), dtype=np.intp)), ArrayMap(np.zeros((2, 3, 1), dtype=np.intp)), ArrayMap(np.zeros((1, 3, 4), dtype=np.intp)), ), ) partition = plan_chunks(transform, dimension_grids_from_chunks((1,) * 4, (1,) * 4)).partition() assert len(partition.joint_sets) == 1 _check_projections(transform, list(partition)) @pytest.mark.parametrize("reader_kind", ["basic", "numpy"]) def test_independent_components_scatter_through_lazy_array(reader_kind: str) -> None: from zarr_indexing import LazyArray source = np.arange(5 * 6 * 7 * 3).reshape(5, 6, 7, 3) a = np.array([4, 0, 4])[:, None] b = np.array([5, 1, 3])[:, None] c = np.array([6, 0, 6, 2])[None, :] wrapped = LazyArray.from_numpy(source) if reader_kind == "numpy" else LazyArray(source) view = wrapped.with_parts((2, 3, 2, 2)).vindex[a, b, c, ...] assert ( len( plan_chunks(view.transform, dimension_grids_from_chunks((2, 3, 2, 2), source.shape)) .partition() .joint_sets ) == 2 ) np.testing.assert_array_equal(view.result(), source[a, b, c, :]) class SignedGrid: """An unbounded grid with translated boundaries and signed chunk identifiers.""" def __init__(self, size: int, origin: int) -> None: self.size = size self.origin = origin def index_to_chunk(self, index: int) -> int: return (index - self.origin) // self.size def indices_to_chunks( self, indices: np.ndarray[Any, np.dtype[np.intp]] ) -> np.ndarray[Any, np.dtype[np.intp]]: return (indices - self.origin) // self.size def chunk_offset(self, chunk: int) -> int: return self.origin + chunk * self.size def chunk_size(self, chunk: int) -> int: return self.size @settings(max_examples=300) @given(data=st.data(), shape=st.lists(st.integers(0, 3), min_size=0, max_size=3)) def test_component_dependency_graph_matches_pointwise_oracle( data: st.DataObject, shape: list[int] ) -> None: """Catch lost duplicates, signed chunk collisions, and incorrect request origins. Enumerate the transform's small domain directly: no planner intersection, grouping, or dependency helpers contribute to the expected mapping. """ origin = tuple(data.draw(st.integers(-4, 4)) for _ in shape) output_rank = data.draw(st.integers(1, 5)) affine_axes = {axis for axis in range(len(shape)) if data.draw(st.booleans())} maps: list[ArrayMap | ConstantMap | DimensionMap] = [ DimensionMap( axis, offset=data.draw(st.integers(-3, 3)), stride=data.draw(st.integers(-2, 2)) ) for axis in sorted(affine_axes) ] for _ in range(output_rank): if data.draw(st.booleans()): maps.append(ConstantMap(data.draw(st.integers(-3, 3)))) continue # Reserve affine axes for one DimensionMap each. Unsupported shared # affine dependencies are exercised explicitly in the error properties. dependencies = [ axis not in affine_axes and data.draw(st.booleans()) for axis in range(len(shape)) ] array_shape = tuple( size if dependent else 1 for size, dependent in zip(shape, dependencies, strict=True) ) count = int(np.prod(array_shape)) # A small value range produces repeated storage points at distinct request positions. values = data.draw(st.lists(st.integers(-2, 2), min_size=count, max_size=count)) maps.append( ArrayMap( np.array(values, dtype=np.intp).reshape(array_shape), offset=data.draw(st.integers(-3, 3)), stride=data.draw(st.integers(-2, 2)), ) ) grids = [SignedGrid(data.draw(st.integers(1, 3)), data.draw(st.integers(-3, 3))) for _ in maps] transform = IndexTransform( IndexDomain(origin, tuple(lo + size for lo, size in zip(origin, shape, strict=True))), tuple(maps), ) expected = {point: _storage_of(transform, point) for point in _points(transform.domain)} expected_chunks = { tuple(grid.index_to_chunk(value) for grid, value in zip(grids, storage, strict=True)) for storage in expected.values() } partition = plan_chunks(transform, tuple(grids)).partition() rows = list(partition) _check_projections(transform, rows) assert {row.chunk_coords for row in rows} == expected_chunks assert len(rows) == len(expected_chunks) assert partition.chunk_coords().tolist() == [list(row.chunk_coords) for row in rows] reconstructed = [] for row in rows: assert row.chunk_domain.inclusive_min == tuple( grid.chunk_offset(chunk) for grid, chunk in zip(grids, row.chunk_coords, strict=True) ) for cell in _points(row.cell_transform.domain): request = _storage_of(row.cell_transform, cell) storage = tuple( local + grid.chunk_offset(chunk) for local, grid, chunk in zip( _storage_of(row.chunk_transform, cell), grids, row.chunk_coords, strict=True ) ) reconstructed.append((request, storage)) # A list comparison retains multiplicity: repeating one position cannot hide a missing one. assert sorted(reconstructed) == sorted(expected.items()) @given(origin=st.integers(-4, 4), size=st.integers(2, 5), stride=st.sampled_from([-2, -1, 1, 2])) def test_generated_shared_affine_dependency_is_rejected( origin: int, size: int, stride: int ) -> None: transform = IndexTransform( IndexDomain((origin,), (origin + size,)), (DimensionMap(0, stride=stride), DimensionMap(0, offset=3)), ) with pytest.raises(ValueError, match="read input axis 0"): list(plan_chunks(transform, (SignedGrid(2, -1), SignedGrid(3, 1)))) @given(origin=st.integers(-4, 4), size=st.integers(2, 5), stride=st.sampled_from([-2, -1, 1, 2])) def test_generated_mixed_affine_array_dependency_is_rejected( origin: int, size: int, stride: int ) -> None: transform = IndexTransform( IndexDomain((origin,), (origin + size,)), (DimensionMap(0, stride=stride), ArrayMap(np.zeros(size, dtype=np.intp))), ) with pytest.raises(NotImplementedError, match="also bound by a slice map"): list(plan_chunks(transform, (SignedGrid(2, -1), SignedGrid(3, 1)))) @pytest.mark.parametrize("stride", [0, 1, 2, -2]) @pytest.mark.parametrize("count", [1, 2]) def test_singleton_coverage(stride: int, count: int) -> None: transform = IndexTransform(IndexDomain.from_shape((count,)), (DimensionMap(0, stride=stride),)) shape = (5,) * transform.output_rank class SingletonGrid: def index_to_chunk(self, index: int) -> int: return index def chunk_offset(self, chunk: int) -> int: return chunk def chunk_size(self, chunk: int) -> int: return 1 def indices_to_chunks(self, indices: Any) -> Any: return indices grids = (SingletonGrid(),) * len(shape) expected = "partial" if stride == 0 and count > 1 else "full" assert all(p.coverage == expected for p in plan_chunks(transform, grids)) partition = plan_chunks(transform, grids).partition() assert all(bool(full) == (expected == "full") for full in partition.sets[0].full) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_composition.py000066400000000000000000000440061527014137600311460ustar00rootroot00000000000000from __future__ import annotations import numpy as np import pytest from zarr_indexing.domain import IndexDomain from zarr_indexing.errors import BoundsCheckError from zarr_indexing.output_map import ArrayMap, ConstantMap, DimensionMap from zarr_indexing.transform import IndexTransform class TestComposeConstantInner: """Inner = constant. Result is always constant.""" def test_constant_inner_any_outer(self) -> None: outer = IndexTransform.from_shape((5,)) inner = IndexTransform( domain=IndexDomain.from_shape((5,)), output=(ConstantMap(offset=42),), ) result = outer.compose(inner) assert isinstance(result.output[0], ConstantMap) assert result.output[0].offset == 42 class TestComposeDimensionInner: """Inner = DimensionMap.""" def test_dimension_inner_constant_outer(self) -> None: outer = IndexTransform( domain=IndexDomain.from_shape((10,)), output=(ConstantMap(offset=5),), ) inner = IndexTransform( domain=IndexDomain.from_shape((10,)), output=(DimensionMap(input_dimension=0, offset=10, stride=3),), ) result = outer.compose(inner) assert isinstance(result.output[0], ConstantMap) assert result.output[0].offset == 25 def test_dimension_inner_constant_outer_rejects_affine_overflow(self) -> None: outer = IndexTransform( domain=IndexDomain.from_shape((1,)), output=(ConstantMap(offset=2**62),), ) inner = IndexTransform( domain=IndexDomain((2**62,), (2**62 + 1,)), output=(DimensionMap(input_dimension=0, stride=4),), ) with pytest.raises(OverflowError, match="outside np.intp"): outer.compose(inner) def test_dimension_inner_dimension_outer(self) -> None: outer = IndexTransform( domain=IndexDomain.from_shape((3,)), output=(DimensionMap(input_dimension=0, offset=5, stride=2),), ) inner = IndexTransform( domain=IndexDomain.from_shape((10,)), output=(DimensionMap(input_dimension=0, offset=10, stride=3),), ) result = outer.compose(inner) assert isinstance(result.output[0], DimensionMap) assert result.output[0].offset == 25 assert result.output[0].stride == 6 assert result.output[0].input_dimension == 0 def test_dimension_inner_array_outer(self) -> None: arr = np.array([0, 1, 2], dtype=np.intp) outer = IndexTransform( domain=IndexDomain.from_shape((3,)), output=(ArrayMap(index_array=arr, offset=5, stride=2),), ) inner = IndexTransform( domain=IndexDomain.from_shape((10,)), output=(DimensionMap(input_dimension=0, offset=10, stride=3),), ) result = outer.compose(inner) assert isinstance(result.output[0], ArrayMap) assert result.output[0].offset == 25 assert result.output[0].stride == 6 np.testing.assert_array_equal(result.output[0].index_array, arr) class TestComposeArrayInner: """Inner = ArrayMap.""" def test_array_inner_constant_outer(self) -> None: inner_arr = np.array([10, 20, 30], dtype=np.intp) outer = IndexTransform( domain=IndexDomain.from_shape((5,)), output=(ConstantMap(offset=1),), ) inner = IndexTransform( domain=IndexDomain.from_shape((3,)), output=(ArrayMap(index_array=inner_arr, offset=0, stride=1),), ) result = outer.compose(inner) assert isinstance(result.output[0], ConstantMap) assert result.output[0].offset == 20 def test_array_inner_constant_outer_rejects_affine_overflow(self) -> None: outer = IndexTransform( domain=IndexDomain.from_shape((1,)), output=(ConstantMap(offset=0),), ) inner = IndexTransform( domain=IndexDomain.from_shape((1,)), output=(ArrayMap(np.array([2**62], dtype=np.intp), stride=4),), ) with pytest.raises(OverflowError, match="outside np.intp"): outer.compose(inner) def test_array_inner_array_outer(self) -> None: outer_arr = np.array([0, 2, 1], dtype=np.intp) inner_arr = np.array([10, 20, 30], dtype=np.intp) outer = IndexTransform( domain=IndexDomain.from_shape((3,)), output=(ArrayMap(index_array=outer_arr, offset=0, stride=1),), ) inner = IndexTransform( domain=IndexDomain.from_shape((3,)), output=(ArrayMap(index_array=inner_arr, offset=0, stride=1),), ) result = outer.compose(inner) assert isinstance(result.output[0], ArrayMap) expected = np.array([10, 30, 20], dtype=np.intp) np.testing.assert_array_equal(result.output[0].index_array, expected) def _storage_of(transform: IndexTransform, point: tuple[int, ...]) -> tuple[int, ...]: """The storage coordinates a transform assigns to one input point. The point is given in the transform's own domain coordinates; an index array carries the full input rank, singleton on the axes it does not vary over, and is addressed positionally from the domain origin. """ coords: list[int] = [] for m in transform.output: if isinstance(m, ConstantMap): coords.append(m.offset) elif isinstance(m, DimensionMap): coords.append(m.offset + m.stride * point[m.input_dimension]) else: origin = transform.domain.inclusive_min idx = tuple( 0 if m.index_array.shape[axis] == 1 else point[axis] - origin[axis] for axis in range(m.index_array.ndim) ) coords.append(m.offset + m.stride * int(m.index_array[idx])) return tuple(coords) def _assert_composes_pointwise(outer: IndexTransform, inner: IndexTransform) -> None: """`outer.compose(inner)` must agree with running the two in sequence.""" composed = outer.compose(inner) assert composed.domain == outer.domain lo = outer.domain.inclusive_min hi = outer.domain.exclusive_max for coord in np.ndindex(*outer.domain.shape): point = tuple(int(c) + int(o) for c, o in zip(coord, lo, strict=True)) assert all(point[d] < hi[d] for d in range(len(hi))) intermediate = _storage_of(outer, point) assert _storage_of(composed, point) == _storage_of(inner, intermediate) class TestComposeOverANonZeroOriginDomain: """The outer domain need not start at 0 — a step-1 slice preserves its literal bounds and a negative step produces a negative origin — so the inner map has to be evaluated over the outer domain's real range.""" def test_array_inner_sliced_outer(self) -> None: inner = IndexTransform.from_shape((10,)).oindex[np.array([3, 1, 4, 1, 5])] outer = IndexTransform.identity(IndexDomain.from_shape((5,)))[1:4] assert outer.domain.inclusive_min == (1,) result = outer.compose(inner) assert isinstance(result.output[0], ArrayMap) np.testing.assert_array_equal( result.output[0].index_array, np.array([1, 4, 1], dtype=np.intp) ) _assert_composes_pointwise(outer, inner) def test_array_inner_reversed_outer(self) -> None: inner = IndexTransform.from_shape((10,)).oindex[np.array([3, 1, 4, 1, 5])] outer = IndexTransform.identity(IndexDomain.from_shape((5,)))[::-1] assert outer.domain.inclusive_min == (-4,) result = outer.compose(inner) assert isinstance(result.output[0], ArrayMap) np.testing.assert_array_equal( result.output[0].index_array, np.array([5, 1, 4, 1, 3], dtype=np.intp) ) _assert_composes_pointwise(outer, inner) def test_array_inner_strided_outer(self) -> None: inner = IndexTransform.from_shape((10,)).oindex[np.array([3, 1, 4, 1, 5, 9])] outer = IndexTransform.identity(IndexDomain.from_shape((6,)))[1::2] _assert_composes_pointwise(outer, inner) def test_array_inner_translated_outer(self) -> None: inner = IndexTransform.from_shape((10,)).oindex[np.array([3, 1, 4, 1, 5])] outer = IndexTransform.identity(IndexDomain.from_shape((5,))).translate_domain_to((-2,)) _assert_composes_pointwise(outer, inner) def test_array_inner_array_outer_over_a_shifted_domain(self) -> None: inner = IndexTransform.from_shape((10,)).oindex[np.array([3, 1, 4, 1, 5])] outer = IndexTransform.from_shape((5,)).oindex[np.array([4, 0, 2])] _assert_composes_pointwise(outer, inner) def test_array_inner_constant_outer_over_a_shifted_domain(self) -> None: inner = IndexTransform.from_shape((10,)).oindex[np.array([3, 1, 4, 1, 5])] outer = IndexTransform( domain=IndexDomain.from_shape((4,)), output=(ConstantMap(offset=3),), ) _assert_composes_pointwise(outer, inner) class TestComposeMultiDim: def test_2d_identity_compose(self) -> None: a = IndexTransform.from_shape((10, 20)) b = IndexTransform.from_shape((10, 20)) result = a.compose(b) assert result.domain.shape == (10, 20) for i in range(2): m = result.output[i] assert isinstance(m, DimensionMap) assert m.input_dimension == i assert m.offset == 0 assert m.stride == 1 def test_mixed_map_types(self) -> None: outer = IndexTransform( domain=IndexDomain.from_shape((10,)), output=( ConstantMap(offset=5), DimensionMap(input_dimension=0, offset=0, stride=1), ), ) inner = IndexTransform( domain=IndexDomain.from_shape((10, 10)), output=( DimensionMap(input_dimension=0, offset=2, stride=3), DimensionMap(input_dimension=1, offset=0, stride=1), ), ) result = outer.compose(inner) assert isinstance(result.output[0], ConstantMap) assert result.output[0].offset == 17 assert isinstance(result.output[1], DimensionMap) assert result.output[1].input_dimension == 0 assert result.output[1].offset == 0 assert result.output[1].stride == 1 def test_rank_mismatch_raises(self) -> None: outer = IndexTransform.from_shape((10,)) inner = IndexTransform.from_shape((10, 20)) with pytest.raises(ValueError, match="rank"): outer.compose(inner) class TestComposeInnerDomainValidation: @pytest.mark.parametrize( ("outer_map", "inner_lower"), [ (ConstantMap(offset=10), 10), (ConstantMap(offset=14), 10), (DimensionMap(input_dimension=0, offset=10, stride=1), 10), (DimensionMap(input_dimension=0, offset=14, stride=-1), 10), (DimensionMap(input_dimension=0, offset=10**100, stride=1), 10**100), ], ids=["lower-bound", "upper-bound", "forward", "reverse", "arbitrary-integer"], ) def test_valid_constant_and_affine_boundaries( self, outer_map: ConstantMap | DimensionMap, inner_lower: int ) -> None: outer = IndexTransform( domain=IndexDomain.from_shape((5,)), output=(outer_map,), ) inner = IndexTransform( domain=IndexDomain((inner_lower,), (inner_lower + 5,)), output=(DimensionMap(input_dimension=0),), ) result = outer.compose(inner) assert result.domain == outer.domain _assert_composes_pointwise(outer, inner) def test_rejects_constant_outer_coordinate_outside_inner_domain(self) -> None: outer = IndexTransform( domain=IndexDomain.from_shape((1,)), output=(ConstantMap(offset=2),), ) inner = IndexTransform.identity(IndexDomain((0,), (2,))) with pytest.raises(BoundsCheckError, match="outside.*inner.*domain"): outer.compose(inner) def test_rejects_affine_outer_range_crossing_inner_domain(self) -> None: outer = IndexTransform( domain=IndexDomain((10**100,), (10**100 + 3,)), output=(DimensionMap(input_dimension=0),), ) inner = IndexTransform.identity(IndexDomain((10**100,), (10**100 + 2,))) with pytest.raises(BoundsCheckError, match="outside.*inner.*domain"): outer.compose(inner) def test_empty_outer_domain_does_not_evaluate_nonexistent_points(self) -> None: outer = IndexTransform( domain=IndexDomain.from_shape((0,)), output=(ConstantMap(offset=99),), ) inner = IndexTransform.identity(IndexDomain((0,), (2,))) result = outer.compose(inner) assert result.domain.shape == (0,) assert result.output == (ConstantMap(offset=99),) class TestComposeMultidimensionalArrayInner: def test_identity_gathers_a_two_dimensional_array_map(self) -> None: inner = IndexTransform( domain=IndexDomain.from_shape((2, 3)), output=( ArrayMap( index_array=np.array([[11, 13, 17], [19, 23, 29]], dtype=np.intp), offset=5, stride=2, ), ), ) outer = IndexTransform.identity(inner.domain) result = outer.compose(inner) assert result == inner _assert_composes_pointwise(outer, inner) def test_mixed_affine_and_constant_outputs_gather_with_metadata(self) -> None: inner = IndexTransform( domain=IndexDomain((4, 8), (7, 10)), output=( ArrayMap( index_array=np.array([[2, 3], [5, 7], [11, 13]], dtype=np.intp), offset=-3, stride=4, ), ), ) outer = IndexTransform( domain=IndexDomain((-2,), (0,)), output=( DimensionMap(input_dimension=0, offset=7, stride=1), ConstantMap(offset=9), ), ) result = outer.compose(inner) assert result.domain == outer.domain assert len(result.output) == 1 array_map = result.output[0] assert isinstance(array_map, ArrayMap) assert array_map.index_array.shape == (2,) assert array_map.offset == -3 assert array_map.stride == 4 np.testing.assert_array_equal(array_map.index_array, np.array([7, 13], dtype=np.intp)) _assert_composes_pointwise(outer, inner) def test_out_of_bounds_affine_output_is_rejected_before_gather(self) -> None: outer = IndexTransform( domain=IndexDomain.from_shape((3,)), output=( DimensionMap(input_dimension=0, offset=1, stride=1), ConstantMap(offset=0), ), ) inner = IndexTransform( domain=IndexDomain.from_shape((3, 2)), output=(ArrayMap(np.arange(6, dtype=np.intp).reshape(3, 2)),), ) with pytest.raises(BoundsCheckError, match="outside.*inner.*domain"): outer.compose(inner) def test_rank_zero_array_map_composition(self) -> None: domain = IndexDomain((), ()) outer = IndexTransform.identity(domain) inner = IndexTransform( domain=domain, output=(ArrayMap(np.array(7, dtype=np.intp), offset=2, stride=3),), ) result = outer.compose(inner) assert result.domain == domain assert result.output == (ConstantMap(offset=23),) class TestComposeChain: def test_three_transforms(self) -> None: a = IndexTransform.from_shape((100,)) b = IndexTransform( domain=IndexDomain.from_shape((100,)), output=(DimensionMap(input_dimension=0, offset=10, stride=1),), ) c = IndexTransform( domain=IndexDomain.from_shape((110,)), output=(DimensionMap(input_dimension=0, offset=5, stride=2),), ) bc = b.compose(c) abc = a.compose(bc) assert isinstance(abc.output[0], DimensionMap) assert abc.output[0].offset == 25 assert abc.output[0].stride == 2 def test_composing_an_inner_array_with_a_broadcast_axis_wider_than_one_cell() -> None: """A non-dependency axis is a singleton that broadcasts, so its coordinate is 0. Indexing it by the raw intermediate coordinate walked off the end of an axis the array only has one entry for. """ outer = IndexTransform( domain=IndexDomain(inclusive_min=(-2,), exclusive_max=(0,)), output=(ConstantMap(offset=1), ConstantMap(offset=0)), ) inner = IndexTransform( domain=IndexDomain(inclusive_min=(0, 0), exclusive_max=(2, 1)), output=(ArrayMap(index_array=np.array([[13]], dtype=np.intp), offset=-2, stride=2),), ) composed = outer.compose(inner) assert composed.output[0] == ConstantMap(offset=24) def test_composing_a_one_dimensional_inner_array_under_a_higher_rank_outer() -> None: """Composed index arrays retain the full input rank of their domain.""" outer = IndexTransform( domain=IndexDomain.from_shape((2, 3)), output=(DimensionMap(input_dimension=0, offset=1, stride=1),), ) inner = IndexTransform( domain=IndexDomain.from_shape((4,)), output=(ArrayMap(index_array=np.array([7, 3, 5, 1], dtype=np.intp)),), ) composed = outer.compose(inner) array_map = composed.output[0] assert isinstance(array_map, ArrayMap) assert array_map.index_array.shape == (2, 1) np.testing.assert_array_equal(array_map.index_array, np.array([[3], [5]])) def test_composing_out_of_the_inner_domain_is_refused() -> None: """An intermediate outside the inner domain wrapped NumPy-style and read a cell.""" outer = IndexTransform( domain=IndexDomain.from_shape((1,)), output=(ConstantMap(offset=-3),), ) inner = IndexTransform( domain=IndexDomain.from_shape((2,)), output=(ArrayMap(index_array=np.array([7, 3], dtype=np.intp)),), ) with pytest.raises(BoundsCheckError, match="outside.*inner.*domain"): outer.compose(inner) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_conformance.py000066400000000000000000000035271527014137600311000ustar00rootroot00000000000000"""ndsel conformance corpus harness. Runs the vendored, language-agnostic ndsel fixtures (see `tests/conformance/PROVENANCE.md`) against this package's message layer (`zarr_indexing.messages`). An implementation is conformant iff: - for every *success* fixture, `normalize_ndsel(input)` equals the fixture's `normalized` value by structural JSON equality; - for every *error* fixture, `normalize_ndsel(input)` is rejected with an `NdselError` carrying the fixture's `error` reason code. """ from __future__ import annotations import json from pathlib import Path from typing import Any import pytest from zarr_indexing.messages import NdselError, normalize_ndsel _CONFORMANCE_DIR = Path(__file__).parent / "conformance" def _load_cases() -> list[tuple[str, dict[str, Any]]]: cases: list[tuple[str, dict[str, Any]]] = [] for path in sorted(_CONFORMANCE_DIR.glob("*.json")): data = json.loads(path.read_text()) cases.extend((f"{path.stem}::{case['name']}", case) for case in data) return cases _CASES = _load_cases() _SUCCESS = [(name, c) for name, c in _CASES if "normalized" in c] _ERROR = [(name, c) for name, c in _CASES if "error" in c] def test_corpus_is_present() -> None: # Guard against an empty/missing vendored corpus silently passing. assert len(_SUCCESS) > 0 assert len(_ERROR) > 0 @pytest.mark.parametrize(("name", "case"), _SUCCESS, ids=[name for name, _ in _SUCCESS]) def test_success_fixture(name: str, case: dict[str, Any]) -> None: result = normalize_ndsel(case["input"]) assert result == case["normalized"] @pytest.mark.parametrize(("name", "case"), _ERROR, ids=[name for name, _ in _ERROR]) def test_error_fixture(name: str, case: dict[str, Any]) -> None: with pytest.raises(NdselError) as excinfo: normalize_ndsel(case["input"]) assert excinfo.value.reason == case["error"] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_doc_examples.py000066400000000000000000000453041527014137600312500ustar00rootroot00000000000000"""Executable documentation contracts. Two kinds of test live here, and nothing else: 1. **Structural.** Every snippet include in the rendered docs resolves to a real file and a balanced, non-empty region — discovered by scanning the markdown, never hand-registered — and every example file executes. The examples carry their own inline assertions, so executing one *is* the value check; expected values are stated once, in the example, beside the prose that narrates them. 2. **Behavioral.** Documented classes are exercised where the example cannot assert the behavior itself: error paths, cache lifecycle invariants, and contracts stated in prose about types the docs define. Editorial choices — section order, exact wording, teaching progression — are deliberately not pinned here; they belong to review. Structural breakage belongs to `mkdocs build --strict`, whose configuration makes it actually fail on the relevant classes: `check_paths: true` for unresolvable includes, and `validation` set to warn (strict turns warnings into errors) for broken link anchors and nav-omitted pages. """ from __future__ import annotations import re import runpy import subprocess import sys from pathlib import Path from typing import Any import numpy as np import pytest import zarr_indexing import zarr_indexing.lazy_array as lazy_array_module from zarr_indexing import IndexTransform, LazyArray, ReadContext from zarr_indexing.domain import IndexDomain DOCS = Path(__file__).parents[1] / "docs" PACKAGE_ROOT = DOCS.parent STANDALONE_EXAMPLES = PACKAGE_ROOT / "examples" DOC_SNIPPETS_DIR = DOCS / "snippets" CACHE_EXAMPLE = STANDALONE_EXAMPLES / "system_memory_chunk_cache" / "system_memory_chunk_cache.py" # Mirrors `pymdownx.snippets: base_path` in ../mkdocs-base.yml. If that list changes, # change this one in the same commit. SNIPPET_BASE_PATHS = (DOCS, STANDALONE_EXAMPLES, PACKAGE_ROOT) _INCLUDE = re.compile(r'--8<--\s+"(?P[^":\n]+?)(?::(?P[^"\n]+))?"') def _markdown_includes() -> tuple[tuple[str, str, str | None], ...]: """Every snippet include in the rendered docs: (page, target, region).""" return tuple( (str(page.relative_to(DOCS)), match["target"], match["region"]) for page in sorted(DOCS.rglob("*.md")) for match in _INCLUDE.finditer(page.read_text()) ) INCLUDES = _markdown_includes() # In-process executables: every snippet, plus the one standalone example that # is importable as a module. The lazy_indexing_* examples are CLI scripts # (they parse argv and call sys.exit), so they run as subprocesses below — # no other test in the repository executes them. EXECUTABLES = (*sorted(DOC_SNIPPETS_DIR.glob("*.py")), CACHE_EXAMPLE) CLI_EXAMPLES = tuple( script for script in sorted(STANDALONE_EXAMPLES.glob("*/*.py")) if script not in EXECUTABLES ) PATTERN_NAMESPACE: dict[str, Any] = runpy.run_path(str(DOC_SNIPPETS_DIR / "indexing_patterns.py")) PATTERN_CASES: tuple[dict[str, Any], ...] = PATTERN_NAMESPACE["PATTERN_CASES"] CACHE_NAMESPACE: dict[str, Any] = runpy.run_path(str(CACHE_EXAMPLE)) # The documented pattern matrix must keep its listed selection families. REQUIRED_PATTERNS = { "basic-slice", "integer-axis-removal", "negative-stride", "empty-selection", "boolean-mask", "orthogonal", "vectorized", "broadcasting", "repeated-out-of-order", } # --------------------------------------------------------------------------- # # Structural: the include graph and the executable examples # --------------------------------------------------------------------------- # def test_docs_reference_snippets_at_all() -> None: """An empty scan means the include regex rotted, not that the docs did.""" assert len(INCLUDES) >= 10 assert any(region for _, _, region in INCLUDES) @pytest.mark.parametrize( ("page", "target", "region"), INCLUDES, ids=[ f"{page}->{target}" + (f":{region}" if region else "") for page, target, region in INCLUDES ], ) def test_markdown_include_resolves(page: str, target: str, region: str | None) -> None: """Each include names exactly one real file; each region is balanced and non-empty.""" resolved = [base / target for base in SNIPPET_BASE_PATHS if (base / target).is_file()] assert len(resolved) == 1, f"{page} includes {target!r}: resolved to {resolved or 'nothing'}" if region is None: return source = resolved[0].read_text() starts = re.findall(rf"^\s*# --8<-- \[start:{re.escape(region)}\]$", source, re.MULTILINE) ends = re.findall(rf"^\s*# --8<-- \[end:{re.escape(region)}\]$", source, re.MULTILINE) assert len(starts) == 1, f"{target}:{region} needs exactly one start marker, has {len(starts)}" assert len(ends) == 1, f"{target}:{region} needs exactly one end marker, has {len(ends)}" body = source.split(starts[0], maxsplit=1)[1].split(ends[0], maxsplit=1)[0] assert body.strip(), f"{target}:{region} is empty" def test_snippet_directories_hold_only_their_kind() -> None: """Rendered pages are markdown; executable snippets are Python.""" assert all(p.suffix == ".md" for p in (DOCS / "examples").iterdir() if p.is_file()) assert all(p.suffix == ".py" for p in DOC_SNIPPETS_DIR.iterdir() if p.is_file()) @pytest.mark.parametrize("example", EXECUTABLES, ids=lambda path: path.stem) def test_documentation_example_executes(example: Path) -> None: """Examples are executable contracts; their inline asserts are the values check.""" runpy.run_path(str(example), run_name="__main__") @pytest.mark.parametrize("script", CLI_EXAMPLES, ids=lambda path: path.stem) def test_cli_example_runs_as_a_subprocess(script: Path) -> None: """CLI examples exit 0 with this test environment's installed dependencies.""" if "dask" in script.stem: pytest.importorskip("dask.array") completed = subprocess.run( [sys.executable, str(script)], capture_output=True, text=True, cwd=script.parent, check=False, ) assert completed.returncode == 0, completed.stderr[-2000:] @pytest.mark.parametrize( "example_dir", sorted(p for p in STANDALONE_EXAMPLES.iterdir() if p.is_dir()), ids=lambda path: path.name, ) def test_standalone_example_is_a_documented_script(example_dir: Path) -> None: """Each standalone example ships a README and a same-named runnable script.""" script = example_dir / f"{example_dir.name}.py" assert script.is_file() assert (example_dir / "README.md").is_file() assert (DOCS / "examples" / f"{example_dir.name}.md").is_file() assert script.read_text().startswith("# /// script\n"), "examples stay PEP 723 runnable" # --------------------------------------------------------------------------- # # Behavioral: the documented pattern matrix # --------------------------------------------------------------------------- # def test_indexing_pattern_matrix_is_complete() -> None: assert {case["name"] for case in PATTERN_CASES} == REQUIRED_PATTERNS def test_pattern_page_tabs_are_the_models() -> None: """Both tabs of every matrix entry are the executable model, in order. The JSON tab must be the model's canonical wire form; the Python tab must evaluate (in the executable matrix's namespace) to the model itself. Either tab drifting from the snippet fails here. """ import json import textwrap page = (DOCS / "guide" / "patterns.md").read_text() json_blocks = re.findall(r"```json\n(.*?)```", page, re.DOTALL) assert len(json_blocks) == len(PATTERN_CASES) for block, case in zip(json_blocks, PATTERN_CASES, strict=True): assert json.loads(block) == case["transform"].to_json(), case["name"] python_blocks = [ textwrap.dedent(block) for block in re.findall(r"```python\n(.*?)```", page, re.DOTALL) if "--8<--" not in block ] assert len(python_blocks) == len(PATTERN_CASES) for block, case in zip(python_blocks, PATTERN_CASES, strict=True): constructed = eval(block, dict(PATTERN_NAMESPACE)) assert constructed == case["transform"], case["name"] @pytest.mark.parametrize("case", PATTERN_CASES, ids=lambda case: case["name"]) def test_indexing_pattern_matrix_matches_numpy(case: dict[str, Any]) -> None: """The wrapper agrees with the matrix the snippet proves at the transform level.""" image = PATTERN_NAMESPACE["image"] lazy = LazyArray.from_numpy(image) accessor = { "basic": lazy, "oindex": lazy.oindex, "vindex": lazy.vindex, }[case["mode"]] view = accessor[case["selection"]] result = view.result() np.testing.assert_array_equal(result, case["expected"]) assert result.shape == case["shape"] assert ("box" if view.is_box else "query") == case["category"] # --------------------------------------------------------------------------- # # Behavioral: contracts the guide states about wrapped sources and parts # --------------------------------------------------------------------------- # def test_default_source_contract_converts_basic_selected_slabs_to_system_memory() -> None: """A source may return Python slabs as long as NumPy can convert each selected slab.""" class ListSlabSource: def __init__(self) -> None: self.data = np.arange(20).reshape(4, 5) self.keys: list[tuple[Any, ...]] = [] @property def shape(self) -> tuple[int, ...]: return self.data.shape @property def dtype(self) -> np.dtype[Any]: return self.data.dtype def __getitem__(self, key: tuple[Any, ...]) -> object: assert all(isinstance(item, (int, slice)) for item in key) self.keys.append(key) return self.data[key].tolist() source = ListSlabSource() result = LazyArray(source).with_parts((2, 3)).oindex[[3, 1, 1], 1:5:2].result() np.testing.assert_array_equal(result, np.array([[16, 18], [6, 8], [6, 8]])) assert len(source.keys) > 0 assert all(all(isinstance(item, (int, slice)) for item in key) for key in source.keys) def test_coordinate_array_example_preserves_order_and_duplicates() -> None: """Coordinate arrays are ordered sequences, not mathematical sets.""" view = LazyArray.from_numpy(np.arange(6)).with_parts((2,)).oindex[[4, 1, 1, 3]] np.testing.assert_array_equal(view.result(), np.array([4, 1, 1, 3])) assembled = np.empty(view.shape, dtype=view.dtype) for part in view.parts(): assembled[part.out_selection] = part.view.result() np.testing.assert_array_equal(assembled, np.array([4, 1, 1, 3])) def test_documented_partition_transform_is_global_and_projection_is_chunk_local() -> None: source = np.arange(8) part = tuple(LazyArray.from_numpy(source).with_parts((4,)).parts())[1] assert part.view.transform.domain == IndexDomain((4,), (8,)) assert part.view.transform.apply((4,)) == (4,) assert part.view.array[part.view.transform.apply((4,))] == 4 assert part.projection.chunk_transform.apply((0,)) == (0,) @pytest.mark.parametrize( ("mode", "parts"), [ ("per-axis", ((0,), (3,))), ("per-axis", ((), (3,))), ("uniform", (1, 1)), ("per-axis", ((0, 0), (3,))), ], ids=["single-zero", "empty-sequence", "positive-uniform", "repeated-zero"], ) def test_documented_zero_length_axis_partition_spellings_execute(mode: str, parts: Any) -> None: data = np.zeros((0, 3)) base = LazyArray.from_numpy(data) view = base.with_parts(parts) if mode == "uniform" else base.with_parts_per_axis(parts) assert view.result().shape == (0, 3) assert tuple(view.parts()) == () def test_projection_example_exposes_paired_directions() -> None: """Both transforms of every projection keep their documented output ranks.""" namespace = runpy.run_path(str(DOC_SNIPPETS_DIR / "chunk_projection.py")) np.testing.assert_array_equal(namespace["ADVANCED_RESULT"], namespace["ADVANCED_EXPECTED"]) assert all(p.chunk_transform.output_rank == 2 for p in namespace["PROJECTIONS"]) assert all(p.cell_transform.output_rank == 1 for p in namespace["PROJECTIONS"]) @pytest.mark.parametrize( "example", [DOC_SNIPPETS_DIR / "integrations.py", CACHE_EXAMPLE], ids=lambda path: path.stem, ) def test_projection_examples_assemble_rank_zero_domains(example: Path) -> None: """The examples' shared assembly helpers handle a zero-rank cell domain.""" namespace = runpy.run_path(str(example)) domain = zarr_indexing.IndexDomain((), ()) cell_points = namespace["_domain_points"](domain) destination = np.empty((), dtype=np.intp) values = namespace["_gather_and_scatter"]( destination, np.array(7, dtype=np.intp), cell_points, cell_points, ) assert cell_points.shape == (1, 0) assert values.shape == (1,) assert destination[()] == 7 # --------------------------------------------------------------------------- # # Behavioral: the documented chunk cache (error paths the example cannot show) # --------------------------------------------------------------------------- # def make_documented_cache() -> tuple[Any, Any]: source_type = CACHE_NAMESPACE["RecordingChunkSource"] cache_type = CACHE_NAMESPACE["SystemMemoryChunkCache"] source = source_type(np.arange(48).reshape(6, 8), chunks=(3, 4)) return source, cache_type(source, capacity=2) def test_system_memory_cache_assembles_and_deduplicates_public_projections() -> None: source, cache = make_documented_cache() np.testing.assert_array_equal(cache.oindex[[1, 1], 2], np.array([10, 10])) assert tuple(source.reads) == ((0, 0),) assert cache.projection_uses == (("chunk_transform", "context.transform"),) def test_chunk_cache_queues_all_parts_before_loading( monkeypatch: pytest.MonkeyPatch, ) -> None: source, cache = make_documented_cache() planning_count = 0 original_plan_chunks = lazy_array_module.plan_chunks def counted_plan_chunks(*args: Any, **kwargs: Any) -> Any: nonlocal planning_count planning_count += 1 return original_plan_chunks(*args, **kwargs) with monkeypatch.context() as request_patch: request_patch.setattr(lazy_array_module, "plan_chunks", counted_plan_chunks) result = cache[1:5, 2] np.testing.assert_array_equal(result, np.array([10, 18, 26, 34])) assert planning_count == 1 assert source.reads == [(0, 0), (1, 0)] assert [ (event.chunk_coords, event.previous.value, event.current.value) for event in cache.events ] == [ ((0, 0), "new", "queued"), ((1, 0), "new", "queued"), ((0, 0), "queued", "loading"), ((0, 0), "loading", "ready"), ((1, 0), "queued", "loading"), ((1, 0), "loading", "ready"), ] def test_chunk_cache_reader_resolves_a_transform_from_cached_chunks() -> None: """The reader boundary consumes the prepared projections of real parts.""" source_type = CACHE_NAMESPACE["RecordingChunkSource"] reader_type = CACHE_NAMESPACE["SystemMemoryChunkReader"] source = source_type(np.arange(48).reshape(6, 8), chunks=(3, 4)) reader = reader_type(capacity=2) view = LazyArray(source).with_reader(reader)[1:5, 2] parts = tuple(view.parts()) out = np.empty(view.shape, dtype=source.dtype) for part in parts: destination = out[part.out_selection] assert ( reader.read_into( source, ReadContext(part.view.transform, part.projection), destination, ) is None ) np.testing.assert_array_equal(out, np.array([10, 18, 26, 34])) assert source.reads == [(0, 0), (1, 0)] assert reader.projection_uses == [ ("chunk_transform", "context.transform"), ("chunk_transform", "context.transform"), ] def test_chunk_cache_reader_requires_a_prepared_projection() -> None: source_type = CACHE_NAMESPACE["RecordingChunkSource"] reader_type = CACHE_NAMESPACE["SystemMemoryChunkReader"] source = source_type(np.arange(48).reshape(6, 8), chunks=(3, 4)) transform = IndexTransform.from_shape(source.shape)[1:3, 2].translate_domain_to((0,)) out = np.empty(transform.domain.shape, dtype=source.dtype) with pytest.raises(ValueError, match="requires context.projection"): reader_type(capacity=2).read_into(source, ReadContext(transform), out) def test_system_memory_cache_separates_basic_and_orthogonal_indexing() -> None: source_type = CACHE_NAMESPACE["RecordingChunkSource"] cache_type = CACHE_NAMESPACE["SystemMemoryChunkCache"] data = np.arange(48).reshape(6, 8) cache = cache_type(source_type(data, chunks=(3, 4)), capacity=4) row = np.array([0, 2]) column = np.array([1, 3]) np.testing.assert_array_equal(cache[0:3, 1:4], data[0:3, 1:4]) np.testing.assert_array_equal( cache.oindex[row, column], data[np.ix_(row, column)], ) def test_system_memory_cache_does_not_treat_array_keys_as_orthogonal() -> None: source, cache = make_documented_cache() row = np.array([0, 2]) column = np.array([1, 3]) with pytest.raises(IndexError, match="unsupported selection type for basic indexing"): cache[row, column] assert tuple(source.reads) == () def test_system_memory_cache_assembles_a_scalar_selection() -> None: source, cache = make_documented_cache() result = cache[1, 2] assert result.shape == () assert result[()] == 10 assert tuple(source.reads) == ((0, 0),) assert cache.projection_uses == (("chunk_transform", "context.transform"),) def test_system_memory_cache_is_documentation_only() -> None: assert not hasattr(zarr_indexing, "SystemMemoryChunkCache") assert not hasattr(zarr_indexing, "ChunkState") def test_chunk_source_failure_is_retained_as_failed_with_its_cause() -> None: source, cache = make_documented_cache() source.failures.add((1, 1)) with pytest.raises(CACHE_NAMESPACE["ChunkLoadError"]) as error: cache[3:5, 4:6] assert isinstance(error.value.__cause__, OSError) assert cache.state((1, 1)).value == "failed" assert tuple(source.reads) == ((1, 1),) def test_failed_chunk_is_not_retried_implicitly() -> None: source, cache = make_documented_cache() source.failures.add((1, 1)) with pytest.raises(CACHE_NAMESPACE["ChunkLoadError"]): cache[3:5, 4:6] with pytest.raises(CACHE_NAMESPACE["ChunkLoadError"]): cache[3:5, 4:6] assert tuple(source.reads) == ((1, 1),) def test_retry_requires_a_failed_chunk() -> None: _, cache = make_documented_cache() with pytest.raises(ValueError, match="retry requires failed chunk .*new"): cache.retry((0, 0)) def test_illegal_chunk_transition_is_rejected() -> None: _, cache = make_documented_cache() with pytest.raises(ValueError, match="illegal chunk transition new -> ready"): cache.reader._transition((0, 0), CACHE_NAMESPACE["ChunkState"].READY, "test") zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_domain.py000066400000000000000000000203331527014137600300470ustar00rootroot00000000000000from __future__ import annotations import pytest from zarr_indexing.domain import IndexDomain from zarr_indexing.errors import BoundsCheckError class TestIndexDomainConstruction: def test_from_shape(self) -> None: d = IndexDomain.from_shape((10, 20)) assert d.inclusive_min == (0, 0) assert d.exclusive_max == (10, 20) assert d.ndim == 2 assert d.origin == (0, 0) assert d.shape == (10, 20) def test_from_shape_0d(self) -> None: d = IndexDomain.from_shape(()) assert d.ndim == 0 assert d.shape == () def test_non_zero_origin(self) -> None: d = IndexDomain(inclusive_min=(5, 10), exclusive_max=(15, 30)) assert d.origin == (5, 10) assert d.shape == (10, 20) assert d.ndim == 2 def test_validation_mismatched_lengths(self) -> None: with pytest.raises(ValueError, match="same length"): IndexDomain(inclusive_min=(0,), exclusive_max=(10, 20)) def test_validation_min_greater_than_max(self) -> None: with pytest.raises(ValueError, match="inclusive_min must be <="): IndexDomain(inclusive_min=(10,), exclusive_max=(5,)) def test_empty_domain(self) -> None: d = IndexDomain(inclusive_min=(5,), exclusive_max=(5,)) assert d.shape == (0,) def test_labels(self) -> None: d = IndexDomain(inclusive_min=(0, 0), exclusive_max=(10, 20), labels=("x", "y")) assert d.labels == ("x", "y") def test_labels_none(self) -> None: d = IndexDomain.from_shape((10,)) assert d.labels is None class TestIndexDomainContains: def test_contains_inside(self) -> None: d = IndexDomain.from_shape((10, 20)) assert d.contains((0, 0)) is True assert d.contains((9, 19)) is True assert d.contains((5, 10)) is True def test_contains_outside(self) -> None: d = IndexDomain.from_shape((10, 20)) assert d.contains((10, 0)) is False assert d.contains((-1, 0)) is False assert d.contains((0, 20)) is False def test_contains_non_zero_origin(self) -> None: d = IndexDomain(inclusive_min=(5,), exclusive_max=(10,)) assert d.contains((5,)) is True assert d.contains((9,)) is True assert d.contains((4,)) is False assert d.contains((10,)) is False def test_contains_wrong_ndim(self) -> None: d = IndexDomain.from_shape((10, 20)) assert d.contains((5,)) is False def test_contains_domain_inside(self) -> None: outer = IndexDomain.from_shape((10, 20)) inner = IndexDomain(inclusive_min=(2, 3), exclusive_max=(8, 15)) assert outer.contains_domain(inner) is True def test_contains_domain_outside(self) -> None: outer = IndexDomain.from_shape((10, 20)) inner = IndexDomain(inclusive_min=(2, 3), exclusive_max=(11, 15)) assert outer.contains_domain(inner) is False def test_contains_domain_wrong_ndim(self) -> None: outer = IndexDomain.from_shape((10, 20)) inner = IndexDomain.from_shape((5,)) assert outer.contains_domain(inner) is False class TestIndexDomainIntersect: def test_overlapping(self) -> None: a = IndexDomain(inclusive_min=(0, 0), exclusive_max=(10, 10)) b = IndexDomain(inclusive_min=(5, 5), exclusive_max=(15, 15)) result = a.intersect(b) assert result is not None assert result.inclusive_min == (5, 5) assert result.exclusive_max == (10, 10) def test_disjoint(self) -> None: a = IndexDomain(inclusive_min=(0,), exclusive_max=(5,)) b = IndexDomain(inclusive_min=(10,), exclusive_max=(15,)) assert a.intersect(b) is None def test_touching_boundary(self) -> None: a = IndexDomain(inclusive_min=(0,), exclusive_max=(5,)) b = IndexDomain(inclusive_min=(5,), exclusive_max=(10,)) assert a.intersect(b) is None def test_contained(self) -> None: a = IndexDomain.from_shape((20,)) b = IndexDomain(inclusive_min=(5,), exclusive_max=(10,)) result = a.intersect(b) assert result is not None assert result.inclusive_min == (5,) assert result.exclusive_max == (10,) def test_wrong_ndim(self) -> None: a = IndexDomain.from_shape((10,)) b = IndexDomain.from_shape((10, 20)) with pytest.raises(ValueError, match="different ranks"): a.intersect(b) class TestIndexDomainTranslate: def test_translate_positive(self) -> None: d = IndexDomain.from_shape((10, 20)) result = d.translate((5, 10)) assert result.inclusive_min == (5, 10) assert result.exclusive_max == (15, 30) def test_translate_negative(self) -> None: d = IndexDomain(inclusive_min=(10, 20), exclusive_max=(30, 40)) result = d.translate((-10, -20)) assert result.inclusive_min == (0, 0) assert result.exclusive_max == (20, 20) def test_translate_wrong_length(self) -> None: d = IndexDomain.from_shape((10,)) with pytest.raises(ValueError, match="same length"): d.translate((1, 2)) class TestIndexDomainNarrow: def test_narrow_slice(self) -> None: d = IndexDomain.from_shape((10, 20)) result = d.narrow((slice(2, 8), slice(5, 15))) assert result.inclusive_min == (2, 5) assert result.exclusive_max == (8, 15) def test_narrow_int(self) -> None: d = IndexDomain.from_shape((10, 20)) result = d.narrow((3, slice(None))) assert result.inclusive_min == (3, 0) assert result.exclusive_max == (4, 20) def test_narrow_ellipsis(self) -> None: d = IndexDomain.from_shape((10, 20, 30)) result = d.narrow((slice(1, 5), ...)) assert result.inclusive_min == (1, 0, 0) assert result.exclusive_max == (5, 20, 30) def test_narrow_slice_none(self) -> None: d = IndexDomain.from_shape((10,)) result = d.narrow((slice(None),)) assert result == d def test_narrow_non_zero_origin(self) -> None: d = IndexDomain(inclusive_min=(10,), exclusive_max=(20,)) result = d.narrow((slice(12, 18),)) assert result.inclusive_min == (12,) assert result.exclusive_max == (18,) def test_narrow_int_out_of_bounds(self) -> None: d = IndexDomain.from_shape((10,)) with pytest.raises(BoundsCheckError, match="out of bounds"): d.narrow((10,)) def test_narrow_int_below_origin(self) -> None: d = IndexDomain(inclusive_min=(5,), exclusive_max=(10,)) with pytest.raises(BoundsCheckError, match="out of bounds"): d.narrow((4,)) def test_narrow_refuses_a_bound_outside_the_domain(self) -> None: """Clamping made two different requests answer alike, and neither well. Indices here are absolute coordinates, so `-5` is a coordinate this domain does not contain rather than NumPy's "five from the end" — and clamping returned the whole axis for it, which is what a caller writing the NumPy spelling would least expect. A stop past the end produced a domain the parent did not contain. """ d = IndexDomain.from_shape((10,)) with pytest.raises(BoundsCheckError, match="absolute coordinates"): d.narrow((slice(-5, 100),)) with pytest.raises(BoundsCheckError, match="out of bounds"): d.narrow((slice(20, 30),)) def test_narrow_accepts_the_bounds_of_the_domain_itself(self) -> None: d = IndexDomain.from_shape((10,)) assert d.narrow((slice(0, 10),)) == d def test_narrow_bare_slice(self) -> None: d = IndexDomain.from_shape((10,)) result = d.narrow(slice(2, 8)) assert result.inclusive_min == (2,) assert result.exclusive_max == (8,) def test_narrow_too_many_indices(self) -> None: d = IndexDomain.from_shape((10,)) with pytest.raises(IndexError, match="too many indices"): d.narrow((1, 2)) def test_narrow_step_not_one(self) -> None: """A stride is not a bounds failure — the rest of the algebra raises `ValueError` for a request it does not implement, and so does this.""" d = IndexDomain.from_shape((10,)) with pytest.raises(ValueError, match="step=1"): d.narrow((slice(0, 10, 2),)) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_json.py000066400000000000000000000553431527014137600275620ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING, Any import numpy as np import pytest from zarr_indexing.domain import IndexDomain from zarr_indexing.messages import NdselError from zarr_indexing.output_map import ( ArrayMap, ConstantMap, DimensionMap, output_index_map_from_json, ) from zarr_indexing.transform import IndexTransform if TYPE_CHECKING: from zarr_indexing.json import IndexTransformJSON def _maps_equal(a: object, b: object) -> bool: if type(a) is not type(b): return False if isinstance(a, ConstantMap): assert isinstance(b, ConstantMap) return a.offset == b.offset if isinstance(a, DimensionMap): assert isinstance(b, DimensionMap) return (a.input_dimension, a.offset, a.stride) == (b.input_dimension, b.offset, b.stride) assert isinstance(a, ArrayMap) assert isinstance(b, ArrayMap) return ( a.offset == b.offset and a.stride == b.stride and np.array_equal(a.index_array, b.index_array) ) def _transforms_equal(a: IndexTransform, b: IndexTransform) -> bool: """Structural equality that compares `ArrayMap` index arrays element-wise (`IndexTransform`'s dataclass `__eq__` cannot, as numpy `==` is ambiguous).""" return ( a.domain == b.domain and len(a.output) == len(b.output) and all(_maps_equal(x, y) for x, y in zip(a.output, b.output, strict=True)) ) class TestIndexDomainJSON: def test_roundtrip(self) -> None: domain = IndexDomain(inclusive_min=(2, 5), exclusive_max=(10, 20)) json = domain.to_json() assert json == { "input_inclusive_min": [2, 5], "input_exclusive_max": [10, 20], "input_labels": ["", ""], } restored = IndexDomain.from_json(json) assert restored == domain def test_with_labels(self) -> None: domain = IndexDomain(inclusive_min=(0, 0), exclusive_max=(10, 20), labels=("x", "y")) json = domain.to_json() assert json["input_labels"] == ["x", "y"] restored = IndexDomain.from_json(json) assert restored.labels == ("x", "y") def test_without_labels_emits_empty_and_round_trips_to_none(self) -> None: domain = IndexDomain.from_shape((5,)) json = domain.to_json() # Canonical form always writes labels; an unlabeled domain gets [""]*rank. assert json["input_labels"] == [""] restored = IndexDomain.from_json(json) assert restored.labels is None def test_zero_origin(self) -> None: domain = IndexDomain.from_shape((10, 20, 30)) json = domain.to_json() assert json == { "input_inclusive_min": [0, 0, 0], "input_exclusive_max": [10, 20, 30], "input_labels": ["", "", ""], } assert IndexDomain.from_json(json) == domain class TestOutputIndexMapJSON: def test_constant(self) -> None: m = ConstantMap(offset=42) json = m.to_json() assert json == {"offset": 42} restored = output_index_map_from_json(json) assert isinstance(restored, ConstantMap) assert restored.offset == 42 def test_constant_zero(self) -> None: m = ConstantMap(offset=0) json = m.to_json() assert json == {"offset": 0} restored = output_index_map_from_json(json) assert isinstance(restored, ConstantMap) assert restored.offset == 0 def test_dimension(self) -> None: m = DimensionMap(input_dimension=1, offset=10, stride=3) json = m.to_json() assert json == {"offset": 10, "stride": 3, "input_dimension": 1} restored = output_index_map_from_json(json) assert isinstance(restored, DimensionMap) assert restored.input_dimension == 1 assert restored.offset == 10 assert restored.stride == 3 def test_dimension_stride_1_written(self) -> None: """Canonical form writes stride even at its default of 1.""" m = DimensionMap(input_dimension=0) json = m.to_json() assert json == {"offset": 0, "stride": 1, "input_dimension": 0} restored = output_index_map_from_json(json) assert isinstance(restored, DimensionMap) assert restored.stride == 1 def test_array(self) -> None: arr = np.array([1, 5, 9], dtype=np.intp) m = ArrayMap(index_array=arr, offset=2, stride=3) json = m.to_json() # Canonical: stride/offset present, index_array_bounds present, and # no input_dimension (ndsel/TensorStore reject it beside index_array). assert json == { "offset": 2, "stride": 3, "index_array": [1, 5, 9], "index_array_bounds": ["-inf", "+inf"], } restored = output_index_map_from_json(json) assert isinstance(restored, ArrayMap) np.testing.assert_array_equal(restored.index_array, arr) assert restored.offset == 2 assert restored.stride == 3 def test_array_stride_1_written(self) -> None: arr = np.array([0, 1, 2], dtype=np.intp) m = ArrayMap(index_array=arr) json = m.to_json() assert json["stride"] == 1 restored = output_index_map_from_json(json) assert isinstance(restored, ArrayMap) assert restored.stride == 1 def test_array_2d(self) -> None: arr = np.array([[1, 2], [3, 4]], dtype=np.intp) m = ArrayMap(index_array=arr) json = m.to_json() assert json["index_array"] == [[1, 2], [3, 4]] restored = output_index_map_from_json(json) assert isinstance(restored, ArrayMap) np.testing.assert_array_equal(restored.index_array, arr) def test_degenerate_singleton_array_collapses_to_constant(self) -> None: """An all-singleton index_array selects one coordinate -> constant map.""" m = ArrayMap(index_array=np.array([[4]], dtype=np.intp), offset=1, stride=2) json = m.to_json() assert json == {"offset": 1 + 2 * 4} restored = output_index_map_from_json(json) assert isinstance(restored, ConstantMap) assert restored.offset == 9 class TestIndexTransformJSON: def test_identity(self) -> None: t = IndexTransform.from_shape((10, 20)) json = t.to_json() assert json == { "input_rank": 2, "input_inclusive_min": [0, 0], "input_exclusive_max": [10, 20], "input_labels": ["", ""], "output": [ {"offset": 0, "stride": 1, "input_dimension": 0}, {"offset": 0, "stride": 1, "input_dimension": 1}, ], } restored = IndexTransform.from_json(json) assert restored.domain == t.domain assert len(restored.output) == 2 for orig, rest in zip(t.output, restored.output, strict=True): assert type(orig) is type(rest) def test_sliced(self) -> None: t = IndexTransform.from_shape((100,))[10:50:2] json = t.to_json() restored = IndexTransform.from_json(json) assert restored.domain.shape == t.domain.shape assert isinstance(restored.output[0], DimensionMap) orig = t.output[0] assert isinstance(orig, DimensionMap) assert restored.output[0].offset == orig.offset assert restored.output[0].stride == orig.stride def test_with_constant(self) -> None: t = IndexTransform.from_shape((10, 20))[3] json = t.to_json() restored = IndexTransform.from_json(json) assert isinstance(restored.output[0], ConstantMap) assert restored.output[0].offset == 3 assert isinstance(restored.output[1], DimensionMap) def test_with_array(self) -> None: idx = np.array([1, 5, 9], dtype=np.intp) t = IndexTransform.from_shape((10, 20)).oindex[idx, :] json = t.to_json() # The oindex array must not carry input_dimension on the wire. assert "input_dimension" not in json["output"][0] restored = IndexTransform.from_json(json) assert isinstance(restored.output[0], ArrayMap) # Orthogonal arrays are normalized to full input rank with a singleton # axis on the dimension they do not vary over. assert restored.output[0].index_array.shape == (3, 1) np.testing.assert_array_equal(restored.output[0].index_array, idx.reshape(3, 1)) assert isinstance(restored.output[1], DimensionMap) def test_roundtrip_preserves_singleton_axes(self) -> None: """Full-rank orthogonal arrays keep their singleton axes across JSON.""" t = IndexTransform.from_shape((10, 20)).oindex[np.array([1, 3]), np.array([2, 4, 6])] restored = IndexTransform.from_json(t.to_json()) orig0, orig1 = t.output[0], t.output[1] rest0, rest1 = restored.output[0], restored.output[1] assert isinstance(orig0, ArrayMap) assert isinstance(orig1, ArrayMap) assert isinstance(rest0, ArrayMap) assert isinstance(rest1, ArrayMap) assert rest0.index_array.shape == (2, 1) assert rest1.index_array.shape == (1, 3) np.testing.assert_array_equal(rest0.index_array, orig0.index_array) np.testing.assert_array_equal(rest1.index_array, orig1.index_array) def test_with_labels(self) -> None: domain = IndexDomain(inclusive_min=(0, 0), exclusive_max=(10, 20), labels=("x", "y")) t = IndexTransform.identity(domain) json = t.to_json() assert json["input_labels"] == ["x", "y"] restored = IndexTransform.from_json(json) assert restored.domain.labels == ("x", "y") def test_tensorstore_compatible_format(self) -> None: """A canonical body loads and round-trips through the engine layer.""" json: IndexTransformJSON = { "input_rank": 3, "input_inclusive_min": [0, 0, 0], "input_exclusive_max": [100, 200, 3], "input_labels": ["x", "y", "channel"], "output": [ {"offset": 5}, {"offset": 10, "stride": 2, "input_dimension": 1}, # Full input rank, which is what TensorStore itself requires: # it rejects a rank-1 array over a rank-3 domain outright. {"offset": 0, "stride": 1, "index_array": [[[1, 2, 0]]]}, ], } t = IndexTransform.from_json(json) assert t.domain.shape == (100, 200, 3) assert t.domain.labels == ("x", "y", "channel") assert isinstance(t.output[0], ConstantMap) assert t.output[0].offset == 5 assert isinstance(t.output[1], DimensionMap) assert t.output[1].offset == 10 assert t.output[1].stride == 2 assert t.output[1].input_dimension == 1 assert isinstance(t.output[2], ArrayMap) np.testing.assert_array_equal(t.output[2].index_array, [[[1, 2, 0]]]) # Roundtrip json_rt = t.to_json() t_rt = IndexTransform.from_json(json_rt) assert t_rt.domain == t.domain class TestCanonicalRoundTrips: """Round-trip `transform == from(to(transform))`, up to the documented degenerate-collapse (all-singleton ArrayMap -> ConstantMap).""" def test_oindex_multi_axis(self) -> None: t = IndexTransform.from_shape((10, 20, 30)).oindex[np.array([1, 3]), :, np.array([2, 4, 6])] rt = IndexTransform.from_json(t.to_json()) assert _transforms_equal(rt, t) def test_oindex_with_slice(self) -> None: t = IndexTransform.from_shape((10, 20))[2:8].oindex[np.array([3, 5, 7]), :] rt = IndexTransform.from_json(t.to_json()) assert _transforms_equal(rt, t) def test_vindex(self) -> None: t = IndexTransform.from_shape((10, 20)).vindex[np.array([1, 3, 5]), np.array([2, 4, 6])] rt = IndexTransform.from_json(t.to_json()) assert _transforms_equal(rt, t) def test_vindex_with_residual_slice(self) -> None: t = IndexTransform.from_shape((10, 20, 30)).vindex[np.array([1, 3]), np.array([2, 4]), :] rt = IndexTransform.from_json(t.to_json()) assert _transforms_equal(rt, t) def test_length1_degenerate_oindex_collapses(self) -> None: """A length-1 oindex selection is the ConstantMap it equals. The selection layer collapses it at construction; a hand-built all-singleton ArrayMap still collapses on serialize, so the canonical wire form is a `constant` map either way. """ t = IndexTransform.from_shape((10, 20)).oindex[np.array([7]), :] m = t.output[0] assert isinstance(m, ConstantMap) assert m.offset == 7 hand_built = IndexTransform( domain=t.domain, output=(ArrayMap(index_array=np.array([[7]], dtype=np.intp)), t.output[1]), ) rt = IndexTransform.from_json(hand_built.to_json()) rm = rt.output[0] assert isinstance(rm, ConstantMap) assert rm.offset == 7 # The size-1 input dimension survives, unconsumed, in the domain. assert rt.domain == t.domain def test_slices_and_constants(self) -> None: t = IndexTransform.from_shape((10, 20, 30))[2:8:2, 5, :] rt = IndexTransform.from_json(t.to_json()) assert _transforms_equal(rt, t) def _index_array_body(index_array: Any, rank: int = 1, extent: int = 2) -> IndexTransformJSON: return { "input_rank": rank, "input_inclusive_min": [0] * rank, "input_exclusive_max": [extent] * rank, "input_labels": [""] * rank, "output": [{"offset": 0, "stride": 1, "index_array": index_array}], } @pytest.mark.parametrize( ("index_array", "detail"), [ ([0.9, 1.9], "float64"), ([0, 1.5], "float64"), ([True, False], "bool"), (["a", "b"], "str"), # Not lists at all, so they are turned away before their content is # looked at: the wire representation requires a nested array rather # than a scalar string or integer. ("abc", "must be an array of integers"), (5, "must be an array of integers"), ([None, None], "object"), ], ids=["floats", "mixed", "bools", "strings", "string", "scalar", "nulls"], ) def test_a_non_integer_index_array_is_rejected(index_array: Any, detail: str) -> None: """Index arrays require integer coordinates and reject float, bool, and string dtypes.""" with pytest.raises(NdselError) as excinfo: IndexTransform.from_json(_index_array_body(index_array)) assert excinfo.value.reason == "invalid_json" assert "index_array" in str(excinfo.value) assert detail in str(excinfo.value) def test_a_ragged_index_array_is_rejected() -> None: """A nested list that is not rectangular is not an array at all.""" with pytest.raises(NdselError) as excinfo: IndexTransform.from_json(_index_array_body([[0, 1], [2]])) assert excinfo.value.reason == "invalid_json" @pytest.mark.parametrize( ("index_array", "rank", "extent"), [([0, 1], 1, 2), ([[0], [1]], 2, 2), ([], 1, 0)], ids=["1d", "2d", "empty"], ) def test_an_integer_index_array_is_accepted(index_array: Any, rank: int, extent: int) -> None: """Integers of any nesting still lower, including an empty selection. An empty array selects nothing, so the domain it is read over is empty too; a domain with room for coordinates the array does not supply is rejected (see `test_an_index_array_that_does_not_span_its_domain_is_rejected`). """ t = IndexTransform.from_json(_index_array_body(index_array, rank, extent)) m = t.output[0] assert isinstance(m, ArrayMap) assert m.index_array.dtype == np.intp def test_a_non_integer_index_array_is_rejected_by_the_map_loader() -> None: """The single-map loader enforces the same constraint as the transform one.""" with pytest.raises(NdselError) as excinfo: output_index_map_from_json({"index_array": [0.5, 1.5]}) assert excinfo.value.reason == "invalid_json" def test_infinite_bound_rejected_on_lowering() -> None: body: IndexTransformJSON = { "input_rank": 1, "input_inclusive_min": [0], "input_exclusive_max": [["+inf"]], "input_labels": [""], "output": [{"offset": 0, "stride": 1, "input_dimension": 0}], } with pytest.raises(ValueError, match="infinite"): IndexTransform.from_json(body) def test_a_lower_rank_index_array_is_widened_on_the_way_in() -> None: """External JSON may broadcast a lower-rank array; the engine never holds one. ndsel leaves index-array rank unvalidated, so a conformant producer may send an array of lower rank. It is widened at the boundary, which keeps the full-rank invariant true of every transform the engine builds. """ # A rank-1 array widens into the trailing axis, so it spans that axis's # extent of four. body: IndexTransformJSON = { "input_inclusive_min": [0, 0], "input_exclusive_max": [3, 4], "output": [{"index_array": [1, 2, 0, 2]}, {"input_dimension": 1}], } transform = IndexTransform.from_json(body) array_map = transform.output[0] assert isinstance(array_map, ArrayMap) assert array_map.index_array.shape == (1, 4) assert array_map.index_array.ndim == transform.domain.ndim def test_an_index_array_of_the_wrong_rank_is_rejected() -> None: """Inside the engine, a rank that does not match the domain is a bug.""" with pytest.raises(ValueError, match="index_array has 1 dims"): IndexTransform( domain=IndexDomain.from_shape((3, 4)), output=(ArrayMap(index_array=np.array([1, 2, 0], dtype=np.intp)),), ) def test_an_index_array_that_does_not_span_its_domain_is_rejected() -> None: """An array with entries for only part of an axis is not a smaller selection. Reading it that way is how a truncated index array turned into a partially written result rather than an error. """ with pytest.raises(ValueError, match="neither 1 nor the domain's extent"): IndexTransform( domain=IndexDomain.from_shape((3, 2)), output=( ArrayMap(index_array=np.zeros((3, 0), dtype=np.intp)), DimensionMap(input_dimension=1), ), ) def test_an_empty_index_array_collapses_to_a_constant() -> None: """Selecting nothing must survive a trip through JSON. An empty index array names no cell, and can only be empty because an input dimension is, so nothing is ever read through it. It is degenerate in exactly the way a size-1 array is, and collapses the same way — which is also what TensorStore emits for `t[ts.d[0][[]]]`. Emitting the array instead produced a document nothing could load: `tolist()` renders every empty array as `[]` once the leading axis is the zero-length one, so the rank went with it, and the loader put the dependency back on a different axis by prepending singletons. """ for shape, selection in ( ((5, 3), (np.array([], dtype=np.intp), slice(None))), ((5, 5), (np.array([], dtype=np.intp), np.array([], dtype=np.intp))), ): transform = IndexTransform.from_shape(shape).oindex[selection] body = transform.to_json() assert all("index_array" not in m for m in body["output"]) reloaded = IndexTransform.from_json(body) assert reloaded.domain == transform.domain assert reloaded.to_json() == body def test_an_empty_index_array_from_elsewhere_is_recovered_from_the_domain() -> None: """A producer that does emit one is still readable when the domain settles it. This package never writes such a document, but ndsel does not forbid it, and the domain names the axis unambiguously when exactly one dimension is empty. """ body: IndexTransformJSON = { "input_inclusive_min": [0, 0], "input_exclusive_max": [0, 4], "output": [{"index_array": []}, {"input_dimension": 1}], } array_map = IndexTransform.from_json(body).output[0] assert isinstance(array_map, ArrayMap) assert array_map.index_array.shape == (0, 1) def test_an_ambiguous_empty_index_array_is_rejected() -> None: """Two zero-length dimensions leave nothing to recover the axis from.""" body: IndexTransformJSON = { "input_inclusive_min": [0, 0], "input_exclusive_max": [0, 0], "output": [{"index_array": []}, {"input_dimension": 1}], } with pytest.raises(NdselError) as excinfo: IndexTransform.from_json(body) assert excinfo.value.reason == "invalid_json" assert "zero-length" in str(excinfo.value) @pytest.mark.parametrize( ("document", "reason", "detail"), [ ( {"input_inclusive_min": [0.0], "input_exclusive_max": [3], "input_labels": [""]}, "invalid_json", "must be an integer", ), ( {"input_inclusive_min": [0], "input_exclusive_max": ["3"], "input_labels": [""]}, "invalid_json", "must be an integer", ), ( {"input_inclusive_min": [False], "input_exclusive_max": [True], "input_labels": [""]}, "invalid_json", "must be an integer", ), ( {"input_inclusive_min": [0], "input_exclusive_max": [3], "input_labels": [5]}, "invalid_json", "must be a string", ), ( {"input_inclusive_min": [0], "input_exclusive_max": [2**200], "input_labels": [""]}, "invalid_json", "64-bit signed range", ), ], ids=["float", "string", "bool", "non-string-label", "out-of-range"], ) def test_a_malformed_domain_document_is_rejected(document: Any, reason: str, detail: str) -> None: """The domain loader enforces coordinate types, integer bounds, and string labels.""" with pytest.raises(NdselError) as excinfo: IndexDomain.from_json(document) assert excinfo.value.reason == reason assert detail in str(excinfo.value) def test_a_transform_body_cannot_reinterpret_itself_as_another_message() -> None: """A `kind` inside the body must not change which message is being read.""" with pytest.raises(NdselError) as excinfo: IndexTransform.from_json({"kind": "points", "coords": [[1, 2], [3, 4]]}) assert excinfo.value.reason == "invalid_json" assert "kind" in str(excinfo.value) def test_an_engine_invariant_failure_leaves_the_loader_as_a_typed_error() -> None: """A document is invalid input however deep the check that catches it lives. The engine's rank and span invariants are the last gate a document passes, and they raised a bare `ValueError` written in the engine's vocabulary. """ body: IndexTransformJSON = { "input_rank": 1, "input_inclusive_min": [0], "input_exclusive_max": [5], "input_labels": [""], "output": [{"index_array": [[1, 2], [3, 4]]}], } with pytest.raises(NdselError) as excinfo: IndexTransform.from_json(body) assert excinfo.value.reason == "rank_mismatch" zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_lazy_array.py000066400000000000000000003116001527014137600307550ustar00rootroot00000000000000"""Tests for `zarr_indexing.grid` grids and the `LazyArray` wrapper. The happy-path suite is a single oracle test: every selection case is applied both to a `LazyArray` and to the NumPy array it wraps, and the results must match. The case list is crossed with four source flavors — an unchunked NumPy array, NumPy with each of the two declared chunk conventions, and a zarr array whose chunking is auto-discovered — so the chunked and unchunked resolution strategies are held to the same answers. """ from __future__ import annotations import operator import pickle from concurrent.futures import ThreadPoolExecutor from dataclasses import replace from typing import TYPE_CHECKING, Any, cast import numpy as np import pytest import zarr_indexing.lazy_array as lazy_array_module from zarr_indexing import ( ArrayMap, ChunkGrid, ChunkProjection, ConstantMap, DimensionMap, EagerArrayAdapter, EdgeDimensionGrid, FixedDimension, IndexDomain, IndexTransform, LazyArray, ReadContext, VaryingDimension, dimension_grids_from_chunks, ) from zarr_indexing.lazy_array import _out_selection_cell_count, _validate_prepared_parts from zarr_indexing.reader import Reader, basic_reader, numpy_reader from zarr_indexing.testing import repartition if TYPE_CHECKING: from collections.abc import Callable, Sequence SHAPE = (7, 5, 4) PART_SHAPE = (3, 2, 3) EXPLICIT_PARTS = ((3, 3, 1), (2, 2, 1), (3, 1)) class IndexLike: """A scalar integer selector implemented only through `__index__`.""" def __init__(self, value: int) -> None: self.value = value def __index__(self) -> int: return self.value class IntOnly: def __int__(self) -> int: return 2 class BadIndex: """An `__index__` that lies: the protocol requires an integer.""" def __index__(self) -> int: return cast("int", 2.5) def reference() -> np.ndarray[Any, np.dtype[np.int64]]: """The array every source flavor holds, and the oracle for every case.""" return np.arange(int(np.prod(SHAPE)), dtype=np.int64).reshape(SHAPE) class DelegatingReader: def __init__(self, inner: Reader) -> None: self.inner = inner def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: self.inner.read_into(source, context, out) def outer(ref: np.ndarray[Any, Any], selections: Sequence[Any]) -> np.ndarray[Any, Any]: """NumPy oracle for orthogonal indexing: the outer product of per-axis selections. Scalar integers are basic indices — NumPy applies them first and drops the axis — so they are peeled off before the outer product is formed. """ scalars = tuple( sel if isinstance(sel, (int, np.integer)) and not isinstance(sel, bool) else slice(None) for sel in selections ) reduced = ref[scalars] axes = [ np.arange(size)[sel] for size, sel in zip( reduced.shape, [ s for s in selections if not (isinstance(s, (int, np.integer)) and not isinstance(s, bool)) ], strict=True, ) ] if len(axes) == 0: return reduced return reduced[np.ix_(*axes)] # --------------------------------------------------------------------------- # EdgeDimensionGrid # --------------------------------------------------------------------------- @pytest.mark.parametrize( ("sizes", "expected_index_to_chunk", "expected_offsets", "expected_sizes"), [ # A clipped edge chunk. ((3, 3, 1), [0, 0, 0, 1, 1, 1, 2], [0, 3, 6], [3, 3, 1]), # A single chunk covering the whole axis. ((4,), [0, 0, 0, 0], [0], [4]), # A size-1 axis. ((1,), [0], [0], [1]), # Irregular sizes: the grid does not have to be regular. ((1, 2, 1), [0, 1, 1, 2], [0, 1, 3], [1, 2, 1]), ], ) def test_edge_dimension_grid( sizes: tuple[int, ...], expected_index_to_chunk: list[int], expected_offsets: list[int], expected_sizes: list[int], ) -> None: """All four `DimensionGridLike` methods agree with hand-computed values.""" grid = EdgeDimensionGrid(sizes) extent = sum(sizes) assert grid.num_chunks == len(sizes) assert grid.extent == extent assert [grid.index_to_chunk(i) for i in range(extent)] == expected_index_to_chunk assert [grid.chunk_offset(c) for c in range(len(sizes))] == expected_offsets assert [grid.chunk_size(c) for c in range(len(sizes))] == expected_sizes np.testing.assert_array_equal( grid.indices_to_chunks(np.arange(extent, dtype=np.intp)), np.asarray(expected_index_to_chunk, dtype=np.intp), ) def test_edge_dimension_grid_rejects_nonpositive_size() -> None: with pytest.raises(ValueError, match="chunk sizes must be positive"): EdgeDimensionGrid((3, 0, 2)) def test_edge_dimension_grid_rejects_out_of_bounds_index() -> None: with pytest.raises(IndexError, match="out of bounds for an axis of extent 4"): EdgeDimensionGrid((3, 1)).index_to_chunk(4) def test_edge_dimension_grid_rejects_out_of_bounds_chunk() -> None: with pytest.raises(IndexError, match="chunk index 2 is out of bounds"): EdgeDimensionGrid((3, 1)).chunk_offset(2) @pytest.mark.parametrize( "grid", [ pytest.param(FixedDimension(size=2, extent=4), id="fixed"), pytest.param(VaryingDimension(edges=(1, 3), extent=4), id="varying"), ], ) def test_compact_dimension_grid_rejects_vector_below_extent(grid: Any) -> None: with pytest.raises(IndexError, match=r"indices must lie in \[0, 4\); got \[-1, 1\]"): grid.indices_to_chunks(np.array([-1, 1], dtype=np.intp)) @pytest.mark.parametrize( "grid", [ pytest.param(FixedDimension(size=2, extent=4), id="fixed"), pytest.param(VaryingDimension(edges=(1, 3), extent=4), id="varying"), ], ) def test_compact_dimension_grid_rejects_vector_above_extent(grid: Any) -> None: with pytest.raises(IndexError, match=r"indices must lie in \[0, 4\); got \[1, 4\]"): grid.indices_to_chunks(np.array([1, 4], dtype=np.intp)) def test_fixed_dimension_rejects_zero_size_for_nonempty_extent() -> None: with pytest.raises(ValueError, match="size must be > 0 when extent is nonzero"): FixedDimension(size=0, extent=4) def test_fixed_dimension_retains_zero_size_for_zero_extent() -> None: grid = FixedDimension(size=0, extent=0) assert grid.nchunks == 0 assert grid.ngridcells == 0 @pytest.mark.parametrize( ("chunks", "shape", "expected"), [ # Uniform chunk shape, tail clipped. ((3, 4), (7, 4), (FixedDimension(size=3, extent=7), FixedDimension(size=4, extent=4))), # Dask-convention per-axis sizes, passed through. ( ((3, 3, 1), (2, 2)), (7, 4), (VaryingDimension(edges=(3, 3, 1), extent=7), VaryingDimension(edges=(2, 2), extent=4)), ), # A chunk longer than the axis collapses to one clipped chunk. ((10,), (4,), (FixedDimension(size=10, extent=4),)), # A zero-length axis has no chunks at all. ((3,), (0,), (FixedDimension(size=3, extent=0),)), ], ) def test_dimension_grids_from_chunks( chunks: Any, shape: tuple[int, ...], expected: tuple[Any, ...] ) -> None: grids = dimension_grids_from_chunks(chunks, shape) assert grids == expected def test_regular_dimension_metadata_is_constant_in_chunk_count() -> None: dimensions = dimension_grids_from_chunks((1,), (1_000_000,)) assert dimensions == (FixedDimension(size=1, extent=1_000_000),) assert dimensions[0].nchunks == 1_000_000 assert dimensions[0].index_to_chunk(999_999) == 999_999 def test_chunk_grid_distinguishes_codec_and_data_shape_at_the_edge() -> None: grid = ChunkGrid(dimensions=(FixedDimension(size=3, extent=7),)) edge = grid[(2,)] assert edge is not None assert edge.slices == (slice(6, 7, 1),) assert edge.shape == (1,) assert edge.codec_shape == (3,) def test_dimension_grids_reject_negative_shape() -> None: with pytest.raises(ValueError, match="shape entries must be non-negative"): dimension_grids_from_chunks((3,), (-1,)) def test_dimension_grids_from_chunks_rejects_wrong_length() -> None: with pytest.raises(ValueError, match="one entry per dimension"): dimension_grids_from_chunks((3,), (7, 4)) def test_dimension_grids_from_chunks_rejects_mixed_conventions() -> None: with pytest.raises(ValueError, match="not a mixture"): dimension_grids_from_chunks((3, (2, 2)), (7, 4)) def test_dimension_grids_from_chunks_rejects_wrong_total() -> None: with pytest.raises(ValueError, match="sum to 5, but the array extent is 7"): dimension_grids_from_chunks(((3, 2), (4,)), (7, 4)) def test_dimension_grids_from_chunks_rejects_non_sequence_entry() -> None: """A float entry is neither convention, and says so instead of raising TypeError.""" with pytest.raises(ValueError, match=r"3\.5 at dimension 0 is neither"): dimension_grids_from_chunks((3.5, (4,)), (7, 4)) def test_array_map_dependent_axis_reports_no_axis() -> None: """A map varying over nothing answers None rather than a stale binding.""" correlated = ArrayMap(index_array=np.array([[1], [2]], dtype=np.intp)) assert correlated.dependent_axis == 0 assert ArrayMap(index_array=np.array([[3]], dtype=np.intp)).dependent_axis is None def test_scalar_on_a_fancy_axis_collapses_to_a_constant() -> None: """Scalar selection collapses an all-singleton ArrayMap to a ConstantMap.""" view = IndexTransform.from_shape((7, 5)).oindex[np.array([3, 1]), slice(None)][0] assert view.output[0] == ConstantMap(offset=3) assert isinstance(view.output[1], DimensionMap) assert view.domain.shape == (5,) # --------------------------------------------------------------------------- # Sources # --------------------------------------------------------------------------- def make_zarr_source() -> Any: """A zarr array holding `reference()`, whose parts are auto-discovered.""" zarr = pytest.importorskip("zarr") array = zarr.create_array({}, shape=SHAPE, chunks=PART_SHAPE, dtype="int64") array[:] = reference() return array def make_source(flavor: str) -> LazyArray: """Build a `LazyArray` over `reference()` with the requested partitioning.""" data = reference() if flavor == "numpy-whole": return LazyArray(data) if flavor == "numpy-explicit-parts": return LazyArray(data).with_parts_per_axis(EXPLICIT_PARTS) if flavor == "numpy-uniform-parts": return LazyArray(data).with_parts(PART_SHAPE) if flavor == "zarr": return LazyArray(make_zarr_source()) if flavor == "zarr-misaligned": # Parts that deliberately straddle the zarr array's own chunks. return LazyArray(make_zarr_source()).with_parts((4, 3, 3)) raise TypeError(f"unknown source flavor {flavor!r}") FLAVORS = [ "numpy-whole", "numpy-explicit-parts", "numpy-uniform-parts", "zarr", "zarr-misaligned", ] @pytest.fixture(params=FLAVORS) def source(request: pytest.FixtureRequest) -> LazyArray: return make_source(request.param) # --------------------------------------------------------------------------- # The oracle # --------------------------------------------------------------------------- MASK = (reference() % 11) == 0 # A mask over the two trailing axes, for the `vindex[..., mask]` idiom. TRAILING_MASK = (reference()[0] % 3) == 0 # (id, lazy view builder, NumPy oracle). Integer scalars are deliberately absent # from the orthogonal cases: the transform algebra keeps an int-selected axis as # a length-1 axis under `oindex`, which `np.ix_` cannot express. CASES: list[tuple[str, Callable[[LazyArray], LazyArray], Callable[[Any], Any]]] = [ ( "basic-strided-and-int-drop", lambda a: a[1:6:2, :, -1], lambda r: r[1:6:2, :, -1], ), ("basic-ellipsis", lambda a: a[..., -2], lambda r: r[..., -2]), ("basic-negative-scalar", lambda a: a[-3], lambda r: r[-3]), ("basic-newaxis", lambda a: a[None, :, :, None], lambda r: r[None, :, :, None]), ("basic-empty", lambda a: a[:, 2:2, :], lambda r: r[:, 2:2, :]), ("basic-all-scalars", lambda a: a[-1, 0, 2], lambda r: r[-1, 0, 2]), ( "oindex-unsorted-duplicates-multi-axis", lambda a: a.oindex[[4, 0, 0, 2], :, [3, 1]], lambda r: outer(r, ([4, 0, 0, 2], slice(None), [3, 1])), ), ( "oindex-negative-and-slice", lambda a: a.oindex[:, [-1, 0], 1:4], lambda r: outer(r, (slice(None), [-1, 0], slice(1, 4))), ), ( "oindex-boolean-axis", lambda a: a.oindex[np.array([True, False, True, False, False, False, True]), :, :], lambda r: outer( r, (np.array([True, False, True, False, False, False, True]), slice(None), slice(None)), ), ), ( "vindex-coordinates", lambda a: a.vindex[np.array([0, 6, 3]), np.array([1, 4, 0]), np.array([2, 0, 1])], lambda r: r[np.array([0, 6, 3]), np.array([1, 4, 0]), np.array([2, 0, 1])], ), ( "vindex-broadcast-pair", lambda a: a.vindex[np.array([[0], [6]]), np.array([1, 4]), np.array([2, 0])], lambda r: r[np.array([[0], [6]]), np.array([1, 4]), np.array([2, 0])], ), ( "vindex-negative-coordinates", lambda a: a.vindex[np.array([-1, -7]), np.array([-2, 0]), np.array([0, -1])], lambda r: r[np.array([-1, -7]), np.array([-2, 0]), np.array([0, -1])], ), ("vindex-mask", lambda a: a.vindex[MASK], lambda r: r[MASK]), ( "compose-basic-then-oindex", lambda a: a[1:6].oindex[[3, 0, 0], [4, 1], :], lambda r: outer(r[1:6], ([3, 0, 0], [4, 1], slice(None))), ), ( "compose-oindex-then-basic-other-axis", lambda a: a.oindex[[4, 0, 2], :, :][:, 1:4, ::2], lambda r: outer(r, ([4, 0, 2], slice(None), slice(None)))[:, 1:4, ::2], ), ( "compose-basic-then-basic", lambda a: a[2:, 1:][::2, -1], lambda r: r[2:, 1:][::2, -1], ), ( "compose-basic-then-vindex", lambda a: a[1:6, :, 1:].vindex[np.array([0, 4]), np.array([2, 0]), np.array([1, 2])], lambda r: r[1:6, :, 1:][np.array([0, 4]), np.array([2, 0]), np.array([1, 2])], ), # Scalar integers are basic indices in the positional dialect: they drop the # axis, in every mode, exactly as NumPy does. ("oindex-scalar-drops-axis", lambda a: a.oindex[0], lambda r: r[0]), ( "oindex-scalar-with-arrays", lambda a: a.oindex[0, [1, 2], :], lambda r: outer(r, (0, [1, 2], slice(None))), ), ( "oindex-scalar-middle-axis", lambda a: a.oindex[[3, 1], -1, :], lambda r: outer(r, ([3, 1], -1, slice(None))), ), ("oindex-all-scalars", lambda a: a.oindex[0, 1, 2], lambda r: r[0, 1, 2]), ("vindex-all-scalars", lambda a: a.vindex[0, 1, 2], lambda r: r[0, 1, 2]), ( "vindex-scalar-with-arrays", lambda a: a.vindex[0, [1, 2], [3, 0]], lambda r: r[0, [1, 2], [3, 0]], ), ( "vindex-scalar-on-middle-axis", lambda a: a.vindex[[1, 2], 0, [3, 0]], lambda r: r[[1, 2], 0, [3, 0]], ), # Scalar applied to a previously fancy-indexed axis, both orders. ( "compose-oindex-then-scalar", lambda a: a.oindex[[3, 1], :, :][0], lambda r: outer(r, ([3, 1], slice(None), slice(None)))[0], ), ( "compose-oindex-then-scalar-negative", lambda a: a.oindex[[3, 1, 1], :, :][-1, 2], lambda r: outer(r, ([3, 1, 1], slice(None), slice(None)))[-1, 2], ), ( "compose-scalar-then-oindex", lambda a: a[0].oindex[[3, 1], :], lambda r: outer(r[0], ([3, 1], slice(None))), ), ( "compose-vindex-then-scalar", lambda a: a.vindex[np.array([0, 6, 3]), np.array([1, 4, 0]), np.array([2, 0, 1])][1], lambda r: r[np.array([0, 6, 3]), np.array([1, 4, 0]), np.array([2, 0, 1])][1], ), # Partial vindex whose coordinate arrays are NOT on the leading axes: NumPy # inserts the gathered axis where the (adjacent) advanced indices sat. ( "vindex-trailing-arrays", lambda a: a.vindex[..., np.array([1, 4, 0]), np.array([2, 0, 1])], lambda r: r[..., np.array([1, 4, 0]), np.array([2, 0, 1])], ), ( "vindex-single-trailing-array", lambda a: a.vindex[..., np.array([3, 0, 1])], lambda r: r[..., np.array([3, 0, 1])], ), ( "vindex-trailing-mask", lambda a: a.vindex[..., TRAILING_MASK], lambda r: r[..., TRAILING_MASK], ), ( "vindex-leading-partial", lambda a: a.vindex[np.array([1, 2, 2])], lambda r: r[np.array([1, 2, 2])], ), ( "compose-basic-then-vindex-trailing", lambda a: a[2:, 1:].vindex[..., np.array([1, 3, 0])], lambda r: r[2:, 1:][..., np.array([1, 3, 0])], ), # A ConstantMap sitting between a slice and the coordinate arrays: NumPy # counts the integer as an advanced index, so the gathered axis moves to the # front of the chunk block even though the arrays are trailing. ( "compose-vindex-trailing-then-scalar", lambda a: a.vindex[..., np.array([3, 0])][2], lambda r: r[..., np.array([3, 0])][2], ), # Two fancy axes, then a scalar on the first: the surviving ArrayMap ends up # behind a ConstantMap and a slice, which is where NumPy's integer-counts-as- # advanced rule bites. ( "compose-oindex-two-axes-then-scalar", lambda a: a.oindex[[3, 1, 0], 3:5, [2, 0, 2]][0], lambda r: outer(r, ([3, 1, 0], slice(3, 5), [2, 0, 2]))[0], ), # Negative steps: the positional dialect is NumPy's, including the empty # cases NumPy allows where the transform algebra alone would object. ("reverse", lambda a: a[::-1], lambda r: r[::-1]), ("reverse-every-axis", lambda a: a[::-1, ::-1, ::-1], lambda r: r[::-1, ::-1, ::-1]), ("reverse-strided", lambda a: a[::-2], lambda r: r[::-2]), ("reverse-nondivisible", lambda a: a[::-3], lambda r: r[::-3]), ("reverse-bounded", lambda a: a[5:1:-1], lambda r: r[5:1:-1]), ("reverse-negative-start", lambda a: a[-1:None:-1], lambda r: r[-1:None:-1]), ("reverse-past-the-start", lambda a: a[:-8:-1], lambda r: r[:-8:-1]), ("reverse-empty", lambda a: a[2:2:-1], lambda r: r[2:2:-1]), # NumPy reads a reversed *positional* interval as empty; only the literal # layer calls it a direction error. ("reverse-inverted-is-empty", lambda a: a[2:5:-1], lambda r: r[2:5:-1]), ("reverse-with-int-drop", lambda a: a[::-2, 2, ::-1], lambda r: r[::-2, 2, ::-1]), ("reverse-trailing-axis", lambda a: a[..., ::-1], lambda r: r[..., ::-1]), ( "compose-reverse-then-reverse", lambda a: a[::-1][::-1], lambda r: r[::-1][::-1], ), ( "compose-strided-then-reverse", lambda a: a[::2][::-1], lambda r: r[::2][::-1], ), ( "compose-reverse-then-strided", lambda a: a[::-1][::2], lambda r: r[::-1][::2], ), ( "compose-reverse-then-oindex", lambda a: a[::-1].oindex[[3, 0, 0], :, :], lambda r: outer(r[::-1], ([3, 0, 0], slice(None), slice(None))), ), ( "compose-oindex-then-reverse", lambda a: a.oindex[[3, 1, 2], :, :][::-1], lambda r: outer(r, ([3, 1, 2], slice(None), slice(None)))[::-1], ), ( "compose-reverse-then-vindex", lambda a: a[::-1].vindex[..., np.array([1, 3, 0])], lambda r: r[::-1][..., np.array([1, 3, 0])], ), ( "compose-vindex-trailing-then-scalar-and-slice", lambda a: a.vindex[..., np.array([3, 0, 1])][-1, 1:4], lambda r: r[..., np.array([3, 0, 1])][-1, 1:4], ), # A downward walk that begins off the front of the axis selects nothing. # Written against an oindex axis and a vindex axis, where the selection is # carried by an index array rather than by the domain. ( "compose-oindex-then-empty-downward-walk", lambda a: a.oindex[np.array([3, 1, 4]), :, :][-8::-1], lambda r: r[np.array([3, 1, 4])][-8::-1], ), ( "compose-vindex-then-empty-downward-walk", lambda a: a.vindex[np.array([3, 1]), np.array([2, 0])][-9::-2], lambda r: r[np.array([3, 1]), np.array([2, 0])][-9::-2], ), ( "empty-downward-walk-on-a-plain-axis", lambda a: a[-11::-1], lambda r: r[-11::-1], ), # A fancy *spelling* whose entries are all slices is not a fancy selection: # it narrows the view's own axes and must compose exactly like basic # indexing. The slices start past 0, so a step that applied them to the # broadcast (singleton) axes of the existing index array would truncate it. ( "compose-oindex-then-oindex-slices-only", lambda a: a.oindex[[4, 0, 0], :, :].oindex[:, 2:5, 1:], lambda r: outer(r, ([4, 0, 0], slice(None), slice(None)))[:, 2:5, 1:], ), ( "compose-oindex-then-oindex-slices-only-strided", lambda a: a.oindex[:, [3, 1, 1], :].oindex[1::2, :, ::-1], lambda r: outer(r, (slice(None), [3, 1, 1], slice(None)))[1::2, :, ::-1], ), ( "compose-vindex-then-oindex-slices-only", lambda a: a.vindex[np.array([4, 0, 2]), np.array([1, 3, 0])].oindex[1:, 2:], lambda r: r[np.array([4, 0, 2]), np.array([1, 3, 0])][1:, 2:], ), ( "compose-oindex-then-oindex-array-on-its-own-axis", lambda a: a.oindex[[4, 0, 0], :, :].oindex[[2, 0], 3:, :], lambda r: outer( outer(r, ([4, 0, 0], slice(None), slice(None))), ([2, 0], slice(3, None), slice(None)), ), ), ] @pytest.mark.parametrize(("build", "oracle"), [c[1:] for c in CASES], ids=[c[0] for c in CASES]) def test_selection_matches_numpy( source: LazyArray, build: Callable[[LazyArray], LazyArray], oracle: Callable[[Any], Any], ) -> None: """Every selection resolves to what NumPy computes positionally on the same data.""" view = build(source) expected = np.asarray(oracle(reference())) assert view.shape == expected.shape assert view.ndim == expected.ndim np.testing.assert_array_equal(np.asarray(view.result()), expected) # NumPy conversion routes through `result()`. np.testing.assert_array_equal(np.asarray(view), expected) # --------------------------------------------------------------------------- # Randomized chain sweep # --------------------------------------------------------------------------- def _random_basic(rng: np.random.Generator, shape: tuple[int, ...]) -> tuple[Any, ...]: selection: list[Any] = [] for size in shape: roll = rng.random() if roll < 0.3: selection.append(int(rng.integers(-size, size))) elif roll < 0.55: start = int(rng.integers(0, size)) stop = int(rng.integers(start, size + 1)) selection.append(slice(start, stop, int(rng.integers(1, 4)))) elif roll < 0.8: # Downward: `start >= stop` and the stop may fall off the front, # which is spelled `None`. The start is drawn from below `-size` as # well, where the walk begins off the front and selects nothing — # a case that reads as an ordinary negative index but is empty. start = int(rng.integers(-2 * size - 1, size)) if size else 0 stop_choice = int(rng.integers(-1, max(start, 0) + 1)) stop = None if stop_choice < 0 else stop_choice selection.append(slice(start, stop, -int(rng.integers(1, 4)))) else: selection.append(slice(None)) return tuple(selection) def _random_oindex(rng: np.random.Generator, shape: tuple[int, ...]) -> tuple[Any, ...]: selection: list[Any] = [] for size in shape: roll = rng.random() if roll < 0.25: selection.append(int(rng.integers(-size, size))) elif roll < 0.65: count = int(rng.integers(1, 5)) selection.append(rng.integers(-size, size, size=count).tolist()) elif roll < 0.8: mask = rng.random(size) < 0.5 mask[int(rng.integers(0, size))] = True selection.append(mask) else: start = int(rng.integers(0, size)) selection.append(slice(start, size)) return tuple(selection) def _broadcast_singleton_axes( rng: np.random.Generator, entries: list[Any], length: int ) -> list[Any]: """Reshape 1-D coordinate arrays so the selection carries singleton axes. A coordinate array of shape `(1, n)` or `(n, 1)` contributes a broadcast axis it does not vary over. That axis stays in the view's domain, and a later basic index that consumes its partner leaves it referenced by no output map at all — the shape that makes a broadcast axis and a genuine extent-1 axis indistinguishable from the index array alone. """ rank = int(rng.integers(2, 4)) reshaped: list[Any] = [] for entry in entries: if not isinstance(entry, np.ndarray): reshaped.append(entry) continue varying = int(rng.integers(0, rank)) reshaped.append( entry.reshape(tuple(length if axis == varying else 1 for axis in range(rank))) ) return reshaped def _random_vindex(rng: np.random.Generator, shape: tuple[int, ...]) -> tuple[Any, ...]: ndim = len(shape) count = int(rng.integers(1, ndim + 1)) trailing = bool(rng.random() < 0.5) axes = range(ndim - count, ndim) if trailing else range(count) sizes = [shape[axis] for axis in axes] entries: list[Any] if rng.random() < 0.2: # A single boolean mask spanning the whole covered block. mask = rng.random(tuple(sizes)) < 0.5 mask.flat[int(rng.integers(0, mask.size))] = True entries = [mask] else: length = int(rng.integers(1, 5)) entries = [ int(rng.integers(-size, size)) if rng.random() < 0.25 else rng.integers(-size, size, size=length) for size in sizes ] if rng.random() < 0.35: entries = _broadcast_singleton_axes(rng, entries, length) return (Ellipsis, *entries) if trailing else tuple(entries) def _apply_oracle( ref: np.ndarray[Any, Any], mode: str, selection: tuple[Any, ...] ) -> np.ndarray[Any, Any]: if mode == "orthogonal": return outer(ref, selection) # NumPy's own semantics *are* basic and vectorized indexing. return ref[selection] def _apply_view(view: LazyArray, mode: str, selection: tuple[Any, ...]) -> LazyArray: if mode == "basic": return view[selection] if mode == "orthogonal": return view.oindex[selection] return view.vindex[selection] def _random_slices_only(rng: np.random.Generator, shape: tuple[int, ...]) -> tuple[Any, ...]: """A selection of slices alone, spelled through `oindex`. `oindex` entries that are all slices are not a fancy selection — they narrow the view's own axes and must compose like basic indexing. A start past 0 is what distinguishes a step that walks the index array's dependency axes from one that walks its broadcast singletons, so slices are drawn to reach past the origin. (`vindex` is coordinate-only and rejects a slice outright, so this spelling has no vectorized counterpart.) """ selection: list[Any] = [] for size in shape: roll = rng.random() if roll < 0.4: start = int(rng.integers(0, size)) if size else 0 selection.append(slice(start, size)) elif roll < 0.7: start = int(rng.integers(0, size)) if size else 0 selection.append(slice(start, size, int(rng.integers(1, 3)))) elif roll < 0.85: selection.append(slice(None, None, -1)) else: selection.append(slice(None)) return tuple(selection) def _random_chain(rng: np.random.Generator) -> list[tuple[str, tuple[Any, ...]]]: """A chain of 2-4 steps, any of which may be fancy. A step spelled through `oindex` but carrying only slices is drawn separately (`slices-only`): it narrows an existing index array by a *slice* rather than by coordinates, a distinct code path kept at full weight. """ n_steps = int(rng.integers(2, 5)) fancy_steps = {int(rng.integers(0, n_steps)) for _ in range(2)} slices_only_at = int(rng.integers(0, n_steps)) if rng.random() < 0.4 else -1 chain: list[tuple[str, tuple[Any, ...]]] = [] running = reference() for step in range(n_steps): if running.ndim == 0 or running.size == 0: break if step in fancy_steps: mode = "orthogonal" if rng.random() < 0.5 else "vectorized" else: mode = "basic" if step == slices_only_at and step not in fancy_steps: mode = "orthogonal" selection = _random_slices_only(rng, running.shape) elif mode == "basic": selection = _random_basic(rng, running.shape) elif mode == "orthogonal": selection = _random_oindex(rng, running.shape) else: selection = _random_vindex(rng, running.shape) chain.append((mode, selection)) running = _apply_oracle(running, mode, selection) return chain @pytest.mark.parametrize("flavor", FLAVORS) def test_random_chains_match_numpy(flavor: str) -> None: """A seeded sweep of composed chains, under every partitioning. The partitioning invariant as a property: `result()` is identical whatever boxes the read is broken into, including boxes deliberately misaligned with the source's own. """ rng = np.random.default_rng(20260730) source = make_source(flavor) partitionings: list[Any] = [None, (2, 2, 2), (7, 5, 4), ((4, 3), (1, 3, 1), (3, 1))] # Reads against a real store cost more per chain; the NumPy flavors carry # the bulk of the sweep and exercise the identical code path. n_chains = 120 if flavor.startswith("zarr") else 400 for _ in range(n_chains): chain = _random_chain(rng) expected = reference() for mode, selection in chain: expected = _apply_oracle(expected, mode, selection) view = source for mode, selection in chain: view = _apply_view(view, mode, selection) assert view.shape == expected.shape, f"{flavor}: {chain}" np.testing.assert_array_equal( np.asarray(view.result()), np.asarray(expected), err_msg=f"{flavor}: {chain}" ) for parts in partitionings: np.testing.assert_array_equal( np.asarray(repartition(view, parts).result()), np.asarray(expected), err_msg=f"{flavor} parts={parts}: {chain}", ) # `result()` can absorb a defect that `parts()` cannot — an empty view assembles # correctly from no parts at all, and a rank-0 one can be reshaped into place — # so the iteration contract needs its own sweep, asserting the documented # assembly literally rather than through `result()`. That sweep is the # `ChainedIndexing` state machine in `test_lazy_array_stateful.py`, which drives # the same operations and shrinks a failure to the chain that caused it. PARTITIONINGS_1D: list[Any] = [None, (1,), (2,), (5,)] def test_a_slice_only_fancy_step_after_a_fancy_step_reads_real_data() -> None: """`oindex[:, 2:8]` after an `oindex` narrows the view, it does not re-index it. The second step carries no coordinates, so it is not fancy-after-fancy: it must compose like basic indexing. Applying its slices to the *broadcast* axes of the first step's index array instead truncates that array to size 0, which leaves the resolver with no parts to read and `result()` handing back an unwritten buffer. """ base = np.arange(24).reshape(3, 8) expected = base[np.ix_([0, 2], range(8))][:, 2:8] for parts in (None, (2, 4), (1, 8), (3, 3)): view = repartition(LazyArray(base), parts).oindex[np.array([0, 2]), :].oindex[:, 2:8] assert view.shape == expected.shape, f"parts={parts}" np.testing.assert_array_equal(np.asarray(view.result()), expected, err_msg=f"{parts}") def test_a_fancy_step_composes_onto_any_axis_of_a_fancy_view() -> None: """A second fancy step may land on axes the first one merely broadcasts along. Composition evaluates the existing index arrays at the new coordinates, so `oindex` after `oindex`, `vindex` after `oindex`, and both orders around a correlated gather all resolve — under every partitioning. """ base = np.arange(24).reshape(3, 8) rows, cols = np.array([0, 2]), np.array([1, 3, 3]) for parts in (None, (2, 4), (1, 8), (3, 3)): view = repartition(LazyArray(base), parts).oindex[rows, :] composed = view.oindex[:, cols] expected = base[np.ix_(rows, cols)] assert composed.shape == expected.shape, f"parts={parts}" np.testing.assert_array_equal(np.asarray(composed.result()), expected, err_msg=f"{parts}") gathered = view.vindex[np.array([0, 1]), np.array([7, 0])] np.testing.assert_array_equal( np.asarray(gathered.result()), base[np.ix_(rows, range(8))][[0, 1], [7, 0]], err_msg=f"{parts}", ) pointwise = repartition(LazyArray(base), parts).vindex[[0, 1, 2], [0, 2, 3]] np.testing.assert_array_equal( np.asarray(pointwise.oindex[[2, 0]].result()), base[[0, 1, 2], [0, 2, 3]][[2, 0]], err_msg=f"{parts}", ) np.testing.assert_array_equal( np.asarray(pointwise.vindex[[1, 1, 0]].result()), base[[0, 1, 2], [0, 2, 3]][[1, 1, 0]], err_msg=f"{parts}", ) def test_a_partial_vindex_after_an_oindex_resolves_the_mixed_transform() -> None: """A composed transform can mix correlated and orthogonal index arrays. `oindex` on the last axis then `vindex` on the first two leaves the orthogonal gather in place while the new coordinate arrays are correlated; resolution takes the pointwise path. """ base = np.arange(60).reshape(3, 4, 5) expected = base[:, :, [4, 0]][[0, 2], [1, 3]] for parts in (None, (2, 2, 2), (3, 4, 5), (1, 1, 1)): view = repartition(LazyArray(base), parts).oindex[:, :, [4, 0]] composed = view.vindex[np.array([0, 2]), np.array([1, 3])] assert composed.shape == expected.shape, f"parts={parts}" np.testing.assert_array_equal(np.asarray(composed.result()), expected, err_msg=f"{parts}") def test_a_boolean_mask_composes_onto_a_fancy_view() -> None: base = np.arange(60).reshape(3, 4, 5) mask = np.array([True, False, True]) expected = base[[0, 1, 2]][mask] for parts in (None, (2, 2, 2), (1, 4, 5)): view = repartition(LazyArray(base), parts).oindex[[0, 1, 2], :, :] np.testing.assert_array_equal( np.asarray(view.oindex[mask].result()), expected, err_msg=f"{parts}" ) def test_an_ellipsis_only_vindex_step_preserves_a_correlated_gather() -> None: """Ellipsis-only vectorized selection preserves correlated coordinates. This also applies after scalar indices have been split off. The selected values must agree across partitionings.""" base = np.arange(16).reshape(4, 4) for parts in (None, (2, 2), (4, 4), (1, 3)): pointwise = repartition(LazyArray(base), parts).vindex[[0, 1, 2], [0, 2, 3]] np.testing.assert_array_equal( np.asarray(pointwise.vindex[...].result()), base[[0, 1, 2], [0, 2, 3]], err_msg=f"{parts}", ) planar = repartition(LazyArray(base), parts).vindex[ np.array([[0], [1]]), np.array([[1], [3]]) ] np.testing.assert_array_equal( np.asarray(planar.vindex[..., np.array(0)].result()), base[[0, 1], [1, 3]], err_msg=f"{parts}", ) # --------------------------------------------------------------------------- # Domain dimensions no output map depends on # --------------------------------------------------------------------------- # # A `vindex` coordinate array with a *singleton* broadcast axis leaves that axis # in the view's domain while the map varies only over its partner. A later basic # index that consumes the partner collapses the map to a `ConstantMap`, and the # singleton axis survives with nothing referencing it. Every stage of resolution # has to keep counting it: the lowered block needs the axis back at its true # extent, and a part has to say where its values belong along it. UNREFERENCED_AXIS_PARTITIONINGS: list[Any] = [None, (1, 1, 1), (2, 2, 2), (3, 4, 5), (3, 1, 2)] # (id, view builder, NumPy oracle) over `np.arange(60).reshape(3, 4, 5)`. UNREFERENCED_AXIS_CASES: list[ tuple[str, Callable[[LazyArray], LazyArray], Callable[[Any], Any]] ] = [ ( "leading-singleton-row", lambda a: a.vindex[np.array([[2, 0]])][:, 0], lambda r: r[np.array([[2, 0]])][:, 0], ), ( "trailing-singleton-column", lambda a: a.vindex[np.array([[2], [0]])][0], lambda r: r[np.array([[2], [0]])][0], ), ( "repeated-coordinates-over-a-singleton", lambda a: a.vindex[np.array([[1, 1]]), np.array([[3, 3]])][:, 0], lambda r: r[np.array([[1, 1]]), np.array([[3, 3]])][:, 0], ), ( "unreferenced-axis-emptied", lambda a: a.vindex[np.array([[2, 0]])][0:0, 0], lambda r: r[np.array([[2, 0]])][0:0, 0], ), ( "partial-vindex-with-a-residual-slice", lambda a: a.vindex[np.array([[2, 0]]), np.array([[1, 3]])][:, 0, 1:4], lambda r: r[np.array([[2, 0]]), np.array([[1, 3]])][:, 0, 1:4], ), ] def unreferenced_axis_reference() -> np.ndarray[Any, np.dtype[np.int64]]: return np.arange(60, dtype=np.int64).reshape(3, 4, 5) @pytest.mark.parametrize( ("build", "oracle"), [case[1:] for case in UNREFERENCED_AXIS_CASES], ids=[case[0] for case in UNREFERENCED_AXIS_CASES], ) @pytest.mark.parametrize("parts", UNREFERENCED_AXIS_PARTITIONINGS) def test_a_domain_axis_no_output_map_depends_on_still_resolves( build: Callable[[LazyArray], LazyArray], oracle: Callable[[Any], Any], parts: Any, ) -> None: """The value, the shape and the tiling all hold when an axis is unreferenced.""" data = unreferenced_axis_reference() expected = np.asarray(oracle(data)) view = build(repartition(LazyArray(data), parts)) assert view.shape == expected.shape np.testing.assert_array_equal(np.asarray(view.result()), expected) hits = np.zeros(view.shape, dtype=np.int64) assembled = np.zeros(view.shape, dtype=view.dtype) for part in view.parts(): assembled[part.out_selection] = np.asarray(part.view.result()) np.add.at(hits, part.out_selection, 1) np.testing.assert_array_equal(assembled, expected) np.testing.assert_array_equal(hits, np.ones(view.shape, dtype=np.int64)) def test_an_unreferenced_domain_axis_of_extent_zero_stays_empty() -> None: """An emptied broadcast axis must not be restored as a fabricated row. The lowered block has no axis for a dimension nothing depends on, so the resolver puts one back. Putting it back at extent 1 invents a row of data for a selection whose own `shape` says it is empty. """ data = np.arange(140, dtype=np.int64).reshape(7, 5, 4) coords = np.array([[6], [3], [0]]) for parts in (None, (1, 1, 1), (3, 2, 3), (7, 5, 4)): view = repartition(LazyArray(data), parts).vindex[coords, -4][0, 0:0] expected = data[coords, -4][0, 0:0] assert view.shape == expected.shape == (0, 4), f"parts={parts}" np.testing.assert_array_equal(np.asarray(view.result()), expected, err_msg=f"{parts}") assert list(view.parts()) == [] def test_a_zero_length_axis_resolves_the_same_way_under_every_partitioning() -> None: """A size-0 axis carries no dependency, so it cannot make a map correlated.""" base = np.zeros((3, 0)) expected = base[np.ix_([1, 0, 0], np.arange(0, dtype=int))] for parts in (None, ((1, 1, 1), ()), ((3,), ())): view = repartition(LazyArray(base), parts).oindex[np.array([1, -3, -3]), :].oindex[:, :] assert view.shape == expected.shape, f"parts={parts}" np.testing.assert_array_equal(np.asarray(view.result()), expected, err_msg=f"{parts}") assert list(view.parts()) == [] def test_an_empty_slice_of_a_length_one_correlated_axis_has_no_parts() -> None: """`parts()` agrees with `result()` that an emptied view selects nothing. A correlated selection of exactly one point normalizes to an all-singleton index array, indistinguishable by shape from an axis the map broadcasts over — so a later slice that empties the domain leaves the array at size 1. """ base = np.arange(5) mask = np.array([False, True, False, False, False]) for parts in PARTITIONINGS_1D: view = repartition(LazyArray(base), parts).vindex[mask][1:-2] assert view.shape == (0,), f"parts={parts}" assert list(view.parts()) == [], f"parts={parts}" np.testing.assert_array_equal(np.asarray(view.result()), base[mask][1:-2]) def test_an_empty_slice_of_a_length_one_vindex_pair_has_no_parts() -> None: """The same emptied view, spelled with explicit coordinates over two axes.""" base = np.arange(35).reshape(7, 5) for parts in (None, (2, 2), (7, 5)): view = repartition(LazyArray(base), parts).vindex[np.array([6]), np.array([0])][0:0] assert view.shape == (0,), f"parts={parts}" assert list(view.parts()) == [], f"parts={parts}" np.testing.assert_array_equal( np.asarray(view.result()), base[np.array([6]), np.array([0])][0:0] ) def test_a_correlated_view_narrowed_to_one_point_has_parts_of_the_views_rank() -> None: """A rank-0 broadcast block survives intersection without gaining an axis. `Partition` documents `out[part.out_selection] = part.view.result()` as the assembly, so a part's values must arrive at the rank the view has — including the rank 0 a correlated selection reaches once every coordinate is a scalar. """ base = np.arange(140).reshape(7, 5, 4) cases = [ # No residual slice: the whole view is the one gathered point. ((np.array([-1]), np.array([-1]), np.array([2])), 0), # One residual slice dimension survives alongside the collapsed block. ((np.array([6, 1]), np.array([4, 0])), 1), ] for parts in (None, (2, 2, 2), (3, 3, 3), (7, 5, 4)): for selection, tail in cases: expected = base[selection][tail] view = repartition(LazyArray(base), parts).vindex[selection][tail] assert view.shape == expected.shape, f"parts={parts}, {selection}" assembled = np.zeros(view.shape, dtype=view.dtype) hits = np.zeros(view.shape, dtype=np.int64) for part in view.parts(): assembled[part.out_selection] = np.asarray(part.view.result()) np.add.at(hits, part.out_selection, 1) err = f"parts={parts}, {selection}" np.testing.assert_array_equal(assembled, expected, err_msg=err) np.testing.assert_array_equal(hits, np.ones(view.shape, dtype=np.int64), err_msg=err) np.testing.assert_array_equal(np.asarray(view.result()), expected, err_msg=err) def test_getitem_returns_a_view(source: LazyArray) -> None: """`arr[...]` returns a view that materializes to the selected values.""" assert isinstance(source[1:3, ::2, -1], LazyArray) np.testing.assert_array_equal(np.asarray(source[1:3, ::2, -1]), reference()[1:3, ::2, -1]) # --------------------------------------------------------------------------- # Attribute forwarding # --------------------------------------------------------------------------- def test_forwards_array_attributes(source: LazyArray) -> None: assert source.shape == SHAPE assert source.ndim == len(SHAPE) assert source.size == int(np.prod(SHAPE)) assert source.dtype == np.dtype("int64") assert len(source) == SHAPE[0] assert "LazyArray" in repr(source) def test_view_shape_comes_from_the_transform(source: LazyArray) -> None: """A view reports its own shape, not the wrapped array's.""" view = source[1:6:2, :, -1] assert view.shape == (3, 5) assert view.ndim == 2 assert view.size == 15 assert view.dtype == source.dtype assert "view=" in repr(view) def test_the_wrapper_has_no_chunks_vocabulary() -> None: """`chunks` is the *source's* word, never the wrapper's.""" assert not hasattr(make_source("numpy-whole"), "chunks") assert not hasattr(make_source("zarr"), "chunks") with pytest.raises(TypeError): LazyArray(reference(), chunks=PART_SHAPE) # type: ignore[call-arg] def test_scalar_is_basic_even_when_a_slice_separates_it_from_the_arrays() -> None: """A documented, deliberate departure from one NumPy corner. NumPy counts an integer as an *advanced* index for its placement rule, so `x[0, :, i]` has shape `(len(i), x.shape[1])` — the arrays lead because the slice separates the integer from them. The positional dialect instead treats every scalar as a basic index applied first, which is the rule the rest of the surface follows and the only one `oindex` can express, so the same selection reads as `x[0][:, i]`. """ data = reference() view = make_source("numpy-uniform-parts").vindex[0, ..., np.array([3, 0])] np.testing.assert_array_equal(np.asarray(view.result()), data[0][..., np.array([3, 0])]) assert view.shape == data[0][..., np.array([3, 0])].shape assert view.shape != data[0, ..., np.array([3, 0])].shape def test_zero_dimensional_result_is_an_array(source: LazyArray) -> None: """Both resolvers agree on the kind of a zero-rank result.""" result = source[0, 1, 2].result() assert isinstance(result, np.ndarray) assert result.ndim == 0 assert result[()] == reference()[0, 1, 2] def test_construction_selects_readers_explicitly() -> None: data = reference() assert LazyArray(data).reader is basic_reader assert LazyArray.from_numpy(data).reader is numpy_reader def test_reader_wrappers_forward_the_read_contract_unchanged() -> None: events: list[tuple[str, Any, ReadContext, Any]] = [] class RecordingDelegatingReader: def __init__(self, name: str, inner: Reader) -> None: self.name = name self.inner = inner def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: events.append((self.name, source, context, out)) self.inner.read_into(source, context, out) data = reference() inner = RecordingDelegatingReader("inner", numpy_reader) outer = RecordingDelegatingReader("outer", inner) view = LazyArray(data).with_reader(outer)[1:6:2, ::-1, 1].unpartitioned() result = view.result() assert [name for name, _, _, _ in events] == ["outer", "inner"] outer_call, inner_call = events assert outer_call[1] is inner_call[1] is data assert outer_call[2] is inner_call[2] assert outer_call[3] is inner_call[3] assert np.shares_memory(outer_call[3], result) np.testing.assert_array_equal(result, data[1:6:2, ::-1, 1]) @pytest.mark.parametrize("value", [object(), [1, 2, 3], "array"]) def test_from_numpy_rejects_non_ndarrays(value: Any) -> None: with pytest.raises(TypeError, match="from_numpy requires a numpy.ndarray"): LazyArray.from_numpy(value) class RecordingReader: def __init__(self) -> None: self.calls: list[tuple[Any, ReadContext, Any]] = [] self.contexts: list[ReadContext] = [] def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: self.calls.append((source, context, out)) self.contexts.append(context) basic_reader.read_into(source, context, out) def test_view_operations_preserve_the_reader_without_reading() -> None: data = reference() reader = RecordingReader() base = LazyArray(data).with_reader(reader) views = ( base[1:5], base.with_parts((2, 2, 2)), base.with_parts_per_axis(((3, 3, 1), (2, 3), (1, 3))), base.unpartitioned(), ) assert reader.calls == [] assert all(view.reader is reader for view in views) assert all(part.view.reader is reader for part in views[1].parts()) @pytest.mark.parametrize("value", [object(), None, lambda: None]) def test_with_reader_requires_callable_read_into(value: Any) -> None: with pytest.raises(TypeError, match="reader.read_into must be callable"): LazyArray(reference()).with_reader(value) class ReturningReader: def read_into(self, source: Any, context: ReadContext, out: Any, /) -> Any: return np.empty(context.transform.domain.shape, dtype=source.dtype) @pytest.mark.parametrize("independent", [False, True]) def test_result_rejects_a_reader_that_returns_a_value(independent: bool) -> None: view = LazyArray(reference()).with_reader(ReturningReader()) target = next(view.parts()).view if independent else view with pytest.raises(TypeError, match="must return None"): target.result() class BufferRecordingReader(RecordingReader): def __init__(self) -> None: super().__init__() self.owns_data: list[bool] = [] def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: self.owns_data.append(bool(out.flags.owndata)) super().read_into(source, context, out) def test_reader_is_called_once_per_touched_part() -> None: data = np.arange(48).reshape(6, 8) reader = RecordingReader() view = LazyArray(data).with_reader(reader).with_parts((3, 4))[1:5, 2] np.testing.assert_array_equal(view.result(), data[1:5, 2]) assert len(reader.calls) == len(tuple(view.parts())) == 2 assert all( call[1].transform.domain.inclusive_min == (0,) * call[1].transform.input_rank for call in reader.calls ) def test_partition_reader_receives_global_transform_and_existing_projection() -> None: reader = RecordingReader() view = LazyArray(np.arange(8)).with_reader(reader).with_parts((4,)) expected = [part.projection for part in view.parts()] np.testing.assert_array_equal(view.result(), np.arange(8)) assert [context.projection for context in reader.contexts] == expected assert [context.transform.apply((0,)) for context in reader.contexts] == [(0,), (4,)] def test_an_empty_result_does_not_call_the_reader() -> None: reader = RecordingReader() result = LazyArray(reference()).with_reader(reader)[:, 0:0, :].result() assert result.shape == (7, 0, 4) assert reader.calls == [] def test_rectangular_parts_write_into_result_views() -> None: reader = BufferRecordingReader() view = LazyArray(reference()).with_reader(reader).with_parts((2, 2, 2))[1:6, 1:4] view.result() assert reader.owns_data assert not any(reader.owns_data) def test_fancy_part_placement_uses_owned_dense_temporaries() -> None: reader = BufferRecordingReader() view = LazyArray(reference()).with_reader(reader).with_parts((2, 2, 2)).oindex[[6, 1, 1], :, :] np.testing.assert_array_equal(view.result(), reference()[np.ix_([6, 1, 1], range(5), range(4))]) assert any(reader.owns_data) @pytest.mark.parametrize("independent", [False, True]) def test_reader_exception_propagates_unchanged(independent: bool) -> None: error = RuntimeError("backend failed") class FailingReader: def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: raise error view = LazyArray(reference()).with_reader(FailingReader()) target = next(view.parts()).view if independent else view with pytest.raises(RuntimeError) as caught: target.result() assert caught.value is error class ForeignArray: """A minimal array-like whose advertised `chunks` we do not control.""" def __init__(self, data: np.ndarray[Any, Any], chunks: Any) -> None: self._data = data self.chunks = chunks @property def shape(self) -> tuple[int, ...]: return self._data.shape @property def dtype(self) -> Any: return self._data.dtype def __getitem__(self, key: Any) -> Any: return self._data[key] def test_malformed_discovered_parts_are_ignored() -> None: """A foreign object's unusable `chunks` falls back to one whole-array part. Discovery parses external input, so an attribute that does not describe a partitioning of the shape means "none I understand", not an error. """ data = reference() for bogus in (((3, 3), (5,), (4,)), (3, 2), "nope", (3.5, 2, 2)): wrapped = LazyArray(ForeignArray(data, bogus)) assert len(list(wrapped.parts())) == 1 np.testing.assert_array_equal(np.asarray(wrapped[1:3].result()), data[1:3]) def test_discovered_parts_come_from_the_source_vocabulary() -> None: """`read_chunk_sizes` wins over `chunks`, and both are read as parts.""" data = reference() wrapped = LazyArray(ForeignArray(data, PART_SHAPE)) assert [part.base_coords for part in wrapped.parts()][:3] == [(0, 0, 0), (0, 0, 1), (0, 1, 0)] assert len(list(wrapped.parts())) == 3 * 3 * 2 # --------------------------------------------------------------------------- # Boxes # --------------------------------------------------------------------------- # (id, build, expected is_box, expected bounding_box). The bounds are storage # intervals of the wrapped (7, 5, 4) array, computed by hand. BOX_CASES: list[tuple[str, Callable[[LazyArray], LazyArray], bool, Any]] = [ ("identity", lambda a: a, True, ((0, 7), (0, 5), (0, 4))), ("basic-slice", lambda a: a[1:6, :, 1:3], True, ((1, 6), (0, 5), (1, 3))), # Stride 2 over axis 1 touches 0, 2, 4; the hull is the closed span. ("strided", lambda a: a[::3, ::2, :], True, ((0, 7), (0, 5), (0, 4))), ("int-drop", lambda a: a[2, :, -1], True, ((2, 3), (0, 5), (3, 4))), ("all-scalars", lambda a: a[0, 1, 2], True, ((0, 1), (1, 2), (2, 3))), ("negative-and-open", lambda a: a[-2:], True, ((5, 7), (0, 5), (0, 4))), ( "oindex", lambda a: a.oindex[[4, 0, 0], :, [3, 1]], False, ((0, 5), (0, 5), (1, 4)), ), ( "vindex", lambda a: a.vindex[np.array([0, 6, 3]), np.array([1, 4, 0]), np.array([2, 0, 1])], False, ((0, 7), (0, 5), (0, 3)), ), ("mask", lambda a: a.vindex[MASK], False, ((0, 7), (0, 5), (0, 4))), # Composition preserves the category in both directions. ("box-of-box", lambda a: a[1:6][:, 1:3], True, ((1, 6), (1, 3), (0, 4))), ( "box-after-fancy", lambda a: a.oindex[[4, 0, 2], :, :][0:2], False, ((0, 5), (0, 5), (0, 4)), ), ("empty", lambda a: a[2:2], True, None), ( "empty-fancy", lambda a: a.oindex[np.array([], dtype=np.intp), :, :], False, None, ), ] @pytest.mark.parametrize( ("build", "expected_is_box", "expected_bounds"), [case[1:] for case in BOX_CASES], ids=[case[0] for case in BOX_CASES], ) def test_is_box_and_bounding_box( source: LazyArray, build: Callable[[LazyArray], LazyArray], expected_is_box: bool, expected_bounds: Any, ) -> None: """The box/query taxonomy, and the hull every selection has either way.""" view = build(source) assert view.is_box is expected_is_box assert view.bounding_box() == expected_bounds def test_a_unit_stride_box_is_dense_in_its_bounding_box() -> None: """Stride 1 everywhere: the hull is exactly what the view selects.""" data = reference() view = make_source("numpy-uniform-parts")[1:6, :, 1:3] assert view.is_box assert view.strides() == (1, 1, 1) bounds = view.bounding_box() assert bounds is not None np.testing.assert_array_equal( np.asarray(view.result()), data[tuple(slice(lo, hi) for lo, hi in bounds)], ) def test_a_strided_box_is_sparse_in_its_bounding_box() -> None: """Stride > 1: the hull is a superset, and `strides()` is what says by how much.""" data = reference() view = make_source("numpy-uniform-parts")[1:6, ::2, :] assert view.is_box assert view.strides() == (1, 2, 1) bounds = view.bounding_box() assert bounds is not None assert bounds == ((1, 6), (0, 5), (0, 4)) hull = data[tuple(slice(lo, hi) for lo, hi in bounds)] assert hull.size == 5 * 5 * 4 assert view.size == 5 * 3 * 4 # A consumer that slabbed the hull and threw the rest away would over-read. assert hull.size > view.size # Applying the strides to the hull recovers the selection exactly. np.testing.assert_array_equal( np.asarray(view.result()), hull[tuple(slice(None, None, step) for step in view.strides() or ())], ) def test_strides_are_none_for_a_query() -> None: view = make_source("numpy-uniform-parts").oindex[[5, 1], :, :] assert not view.is_box assert view.strides() is None def test_an_integer_indexed_dimension_has_stride_one() -> None: view = make_source("numpy-uniform-parts")[2, ::3, :] assert view.strides() == (1, 3, 1) assert view.bounding_box() == ((2, 3), (0, 4), (0, 4)) def test_a_query_bounding_box_is_only_a_hull() -> None: """For a fancy selection the box is a superset, and `is_box` says so.""" view = make_source("numpy-uniform-parts").oindex[[5, 1], :, :] assert not view.is_box assert view.bounding_box() == ((1, 6), (0, 5), (0, 4)) # The hull spans 5 rows; the selection touches 2 of them. assert view.shape[0] == 2 # --------------------------------------------------------------------------- # Parts # --------------------------------------------------------------------------- class _ReadMustNotRun: def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: raise AssertionError("prepared-plan validation must happen before any read") def test_prepared_part_validation_allocates_boolean_coverage_bitmap( monkeypatch: pytest.MonkeyPatch, ) -> None: view = LazyArray.from_numpy(reference()).with_parts(PART_SHAPE)[1:6, ::2, 1:] parts = tuple(view.parts()) allocations: list[tuple[tuple[int, ...], np.dtype[Any]]] = [] real_zeros = np.zeros def recording_zeros(shape: Any, *args: Any, **kwargs: Any) -> np.ndarray[Any, Any]: result = real_zeros(shape, *args, **kwargs) allocations.append((tuple(shape), result.dtype)) return result monkeypatch.setattr(lazy_array_module.np, "zeros", recording_zeros) _validate_prepared_parts(parts, view.shape) assert allocations == [(view.shape, np.dtype(np.bool_))] @pytest.mark.parametrize( ("data", "part_shape", "build", "expected"), [ pytest.param( np.arange(8), (3,), lambda array: array[1:7:2], np.array([1, 3, 5]), id="basic-reordered-parts", ), pytest.param( np.arange(8), (3,), lambda array: array[3], np.array(3), id="scalar", ), pytest.param( np.arange(20).reshape(4, 5), (2, 3), lambda array: array.oindex[[3, 1, 1], [4, 0]], np.array([[19, 15], [9, 5], [9, 5]]), id="orthogonal-fancy", ), pytest.param( np.arange(20).reshape(4, 5), (2, 3), lambda array: array.vindex[[3, 1, 1], [4, 0, 4]], np.array([19, 5, 9]), id="correlated-fancy", ), pytest.param( np.arange(8), (3,), lambda array: array[2:2], np.array([], dtype=np.int64), id="empty", ), ], ) def test_result_accepts_prepared_parts_from_the_same_view( monkeypatch: pytest.MonkeyPatch, data: np.ndarray[Any, Any], part_shape: tuple[int, ...], build: Callable[[LazyArray], LazyArray], expected: np.ndarray[Any, Any], ) -> None: view = build(LazyArray.from_numpy(data).with_parts(part_shape)) parts = tuple(reversed(tuple(view.parts()))) def unexpected_replan(self: LazyArray) -> Any: raise AssertionError("result(parts=...) must not construct another partition plan") monkeypatch.setattr(LazyArray, "parts", unexpected_replan) np.testing.assert_array_equal(view.result(parts=parts), expected) def test_result_rejects_prepared_parts_owned_by_another_view() -> None: data = reference() base = LazyArray.from_numpy(data).with_reader(_ReadMustNotRun()).with_parts(PART_SHAPE) view = base[1:6, ::2, 1:] owned_parts = tuple(view.parts()) foreign_parts = tuple(base[1:6, ::2, 1:].parts()) mixed_parts = (owned_parts[0], *foreign_parts[1:]) with pytest.raises(ValueError, match="prepared parts do not belong to this view"): view.result(parts=mixed_parts) def test_result_rejects_prepared_parts_that_do_not_tile_the_view() -> None: data = reference() view = ( LazyArray.from_numpy(data) .with_reader(_ReadMustNotRun()) .with_parts(PART_SHAPE)[1:6, ::2, 1:] ) parts = tuple(view.parts()) with pytest.raises(ValueError, match="prepared parts do not tile the view exactly"): view.result(parts=parts[:-1]) def test_result_rejects_prepared_parts_with_a_duplicate_and_omission() -> None: view = LazyArray.from_numpy(np.arange(8)).with_reader(_ReadMustNotRun()).with_parts((4,)) first, _second = tuple(view.parts()) with pytest.raises(ValueError, match="prepared parts do not tile the view exactly"): view.result(parts=(first, first)) def test_result_rejects_complete_prepared_parts_plus_a_duplicate() -> None: view = LazyArray.from_numpy(np.arange(8)).with_reader(_ReadMustNotRun()).with_parts((4,)) first, second = tuple(view.parts()) with pytest.raises(ValueError, match="prepared parts do not tile the view exactly"): view.result(parts=(first, second, first)) def test_result_rejects_overlapping_prepared_parts() -> None: view = LazyArray.from_numpy(np.arange(8)).with_reader(_ReadMustNotRun()).with_parts((4,)) first, second = tuple(view.parts()) overlapping = replace(second, out_selection=(slice(2, 6),)) with pytest.raises(ValueError, match="prepared parts do not tile the view exactly"): view.result(parts=(first, overlapping)) def test_result_rejects_wrong_rank_prepared_parts() -> None: view = LazyArray.from_numpy(np.arange(8)).with_reader(_ReadMustNotRun()).with_parts((4,)) first, second = tuple(view.parts()) wrong_rank = replace(first, out_selection=()) with pytest.raises(ValueError, match="prepared parts do not tile the view exactly"): view.result(parts=(wrong_rank, second)) def test_result_rejects_empty_prepared_parts_for_a_nonempty_view() -> None: view = LazyArray.from_numpy(np.arange(8)).with_reader(_ReadMustNotRun()).with_parts((4,)) with pytest.raises(ValueError, match="prepared parts do not tile the view exactly"): view.result(parts=()) def test_result_accepts_empty_prepared_parts_for_an_empty_view() -> None: view = LazyArray.from_numpy(np.arange(8)).with_reader(_ReadMustNotRun()).with_parts((4,))[0:0] np.testing.assert_array_equal(view.result(parts=()), np.array([], dtype=np.int64)) @pytest.mark.parametrize("flavor", FLAVORS) @pytest.mark.parametrize( "build", [ lambda a: a, lambda a: a[1:6, :, 1:], lambda a: a.oindex[[4, 0, 0], :, [3, 1]], lambda a: a.vindex[..., np.array([1, 4, 0]), np.array([2, 0, 1])], # A reversing view drives the negative-stride branches of # `_intersect_dimension_map` and chunk projection, which were # written defensively long before anything could reach them. lambda a: a[::-1, ::-2, :], lambda a: a[5:1:-1, :, ::-1], ], ids=["identity", "basic", "oindex", "vindex", "reversed", "reversed-bounded"], ) def test_parts_tile_the_view_exactly_and_disjointly( flavor: str, build: Callable[[LazyArray], LazyArray] ) -> None: """Assembling the parts reproduces `result()`; the placements cover with no overlap.""" view = build(make_source(flavor)) expected = np.asarray(view.result()) assembled = np.zeros(view.shape, dtype=view.dtype) hits = np.zeros(view.shape, dtype=np.int64) for part in view.parts(): assembled[part.out_selection] = np.asarray(part.view.result()) # `np.add.at` accumulates per element; `+= 1` on a fancy index would # count a repeated position once and hide a real overlap. np.add.at(hits, part.out_selection, 1) np.testing.assert_array_equal(assembled, expected) np.testing.assert_array_equal(hits, np.ones(view.shape, dtype=np.int64)) def _transform_point(transform: IndexTransform, point: tuple[int, ...]) -> tuple[int, ...]: """Evaluate a transform pointwise for projection placement assertions.""" result: list[int] = [] for output_map in transform.output: if isinstance(output_map, ConstantMap): result.append(output_map.offset) elif isinstance(output_map, DimensionMap): result.append(output_map.offset + output_map.stride * point[output_map.input_dimension]) else: index = tuple( 0 if output_map.index_array.shape[axis] == 1 else point[axis] - transform.domain.inclusive_min[axis] for axis in range(output_map.index_array.ndim) ) result.append( output_map.offset + output_map.stride * int(output_map.index_array[index]) ) return tuple(result) @pytest.mark.parametrize( "build", [ lambda array: array.oindex[[6, 0, 2], :, [3, 1]], lambda array: array.vindex[..., np.array([4, 0, 4]), np.array([3, 1, 1])], ], ids=["orthogonal", "vectorized"], ) def test_partition_exposes_projection_as_its_placement_authority( build: Callable[[LazyArray], LazyArray], ) -> None: """A part's NumPy placement addresses exactly its cell-transform range.""" view = build(LazyArray(reference()).with_parts(PART_SHAPE)) assembled = np.zeros(view.shape, dtype=view.dtype) for part in view.parts(): projection = part.projection assert isinstance(projection, ChunkProjection) assert part.view.transform == projection.chunk_transform.translate( tuple(lo for lo, _ in part.box) ) assert part.base_coords == projection.chunk_coords assert part.box == tuple( zip( projection.chunk_domain.inclusive_min, projection.chunk_domain.exclusive_max, strict=True, ) ) expected_hits = np.zeros(view.shape, dtype=np.int8) cell_domain = projection.cell_transform.domain for positional_point in np.ndindex(*cell_domain.shape): cell_point = tuple( coordinate + origin for coordinate, origin in zip( positional_point, cell_domain.inclusive_min, strict=True ) ) expected_hits[_transform_point(projection.cell_transform, cell_point)] = 1 actual_hits = np.zeros(view.shape, dtype=np.int8) actual_hits[part.out_selection] = 1 np.testing.assert_array_equal(actual_hits, expected_hits) assembled[part.out_selection] = np.asarray(part.view.result()) np.testing.assert_array_equal(assembled, np.asarray(view.result())) def test_nonfirst_partition_transform_directly_addresses_its_array() -> None: source = np.arange(8) part = list(LazyArray.from_numpy(source).with_parts((4,)).parts())[1] assert part.box == ((4, 8),) # A box part keeps the request's literal coordinates, so its domain says # where it sits; the transform addresses the raw source from there. assert part.view.transform.domain == IndexDomain((4,), (8,)) assert part.view.transform.apply((4,)) == (4,) assert part.view.array[part.view.transform.apply((4,))] == 4 assert part.view.result()[0] == 4 assert part.view[0].result() == 4 assert part.projection.chunk_transform.apply((0,)) == (0,) def test_partition_token_encodes_its_public_global_transform() -> None: pytest.importorskip("dask.base") source = np.arange(8) base = LazyArray.from_numpy(source) partition_view = list(base.with_parts((4,)).parts())[1].view direct_view = base[4:8] assert partition_view.__dask_tokenize__() == direct_view.__dask_tokenize__() def test_parts_resolve_independently_and_concurrently() -> None: """Each part's `array` is a standalone `LazyArray` with no shared mutable state.""" view = make_source("zarr")[1:7, :, 1:].with_parts((2, 2, 2)) parts = list(view.parts()) assert len(parts) > 1 with ThreadPoolExecutor(max_workers=4) as pool: values = list(pool.map(lambda part: np.asarray(part.view.result()), parts)) assembled = np.zeros(view.shape, dtype=view.dtype) for part, value in zip(parts, values, strict=True): assembled[part.out_selection] = value np.testing.assert_array_equal(assembled, np.asarray(view.result())) def test_parts_report_completeness() -> None: """`is_complete` distinguishes a fully-covered box from a partial one.""" array = make_source("numpy-uniform-parts") assert all(part.is_complete for part in array.parts()) # Boxes along axis 2 are [0, 3) and [3, 4); dropping column 0 leaves the # first partially covered and the second whole. trimmed = {part.base_coords[2]: part.is_complete for part in array[:, :, 1:].parts()} assert trimmed == {0: False, 1: True} # A fancy axis is always reported incomplete. assert not any(part.is_complete for part in array.oindex[[4, 0, 0], :, :].parts()) def test_partition_boxes_are_global_and_tile_the_base() -> None: """Both the selected hull and the whole partition box use global coordinates.""" view = make_source("numpy-uniform-parts")[1:6, :, 1:] parts = {part.base_coords: part for part in view.parts()} # The reviewer's repro: two parts of the same view now expose distinct # source-global selected hulls as well as distinct whole partition boxes. first, second = parts[0, 0, 0], parts[0, 1, 0] assert first.view.bounding_box() == ((1, 3), (0, 2), (1, 3)) assert second.view.bounding_box() == ((1, 3), (2, 4), (1, 3)) assert first.box != second.box assert first.box == ((0, 3), (0, 2), (0, 3)) assert second.box == ((0, 3), (2, 4), (0, 3)) # Every box is the base partitioning's own box for those coordinates, and # the touched boxes tile the region the view reads without overlapping. grids = dimension_grids_from_chunks(PART_SHAPE, SHAPE) for coords, part in parts.items(): expected = tuple( (grid.chunk_offset(c), grid.chunk_offset(c) + grid.data_size(c)) for grid, c in zip(grids, coords, strict=True) ) assert part.box == expected covered = np.zeros(SHAPE, dtype=np.int64) for part in parts.values(): covered[tuple(slice(lo, hi) for lo, hi in part.box)] += 1 assert covered.max() == 1 # The view reads rows 1..5 and columns 1.., so every box it touches # intersects that region and no box outside it is visited. assert covered[1:6, :, 1:].sum() > 0 assert covered[6:, :, :].sum() == 0 def test_partition_box_of_a_whole_array_part() -> None: part = next(iter(make_source("numpy-whole").parts())) assert part.box == tuple((0, extent) for extent in SHAPE) def test_an_unpartitioned_wrapper_has_a_single_whole_array_part() -> None: view = make_source("numpy-whole") parts = list(view.parts()) assert len(parts) == 1 assert parts[0].base_coords == (0, 0, 0) assert parts[0].view.shape == SHAPE assert parts[0].is_complete np.testing.assert_array_equal(np.asarray(parts[0].view.result()), reference()) def test_with_parts_keeps_the_view_and_the_base() -> None: view = make_source("zarr")[1:6, ::2] repartitioned = view.with_parts((2, 1, 4)) assert repartitioned.shape == view.shape assert repartitioned.array is view.array assert repartitioned.transform == view.transform # A different partitioning really is a different set of boxes. assert [part.base_coords for part in repartitioned.parts()] != [ part.base_coords for part in view.parts() ] np.testing.assert_array_equal(np.asarray(repartitioned.result()), np.asarray(view.result())) def test_with_parts_none_forces_one_shot_resolution() -> None: view = make_source("zarr").oindex[[4, 0, 0], :, :] whole = view.unpartitioned() assert len(list(whole.parts())) == 1 np.testing.assert_array_equal(np.asarray(whole.result()), np.asarray(view.result())) @pytest.mark.parametrize( ("parts", "match"), [ ((3,), "one entry per dimension"), ((3, (2, 2), 4), "not a mixture"), (((3, 3), (2, 2, 1), (3, 1)), "sum to 6, but the array extent is 7"), ((3, 0, 3), "chunk shape entries must be positive"), # A float or a None belongs to neither convention; saying "not a # mixture" would send the reader looking for the wrong mistake. ((3.5, 2, 2), r"3\.5 at dimension 0 is neither"), ((None, 5, 4), "None at dimension 0 is neither"), ((3.5, 2.5, 2.5), r"3\.5 at dimension 0, 2\.5 at dimension 1, .* are neither"), (((1.5, 5.5), (5,), (4,)), "per-axis chunk sizes must be integers; dimension 0"), ], ) def test_with_parts_validates_strictly(parts: Any, match: str) -> None: """`with_parts` is our own API, so a malformed partitioning raises.""" with pytest.raises(ValueError, match=match): repartition(make_source("numpy-whole"), parts) # --------------------------------------------------------------------------- # Protocols # --------------------------------------------------------------------------- def test_dask_token_is_deterministic_and_discriminating() -> None: """Same data and same view token alike; a different selection differs.""" pytest.importorskip("dask.base") data = reference() base = LazyArray(data) assert base.__dask_tokenize__() == LazyArray(reference()).__dask_tokenize__() tokens = { "base": base.__dask_tokenize__(), "view": base[1:3].__dask_tokenize__(), "other view": base[2:4].__dask_tokenize__(), "other data": LazyArray(data + 1).__dask_tokenize__(), } assert len({repr(token) for token in tokens.values()}) == len(tokens) # Equivalent transforms reached different ways still token alike. assert base[1:5][0:2].__dask_tokenize__() == base[1:3].__dask_tokenize__() def test_reader_and_partitioning_do_not_change_dask_identity() -> None: pytest.importorskip("dask.base") base = LazyArray(reference()) token = base.__dask_tokenize__() assert base.with_reader(numpy_reader).__dask_tokenize__() == token assert base.with_reader(basic_reader).__dask_tokenize__() == token assert base.with_parts((2, 2, 2)).__dask_tokenize__() == token @pytest.mark.parametrize("reader", [basic_reader, numpy_reader, DelegatingReader(numpy_reader)]) def test_reader_survives_pickle(reader: Reader) -> None: view = LazyArray(reference()).with_reader(reader)[1:5, ::2] restored = pickle.loads(pickle.dumps(view)) assert type(restored.reader) is type(reader) np.testing.assert_array_equal(restored.result(), view.result()) def test_iteration_yields_lazy_slices(source: LazyArray) -> None: rows = list(source[2:5]) assert len(rows) == 3 for row, expected in zip(rows, reference()[2:5], strict=True): assert isinstance(row, LazyArray) np.testing.assert_array_equal(np.asarray(row), expected) def test_iteration_over_a_zero_dimensional_view_is_rejected() -> None: with pytest.raises(TypeError, match="iteration over a 0-d array"): iter(make_source("numpy-uniform-parts")[0, 0, 0]) def test_len_of_a_zero_dimensional_view_is_rejected() -> None: with pytest.raises(TypeError, match="len\\(\\) of unsized object"): len(make_source("numpy-uniform-parts")[0, 0, 0]) @pytest.mark.parametrize( ("convert", "selection"), [ (bool, (1, 1, 1)), (int, (1, 1, 1)), (float, (1, 1, 1)), (operator.index, (1, 1, 1)), (bool, (1, 1, slice(0, 1))), ], ids=["bool-0d", "int-0d", "float-0d", "index-0d", "bool-size-1"], ) def test_scalar_conversions_match_numpy(convert: Any, selection: Any) -> None: """Size-1 conversions delegate to NumPy, values and all.""" data = reference() view = make_source("numpy-uniform-parts")[selection] assert convert(view) == convert(data[selection]) @pytest.mark.parametrize( ("convert", "selection", "error"), [ (bool, (slice(0, 2), 0, 0), ValueError), (bool, (slice(0, 0), 0, 0), ValueError), (int, (slice(0, 1), 0, 0), TypeError), (float, (slice(0, 2), 0, 0), TypeError), (operator.index, (slice(0, 1), 0, 0), TypeError), ], ids=["bool-many", "bool-empty", "int-1d", "float-many", "index-1d"], ) def test_scalar_conversions_raise_what_numpy_raises( convert: Any, selection: Any, error: type[Exception] ) -> None: data = reference() view = make_source("numpy-uniform-parts")[selection] with pytest.raises(error): convert(view) with pytest.raises(error): convert(data[selection]) def test_pickle_round_trip() -> None: """A wrapper over a picklable base survives a round trip, view and parts intact.""" view = LazyArray(reference()).with_parts((2, 2, 2))[1:6, ::2].oindex[[3, 0, 0], :, :] restored = pickle.loads(pickle.dumps(view)) assert restored.shape == view.shape np.testing.assert_array_equal(np.asarray(restored.result()), np.asarray(view.result())) # --------------------------------------------------------------------------- # dask interop # --------------------------------------------------------------------------- def test_dask_from_array_roundtrip() -> None: """Dask can tokenize and read a wrapper over a Zarr source.""" da = pytest.importorskip("dask.array") source = make_source("zarr") lazy = da.from_array(EagerArrayAdapter(source)) np.testing.assert_array_equal(lazy.compute(), reference()) # dask chooses its own blocks; the wrapper reads each of them through its # own parts, so the two partitionings need not agree. blocked = da.from_array(EagerArrayAdapter(source), chunks=(4, 3, 3)) assert blocked.chunks == ((4, 3), (3, 2), (3, 1)) np.testing.assert_array_equal(blocked[2:, ::2].compute(), reference()[2:, ::2]) # --------------------------------------------------------------------------- # Errors # --------------------------------------------------------------------------- def test_boolean_scalar_is_rejected() -> None: with pytest.raises(IndexError, match="boolean scalars are not valid indices"): make_source("numpy-uniform-parts")[True] @pytest.mark.parametrize( ("mode", "selection", "expected"), [ pytest.param("basic", IndexLike(2), np.array(2), id="basic-scalar"), pytest.param( "basic", slice(IndexLike(1), IndexLike(7), IndexLike(2)), np.array([1, 3, 5]), id="basic-slice-components", ), pytest.param("orthogonal", IndexLike(2), np.array(2), id="orthogonal-scalar"), pytest.param("vectorized", IndexLike(2), np.array(2), id="vectorized-scalar"), ], ) def test_positional_selectors_support_the_index_protocol( mode: str, selection: Any, expected: np.ndarray[Any, Any] ) -> None: source = LazyArray.from_numpy(np.arange(8)) if mode == "basic": view = source[selection] else: view = getattr(source, "oindex" if mode == "orthogonal" else "vindex")[selection] result = np.asarray(view.result()) assert result.shape == expected.shape np.testing.assert_array_equal(result, expected) def test_positional_selector_rejects_int_only_objects() -> None: with pytest.raises(IndexError, match="unsupported selection type"): LazyArray.from_numpy(np.arange(8))[IntOnly()] def test_positional_selector_propagates_malformed_index_protocol() -> None: with pytest.raises(TypeError, match="__index__ returned non-int"): LazyArray.from_numpy(np.arange(8))[BadIndex()] def test_positional_slice_propagates_malformed_index_protocol() -> None: with pytest.raises(TypeError, match="__index__ returned non-int"): LazyArray.from_numpy(np.arange(8))[:: BadIndex()] def test_protocol_objects_inside_an_index_array_remain_invalid() -> None: selection = np.array([IndexLike(2)], dtype=object) with pytest.raises(IndexError, match="integer or boolean"): LazyArray.from_numpy(np.arange(8)).oindex[selection] def test_mask_shape_must_match_the_view() -> None: array = make_source("numpy-uniform-parts") with pytest.raises(IndexError, match="boolean index has shape"): array.vindex[np.ones((2, 2, 2), dtype=bool)] def test_scalar_index_out_of_bounds() -> None: with pytest.raises(IndexError, match="index 7 is out of bounds for axis 0 with size 7"): make_source("numpy-uniform-parts")[7] def test_scalar_index_out_of_bounds_in_a_view() -> None: """Bounds are the *view's*, not the wrapped array's.""" array = make_source("numpy-uniform-parts")[1:4] with pytest.raises(IndexError, match="index 3 is out of bounds for axis 0 with size 3"): array[3] def test_index_array_out_of_bounds() -> None: array = make_source("numpy-uniform-parts") with pytest.raises(IndexError, match="index 99 is out of bounds for axis 0 with size 7"): array.oindex[[0, 99], :, :] @pytest.mark.parametrize("mode", ["oindex", "vindex"]) def test_unsigned_index_array_beyond_intp_is_out_of_bounds(mode: str) -> None: """A uint64 value past the intp range must not wrap to a negative index.""" array = make_source("numpy-uniform-parts") index = np.array([2**64 - 1], dtype=np.uint64) selection = (index, 0, 0) if mode == "vindex" else (index, slice(None), slice(None)) with pytest.raises(IndexError, match=f"index {2**64 - 1} is out of bounds for axis 0"): getattr(array, mode)[selection] def test_too_many_indices() -> None: with pytest.raises(IndexError, match="too many indices"): make_source("numpy-uniform-parts")[0, 0, 0, 0] def test_copy_false_conversion_is_rejected() -> None: array = make_source("numpy-uniform-parts") with pytest.raises(ValueError, match="cannot be converted to a NumPy array without a copy"): np.array(array, copy=False) # --------------------------------------------------------------------------- # Negative steps # --------------------------------------------------------------------------- def test_reversed_box_reports_a_positive_stride(source: LazyArray) -> None: """A reversal is still a box; `strides()` is magnitudes, so it matches the forward twin.""" reversed_view = source[::-2] forward = source[::2] assert reversed_view.is_box assert reversed_view.strides() == forward.strides() == (2, 1, 1) assert reversed_view.bounding_box() == ((0, 7), (0, 5), (0, 4)) def test_a_reversed_view_keeps_its_negative_literal_domain() -> None: """The literal domain of a reversal is negative; positional keys hide it.""" view = make_source("numpy-whole")[::-1] # The algebra's answer keeps the source frame, and so does the wrapper. assert IndexTransform.from_shape(SHAPE)[::-1].domain.inclusive_min[0] == -6 assert view.transform.domain.inclusive_min == (-6, 0, 0) assert view.shape == SHAPE np.testing.assert_array_equal(np.asarray(view.result()), reference()[::-1]) # Position 0 is still the first element, whatever the domain says. np.testing.assert_array_equal(view[0].result(), reference()[-1]) def test_zero_step_is_rejected() -> None: with pytest.raises(ValueError, match="step cannot be zero"): make_source("numpy-whole")[::0] def test_reversed_positional_interval_is_empty_not_an_error() -> None: """NumPy's rule at the boundary; the literal layer keeps TensorStore's.""" view = make_source("numpy-uniform-parts")[2:5:-1] assert view.shape == (0, 5, 4) np.testing.assert_array_equal(np.asarray(view.result()), reference()[2:5:-1]) # Literal coordinates, on the other hand, call it a direction error. with pytest.raises(IndexError, match="valid interval"): IndexTransform.from_shape(SHAPE)[2:5:-1] def test_negative_step_over_a_fancy_axis_reverses_the_coordinates(source: LazyArray) -> None: """Reversing a gathered axis materializes, rather than attaching a stride.""" view = source.oindex[[3, 1, 2], :, :][::-1] expected = outer(reference(), ([3, 1, 2], slice(None), slice(None)))[::-1] np.testing.assert_array_equal(np.asarray(view.result()), expected) m = view.transform.output[0] assert isinstance(m, ArrayMap) np.testing.assert_array_equal(m.index_array.reshape(-1), np.array([2, 1, 3])) # --------------------------------------------------------------------------- # Minimal sources # --------------------------------------------------------------------------- class MinimalSource: """The floor of the wrapped-array protocol: `shape`, `dtype`, `__getitem__`.""" def __init__(self, data: np.ndarray[Any, Any]) -> None: self._data = data @property def shape(self) -> tuple[int, ...]: return self._data.shape @property def dtype(self) -> Any: return self._data.dtype def __getitem__(self, key: Any) -> Any: return self._data[key] class RecordingSource(MinimalSource): """A minimal source that records every attempted data read.""" def __init__(self, data: np.ndarray[Any, Any]) -> None: super().__init__(data) self.reads: list[Any] = [] def __getitem__(self, key: Any) -> Any: self.reads.append(key) return super().__getitem__(key) @pytest.mark.parametrize( "build", [ lambda a: a[:, 2:2, :], lambda a: a.vindex[np.array([[6], [3], [0]]), -4][0, 0:0], ], ids=["ordinary", "unreferenced-axis"], ) def test_an_empty_view_does_not_read_its_source( build: Callable[[LazyArray], LazyArray], ) -> None: source = RecordingSource(reference()) result = build(LazyArray(source)).result() assert result.size == 0 assert source.reads == [] # --------------------------------------------------------------------------- # Materializing # --------------------------------------------------------------------------- class NumpyBackedSource: """A duck array that stores its data in NumPy and returns views from reads. Not an `np.ndarray`, so nothing about the source can be compared against the result's memory — but its blocks are NumPy views of storage the caller must not be handed. The `BASIC` source this package invites people to wrap. """ def __init__(self, data: np.ndarray[Any, Any]) -> None: self.data = data @property def shape(self) -> tuple[int, ...]: return self.data.shape @property def dtype(self) -> Any: return self.data.dtype def __getitem__(self, selection: Any) -> Any: return self.data[selection] @pytest.mark.parametrize("parts", [None, (2, 2, 2), SHAPE]) @pytest.mark.parametrize("wrap", [lambda d: d, NumpyBackedSource], ids=["ndarray", "duck"]) @pytest.mark.parametrize( ("build", "description"), [ (lambda a: a[1:3, :, :], "a basic slice"), (lambda a: a[:, :, :], "the whole array"), (lambda a: a[::-1, :, :], "a reversal"), (lambda a: a.oindex[[2, 0], :, :], "a gather"), ], ) def test_materializing_never_hands_back_the_wrapped_array( parts: Any, wrap: Callable[[np.ndarray[Any, Any]], Any], build: Callable[[LazyArray], LazyArray], description: str, ) -> None: """Writing to a materialized result must never reach the source. An unpartitioned read of a basic selection can be answered with a *view* of the wrapped array, and NumPy 2 hands whatever `__array__` returns straight to the caller. Every route out of the wrapper detaches, so the answer does not depend on how the read happened to be divided. Run over both a raw `ndarray` and a duck array that merely stores its data in NumPy: the second is the case where the source cannot be compared against the result, so detaching has to decide from the result's own buffer instead. """ data = reference() view = build(repartition(LazyArray(wrap(data)), parts)) for materialize in ( lambda v: v.result(), lambda v: np.array(v, copy=True), lambda v: np.asarray(v), lambda v: np.array(v), ): before = data.copy() materialized = np.asarray(materialize(view)) assert not np.shares_memory(materialized, data), description materialized[...] = -1 np.testing.assert_array_equal(data, before, err_msg=description) def test_eager_adapter_getitem_never_hands_back_the_wrapped_array() -> None: data = reference() block = EagerArrayAdapter(LazyArray(data))[1:3] block[...] = -1 np.testing.assert_array_equal(data, reference()) def test_result_refuses_to_return_a_partly_written_buffer( monkeypatch: pytest.MonkeyPatch, ) -> None: """A partition walk that leaves a gap must raise, not return process memory. `result()` scatters into an uninitialized buffer, which is only safe because the parts tile the view. This is the guard that turns any future break of that contract into a failure instead of into plausible-looking numbers. """ view = LazyArray(reference()).with_parts(PART_SHAPE)[:, 1:, :] complete = LazyArray.parts def drop_one(self: LazyArray) -> Any: return list(complete(self))[:-1] monkeypatch.setattr(LazyArray, "parts", drop_one) with pytest.raises(AssertionError, match="partition walk addressed"): view.result() def test_result_refuses_a_partition_of_the_wrong_rank(monkeypatch: pytest.MonkeyPatch) -> None: """A part addressing fewer axes than the view has is caught by name.""" view = LazyArray(reference()).with_parts(PART_SHAPE)[:, 1:, :] complete = LazyArray.parts def truncate(self: LazyArray) -> Any: return [replace(part, out_selection=part.out_selection[:-1]) for part in complete(self)] monkeypatch.setattr(LazyArray, "parts", truncate) with pytest.raises(AssertionError, match="of the view's 3 dimensions"): view.result() class DuckBlock: """An array-like meeting exactly the documented `BASIC` floor and no more. `shape`, `dtype`, and a `__getitem__` that understands integers and slices. Anything else — an integer array, `take`, `reshape` — raises, and indexing it yields another one of itself, so a block that comes back from it is as limited as the source was. """ def __init__(self, data: np.ndarray[Any, Any]) -> None: self._data = data @property def shape(self) -> tuple[int, ...]: return self._data.shape @property def dtype(self) -> Any: return self._data.dtype def __array__(self, dtype: Any = None, copy: bool | None = None) -> Any: return np.array(self._data, dtype=dtype, copy=True if copy is None else copy) def __getitem__(self, key: Any) -> DuckBlock: selectors = key if isinstance(key, tuple) else (key,) for selector in selectors: if not isinstance(selector, (int, np.integer, slice)): raise TypeError(f"basic indexing only, got {selector!r}") return DuckBlock(self._data[key]) @pytest.mark.parametrize( ("build", "oracle"), [ (lambda a: a[1:5, ::2, :], lambda r: r[1:5, ::2, :]), ( lambda a: a.oindex[[4, 0, 0], :, :], lambda r: r[np.ix_([4, 0, 0], range(5), range(4))], ), (lambda a: a.vindex[[4, 0], [1, 1]], lambda r: r[[4, 0], [1, 1]]), (lambda a: a[::-1, :, :], lambda r: r[::-1, :, :]), ], ids=["basic", "oindex", "vindex", "reversal"], ) @pytest.mark.parametrize("parts", [None, PART_SHAPE]) def test_a_source_meeting_only_the_basic_floor_resolves_any_selection( build: Callable[[LazyArray], LazyArray], oracle: Callable[[Any], Any], parts: Any ) -> None: """The floor is a promise about the source; its blocks are coerced, not trusted.""" expected = np.asarray(oracle(reference())) view = build(repartition(LazyArray(DuckBlock(reference())), parts)) assert view.shape == expected.shape np.testing.assert_array_equal(np.asarray(view.result()), expected) def test_the_duck_block_double_refuses_a_fancy_key() -> None: """A negative control: the floor test only means something if the double bites.""" with pytest.raises(TypeError, match="basic indexing only"): DuckBlock(reference())[np.array([1, 0])] # --------------------------------------------------------------------------- # Sources with their own opinions # --------------------------------------------------------------------------- @pytest.mark.filterwarnings("ignore::PendingDeprecationWarning") def test_numpy_matrix_is_refused() -> None: """`np.matrix` never reduces rank, so a view's shape could not be honored.""" with pytest.raises(TypeError, match="numpy.matrix cannot be wrapped"): LazyArray(np.matrix(np.arange(12).reshape(3, 4))) @pytest.mark.parametrize("parts", [None, (2, 2), (1, 4), (3, 4)]) def test_a_masked_source_keeps_its_mask_under_every_partitioning(parts: Any) -> None: data = np.ma.masked_greater(np.arange(12).reshape(3, 4), 7) view = repartition(LazyArray(data), parts)[:, 1:] got = view.result() expected = data[:, 1:] assert isinstance(got, np.ma.MaskedArray), parts np.testing.assert_array_equal(np.ma.getmaskarray(got), np.ma.getmaskarray(expected)) np.testing.assert_array_equal(np.ma.filled(got, 0), np.ma.filled(expected, 0)) assembled = np.ma.masked_all(view.shape, dtype=view.dtype) for part in view.parts(): assembled[part.out_selection] = part.view.result() np.testing.assert_array_equal(np.ma.getmaskarray(assembled), np.ma.getmaskarray(expected)) np.testing.assert_array_equal(np.ma.filled(assembled, 0), np.ma.filled(expected, 0)) @pytest.mark.parametrize("parts", [None, (2, 2), (3, 4)]) def test_a_masked_source_keeps_its_mask_when_the_view_is_empty(parts: Any) -> None: """Empty views of masked sources return masked arrays for every partitioning.""" data = np.ma.masked_greater(np.arange(12).reshape(3, 4), 7) got = repartition(LazyArray(data), parts)[:, 2:2].result() assert isinstance(got, np.ma.MaskedArray), parts assert np.asarray(got).shape == (3, 0), parts # --------------------------------------------------------------------------- # Completeness and partition spellings # --------------------------------------------------------------------------- def test_a_reversing_view_covers_its_parts() -> None: """A reversal reads every cell of every box, back to front.""" data = np.arange(48).reshape(8, 6) forward = LazyArray(data).with_parts((2, 2))[:, :] reversed_view = LazyArray(data).with_parts((2, 2))[::-1, ::-1] assert [part.is_complete for part in reversed_view.parts()] == [ part.is_complete for part in forward.parts() ] assert all(part.is_complete for part in reversed_view.parts()) def test_a_strided_reversal_is_still_incomplete() -> None: data = np.arange(48).reshape(8, 6) view = LazyArray(data).with_parts((2, 2))[::-2, :] assert not any(part.is_complete for part in view.parts()) @pytest.mark.parametrize("parts", [((0,), (3,)), ((), (3,)), (1, 1), ((0, 0), (3,))]) def test_a_zero_length_axis_accepts_every_spelling_of_no_chunks(parts: Any) -> None: """`(0,)`, `(0, 0)`, `()` and a uniform shape all describe an axis with no cells.""" data = np.zeros((0, 3)) view = repartition(LazyArray(data), parts) assert view.result().shape == (0, 3) assert list(view.parts()) == [] def test_a_zero_chunk_on_a_nonempty_axis_is_still_rejected() -> None: with pytest.raises(ValueError, match="chunk sizes must be positive"): LazyArray(np.zeros((4, 3))).with_parts_per_axis(((0, 4), (3,))) @pytest.mark.parametrize( "selection", [ (slice(None, None, -1), slice(None), slice(None)), (slice(3, 1, -1), slice(None), slice(None)), (slice(None, None, -2), slice(None, None, -1), slice(None)), ], ids=["reversed", "reversed-partial", "reversed-strided"], ) def test_the_coverage_count_agrees_with_numpy_for_reversed_selections( selection: tuple[Any, ...], ) -> None: """Coverage counts match NumPy selection sizes for reversed slices.""" data = reference() view = LazyArray(data).with_parts((2, 2, 2))[selection] out_shape = view.shape for part in view.parts(): counted = _out_selection_cell_count(part.out_selection, out_shape) assert counted == np.empty(out_shape)[part.out_selection].size @pytest.mark.parametrize( ("selection", "out_shape", "expected"), [ (((slice(2, 5)),), (10,), 3), # A backwards interval selects nothing. The fast path subtracts, which # would make this negative and let an incomplete walk sum to the view's # own size — so the count falls back to `range` whenever the interval is # not a forward, in-bounds one. (((slice(5, 2)),), (10,), 0), # Counts from the end, to index 8 — past the stop, so nothing. (((slice(-2, 5)),), (10,), 0), ((slice(5, 2), slice(0, 3)), (10, 10), 0), ], ids=["forward", "backwards", "negative-start", "backwards-in-a-pair"], ) def test_the_coverage_count_matches_numpy_for_intervals_the_fast_path_declines( selection: tuple[Any, ...], out_shape: tuple[int, ...], expected: int ) -> None: """Coverage counts match NumPy for intervals outside the subtraction fast path.""" counted = _out_selection_cell_count(selection, out_shape) assert counted == np.empty(out_shape)[selection].size assert counted == expected def test_a_zero_dimensional_index_array_drops_its_axis_like_a_scalar() -> None: """Zero-dimensional integer arrays drop axes in orthogonal and vectorized selections.""" data = np.arange(20).reshape(4, 5) for mode, expected in ( ("oindex", data[np.array(2), :]), ("vindex", data[np.array(2), np.array(3)]), ): view = ( LazyArray(data).oindex[np.array(2), slice(None)] if mode == "oindex" else LazyArray(data).vindex[np.array(2), np.array(3)] ) assert view.shape == expected.shape, mode np.testing.assert_array_equal(np.asarray(view.result()), expected, err_msg=mode) def test_a_multidimensional_array_in_an_orthogonal_selection_is_refused() -> None: """Reject multidimensional orthogonal index arrays with a selection-level error.""" with pytest.raises(IndexError, match="must be 1-dimensional"): LazyArray(np.arange(20).reshape(4, 5)).oindex[[[0, 1], [2, 3]], slice(None)] def test_with_parts_rejects_a_bare_integer() -> None: """Both partitioning methods document ValueError for malformed input.""" view = LazyArray.from_numpy(np.arange(12).reshape(3, 4)) with pytest.raises(ValueError, match="one entry per dimension"): view.with_parts(3) # type: ignore[arg-type] with pytest.raises(ValueError, match="one entry per dimension"): view.with_parts_per_axis(3) # type: ignore[arg-type] def test_fancy_composition_over_an_empty_axis() -> None: """Fancy composition over an empty axis preserves shape through later selections.""" base = np.empty((3, 0, 6), dtype=np.int64) view = LazyArray(base).oindex[[2, 1], :, [5, 0, 3]] assert view.shape == (2, 0, 3) composed = view.oindex[[1, 0], :, [2, 2]] assert composed.shape == (2, 0, 2) scalar = composed.vindex[..., np.array(1)] assert scalar.shape == (2, 0) assert np.asarray(scalar.result()).shape == (2, 0) @pytest.mark.parametrize("kind", ["numpy", "object", "masked", "registered", "hook"]) def test_source_token_uses_dask_policy(kind: str) -> None: dask_base = pytest.importorskip("dask.base") class RegisteredArray(ForeignArray): pass class VersionedArray(ForeignArray): def __dask_tokenize__(self) -> Any: return ("versioned-source", 1) dask_base.normalize_token.register(RegisteredArray, lambda source: ("registered-source", 1)) data = np.arange(4) sources = { "numpy": data, "object": data.astype(object), "masked": np.ma.masked_greater(data, 2), "registered": RegisteredArray(data, None), "hook": VersionedArray(data, None), } source = sources[kind] assert LazyArray(source).__dask_tokenize__()[1] == dask_base.tokenize(source) def test_source_token_preserves_dask_determinism_requirement() -> None: dask_base = pytest.importorskip("dask.base") dask_tokenize = pytest.importorskip("dask.tokenize") class UnserializableArray(ForeignArray): def __reduce_ex__(self, protocol: int) -> Any: raise TypeError("cannot serialize source") source = UnserializableArray(np.arange(4), None) for value in (source, LazyArray(source)): with pytest.raises(dask_tokenize.TokenizationError): dask_base.tokenize(value, ensure_deterministic=True) def test_source_token_preserves_hook_failure() -> None: pytest.importorskip("dask.base") class RefusingArray(ForeignArray): def __dask_tokenize__(self) -> Any: raise RuntimeError("source version unavailable") with pytest.raises(RuntimeError, match="source version unavailable"): LazyArray(RefusingArray(np.arange(4), None)).__dask_tokenize__() def test_dask_is_only_required_for_tokenization(monkeypatch: pytest.MonkeyPatch) -> None: import sys monkeypatch.setitem(sys.modules, "dask.base", None) data = np.arange(4) view = LazyArray(data)[1:] np.testing.assert_array_equal(view.result(), data[1:]) with pytest.raises(ModuleNotFoundError): view.__dask_tokenize__() zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_lazy_array_stateful.py000066400000000000000000000072371527014137600326740ustar00rootroot00000000000000"""This package's own use of the state machine it exports. `ChainedIndexingStateMachine` composes indexing steps onto a `LazyArray` and checks each step against NumPy — see `zarr_indexing.testing.stateful` for what the invariants assert and why. Two sources: a NumPy array, which exercises both built-in readers, and a real zarr array, whose partitioning is discovered from the store rather than declared. The zarr case runs a smaller budget: it reads through a store, and it exercises the same code paths. The state machine checks both result() and explicit partition assembly against NumPy, including the shape of each partition's values. test_lazy_array also provides deterministic selection-chain coverage across source flavors. """ from __future__ import annotations from typing import Any, ClassVar import numpy as np import pytest from hypothesis import settings from zarr_indexing import LazyArray, ReadContext from zarr_indexing.reader import basic_reader, numpy_reader from zarr_indexing.testing import ( DEFAULT_SETTINGS, ChainedIndexingStateMachine, state_machine_test, stateful, ) class NumpyIndexing(ChainedIndexingStateMachine): readers = (numpy_reader,) TestNumpyIndexing = state_machine_test(NumpyIndexing) class UnhashableReader: __hash__ = None def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: basic_reader.read_into(source, context, out) def __eq__(self, other: object) -> bool: return isinstance(other, UnhashableReader) def test_reader_set_deduplicates_by_identity_without_hashing() -> None: first = UnhashableReader() second = UnhashableReader() readers = stateful._reader_set(LazyArray(np.arange(3)), (first, first, second)) assert readers[0] is basic_reader assert readers[1] is first assert readers[2] is second assert len(readers) == 3 class OneDimensionalIndexing(ChainedIndexingStateMachine): """The same chains over a rank-1 source. The sorted one-dimensional fancy path in `chunk_resolution` is entered only when both the input and output ranks are 1, and the output rank is the *source's* — so the rank-3 default walls that path off from the machine entirely, and from every downstream project told to subclass it. That path is where reordering and duplicate coordinates are partitioned, which is the corruption class this whole harness exists to catch. """ data = np.arange(30, dtype=np.int64) partitionings: ClassVar[tuple[Any, ...]] = (None, (4,), (30,), ((7, 8, 15),)) TestOneDimensionalIndexing = state_machine_test(OneDimensionalIndexing) class SingletonAxisIndexing(ChainedIndexingStateMachine): """A source with an extent-1 axis, which the code must not read as a broadcast one. An index array's axis is a singleton either because the map broadcasts over it or because the domain is genuinely one cell wide there, and the two are told apart by the domain rather than the array. Nothing generated the second kind. """ data = np.arange(2 * 1 * 3, dtype=np.int64).reshape(2, 1, 3) partitionings: ClassVar[tuple[Any, ...]] = (None, (1, 1, 1), (2, 1, 2)) TestSingletonAxisIndexing = state_machine_test(SingletonAxisIndexing) class ZarrIndexing(ChainedIndexingStateMachine): """The same chains against a zarr array through the universal basic reader.""" def make_source(self, data: Any) -> Any: zarr = pytest.importorskip("zarr") array = zarr.create_array({}, shape=data.shape, chunks=(3, 2, 3), dtype=data.dtype) array[:] = data return array TestZarrIndexing = state_machine_test( ZarrIndexing, config=settings(DEFAULT_SETTINGS, max_examples=50) ) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_lazy_default.py000066400000000000000000000216471527014137600312740ustar00rootroot00000000000000"""The wrapper defers selection; explicit execution touches the source.""" from collections.abc import Callable from typing import Any import numpy as np import pytest from zarr_indexing import EagerArrayAdapter, IndexDomain, IndexTransform, LazyArray from zarr_indexing.errors import BoundsCheckError class RecordingArray: def __init__(self) -> None: self.data = np.arange(30).reshape(5, 6) self.shape = self.data.shape self.dtype = self.data.dtype self.reads: list[Any] = [] def __getitem__(self, key: Any) -> Any: self.reads.append(key) return self.data[key] @pytest.mark.parametrize("mode", ["basic", "oindex", "vindex", "iteration"]) def test_selection_defers_reads(mode: str) -> None: source = RecordingArray() array = LazyArray(source) if mode == "basic": view = array[1:][::-1, ::2] expected = source.data[1:][::-1, ::2] elif mode == "oindex": view = array.oindex[[4, 1], :][:, ::2] expected = source.data[[4, 1]][:, ::2] elif mode == "vindex": view = array.vindex[[4, 1], [2, 0]][::-1] expected = source.data[[4, 1], [2, 0]][::-1] else: view = list(array)[2] expected = source.data[2] assert isinstance(view, LazyArray) assert source.reads == [] np.testing.assert_array_equal(view.result(), expected) assert source.reads def test_no_redundant_lazy_accessor() -> None: assert not hasattr(LazyArray(np.arange(3)), "lazy") @pytest.mark.parametrize("selection", [slice(None), slice(0, 0), 0]) def test_eager_adapter_executes_selection(selection: Any) -> None: source = RecordingArray() view = LazyArray(source)[1:, ::2] adapter = EagerArrayAdapter(view) assert (adapter.shape, adapter.ndim, adapter.dtype) == (view.shape, view.ndim, view.dtype) assert source.reads == [] result = adapter[selection] assert isinstance(result, np.ndarray) np.testing.assert_array_equal(result, source.data[1:, ::2][selection]) assert not np.shares_memory(result, source.data) def test_eager_adapter_distinguishes_token_semantics() -> None: pytest.importorskip("dask") view = LazyArray(np.arange(8))[::2] assert EagerArrayAdapter(view).__dask_tokenize__() != view.__dask_tokenize__() def test_eager_adapter_refuses_no_copy_conversion() -> None: with pytest.raises(ValueError, match="copy"): EagerArrayAdapter(LazyArray(np.arange(3))).__array__(copy=False) def test_views_keep_literal_domains_while_keys_stay_positional() -> None: source = np.arange(30) view = LazyArray(source)[10:20] assert view.transform.domain == IndexDomain((10,), (20,)) nested = view[2:5] assert nested.transform.domain == IndexDomain((12,), (15,)) np.testing.assert_array_equal(nested.result(), source[12:15]) # NumPy keys are positions in the current view, not literal coordinates. assert view[0].result() == source[10] assert view[-1].result() == source[19] assert nested[-1].result() == source[14] np.testing.assert_array_equal(view[::-1][:3].result(), source[10:20][::-1][:3]) # Equivalent selections built by different routes share one domain. assert LazyArray(source)[0:20][5:10].transform == LazyArray(source)[5:10].transform def test_domain_key_selects_literal_coordinates() -> None: source = np.arange(30) view = LazyArray(source)[10:20] inner = view[IndexDomain((12,), (15,))] assert inner.transform.domain == IndexDomain((12,), (15,)) np.testing.assert_array_equal(inner.result(), source[12:15]) np.testing.assert_array_equal(LazyArray(source)[IndexDomain((2,), (5,))].result(), source[2:5]) with pytest.raises(BoundsCheckError): view[IndexDomain((0,), (3,))] # An empty domain outside the view is refused too: the key names the view's coordinates. with pytest.raises(BoundsCheckError): view[IndexDomain((50,), (50,))] assert view[IndexDomain((15,), (15,))].shape == (0,) with pytest.raises(ValueError, match="rank"): view[IndexDomain((0, 0), (1, 1))] view[IndexDomain((12,), (15,))] = [-1, -2, -3] np.testing.assert_array_equal(source[12:15], [-1, -2, -3]) def test_transform_key_composes_onto_the_view() -> None: source = np.arange(30) view = LazyArray(source)[10:20] reversed_key = IndexTransform.identity(view.transform.domain)[::-1] np.testing.assert_array_equal(view[reversed_key].result(), source[10:20][::-1]) rebased = IndexTransform.identity(IndexDomain((12,), (15,))).translate_domain_to((0,)) composed = view[rebased] assert composed.transform.domain == IndexDomain((0,), (3,)) np.testing.assert_array_equal(composed.result(), source[12:15]) with pytest.raises((ValueError, BoundsCheckError)): view[IndexTransform.identity(IndexDomain((0,), (3,)))] def test_box_part_views_are_sub_domains_of_their_parent() -> None: source = np.arange(40).reshape(4, 10) view = LazyArray(source).with_parts((2, 4))[1:, 3:9] origin = view.transform.domain.inclusive_min for part in view.parts(): domain = part.view.transform.domain assert all( lo >= parent_lo and hi <= parent_hi for lo, hi, parent_lo, parent_hi in zip( domain.inclusive_min, domain.exclusive_max, view.transform.domain.inclusive_min, view.transform.domain.exclusive_max, strict=True, ) ) # Placement is readable from the domain: it is `out_selection`. assert part.out_selection == tuple( slice(lo - o, hi - o, 1) for lo, hi, o in zip(domain.inclusive_min, domain.exclusive_max, origin, strict=True) ) np.testing.assert_array_equal(part.view.result(), source[1:, 3:9][part.out_selection]) for child in part.view.parts(): assert child.view.transform.domain == domain # A part placed by index arrays cannot carry the request's coordinates: its # domain is fresh and zero-origin, and `out_selection` is the placement. fancy = LazyArray(source).with_parts((2, 4))[1:, 3:9].oindex[[2, 0], :] assembled = np.empty(fancy.shape, dtype=fancy.dtype) for part in fancy.parts(): assert part.view.transform.domain.inclusive_min == (0,) * part.view.ndim assembled[part.out_selection] = part.view.result() np.testing.assert_array_equal(assembled, source[1:, 3:9][[2, 0], :]) def test_reader_contexts_are_re_based_whatever_the_view_domain() -> None: from zarr_indexing.reader import ReadContext view = LazyArray(np.arange(30))[10:20][::-1] context = ReadContext(view.transform) assert context.transform.domain == IndexDomain((0,), (10,)) assert context.transform.apply((0,)) == view.transform.apply( view.transform.domain.inclusive_min ) DOMAIN_CASES: list[ tuple[str, Callable[[LazyArray], LazyArray], tuple[int, ...], tuple[int, ...]] ] = [ ("whole", lambda v: v[:], (0, 0), (6, 8)), ("ellipsis", lambda v: v[...], (0, 0), (6, 8)), ("slice", lambda v: v[2:5], (2, 0), (5, 8)), ("nested slice", lambda v: v[2:5][1:], (3, 0), (5, 8)), ("negative stop", lambda v: v[2:5][:-1], (2, 0), (4, 8)), ("empty", lambda v: v[2:5][1:1], (3, 0), (3, 8)), ("int drops axis", lambda v: v[2:5][-1], (0,), (8,)), ("strided", lambda v: v[:, 1:7:2], (0, 0), (6, 3)), ("reversed", lambda v: v[::-1], (-5, 0), (1, 8)), ("reversed then slice", lambda v: v[::-1][1:3], (-4, 0), (-2, 8)), ("reversed twice", lambda v: v[1:4, 2:6][::-1, ::-1], (-3, -5), (0, -1)), ("newaxis", lambda v: v[None, 2:4], (0, 2, 0), (1, 4, 8)), ("oindex", lambda v: v.oindex[[3, 1], 2:6], (0, 2), (2, 6)), ("oindex on slice", lambda v: v[2:5].oindex[[1, 0], :], (0, 0), (2, 8)), ("vindex", lambda v: v.vindex[[1, 2], [3, 4]], (0,), (2,)), ("domain key", lambda v: v[2:5][IndexDomain((3, 0), (4, 8))], (3, 0), (4, 8)), ] @pytest.mark.parametrize( ("case", "select", "lo", "hi"), DOMAIN_CASES, ids=[c[0] for c in DOMAIN_CASES] ) def test_indexing_preserves_the_literal_domain( case: str, select: Callable[[LazyArray], LazyArray], lo: tuple[int, ...], hi: tuple[int, ...] ) -> None: """A view's domain is the literal domain TensorStore-style composition gives it. Slices keep their coordinates, nested slices compose them, integers drop axes, `None` and index arrays open fresh zero-origin axes, and reversals run negative. Values are unaffected: the same positional keys select the same NumPy elements. """ source = np.arange(48).reshape(6, 8) view = select(LazyArray(source)) expected = IndexDomain(lo, hi) assert view.transform.domain == expected, case assert view.shape == expected.shape # The literal frame is live: the domain's own first coordinate addresses the # view's first element, and `ReadContext` still hands readers a zero origin. if view.size: np.testing.assert_array_equal( view.result().flat[0], source[view.transform.apply(expected.inclusive_min)] ) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_messages.py000066400000000000000000000122521527014137600304100ustar00rootroot00000000000000"""Message-layer tests beyond the vendored conformance corpus. The corpus (see `test_conformance.py`) covers the desugaring matrix and error codes. These tests pin behaviors the corpus does not: `normalize` idempotence, `parse_ndsel`, 64-bit boundary handling, and schema-valid-but-redundant maps. """ from __future__ import annotations from typing import Any import pytest from zarr_indexing.messages import NdselError, normalize_ndsel, parse_ndsel _MESSAGES = [ {"kind": "point", "coords": [4, 7]}, {"kind": "box", "inclusive_min": [0, 0], "exclusive_max": [3, 4]}, {"kind": "box", "inclusive_min": [["-inf"], 0], "exclusive_max": [["+inf"], 4]}, {"kind": "slice", "start": [5], "stop": [10], "step": [2]}, {"kind": "points", "coords": [[1, 10], [2, 20]]}, { "kind": "transform", "input_inclusive_min": [0], "input_exclusive_max": [3], "output": [{"offset": 7}, {"input_dimension": 0, "stride": 2}, {"index_array": [1, 2, 3]}], }, ] @pytest.mark.parametrize("message", _MESSAGES) def test_normalize_is_idempotent(message: dict[str, Any]) -> None: once = normalize_ndsel(message) twice = normalize_ndsel({"kind": "transform", **once}) assert twice == once @pytest.mark.parametrize("message", _MESSAGES) def test_parse_returns_message_unchanged(message: dict[str, Any]) -> None: assert parse_ndsel(message) == message def test_parse_rejects_invalid() -> None: with pytest.raises(NdselError) as excinfo: parse_ndsel({"kind": "slice", "start": [0]}) assert excinfo.value.reason == "invalid_json" def test_constant_map_drops_redundant_stride() -> None: # A constant map (no input_dimension, no index_array) is schema-valid even # with a stray stride; it canonicalizes to offset-only. result = normalize_ndsel( {"kind": "transform", "input_rank": 0, "output": [{"offset": 5, "stride": 9}]} ) assert result["output"] == [{"offset": 5}] def test_i64_min_and_max_round_trip() -> None: i64_min, i64_max = -(2**63), 2**63 - 1 result = normalize_ndsel({"kind": "point", "coords": [i64_min, i64_max]}) assert result["output"] == [{"offset": i64_min}, {"offset": i64_max}] def test_i64_overflow_rejected() -> None: with pytest.raises(NdselError) as excinfo: normalize_ndsel({"kind": "point", "coords": [2**63]}) assert excinfo.value.reason == "invalid_json" def test_bool_in_output_offset_rejected() -> None: with pytest.raises(NdselError) as excinfo: normalize_ndsel({"kind": "transform", "input_rank": 0, "output": [{"offset": True}]}) assert excinfo.value.reason == "invalid_json" def test_sentinel_not_allowed_in_plain_integer_position() -> None: with pytest.raises(NdselError) as excinfo: normalize_ndsel({"kind": "point", "coords": ["+inf"]}) assert excinfo.value.reason == "invalid_json" def test_not_an_object_rejected() -> None: with pytest.raises(NdselError) as excinfo: normalize_ndsel([1, 2, 3]) assert excinfo.value.reason == "invalid_json" def test_empty_string_kind_is_unknown_kind() -> None: with pytest.raises(NdselError) as excinfo: normalize_ndsel({"kind": ""}) assert excinfo.value.reason == "unknown_kind" class TestNegativeStep: """ndsel 1.0-draft.2 section 5.3: one desugaring rule, both signs.""" def test_full_reverse(self) -> None: """The spec's own worked example: reversing a length-20 axis.""" body = normalize_ndsel({"kind": "slice", "start": [19], "stop": [-1], "step": [-1]}) assert body["input_inclusive_min"] == [-19] assert body["input_exclusive_max"] == [1] assert body["output"] == [{"offset": 0, "stride": -1, "input_dimension": 0}] def test_trunc_origin_for_a_negative_step(self) -> None: """`trunc(15 / -2) == -7`; `floor` would give -8.""" body = normalize_ndsel({"kind": "slice", "start": [15], "stop": [5], "step": [-2]}) assert body["input_inclusive_min"] == [-7] assert body["input_exclusive_max"] == [-2] assert body["output"] == [{"offset": 1, "stride": -2, "input_dimension": 0}] def test_empty_is_legal_at_any_coordinate(self) -> None: body = normalize_ndsel({"kind": "slice", "start": [5], "stop": [5], "step": [-1]}) assert body["input_inclusive_min"] == body["input_exclusive_max"] == [-5] @pytest.mark.parametrize( "message", [ {"kind": "slice", "start": [9], "stop": [0]}, {"kind": "slice", "start": [9], "stop": [0], "step": [2]}, {"kind": "slice", "start": [5], "stop": [6], "step": [-1]}, ], ids=["unit-step", "positive-step", "negative-step"], ) def test_a_reversed_interval_is_an_error(self, message: dict[str, object]) -> None: """Not clamped to empty: travelling the wrong way is a mistake, either sign.""" with pytest.raises(NdselError) as excinfo: normalize_ndsel(message) assert excinfo.value.reason == "bounds_out_of_order" def test_zero_step_still_errors(self) -> None: with pytest.raises(NdselError) as excinfo: normalize_ndsel({"kind": "slice", "start": [0], "stop": [4], "step": [0]}) assert excinfo.value.reason == "step_zero" zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_ndsel_tensorstore.py000066400000000000000000000040071527014137600323540ustar00rootroot00000000000000"""Cross-check canonical ndsel bodies against a real TensorStore. Canonical ndsel bodies use TensorStore's `IndexTransform` field vocabulary, but the consumers have different validation constraints. This test loads a handful of finite-bound canonical bodies supported by both implementations into `tensorstore.IndexTransform(json=...)` and confirms that TensorStore's own `to_json()` re-loads, through our engine layer, into an equivalent transform. Skipped when tensorstore is not installed. Run it explicitly with: hatch run test.py3.12-optional:pytest \ packages/zarr-indexing/tests/test_ndsel_tensorstore.py -q """ from __future__ import annotations import numpy as np import pytest from zarr_indexing.transform import IndexTransform ts = pytest.importorskip("tensorstore") def _canonical_transforms() -> list[IndexTransform]: base = IndexTransform.from_shape((10, 20)) return [ base, # identity base[2:8:2, :], # strided DimensionMap + identity base[3, :], # integer index -> ConstantMap + DimensionMap base.oindex[np.array([1, 5, 9]), :], # orthogonal index_array IndexTransform.from_shape((10, 20, 30)).vindex[ np.array([1, 3]), np.array([2, 4]), : ], # correlated index_arrays + residual slice ] @pytest.mark.parametrize("transform", _canonical_transforms()) def test_body_loads_in_tensorstore_and_round_trips(transform: IndexTransform) -> None: body = transform.to_json() # (1) The canonical body loads directly as a TensorStore IndexTransform. ts_transform = ts.IndexTransform(json=body) # (2) TensorStore's own JSON re-loads, through our engine, to an equivalent # transform. Comparing via our canonical form normalizes away # representational choices (index_array_bounds, default omissions) that # both sides make differently but that denote the same selection. ts_json = ts_transform.to_json() reloaded = IndexTransform.from_json(ts_json) assert reloaded.to_json() == transform.to_json() zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_output_map.py000066400000000000000000000062661527014137600310060ustar00rootroot00000000000000from __future__ import annotations import pickle import numpy as np import pytest from zarr_indexing.output_map import ArrayMap, ConstantMap, DimensionMap class TestConstantMap: def test_construction(self) -> None: m = ConstantMap(offset=42) assert m.offset == 42 def test_default_offset(self) -> None: m = ConstantMap() assert m.offset == 0 def test_frozen(self) -> None: m = ConstantMap(offset=5) assert isinstance(m, ConstantMap) class TestDimensionMap: def test_construction(self) -> None: m = DimensionMap(input_dimension=3, offset=5, stride=2) assert m.input_dimension == 3 assert m.offset == 5 assert m.stride == 2 def test_defaults(self) -> None: m = DimensionMap(input_dimension=0) assert m.offset == 0 assert m.stride == 1 def test_frozen(self) -> None: m = DimensionMap(input_dimension=0) assert isinstance(m, DimensionMap) class TestArrayMap: def test_construction(self) -> None: arr = np.array([1, 3, 5], dtype=np.intp) m = ArrayMap(index_array=arr, offset=10, stride=2) assert m.offset == 10 assert m.stride == 2 np.testing.assert_array_equal(m.index_array, arr) def test_defaults(self) -> None: arr = np.array([0, 1], dtype=np.intp) m = ArrayMap(index_array=arr) assert m.offset == 0 assert m.stride == 1 def test_frozen(self) -> None: arr = np.array([0], dtype=np.intp) m = ArrayMap(index_array=arr) assert isinstance(m, ArrayMap) def test_owns_index_array_and_keeps_hash_stable(self) -> None: arr = np.array([1, 3, 5], dtype=np.intp) m = ArrayMap(index_array=arr) lookup = {m: "value"} arr[:] = 9 np.testing.assert_array_equal(m.index_array, [1, 3, 5]) assert lookup[m] == "value" def test_pickle_round_trip_keeps_index_array_read_only(self) -> None: restored = pickle.loads(pickle.dumps(ArrayMap(index_array=np.array([1, 3, 5])))) assert not restored.index_array.flags.writeable with pytest.raises(ValueError, match="read-only"): restored.index_array[0] = 9 def test_index_array_cannot_be_made_writeable(self) -> None: m = ArrayMap(index_array=np.array([1, 3, 5])) with pytest.raises(ValueError): m.index_array.flags.writeable = True def test_equal_array_maps_have_equal_hashes_across_integer_dtypes(self) -> None: left = ArrayMap(np.array([1, 2], dtype=np.int32)) right = ArrayMap(np.array([1, 2], dtype=np.int64)) assert left == right assert hash(left) == hash(right) assert left.index_array.dtype == np.dtype(np.intp) def test_rejects_non_integer_index_array(self) -> None: with pytest.raises(TypeError, match="index_array must have an integer dtype"): ArrayMap(np.array([1.5, 2.0], dtype=np.float64)) def test_rejects_unsigned_index_array_value_outside_intp(self) -> None: outside_intp = np.array([np.iinfo(np.uint64).max], dtype=np.uint64) with pytest.raises(OverflowError, match="outside np.intp range"): ArrayMap(outside_intp) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_reader.py000066400000000000000000000504211527014137600300430ustar00rootroot00000000000000from __future__ import annotations import math from typing import TYPE_CHECKING, Any, get_type_hints import numpy as np import numpy.typing as npt import pytest import zarr_indexing.reader as reader_module from zarr_indexing import ( ArrayMap, ChunkProjection, ConstantMap, DimensionMap, IndexDomain, IndexTransform, LazyArray, ReadContext, plan_chunks, ) from zarr_indexing.grid import dimension_grids_from_chunks from zarr_indexing.reader import basic_reader, numpy_reader, unit_step_reader if TYPE_CHECKING: from collections.abc import Callable class BasicOnlySource: def __init__(self, data: np.ndarray[Any, Any]) -> None: self.data = data self.keys: list[tuple[Any, ...]] = [] @property def shape(self) -> tuple[int, ...]: return self.data.shape @property def dtype(self) -> np.dtype[Any]: return self.data.dtype def __getitem__(self, key: tuple[Any, ...]) -> np.ndarray[Any, Any]: assert all(isinstance(item, slice) and (item.step or 1) > 0 for item in key) self.keys.append(key) return self.data[key] class UnitStepOnlySource(BasicOnlySource): """A source that rejects everything but ascending unit-step slices.""" def __getitem__(self, key: tuple[Any, ...]) -> np.ndarray[Any, Any]: assert all( isinstance(item, slice) and item.step == 1 and 0 <= item.start <= item.stop <= size for item, size in zip(key, self.data.shape, strict=True) ) self.keys.append(key) return self.data[key] SOURCE = np.arange(6 * 7 * 8).reshape(6, 7, 8) BASE = IndexTransform.from_shape(SOURCE.shape) SUCCESSFUL_CONTRACT_CASES = ( pytest.param( np.arange(8), IndexTransform.from_shape((8,))[3], (3,), np.zeros((1, 0), dtype=np.intp), np.array([[3]], dtype=np.intp), np.array(3), lambda array: array[3], id="constant", ), pytest.param( np.arange(8), IndexTransform.from_shape((8,))[1:8:2], (3,), np.array([[0], [1], [2], [3]], dtype=np.intp), np.array([[1], [3], [5], [7]], dtype=np.intp), np.array([1, 3, 5, 7]), lambda array: array[1:8:2], id="positive-affine", ), pytest.param( np.arange(8), IndexTransform.from_shape((8,))[::-2], (3,), np.array([[-3], [-2], [-1], [0]], dtype=np.intp), np.array([[7], [5], [3], [1]], dtype=np.intp), np.array([7, 5, 3, 1]), lambda array: array[::-2], id="negative-affine", ), pytest.param( np.arange(8), IndexTransform( domain=IndexDomain.from_shape((3,)), output=(DimensionMap(input_dimension=0, offset=2, stride=0),), ), (3,), np.array([[0], [1], [2]], dtype=np.intp), np.array([[2], [2], [2]], dtype=np.intp), np.array([2, 2, 2]), lambda array: array.oindex[[2, 2, 2]], id="zero-affine", ), pytest.param( np.arange(20).reshape(4, 5), IndexTransform.from_shape((4, 5)).oindex[[3, 1, 1], [4, 0]], (2, 3), np.array([[0, 0], [0, 1], [1, 0], [1, 1], [2, 0], [2, 1]], dtype=np.intp), np.array([[3, 4], [3, 0], [1, 4], [1, 0], [1, 4], [1, 0]], dtype=np.intp), np.array([[19, 15], [9, 5], [9, 5]]), lambda array: array.oindex[[3, 1, 1], [4, 0]], id="orthogonal-array", ), pytest.param( np.arange(20).reshape(4, 5), IndexTransform.from_shape((4, 5)).vindex[[3, 1, 1], [4, 0, 4]], (2, 3), np.array([[0], [1], [2]], dtype=np.intp), np.array([[3, 4], [1, 0], [1, 4]], dtype=np.intp), np.array([19, 5, 9]), lambda array: array.vindex[[3, 1, 1], [4, 0, 4]], id="correlated-array", ), pytest.param( np.arange(8), IndexTransform( domain=IndexDomain((4,), (7,)), output=(DimensionMap(input_dimension=0, offset=-3, stride=1),), ), (3,), np.array([[4], [5], [6]], dtype=np.intp), np.array([[1], [2], [3]], dtype=np.intp), np.array([1, 2, 3]), None, id="non-zero-origin", ), pytest.param( np.arange(8), IndexTransform.from_shape((8,))[2:2], (3,), np.empty((0, 1), dtype=np.intp), np.empty((0, 1), dtype=np.intp), np.array([], dtype=np.intp), lambda array: array[2:2], id="empty", ), ) @pytest.mark.parametrize( ( "source_data", "transform", "chunk_shape", "request_coordinates", "storage_coordinates", "expected_values", "lazy_selection", ), SUCCESSFUL_CONTRACT_CASES, ) def test_successful_transform_contract_across_planning_readers_and_lazy_array( source_data: np.ndarray[Any, Any], transform: IndexTransform, chunk_shape: tuple[int, ...], request_coordinates: npt.NDArray[np.intp], storage_coordinates: npt.NDArray[np.intp], expected_values: np.ndarray[Any, Any], lazy_selection: Callable[[LazyArray], LazyArray] | None, ) -> None: """One literal matrix keeps transform, planning, readers, and wrappers aligned.""" np.testing.assert_array_equal(transform.apply_many(request_coordinates), storage_coordinates) grids = dimension_grids_from_chunks(chunk_shape, source_data.shape) reconstructed_pairs = [ ( tuple(projection.cell_transform.apply(cell_coordinate)), tuple( local_coordinate + chunk_origin for local_coordinate, chunk_origin in zip( projection.chunk_transform.apply(cell_coordinate), projection.chunk_domain.inclusive_min, strict=True, ) ), ) for projection in plan_chunks(transform, grids) for cell_coordinate in _domain_coordinates(projection.cell_transform.domain) ] expected_pairs = list( zip( map(tuple, request_coordinates.tolist()), map(tuple, storage_coordinates.tolist()), strict=True, ) ) assert sorted(reconstructed_pairs) == sorted(expected_pairs) for reader, source in ( (basic_reader, BasicOnlySource(source_data)), (numpy_reader, source_data), (unit_step_reader, UnitStepOnlySource(source_data)), ): out = np.empty(transform.domain.shape, dtype=source_data.dtype) assert reader.read_into(source, ReadContext(transform), out) is None np.testing.assert_array_equal(out, expected_values) if lazy_selection is not None: reader = ProjectionRequiredReader() view = lazy_selection( LazyArray.from_numpy(source_data).with_parts(chunk_shape).with_reader(reader) ) parts = tuple(view.parts()) np.testing.assert_array_equal(view.result(parts=parts), expected_values) parent_contexts = tuple(reader.contexts) reader.contexts.clear() assembled = np.empty(view.shape, dtype=view.dtype) for part in reversed(parts): values = part.view.result() assert not np.shares_memory(values, source_data) assert values.dtype == expected_values.dtype assembled[part.out_selection] = values np.testing.assert_array_equal(assembled, expected_values) for independent, parent in zip(reversed(reader.contexts), parent_contexts, strict=True): assert independent.transform == parent.transform assert independent.projection is not None assert parent.projection is not None assert independent.projection.chunk_coords == parent.projection.chunk_coords assert independent.projection.chunk_domain == parent.projection.chunk_domain assert independent.projection.chunk_transform == parent.projection.chunk_transform for candidate in (view, view.unpartitioned()): np.testing.assert_array_equal(candidate.result(), expected_values) for part in parts: assert part.view.base_shape == source_data.shape for child in part.view.parts(): np.testing.assert_array_equal(child.view.result(), part.view.result()) reverse = (slice(None, None, -1),) * part.view.ndim np.testing.assert_array_equal(part.view[reverse].result(), part.view.result()[reverse]) for candidate in ( part.view.unpartitioned(), part.view.with_parts((1,) * source_data.ndim), ): np.testing.assert_array_equal(candidate.result(), part.view.result()) class ProjectionRequiredReader: """Read through chunk-local coordinates so losing the projection breaks execution.""" def __init__(self) -> None: self.contexts: list[ReadContext] = [] def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: projection = context.projection assert projection is not None self.contexts.append(context) for position, cell in zip( np.ndindex(out.shape), _domain_coordinates(projection.chunk_transform.domain), strict=True, ): local = projection.chunk_transform.apply(cell) global_position = tuple( coord + origin for coord, origin in zip(local, projection.chunk_domain.inclusive_min, strict=True) ) assert context.transform.apply(position) == global_position out[position] = source[global_position] def _domain_coordinates(domain: IndexDomain) -> list[tuple[int, ...]]: return [ tuple( position + origin for position, origin in zip(index, domain.inclusive_min, strict=True) ) for index in np.ndindex(*domain.shape) ] def test_read_context_public_annotations_resolve() -> None: assert get_type_hints(ReadContext)["projection"] == ChunkProjection | None READER_CASES = ( pytest.param( SOURCE, BASE[1:6:2, 2, ::-2], np.array( [ [79, 77, 75, 73], [191, 189, 187, 185], [303, 301, 299, 297], ] ), id="strided-reversed-nonzero-origin", ), pytest.param( SOURCE, BASE.oindex[[5, 1, 1], slice(1, 6), [7, 2]], np.array( [ [[295, 290], [303, 298], [311, 306], [319, 314], [327, 322]], [[71, 66], [79, 74], [87, 82], [95, 90], [103, 98]], [[71, 66], [79, 74], [87, 82], [95, 90], [103, 98]], ] ), id="orthogonal-nonzero-origin", ), pytest.param( SOURCE, BASE.vindex[np.array([[5], [1]]), np.array([[2, 4, 0]])], np.array( [ [ [296, 297, 298, 299, 300, 301, 302, 303], [312, 313, 314, 315, 316, 317, 318, 319], [280, 281, 282, 283, 284, 285, 286, 287], ], [ [72, 73, 74, 75, 76, 77, 78, 79], [88, 89, 90, 91, 92, 93, 94, 95], [56, 57, 58, 59, 60, 61, 62, 63], ], ] ), id="correlated-broadcast", ), pytest.param( SOURCE, BASE.vindex[[5, 1], [2, 2]], np.array( [ [296, 297, 298, 299, 300, 301, 302, 303], [72, 73, 74, 75, 76, 77, 78, 79], ] ), id="correlated-vector", ), pytest.param( SOURCE, BASE[:, 0:0, :], np.empty((6, 0, 8), dtype=SOURCE.dtype), id="empty", ), pytest.param(SOURCE, BASE[2, 3, 4], np.array(140), id="scalar"), pytest.param( np.arange(8), IndexTransform( domain=IndexDomain((4,), (7,)), output=(DimensionMap(input_dimension=0, offset=2, stride=0),), ), np.array([2, 2, 2]), id="zero-stride-nonzero-origin", ), ) @pytest.mark.parametrize(("source_data", "transform", "expected"), READER_CASES) @pytest.mark.parametrize("reader_name", ["basic", "numpy", "unit-step"]) def test_builtin_readers_match_the_transform( source_data: np.ndarray[Any, Any], transform: IndexTransform, expected: np.ndarray[Any, Any], reader_name: str, ) -> None: out = np.empty(transform.domain.shape, dtype=source_data.dtype) if reader_name == "basic": source = BasicOnlySource(source_data) result = basic_reader.read_into(source, ReadContext(transform), out) assert len(source.keys) == 1 elif reader_name == "unit-step": source = UnitStepOnlySource(source_data) result = unit_step_reader.read_into(source, ReadContext(transform), out) assert len(source.keys) == 1 else: result = numpy_reader.read_into(source_data, ReadContext(transform), out) assert result is None np.testing.assert_array_equal(out, expected) @pytest.mark.parametrize("reader_name", ["basic", "numpy", "unit-step"]) def test_builtin_readers_share_transform_affine_overflow(reader_name: str) -> None: transform = IndexTransform( domain=IndexDomain.from_shape((1,)), output=(ArrayMap(np.array([2**62], dtype=np.intp), stride=4),), ) with pytest.raises(OverflowError, match="outside np.intp"): transform.apply_many(np.array([[0]], dtype=np.intp)) out = np.empty(transform.domain.shape, dtype=np.intp) if reader_name == "basic": with pytest.raises(OverflowError, match="outside np.intp"): basic_reader.read_into(BasicOnlySource(np.arange(1)), ReadContext(transform), out) elif reader_name == "unit-step": with pytest.raises(OverflowError, match="outside np.intp"): unit_step_reader.read_into( UnitStepOnlySource(np.arange(1)), ReadContext(transform), out ) else: with pytest.raises(OverflowError, match="outside np.intp"): numpy_reader.read_into(np.arange(1), ReadContext(transform), out) def test_empty_domain_composed_fancy_transform_reads_as_empty() -> None: """An empty composed domain reads as empty for every built-in reader. Empty index arrays are valid when the domain selects no elements.""" source_data = np.arange(6).reshape(2, 3) view = LazyArray.from_numpy(source_data).oindex[slice(0, 0), np.array([2, 1, 2, 0])] transform = view.oindex[slice(None), np.array([1, 3, 1])].transform assert transform.domain.shape == (0, 3) for reader, source in ( (basic_reader, BasicOnlySource(source_data)), (numpy_reader, source_data), (unit_step_reader, UnitStepOnlySource(source_data)), ): out = np.empty(transform.domain.shape, dtype=source_data.dtype) assert reader.read_into(source, ReadContext(transform), out) is None def test_unit_step_reader_reads_through_lazy_array() -> None: """The parametrized selections resolve through a unit-step-only source. `UnitStepOnlySource` asserts the shape of every key it receives, so each selection here also proves no strided, descending, or non-slice key reached the source — partitioned and unpartitioned alike. """ selections: tuple[Callable[[LazyArray], LazyArray], ...] = ( lambda v: v[1:5, ::2, ::-1], lambda v: v[5:1:-2, None, 3, ::3], lambda v: v.oindex[[3, 0, 3], ::-2, [7, 7]], lambda v: v.vindex[np.array([[0, 5]]), np.array([[6], [0]]), 2], lambda v: v[2:2, :, ::-1], lambda v: v[::5, 6, 1:8:4], ) for select in selections: for parts in (None, (2, 3, 8), (6, 7, 1)): source = UnitStepOnlySource(SOURCE) view = LazyArray(source).with_reader(unit_step_reader) if parts is not None: view = view.with_parts(parts) expected = select(LazyArray.from_numpy(SOURCE)).result() np.testing.assert_array_equal(select(view).result(), expected) # An empty view allocates without reading; every other one must read. assert (len(source.keys) > 0) == (expected.size > 0) def test_constant_outside_intp_has_transform_planner_reader_error_parity() -> None: outside_intp = int(np.iinfo(np.intp).max) + 1 transform = IndexTransform( domain=IndexDomain((), ()), output=(ConstantMap(offset=outside_intp),), ) with pytest.raises(OverflowError, match="outside np.intp range"): transform.apply_many(np.zeros((1, 0), dtype=np.intp)) grids = dimension_grids_from_chunks((1,), (1,)) with pytest.raises(OverflowError, match="outside np.intp range"): list(plan_chunks(transform, grids)) for reader, source in ( (basic_reader, BasicOnlySource(np.arange(1))), (numpy_reader, np.arange(1)), ): with pytest.raises(OverflowError, match="outside np.intp range"): reader.read_into(source, ReadContext(transform), np.empty((), dtype=np.intp)) def test_numpy_reader_narrows_basic_slab_before_gather( monkeypatch: pytest.MonkeyPatch, ) -> None: source = np.arange(1_000_000).reshape(1000, 1000) transform = IndexTransform.from_shape(source.shape).oindex[:10, [0, 999]] seen: list[tuple[int, ...]] = [] real_take = reader_module._take # pyright: ignore[reportPrivateUsage] def recording_take(array: Any, indices: npt.NDArray[np.intp], axis: int) -> Any: seen.append(tuple(int(value) for value in array.shape)) return real_take(array, indices, axis) monkeypatch.setattr(reader_module, "_take", recording_take) out = np.empty(transform.domain.shape, dtype=source.dtype) assert numpy_reader.read_into(source, ReadContext(transform), out) is None assert out.shape == (10, 2) np.testing.assert_array_equal( out, np.array( [ [0, 999], [1000, 1999], [2000, 2999], [3000, 3999], [4000, 4999], [5000, 5999], [6000, 6999], [7000, 7999], [8000, 8999], [9000, 9999], ] ), ) assert seen assert all(math.prod(shape) <= 20_000 for shape in seen), seen def test_numpy_reader_preserves_a_mask_in_the_supplied_buffer() -> None: source = np.ma.masked_greater(np.arange(12).reshape(3, 4), 7) transform = IndexTransform.from_shape(source.shape)[:, 1:] out = np.ma.masked_all(transform.domain.shape, dtype=source.dtype) assert numpy_reader.read_into(source, ReadContext(transform), out) is None np.testing.assert_array_equal(np.ma.getmaskarray(out), np.ma.getmaskarray(source[:, 1:])) np.testing.assert_array_equal(np.ma.filled(out, 0), np.ma.filled(source[:, 1:], 0)) # --------------------------------------------------------------------------- # Diagonal gathers — output maps sharing an input axis # --------------------------------------------------------------------------- def test_reading_a_diagonal_gather_transform() -> None: """Two index arrays bound to the same input axis resolve pointwise. No selection dialect produces this transform — it is the hand-built diagonal-extraction form — but the reader resolves it through the same pointwise path as a correlated gather. """ data = np.arange(30).reshape(5, 6) rows = np.array([4, 0, 2]) cols = np.array([1, 5, 2]) transform = IndexTransform( domain=IndexDomain.from_shape((3,)), output=( ArrayMap(index_array=rows), ArrayMap(index_array=cols), ), ) out = np.empty((3,), dtype=data.dtype) basic_reader.read_into(data, ReadContext(transform), out) np.testing.assert_array_equal(out, data[rows, cols]) out = np.empty((3,), dtype=data.dtype) numpy_reader.read_into(data, ReadContext(transform), out) np.testing.assert_array_equal(out, data[rows, cols]) def test_reading_a_diagonal_gather_with_a_residual_slice_axis() -> None: data = np.arange(60).reshape(5, 6, 2) rows = np.array([[4], [0], [2]]) cols = np.array([[1], [5], [2]]) transform = IndexTransform( domain=IndexDomain.from_shape((3, 2)), output=( ArrayMap(index_array=rows), ArrayMap(index_array=cols), DimensionMap(input_dimension=1), ), ) out = np.empty((3, 2), dtype=data.dtype) basic_reader.read_into(data, ReadContext(transform), out) np.testing.assert_array_equal(out, data[rows[:, 0], cols[:, 0], :]) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_tensorstore_parity.py000066400000000000000000000420571527014137600325660ustar00rootroot00000000000000"""TensorStore-parity oracle tests for IndexTransform semantics. Every case in this module was executed against tensorstore 0.1.84 (see the lazy-indexing design notes): the expected domains, values, and error conditions are TensorStore's observed behavior, which zarr's lazy indexing matches by design. Core rules pinned here: - **Domain preservation**: a step-1 slice keeps the literal coordinates of the selected interval (`a[2:10]` has domain `[2, 10)`); nothing re-zeros implicitly. Re-zeroing is explicit via `translate_to`. - **Strided-domain rule**: for step ``k``, ``origin = trunc(start/k)`` (rounded toward zero), ``shape = ceil((stop - start)/k)``, and coordinate ``origin + i`` maps to base cell ``start + i*k``. - **Strict containment**: non-empty slice intervals must lie within the domain — no clamping, no negative-wrapping; empty intervals are valid anywhere; reversed non-empty bounds are an error, not an empty result. - **Fancy-dim rule**: index-array dims get fresh explicit ``[0, n)`` domains; index-array values are absolute domain coordinates. - **Translate rules**: ``translate_by``/``translate_to`` shift the input domain while preserving which cells are addressed. """ from __future__ import annotations from typing import ClassVar import numpy as np import pytest from zarr_indexing.domain import IndexDomain from zarr_indexing.errors import BoundsCheckError from zarr_indexing.output_map import ArrayMap, ConstantMap, DimensionMap from zarr_indexing.transform import IndexTransform def _identity(lo: int, hi: int) -> IndexTransform: """Identity transform over the 1-D domain [lo, hi).""" return IndexTransform.identity(IndexDomain(inclusive_min=(lo,), exclusive_max=(hi,))) def _a() -> IndexTransform: """The oracle's base fixture: identity over [0, 12).""" return _identity(0, 12) def _w() -> IndexTransform: """The oracle's translated fixture: identity over [-10, 2), cell c -> base c + 10.""" return _a().translate_domain_by((-10,)) def _dim(t: IndexTransform) -> DimensionMap: m = t.output[0] assert isinstance(m, DimensionMap) return m def _base_cells(t: IndexTransform) -> list[int]: """The base cells a 1-D single-DimensionMap transform addresses, in order.""" m = _dim(t) lo, hi = t.domain.inclusive_min[0], t.domain.exclusive_max[0] return [m.offset + m.stride * c for c in range(lo, hi)] class TestDomainPreservation: """Oracle section 1-2: step-1 slices keep literal coordinates.""" def test_slice_preserves_domain(self) -> None: t = _a()[2:10] assert (t.domain.inclusive_min, t.domain.exclusive_max) == ((2,), (10,)) assert _base_cells(t) == list(range(2, 10)) def test_integer_on_preserved_domain_is_a_coordinate(self) -> None: v = _a()[2:10] assert isinstance(v[3].output[0], ConstantMap) assert v[3].output[0].offset == 3 # coordinate 3 = base cell 3 assert v[2].output[0].offset == 2 assert v[9].output[0].offset == 9 @pytest.mark.parametrize("bad", [0, -1, 10]) def test_out_of_domain_integer_raises(self, bad: int) -> None: with pytest.raises(BoundsCheckError, match=r"valid indices \[2, 10\)"): _a()[2:10][bad] def test_slice_of_slice_is_literal(self) -> None: v = _a()[2:10] t = v[3:7] assert (t.domain.inclusive_min, t.domain.exclusive_max) == ((3,), (7,)) assert _base_cells(t) == [3, 4, 5, 6] def test_ellipsis_preserves_domain(self) -> None: v = _a()[2:10] t = v[...] assert (t.domain.inclusive_min, t.domain.exclusive_max) == ((2,), (10,)) class TestNegativeOriginDomain: """Oracle section 3: on domain [-10, 2), -1 is just another index.""" def test_translated_domain(self) -> None: w = _w() assert (w.domain.inclusive_min, w.domain.exclusive_max) == ((-10,), (2,)) assert _base_cells(w) == list(range(12)) @pytest.mark.parametrize(("coord", "base"), [(-5, 5), (-10, 0), (-1, 9), (1, 11)]) def test_negative_coordinates_address_cells(self, coord: int, base: int) -> None: t = _w()[coord] assert isinstance(t.output[0], ConstantMap) assert t.output[0].offset == base @pytest.mark.parametrize("bad", [-11, 2]) def test_out_of_domain_raises(self, bad: int) -> None: with pytest.raises(BoundsCheckError, match=r"valid indices \[-10, 2\)"): _w()[bad] def test_negative_slice_bounds_are_coordinates(self) -> None: t = _w()[-5:] assert (t.domain.inclusive_min, t.domain.exclusive_max) == ((-5,), (2,)) assert _base_cells(t) == [5, 6, 7, 8, 9, 10, 11] t2 = _w()[-5:-2] assert (t2.domain.inclusive_min, t2.domain.exclusive_max) == ((-5,), (-2,)) assert _base_cells(t2) == [5, 6, 7] class TestStridedDomains: """Oracle section 5: origin = trunc(start/step), coord origin+i -> start + i*step.""" # (slice, expected (lo, hi), expected base cells) — verbatim oracle rows. CASES: ClassVar[list[tuple[slice, tuple[int, int], list[int]]]] = [ (slice(1, 10, 3), (0, 3), [1, 4, 7]), (slice(None, None, 2), (0, 6), [0, 2, 4, 6, 8, 10]), (slice(2, 11, 3), (0, 3), [2, 5, 8]), (slice(0, 12, 4), (0, 3), [0, 4, 8]), (slice(5, 12, 2), (2, 6), [5, 7, 9, 11]), (slice(6, 12, 2), (3, 6), [6, 8, 10]), (slice(7, 12, 3), (2, 4), [7, 10]), ] @pytest.mark.parametrize(("sel", "dom", "cells"), CASES) def test_strided_domain_and_cells( self, sel: slice, dom: tuple[int, int], cells: list[int] ) -> None: t = _a()[sel] assert (t.domain.inclusive_min[0], t.domain.exclusive_max[0]) == dom assert _base_cells(t) == cells def test_strided_on_negative_origin(self) -> None: # w[-9:2:2] -> domain [-4, 2), base cells 1,3,5,7,9,11 t = _w()[-9:2:2] assert (t.domain.inclusive_min[0], t.domain.exclusive_max[0]) == (-4, 2) assert _base_cells(t) == [1, 3, 5, 7, 9, 11] # w[::2] -> domain [-5, 1), base cells 0,2,4,6,8,10 t2 = _w()[::2] assert (t2.domain.inclusive_min[0], t2.domain.exclusive_max[0]) == (-5, 1) assert _base_cells(t2) == [0, 2, 4, 6, 8, 10] def test_strided_composition(self) -> None: s = _a()[1:10:3] # domain [0, 3), cells 1,4,7 assert [s[k].output[0].offset for k in range(3)] == [1, 4, 7] t = s[1:3] assert (t.domain.inclusive_min[0], t.domain.exclusive_max[0]) == (1, 3) assert _base_cells(t) == [4, 7] t2 = _a()[::2][1:4] assert (t2.domain.inclusive_min[0], t2.domain.exclusive_max[0]) == (1, 4) assert _base_cells(t2) == [2, 4, 6] t3 = _a()[::2][::2] assert (t3.domain.inclusive_min[0], t3.domain.exclusive_max[0]) == (0, 3) assert _base_cells(t3) == [0, 4, 8] @pytest.mark.parametrize("bad", [-2, -1, 3, 4]) def test_strided_bounds(self, bad: int) -> None: with pytest.raises(BoundsCheckError, match=r"valid indices \[0, 3\)"): _a()[1:10:3][bad] class TestStrictContainment: """Oracle section 11: no clamping, no wrapping; empty intervals valid anywhere.""" @pytest.mark.parametrize( "sel", [ slice(5, 100), slice(-3, None), slice(-3, -1), slice(0, 13), slice(12, 14), slice(100, 200), ], ) def test_uncontained_interval_raises(self, sel: slice) -> None: with pytest.raises(BoundsCheckError, match="not contained"): _a()[sel] def test_uncontained_on_negative_origin(self) -> None: with pytest.raises(BoundsCheckError, match="not contained"): _w()[-20:] @pytest.mark.parametrize( ("sel", "pos"), [(slice(5, 5), 5), (slice(0, 0), 0), (slice(13, 13), 13)] ) def test_empty_interval_valid_anywhere(self, sel: slice, pos: int) -> None: t = _a()[sel] assert t.domain.shape == (0,) assert t.domain.inclusive_min[0] == pos @pytest.mark.parametrize("sel", [slice(5, 2), slice(100, 50)]) def test_reversed_bounds_raise(self, sel: slice) -> None: with pytest.raises(IndexError, match="valid.*interval|interval"): _a()[sel] class TestNegativeStep: """Section 1 of the negative-step study: one desugaring rule, both signs. Every expectation below is TensorStore 0.1.84's recorded output (study sections 1.3-1.5), which an exhaustive 32,980-case sweep found zero disagreements with. """ # (domain, slice, expected domain, expected offset, expected stride, cells) RECORDED: ClassVar[list[tuple[tuple[int, int], slice, tuple[int, int], int, int]]] = [ ((0, 20), slice(15, 5, -1), (-15, -5), 0, -1), ((0, 20), slice(15, 5, -2), (-7, -2), 1, -2), ((0, 20), slice(15, 4, -2), (-7, -1), 1, -2), ((0, 20), slice(None, None, -1), (-19, 1), 0, -1), ((0, 20), slice(None, None, -2), (-9, 1), 1, -2), ((0, 20), slice(5, None, -1), (-5, 1), 0, -1), ((0, 20), slice(None, 5, -1), (-19, -5), 0, -1), ((0, 20), slice(5, 5, -1), (-5, -5), 0, -1), ((0, 20), slice(5, 4, -1), (-5, -4), 0, -1), ((0, 20), slice(5, 4, -3), (-1, 0), 2, -3), ((0, 20), slice(15, 5, -4), (-3, 0), 3, -4), ((0, 20), slice(15, 5, -7), (-2, 0), 1, -7), ((5, 25), slice(None, None, -2), (-12, -2), 0, -2), ((-10, 10), slice(-1, -6, -2), (0, 3), -1, -2), ((-10, 10), slice(-2, -9, -3), (0, 3), -2, -3), ] @pytest.mark.parametrize( ("domain", "sel", "expected_domain", "offset", "stride"), RECORDED, ids=[f"{d}{s}" for d, s, _, _, _ in RECORDED], ) def test_recorded_desugaring( self, domain: tuple[int, int], sel: slice, expected_domain: tuple[int, int], offset: int, stride: int, ) -> None: t = _identity(*domain)[sel] assert (t.domain.inclusive_min[0], t.domain.exclusive_max[0]) == expected_domain m = _dim(t) assert (m.offset, m.stride) == (offset, stride) @pytest.mark.parametrize( ("domain", "sel", "cells"), [ ((0, 20), slice(15, 5, -1), list(range(15, 5, -1))), ((0, 20), slice(15, 5, -2), [15, 13, 11, 9, 7]), ((0, 20), slice(None, None, -1), list(range(19, -1, -1))), ((0, 20), slice(15, 5, -7), [15, 8]), ((5, 25), slice(None, None, -2), list(range(24, 4, -2))), ((-10, 10), slice(-1, -6, -2), [-1, -3, -5]), ((-10, 10), slice(-2, -9, -3), [-2, -5, -8]), ], ) def test_recorded_cells(self, domain: tuple[int, int], sel: slice, cells: list[int]) -> None: assert _base_cells(_identity(*domain)[sel]) == cells def test_trunc_not_floor_or_ceil(self) -> None: """The three rows of study section 1.2 that discriminate the rounding.""" # floor would give -8 here, trunc gives -7. assert _identity(0, 20)[15:5:-2].domain.inclusive_min[0] == -7 # ceil would give 1 for both of these; trunc gives 0. assert _identity(-10, 10)[-1:-6:-2].domain.inclusive_min[0] == 0 assert _identity(-10, 10)[-2:-9:-3].domain.inclusive_min[0] == 0 @pytest.mark.parametrize("sel", [slice(5, 15, -1), slice(5, 6, -1)]) def test_inverted_interval_raises(self, sel: slice) -> None: with pytest.raises(IndexError, match="valid.*interval"): _identity(0, 20)[sel] @pytest.mark.parametrize("sel", [slice(20, 0, -1), slice(20, 19, -1), slice(15, -5, -1)]) def test_uncontained_interval_raises(self, sel: slice) -> None: with pytest.raises(BoundsCheckError, match="not contained"): _identity(0, 20)[sel] def test_zero_step_raises(self) -> None: with pytest.raises(IndexError, match="step must not be zero"): _identity(0, 20)[15:5:0] def test_empty_interval_legal_outside_the_domain(self) -> None: t = _identity(0, 20)[25:25:-1] assert t.domain.shape == (0,) assert t.domain.inclusive_min[0] == -25 @pytest.mark.parametrize( ("domain", "first", "second", "expected_domain", "offset", "stride", "cells"), [ # Study section 1.5, recorded verbatim. Each row applies `second` # to the view `first` produced — strides multiply, and a double # reverse recovers the identity. ( (0, 20), slice(0, 20, 2), slice(None, None, -1), (-9, 1), 0, -2, list(range(18, -2, -2)), ), ((0, 20), slice(0, 20, 2), slice(None, None, -2), (-4, 1), 2, -4, [18, 14, 10, 6, 2]), ( (0, 20), slice(None, None, -1), slice(None, None, -1), (0, 20), 0, 1, list(range(20)), ), ( (0, 20), slice(None, None, -1), slice(None, None, 2), (-9, 1), 1, -2, list(range(19, -1, -2)), ), ( (-10, 10), slice(None, None, -1), slice(None, None, -3), (-3, 4), -1, 3, [-10, -7, -4, -1, 2, 5, 8], ), ], ids=[ "strided-then-reverse", "strided-then-reverse-by-two", "double-reverse-is-identity", "reverse-then-strided", "reverse-then-strided-on-negative-origin", ], ) def test_recorded_composition( self, domain: tuple[int, int], first: slice, second: slice, expected_domain: tuple[int, int], offset: int, stride: int, cells: list[int], ) -> None: t = _identity(*domain)[first][second] assert (t.domain.inclusive_min[0], t.domain.exclusive_max[0]) == expected_domain m = _dim(t) assert (m.offset, m.stride) == (offset, stride) assert _base_cells(t) == cells def test_negative_step_over_an_index_array_reverses_it(self) -> None: """Study section 1.5: a reversing step materializes, it does not stride. Recorded: `oindex[[3, 1, 2]]` then `[2:-1:-1]` gives index array `[[2], [1], [3]]` over domain `[-2, 1)`. """ base = IndexTransform.identity(IndexDomain(inclusive_min=(0, 0), exclusive_max=(4, 5))) gathered = base.oindex[np.array([3, 1, 2]), slice(None)] reversed_view = gathered[2:-1:-1, :] assert reversed_view.domain.inclusive_min[0] == -2 assert reversed_view.domain.exclusive_max[0] == 1 m = reversed_view.output[0] assert isinstance(m, ArrayMap) np.testing.assert_array_equal(m.index_array.reshape(-1), np.array([2, 1, 3])) class TestTranslate: """Oracle sections 4 and 12: translate_by / translate_to preserve the cell mapping.""" def test_translate_to_zero(self) -> None: t = _a()[2:10].translate_domain_to((0,)) assert (t.domain.inclusive_min, t.domain.exclusive_max) == ((0,), (8,)) assert _base_cells(t) == list(range(2, 10)) def test_translate_to_offset(self) -> None: t = _a().translate_domain_to((5,)) assert (t.domain.inclusive_min, t.domain.exclusive_max) == ((5,), (17,)) assert _base_cells(t) == list(range(12)) def test_translate_by_composes_with_stride(self) -> None: # a[::2].translate_by[5] -> domain [5, 11), base = 2*(coord-5) t = _a()[::2].translate_domain_by((5,)) assert (t.domain.inclusive_min, t.domain.exclusive_max) == ((5,), (11,)) assert _base_cells(t) == [0, 2, 4, 6, 8, 10] assert t[5].output[0].offset == 0 assert t[10].output[0].offset == 10 with pytest.raises(BoundsCheckError, match=r"valid indices \[5, 11\)"): t[0] def test_translate_strided_to(self) -> None: t = _a()[1:10:3].translate_domain_to((100,)) assert (t.domain.inclusive_min, t.domain.exclusive_max) == ((100,), (103,)) assert _base_cells(t) == [1, 4, 7] class TestFancyDims: """Oracle section 7: fancy dims get fresh [0, n); values are absolute coordinates.""" def test_index_array_values_are_coordinates(self) -> None: v = _a()[2:10] t = v.oindex[(np.array([3, 5], dtype=np.intp),)] assert (t.domain.inclusive_min, t.domain.exclusive_max) == ((0,), (2,)) m = t.output[0] assert isinstance(m, ArrayMap) storage = m.offset + m.stride * m.index_array np.testing.assert_array_equal(np.asarray(storage).ravel(), [3, 5]) def test_index_array_on_negative_origin(self) -> None: t = _w().oindex[(np.array([-10, -1], dtype=np.intp),)] m = t.output[0] assert isinstance(m, ArrayMap) storage = m.offset + m.stride * m.index_array np.testing.assert_array_equal(np.asarray(storage).ravel(), [0, 9]) def test_index_array_out_of_domain_raises(self) -> None: v = _a()[2:10] for bad in ([0, 3], [-1, 3], [3, 10]): with pytest.raises(BoundsCheckError): v.oindex[(np.array(bad, dtype=np.intp),)] zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_transform.py000066400000000000000000001463661527014137600306320ustar00rootroot00000000000000from __future__ import annotations from typing import cast import numpy as np import pytest from zarr_indexing.domain import IndexDomain from zarr_indexing.errors import BoundsCheckError from zarr_indexing.lazy_array import LazyArray from zarr_indexing.output_map import ArrayMap, ConstantMap, DimensionMap from zarr_indexing.transform import ( IndexTransform, ) class IndexLike: """A scalar integer selector implemented only through `__index__`.""" def __init__(self, value: int) -> None: self.value = value def __index__(self) -> int: return self.value class IntOnly: def __int__(self) -> int: return 2 class BadIndex: """An `__index__` that lies: the protocol requires an integer.""" def __index__(self) -> int: return cast("int", 2.5) class TestIndexTransformConstruction: def test_from_shape(self) -> None: t = IndexTransform.from_shape((10, 20)) assert t.input_rank == 2 assert t.output_rank == 2 assert t.domain.shape == (10, 20) assert t.domain.origin == (0, 0) for i, m in enumerate(t.output): assert isinstance(m, DimensionMap) assert m.input_dimension == i assert m.offset == 0 assert m.stride == 1 def test_identity(self) -> None: domain = IndexDomain(inclusive_min=(5,), exclusive_max=(15,)) t = IndexTransform.identity(domain) assert t.input_rank == 1 assert t.output_rank == 1 assert t.domain == domain assert isinstance(t.output[0], DimensionMap) assert t.output[0].input_dimension == 0 def test_from_shape_0d(self) -> None: t = IndexTransform.from_shape(()) assert t.input_rank == 0 assert t.output_rank == 0 assert t.domain.shape == () def test_custom_output_maps(self) -> None: domain = IndexDomain.from_shape((10,)) maps = (ConstantMap(offset=42), DimensionMap(input_dimension=0, offset=5, stride=2)) t = IndexTransform(domain=domain, output=maps) assert t.input_rank == 1 assert t.output_rank == 2 def test_validation_input_dimension_out_of_range(self) -> None: domain = IndexDomain.from_shape((10,)) maps = (DimensionMap(input_dimension=5),) with pytest.raises(ValueError, match="input_dimension"): IndexTransform(domain=domain, output=maps) class TestIndexTransformApply: @pytest.mark.parametrize( ("transform", "points", "expected"), [ pytest.param( IndexTransform.identity(IndexDomain((-2, 5), (1, 8))), np.array([-2, 7], dtype=np.int64), np.array([-2, 7], dtype=np.intp), id="identity-negative-and-nonzero-origins", ), pytest.param( IndexTransform( domain=IndexDomain((3,), (6,)), output=( ConstantMap(41), DimensionMap(0, offset=10, stride=-2), ), ), np.array([[3], [5]], dtype=np.int16), np.array([[41, 4], [41, 0]], dtype=np.intp), id="constant-and-negative-stride", ), pytest.param( IndexTransform( domain=IndexDomain((-2, 5), (1, 8)), output=( ArrayMap( np.array([[7], [11], [13]], dtype=np.intp), offset=-1, stride=2, ), ), ), np.array( [ [[-2, 5], [-1, 7]], [[0, 6], [-2, 6]], ], dtype=np.intp, ), np.array([[[13], [21]], [[25], [13]]], dtype=np.intp), id="array-map-singleton-broadcast-multidimensional-batch", ), pytest.param( IndexTransform(IndexDomain((), ()), (ConstantMap(42),)), np.empty((2, 0), dtype=np.intp), np.array([[42], [42]], dtype=np.intp), id="rank-zero-input", ), pytest.param( IndexTransform(IndexDomain((-1,), (2,)), ()), np.array([[-1], [1]], dtype=np.intp), np.empty((2, 0), dtype=np.intp), id="rank-zero-output", ), pytest.param( IndexTransform.identity(IndexDomain.from_shape((2,))), np.empty((0, 1), dtype=np.intp), np.empty((0, 1), dtype=np.intp), id="empty-batch", ), ], ) def test_apply_many_maps_integer_point_batches( self, transform: IndexTransform, points: np.ndarray, expected: np.ndarray, ) -> None: result = transform.apply_many(points) np.testing.assert_array_equal(result, expected) assert result.dtype == np.dtype(np.intp) assert result.flags.owndata def test_apply_maps_one_point(self) -> None: transform = IndexTransform( IndexDomain((-2, 4), (1, 7)), ( DimensionMap(1, offset=3, stride=-1), DimensionMap(0, offset=2, stride=2), ), ) assert transform.apply((-1, 6)) == (-3, 0) def test_apply_rejects_a_point_with_the_wrong_rank(self) -> None: with pytest.raises(ValueError, match=r"point must have shape \(2,\), got \(1,\)"): IndexTransform.from_shape((2, 3)).apply((1,)) def test_apply_rejects_an_explicitly_floating_rank_zero_point(self) -> None: transform = IndexTransform(IndexDomain((), ()), ()) with pytest.raises(TypeError, match="integer dtype"): transform.apply(np.array([], dtype=np.float64)) @pytest.mark.parametrize( "points", [ pytest.param(np.array(1, dtype=np.intp), id="no-coordinate-axis"), pytest.param(np.zeros((4, 3), dtype=np.intp), id="wrong-trailing-size"), ], ) def test_apply_many_rejects_an_invalid_coordinate_axis(self, points: np.ndarray) -> None: with pytest.raises(ValueError, match="trailing coordinate axis"): IndexTransform.from_shape((2, 3)).apply_many(points) @pytest.mark.parametrize( "points", [ pytest.param(np.array([[True]], dtype=np.bool_), id="bool"), pytest.param(np.array([[1.0]], dtype=np.float64), id="float"), pytest.param(np.array([["1"]], dtype=np.str_), id="string"), pytest.param(np.array([[1]], dtype=object), id="object"), ], ) def test_apply_many_rejects_non_integer_coordinates(self, points: np.ndarray) -> None: with pytest.raises(TypeError, match="integer dtype"): IndexTransform.from_shape((2,)).apply_many(points) def test_apply_many_reports_the_first_out_of_bounds_coordinate(self) -> None: transform = IndexTransform.identity(IndexDomain((-2, 10), (2, 13))) points = np.array( [ [[-2, 10], [-1, 20]], [[9, 11], [0, 12]], ], dtype=np.intp, ) with pytest.raises(BoundsCheckError) as error: transform.apply_many(points) assert str(error.value) == ( "point at batch position (0, 1) has input dimension 1 coordinate 20 outside [10, 13)" ) def test_apply_reports_a_single_point_error_without_batch_vocabulary(self) -> None: transform = IndexTransform.identity(IndexDomain((-10,), (10,))) with pytest.raises(BoundsCheckError) as error: transform.apply((11,)) assert str(error.value) == ( "coordinate 11 on input dimension 0 is outside the domain [-10, 10)" ) assert error.value.__cause__ is None assert error.value.__suppress_context__ @pytest.mark.parametrize( "beyond_intp", [ pytest.param(int(np.iinfo(np.intp).max) + 1, id="first-uint64-coordinate"), pytest.param(int(np.iinfo(np.uint64).max), id="maximum-uint64-coordinate"), ], ) def test_apply_many_maps_large_literal_coordinates_exactly(self, beyond_intp: int) -> None: transform = IndexTransform( IndexDomain((beyond_intp,), (beyond_intp + 1,)), (DimensionMap(0, offset=-beyond_intp),), ) result = transform.apply_many(np.array([[beyond_intp]], dtype=np.uint64)) np.testing.assert_array_equal(result, np.array([[0]], dtype=np.intp)) assert result.dtype == np.dtype(np.intp) assert result.flags.owndata @pytest.mark.parametrize( ("transform", "points"), [ pytest.param( IndexTransform( IndexDomain.from_shape((1,)), (ConstantMap(np.iinfo(np.intp).max + 1),), ), [[0]], id="constant", ), pytest.param( IndexTransform( IndexDomain.from_shape((2,)), (DimensionMap(0, offset=np.iinfo(np.intp).max),), ), [[1]], id="dimension", ), pytest.param( IndexTransform( IndexDomain.from_shape((1,)), ( ArrayMap( np.array([1], dtype=np.intp), offset=np.iinfo(np.intp).max, ), ), ), [[0]], id="array", ), pytest.param( IndexTransform.identity( IndexDomain( (int(np.iinfo(np.intp).max) + 1,), (int(np.iinfo(np.intp).max) + 2,), ) ), np.array([[int(np.iinfo(np.intp).max) + 1]], dtype=np.uint64), id="large-input-identity", ), ], ) def test_apply_many_rejects_mapped_coordinates_outside_intp( self, transform: IndexTransform, points: list[list[int]] | np.ndarray ) -> None: with pytest.raises(OverflowError, match="output coordinate.*np.intp"): transform.apply_many(points) def test_apply_many_rejects_affine_coordinate_overflow(self) -> None: transform = IndexTransform( domain=IndexDomain.from_shape((1,)), output=(ArrayMap(np.array([2**62], dtype=np.intp), stride=4),), ) with pytest.raises(OverflowError, match="outside np.intp"): transform.apply_many(np.array([[0]], dtype=np.intp)) def test_apply_rejects_affine_coordinate_overflow(self) -> None: transform = IndexTransform( domain=IndexDomain.from_shape((1,)), output=(ArrayMap(np.array([2**62], dtype=np.intp), stride=4),), ) with pytest.raises(OverflowError, match="outside np.intp"): transform.apply((0,)) class TestIndexTransformInverted: @pytest.mark.parametrize( ("transform", "points"), [ pytest.param( IndexTransform( IndexDomain((-3, 4), (1, 7)), ( DimensionMap(1, offset=10), DimensionMap(0, offset=2, stride=-1), ), ), np.array([[-3, 4], [0, 6]], dtype=np.intp), id="permutation-translation-reversal-nonzero-origin", ), pytest.param( IndexTransform( IndexDomain((5, -2), (8, -1)), (DimensionMap(0, offset=3), ConstantMap(99)), ), np.array([[5, -2], [7, -2]], dtype=np.intp), id="constant-and-unreferenced-singleton", ), pytest.param( IndexTransform(IndexDomain((), ()), ()), np.empty((1, 0), dtype=np.intp), id="rank-zero", ), ], ) def test_inverted_round_trips_points( self, transform: IndexTransform, points: np.ndarray ) -> None: inverse = transform.inverted() mapped = transform.apply_many(points) np.testing.assert_array_equal(inverse.apply_many(mapped), points) assert inverse.apply(transform.apply(tuple(points[0]))) == tuple(points[0]) assert inverse.inverted() == transform def test_inverted_rejects_unequal_ranks(self) -> None: transform = IndexTransform(IndexDomain.from_shape((2,)), (ConstantMap(1), ConstantMap(2))) with pytest.raises(ValueError, match="input rank must equal output rank"): transform.inverted() def test_inverted_rejects_an_array_map(self) -> None: transform = IndexTransform( IndexDomain.from_shape((2,)), (ArrayMap(np.array([1, 0], dtype=np.intp)),), ) with pytest.raises(ValueError, match="ArrayMap"): transform.inverted() def test_inverted_rejects_a_non_unit_stride(self) -> None: transform = IndexTransform( IndexDomain.from_shape((2,)), (DimensionMap(0, stride=2),), ) with pytest.raises(ValueError, match=r"stride must be \+1 or -1"): transform.inverted() def test_inverted_rejects_a_repeated_input_dimension(self) -> None: transform = IndexTransform( IndexDomain.from_shape((2, 1)), (DimensionMap(0), DimensionMap(0, offset=5)), ) with pytest.raises(ValueError, match="referenced more than once"): transform.inverted() def test_inverted_rejects_an_unreferenced_non_singleton_dimension(self) -> None: transform = IndexTransform( IndexDomain.from_shape((2, 2)), (DimensionMap(0), ConstantMap(7)), ) with pytest.raises(ValueError, match="unreferenced input dimension 1.*extent 2"): transform.inverted() def test_inverted_rejects_input_labels_that_cannot_be_preserved(self) -> None: transform = IndexTransform.identity(IndexDomain((0,), (2,), labels=("row",))) with pytest.raises(ValueError, match="input labels cannot be represented"): transform.inverted() class TestIndexTransformBasicIndexing: def test_slice_identity(self) -> None: """slice(None) on identity transform is a no-op.""" t = IndexTransform.from_shape((10, 20)) result = t[slice(None), slice(None)] assert result.domain.shape == (10, 20) assert result.input_rank == 2 assert result.output_rank == 2 def test_slice_narrows(self) -> None: t = IndexTransform.from_shape((10, 20)) result = t[2:8, 5:15] # Domains are preserved (TensorStore): the slice keeps its literal # coordinates, so the map stays the identity (out = in). assert result.domain.shape == (6, 10) assert result.domain.origin == (2, 5) assert isinstance(result.output[0], DimensionMap) assert result.output[0].offset == 0 assert result.output[0].stride == 1 assert result.output[0].input_dimension == 0 assert isinstance(result.output[1], DimensionMap) assert result.output[1].offset == 0 assert result.output[1].input_dimension == 1 def test_strided_slice(self) -> None: t = IndexTransform.from_shape((10,)) result = t[::2] assert result.domain.shape == (5,) assert isinstance(result.output[0], DimensionMap) assert result.output[0].offset == 0 assert result.output[0].stride == 2 def test_strided_slice_with_start(self) -> None: t = IndexTransform.from_shape((10,)) result = t[1:9:3] # indices: 1, 4, 7 -> 3 elements assert result.domain.shape == (3,) assert isinstance(result.output[0], DimensionMap) assert result.output[0].offset == 1 assert result.output[0].stride == 3 def test_int_drops_dimension(self) -> None: t = IndexTransform.from_shape((10, 20)) result = t[3] assert result.input_rank == 1 assert result.output_rank == 2 assert isinstance(result.output[0], ConstantMap) assert result.output[0].offset == 3 assert isinstance(result.output[1], DimensionMap) assert result.output[1].input_dimension == 0 def test_int_middle_dimension(self) -> None: t = IndexTransform.from_shape((10, 20, 30)) result = t[:, 5, :] assert result.input_rank == 2 assert result.output_rank == 3 assert isinstance(result.output[0], DimensionMap) assert result.output[0].input_dimension == 0 assert isinstance(result.output[1], ConstantMap) assert result.output[1].offset == 5 assert isinstance(result.output[2], DimensionMap) assert result.output[2].input_dimension == 1 def test_ellipsis(self) -> None: t = IndexTransform.from_shape((10, 20, 30)) result = t[2:8, ...] assert result.input_rank == 3 assert result.domain.shape == (6, 20, 30) def test_newaxis(self) -> None: t = IndexTransform.from_shape((10, 20)) result = t[np.newaxis, :, :] assert result.input_rank == 3 assert result.domain.shape == (1, 10, 20) assert result.output_rank == 2 assert isinstance(result.output[0], DimensionMap) assert result.output[0].input_dimension == 1 assert isinstance(result.output[1], DimensionMap) assert result.output[1].input_dimension == 2 def test_int_out_of_bounds(self) -> None: t = IndexTransform.from_shape((10,)) with pytest.raises(IndexError): t[10] def test_negative_int_is_literal(self) -> None: """Negative indices are literal coordinates (TensorStore convention), not 'from the end' like NumPy.""" t = IndexTransform.from_shape((10,)) with pytest.raises(IndexError): t[-1] # -1 is out of bounds for domain [0, 10) def test_negative_int_valid_with_negative_origin(self) -> None: """Negative index is valid if the domain includes negative coordinates.""" domain = IndexDomain(inclusive_min=(-5,), exclusive_max=(5,)) t = IndexTransform.identity(domain) result = t[-3] assert isinstance(result.output[0], ConstantMap) assert result.output[0].offset == -3 def test_composition_of_slices(self) -> None: """Slicing a sliced transform re-selects in literal domain coordinates.""" t = IndexTransform.from_shape((100,)) result = t[10:50][15:30] assert result.domain.shape == (15,) assert result.domain.origin == (15,) assert isinstance(result.output[0], DimensionMap) assert result.output[0].offset == 0 assert result.output[0].stride == 1 def test_composition_of_strides(self) -> None: t = IndexTransform.from_shape((100,)) result = t[::2][::3] # t[::2] -> shape (50,), offset=0, stride=2 # [::3] -> shape ceil(50/3)=17, offset=0, stride=2*3=6 assert result.domain.shape == (17,) assert isinstance(result.output[0], DimensionMap) assert result.output[0].stride == 6 def test_bare_int(self) -> None: """Non-tuple selection.""" t = IndexTransform.from_shape((10, 20)) result = t[3] assert result.input_rank == 1 def test_bare_slice(self) -> None: t = IndexTransform.from_shape((10, 20)) result = t[2:8] assert result.domain.shape == (6, 20) @pytest.mark.parametrize( ("mode", "selection", "expected_selection"), [ pytest.param("basic", IndexLike(2), 2, id="basic-scalar"), pytest.param( "basic", slice(IndexLike(1), IndexLike(7), IndexLike(2)), slice(1, 7, 2), id="basic-slice-components", ), pytest.param("oindex", IndexLike(2), 2, id="orthogonal-scalar"), pytest.param("vindex", IndexLike(2), 2, id="vectorized-scalar"), ], ) def test_literal_selectors_support_the_index_protocol( self, mode: str, selection: object, expected_selection: object ) -> None: transform = IndexTransform.from_shape((8,)) if mode == "basic": result = transform[selection] expected = transform[expected_selection] else: result = getattr(transform, mode)[selection] expected = getattr(transform, mode)[expected_selection] assert result == expected def test_literal_selector_rejects_int_only_objects(self) -> None: with pytest.raises(IndexError, match="unsupported selection type"): IndexTransform.from_shape((8,))[IntOnly()] def test_literal_selector_propagates_malformed_index_protocol(self) -> None: with pytest.raises(TypeError, match="__index__ returned non-int"): IndexTransform.from_shape((8,))[BadIndex()] def test_literal_slice_propagates_malformed_index_protocol(self) -> None: with pytest.raises(TypeError, match="__index__ returned non-int"): IndexTransform.from_shape((8,))[:: BadIndex()] class TestBasicIndexingOnArrayMaps: """When a transform already has ArrayMap outputs, basic indexing must apply the corresponding operation to the index_array's axes.""" def test_int_on_array_map_drops_axis(self) -> None: """Integer index on a dimension referenced by an ArrayMap should index into the array on that axis.""" arr = np.array([[10, 20], [30, 40], [50, 60]], dtype=np.intp) # 2D input domain (3, 2), one ArrayMap output t = IndexTransform( domain=IndexDomain.from_shape((3, 2)), output=(ArrayMap(index_array=arr),), ) # Index with int on dim 0 -> pick row 1 -> arr[1, :] = [30, 40] result = t[1] assert result.input_rank == 1 assert result.domain.shape == (2,) assert isinstance(result.output[0], ArrayMap) np.testing.assert_array_equal(result.output[0].index_array, np.array([30, 40])) def test_slice_on_array_map(self) -> None: """Slice on a dimension referenced by an ArrayMap should slice the array.""" arr = np.array([10, 20, 30, 40, 50], dtype=np.intp) t = IndexTransform( domain=IndexDomain.from_shape((5,)), output=(ArrayMap(index_array=arr),), ) result = t[1:4] assert result.domain.shape == (3,) assert isinstance(result.output[0], ArrayMap) np.testing.assert_array_equal(result.output[0].index_array, np.array([20, 30, 40])) def test_strided_slice_on_array_map(self) -> None: """Strided slice on ArrayMap should stride the array.""" arr = np.array([10, 20, 30, 40, 50], dtype=np.intp) t = IndexTransform( domain=IndexDomain.from_shape((5,)), output=(ArrayMap(index_array=arr),), ) result = t[::2] assert result.domain.shape == (3,) assert isinstance(result.output[0], ArrayMap) np.testing.assert_array_equal(result.output[0].index_array, np.array([10, 30, 50])) def test_newaxis_on_array_map(self) -> None: """Newaxis should insert an axis in the index_array.""" arr = np.array([10, 20, 30], dtype=np.intp) t = IndexTransform( domain=IndexDomain.from_shape((3,)), output=(ArrayMap(index_array=arr),), ) result = t[np.newaxis, :] assert result.input_rank == 2 assert result.domain.shape == (1, 3) assert isinstance(result.output[0], ArrayMap) assert result.output[0].index_array.shape == (1, 3) np.testing.assert_array_equal(result.output[0].index_array, np.array([[10, 20, 30]])) def test_int_drops_one_of_two_array_dims(self) -> None: """2D array map, int on dim 0, slice on dim 1.""" arr = np.array([[10, 20, 30], [40, 50, 60]], dtype=np.intp) t = IndexTransform( domain=IndexDomain.from_shape((2, 3)), output=(ArrayMap(index_array=arr),), ) result = t[0, 1:3] assert result.input_rank == 1 assert result.domain.shape == (2,) assert isinstance(result.output[0], ArrayMap) # arr[0, 1:3] = [20, 30] np.testing.assert_array_equal(result.output[0].index_array, np.array([20, 30])) class TestIndexTransformOindex: def test_oindex_int_array(self) -> None: t = IndexTransform.from_shape((10, 20)) idx = np.array([1, 3, 5], dtype=np.intp) result = t.oindex[idx, :] assert result.input_rank == 2 assert result.domain.shape == (3, 20) assert isinstance(result.output[0], ArrayMap) # Full input rank: the array varies along its own axis (0), singleton on 1. assert result.output[0].index_array.shape == (3, 1) np.testing.assert_array_equal(result.output[0].index_array, idx.reshape(3, 1)) assert result.output[0].offset == 0 assert result.output[0].stride == 1 assert isinstance(result.output[1], DimensionMap) assert result.output[1].input_dimension == 1 def test_oindex_bool_array(self) -> None: t = IndexTransform.from_shape((5,)) mask = np.array([True, False, True, False, True]) result = t.oindex[mask] assert result.domain.shape == (3,) assert isinstance(result.output[0], ArrayMap) np.testing.assert_array_equal( result.output[0].index_array, np.array([0, 2, 4], dtype=np.intp) ) def test_oindex_mixed(self) -> None: t = IndexTransform.from_shape((10, 20)) idx = np.array([2, 4], dtype=np.intp) result = t.oindex[idx, 5:15] assert result.input_rank == 2 assert result.domain.shape == (2, 10) # fancy dim: fresh zero-origin; slice dim: preserved literal coords assert result.domain.origin == (0, 5) assert isinstance(result.output[0], ArrayMap) assert isinstance(result.output[1], DimensionMap) assert result.output[1].offset == 0 def test_oindex_multiple_arrays(self) -> None: t = IndexTransform.from_shape((10, 20, 30)) idx0 = np.array([1, 3], dtype=np.intp) idx1 = np.array([5, 10, 15], dtype=np.intp) result = t.oindex[idx0, :, idx1] assert result.input_rank == 3 assert result.domain.shape == (2, 20, 3) assert isinstance(result.output[0], ArrayMap) assert isinstance(result.output[1], DimensionMap) assert isinstance(result.output[2], ArrayMap) def test_oindex_multiple_arrays_preserves_independent_axes(self) -> None: t = IndexTransform.from_shape((10, 20)) result = t.oindex[np.array([1, 3]), np.array([2, 4, 6])] assert result.domain.shape == (2, 3) assert isinstance(result.output[0], ArrayMap) assert isinstance(result.output[1], ArrayMap) assert result.output[0].index_array.shape == (2, 1) assert result.output[1].index_array.shape == (1, 3) class TestIndexTransformVindex: def test_vindex_single_array(self) -> None: t = IndexTransform.from_shape((10,)) idx = np.array([1, 3, 5], dtype=np.intp) result = t.vindex[idx] assert result.input_rank == 1 assert result.domain.shape == (3,) assert isinstance(result.output[0], ArrayMap) np.testing.assert_array_equal(result.output[0].index_array, idx) def test_vindex_broadcast(self) -> None: t = IndexTransform.from_shape((10, 20)) idx0 = np.array([[1, 2], [3, 4]], dtype=np.intp) idx1 = np.array([[10, 11], [12, 13]], dtype=np.intp) result = t.vindex[idx0, idx1] assert result.input_rank == 2 assert result.domain.shape == (2, 2) assert isinstance(result.output[0], ArrayMap) assert isinstance(result.output[1], ArrayMap) np.testing.assert_array_equal(result.output[0].index_array, idx0) np.testing.assert_array_equal(result.output[1].index_array, idx1) def test_vindex_with_slice(self) -> None: t = IndexTransform.from_shape((10, 20, 30)) idx = np.array([1, 3, 5], dtype=np.intp) result = t.vindex[idx, :, :] assert result.input_rank == 3 assert result.domain.shape == (3, 20, 30) assert isinstance(result.output[0], ArrayMap) def test_vindex_bool_mask(self) -> None: t = IndexTransform.from_shape((5,)) mask = np.array([True, False, True, False, True]) result = t.vindex[mask] assert result.domain.shape == (3,) assert isinstance(result.output[0], ArrayMap) def test_vindex_multidimensional_boolean_list_mask(self) -> None: result = IndexTransform.from_shape((2, 3)).vindex[ [[True, False, True], [False, True, False]] ] assert result.domain.shape == (3,) np.testing.assert_array_equal( result.apply_many(np.array([[0], [1], [2]], dtype=np.intp)), np.array([[0, 0], [0, 2], [1, 1]], dtype=np.intp), ) def test_vindex_broadcast_different_shapes(self) -> None: t = IndexTransform.from_shape((10, 20)) idx0 = np.array([1, 2, 3], dtype=np.intp) idx1 = np.array([[10], [11]], dtype=np.intp) result = t.vindex[idx0, idx1] assert result.input_rank == 2 assert result.domain.shape == (2, 3) def test_vindex_multiple_arrays_preserves_shared_axes(self) -> None: t = IndexTransform.from_shape((10, 20)) result = t.vindex[np.array([1, 3]), np.array([2, 4])] assert result.domain.shape == (2,) assert isinstance(result.output[0], ArrayMap) assert isinstance(result.output[1], ArrayMap) assert result.output[0].index_array.shape == (2,) assert result.output[1].index_array.shape == (2,) @pytest.mark.parametrize("mode", ["oindex", "vindex"]) @pytest.mark.parametrize("as_list", [False, True]) @pytest.mark.parametrize("value", [2**63, 2**64 - 1]) def test_direct_advanced_index_rejects_unsigned_overflow( mode: str, as_list: bool, value: int ) -> None: """Narrowing must not turn huge positive coordinates into valid negative ones.""" transform = IndexTransform.identity( IndexDomain(inclusive_min=(np.iinfo(np.intp).min,), exclusive_max=(0,)) ) selector = [value] if as_list else np.array([value], dtype=np.uint64) with pytest.raises(OverflowError, match="outside np.intp range"): getattr(transform, mode)[selector] @pytest.mark.parametrize("mode", ["oindex", "vindex"]) @pytest.mark.parametrize("dtype", ["intp", "uint8", "uint64"]) def test_direct_advanced_index_preserves_valid_coordinates(mode: str, dtype: str) -> None: transform = IndexTransform.from_shape((5,)) selector = np.array([3, 0, 3], dtype=dtype) selector.flags.writeable = False result = getattr(transform, mode)[selector] np.testing.assert_array_equal(result.apply_many(np.arange(3).reshape(3, 1)), [[3], [0], [3]]) np.testing.assert_array_equal(selector, [3, 0, 3]) @pytest.mark.parametrize("mode", ["oindex", "vindex"]) def test_direct_advanced_index_rejects_float_arrays(mode: str) -> None: helper = getattr(IndexTransform.from_shape((5,)), mode) with pytest.raises(IndexError, match="integer or boolean"): helper[np.array([1.9, 3.2])] @pytest.mark.parametrize("mode", ["oindex", "vindex"]) def test_direct_advanced_index_rejects_wrong_length_boolean_mask(mode: str) -> None: helper = getattr(IndexTransform.from_shape((5,)), mode) with pytest.raises(IndexError, match="boolean index.*dimension 5"): helper[np.array([True, False])] class TestSelectionToTransform: def test_basic_slice(self) -> None: t = IndexTransform.from_shape((10, 20)) result = t.select((slice(2, 8), slice(5, 15)), "basic") assert result.domain.shape == (6, 10) assert result.domain.origin == (2, 5) # preserved literal coordinates assert isinstance(result.output[0], DimensionMap) assert result.output[0].offset == 0 def test_basic_int(self) -> None: t = IndexTransform.from_shape((10, 20)) result = t.select((3, slice(None)), "basic") assert result.input_rank == 1 assert isinstance(result.output[0], ConstantMap) assert result.output[0].offset == 3 def test_basic_ellipsis(self) -> None: t = IndexTransform.from_shape((10, 20)) result = t.select(Ellipsis, "basic") assert result.domain.shape == (10, 20) def test_orthogonal(self) -> None: t = IndexTransform.from_shape((10, 20)) idx = np.array([1, 3, 5], dtype=np.intp) result = t.select((idx, slice(None)), "orthogonal") assert result.domain.shape == (3, 20) assert isinstance(result.output[0], ArrayMap) def test_vectorized(self) -> None: t = IndexTransform.from_shape((10, 20)) idx0 = np.array([1, 3], dtype=np.intp) idx1 = np.array([5, 7], dtype=np.intp) result = t.select((idx0, idx1), "vectorized") assert result.domain.shape == (2,) assert isinstance(result.output[0], ArrayMap) assert isinstance(result.output[1], ArrayMap) def test_composition_with_non_identity(self) -> None: """Indexing a sliced transform uses literal domain coordinates. The slice [10:50] preserves its domain, so a follow-up [15:30] re-selects coordinates 15..29 of the base (TensorStore semantics), and the composed map stays the identity (out = in). """ t = IndexTransform.from_shape((100,))[10:50] result = t.select(slice(15, 30), "basic") assert (result.domain.inclusive_min, result.domain.exclusive_max) == ((15,), (30,)) assert isinstance(result.output[0], DimensionMap) assert result.output[0].offset == 0 assert result.output[0].stride == 1 class TestIndexTransformIntersect: def test_constant_inside(self) -> None: t = IndexTransform( domain=IndexDomain.from_shape((10,)), output=(ConstantMap(offset=5),), ) result = t.intersect(IndexDomain(inclusive_min=(0,), exclusive_max=(10,))) assert result is not None restricted, surviving = result assert isinstance(restricted.output[0], ConstantMap) assert restricted.output[0].offset == 5 assert surviving is None def test_constant_outside(self) -> None: t = IndexTransform( domain=IndexDomain.from_shape((10,)), output=(ConstantMap(offset=5),), ) result = t.intersect(IndexDomain(inclusive_min=(10,), exclusive_max=(20,))) assert result is None def test_dimension_partial(self) -> None: """DimensionMap over [0,10) intersected with [5,15) narrows input to [5,10).""" t = IndexTransform.from_shape((10,)) result = t.intersect(IndexDomain(inclusive_min=(5,), exclusive_max=(15,))) assert result is not None restricted, surviving = result assert restricted.domain.inclusive_min == (5,) assert restricted.domain.exclusive_max == (10,) assert surviving is None def test_dimension_no_overlap(self) -> None: t = IndexTransform.from_shape((10,)) result = t.intersect(IndexDomain(inclusive_min=(20,), exclusive_max=(30,))) assert result is None def test_dimension_strided(self) -> None: """stride=2, offset=1 over [0,5): storage 1,3,5,7,9. Chunk [4,8).""" t = IndexTransform( domain=IndexDomain.from_shape((5,)), output=(DimensionMap(input_dimension=0, offset=1, stride=2),), ) result = t.intersect(IndexDomain(inclusive_min=(4,), exclusive_max=(8,))) assert result is not None restricted, _surviving = result # input 2->5, input 3->7. Both in [4,8). assert restricted.domain.inclusive_min == (2,) assert restricted.domain.exclusive_max == (4,) @pytest.mark.parametrize( ("input_domain", "output_domain", "output_map", "expected"), [ ( (2**53, 2**53 + 3), (2**53 + 1, 2**53 + 2), DimensionMap(input_dimension=0), (2**53 + 1, 2**53 + 2), ), ( (-(2**53) - 2, -(2**53) + 1), (-(2**53) - 1, -(2**53)), DimensionMap(input_dimension=0), (-(2**53) - 1, -(2**53)), ), ( (-(2**53) - 2, -(2**53) + 1), (2**53 + 1, 2**53 + 2), DimensionMap(input_dimension=0, stride=-1), (-(2**53) - 1, -(2**53)), ), ( (2**53, 2**53 + 3), (-(2**53) - 2, -(2**53) - 1), DimensionMap(input_dimension=0, stride=-1), (2**53 + 2, 2**53 + 3), ), ], ids=[ "positive-coordinates-positive-stride", "negative-coordinates-positive-stride", "positive-coordinates-negative-stride", "negative-coordinates-negative-stride", ], ) def test_dimension_intersection_is_exact_above_float_precision( self, input_domain: tuple[int, int], output_domain: tuple[int, int], output_map: DimensionMap, expected: tuple[int, int], ) -> None: transform = IndexTransform( domain=IndexDomain((input_domain[0],), (input_domain[1],)), output=(output_map,), ) result = transform.intersect(IndexDomain((output_domain[0],), (output_domain[1],))) assert result is not None restricted, _surviving = result assert restricted.domain == IndexDomain((expected[0],), (expected[1],)) def test_dimension_intersection_accepts_unbounded_python_integer_precision(self) -> None: huge = 10**400 transform = IndexTransform( domain=IndexDomain((huge,), (huge + 2,)), output=(DimensionMap(input_dimension=0),), ) result = transform.intersect(IndexDomain((huge + 1,), (huge + 2,))) assert result is not None restricted, _surviving = result assert restricted.domain == IndexDomain((huge + 1,), (huge + 2,)) def test_array_partial(self) -> None: arr = np.array([3, 8, 15, 22], dtype=np.intp) t = IndexTransform( domain=IndexDomain.from_shape((4,)), output=(ArrayMap(index_array=arr),), ) result = t.intersect(IndexDomain(inclusive_min=(5,), exclusive_max=(20,))) assert result is not None restricted, surviving = result assert isinstance(restricted.output[0], ArrayMap) np.testing.assert_array_equal(restricted.output[0].index_array, np.array([8, 15])) assert surviving is not None np.testing.assert_array_equal(surviving, np.array([1, 2])) def test_array_none_inside(self) -> None: arr = np.array([1, 2, 3], dtype=np.intp) t = IndexTransform( domain=IndexDomain.from_shape((3,)), output=(ArrayMap(index_array=arr),), ) assert t.intersect(IndexDomain(inclusive_min=(10,), exclusive_max=(20,))) is None def test_2d_mixed(self) -> None: """2D: ConstantMap on dim 0, DimensionMap on dim 1.""" t = IndexTransform( domain=IndexDomain.from_shape((10,)), output=( ConstantMap(offset=5), DimensionMap(input_dimension=0, offset=0, stride=1), ), ) chunk = IndexDomain(inclusive_min=(0, 5), exclusive_max=(10, 15)) result = t.intersect(chunk) assert result is not None restricted, _ = result assert isinstance(restricted.output[0], ConstantMap) assert restricted.output[0].offset == 5 assert isinstance(restricted.output[1], DimensionMap) assert restricted.domain.inclusive_min == (5,) assert restricted.domain.exclusive_max == (10,) class TestIndexTransformTranslate: def test_translate_constant(self) -> None: t = IndexTransform( domain=IndexDomain.from_shape((10,)), output=(ConstantMap(offset=5),), ) result = t.translate((-5,)) assert isinstance(result.output[0], ConstantMap) assert result.output[0].offset == 0 def test_translate_dimension(self) -> None: t = IndexTransform.from_shape((10,)) result = t.translate((-3,)) assert isinstance(result.output[0], DimensionMap) assert result.output[0].offset == -3 assert result.output[0].stride == 1 def test_translate_array(self) -> None: arr = np.array([5, 10], dtype=np.intp) t = IndexTransform( domain=IndexDomain.from_shape((2,)), output=(ArrayMap(index_array=arr, offset=3),), ) result = t.translate((-3,)) assert isinstance(result.output[0], ArrayMap) assert result.output[0].offset == 0 np.testing.assert_array_equal(result.output[0].index_array, arr) def test_translate_2d(self) -> None: t = IndexTransform.from_shape((10, 20)) result = t.translate((-5, -10)) assert isinstance(result.output[0], DimensionMap) assert result.output[0].offset == -5 assert isinstance(result.output[1], DimensionMap) assert result.output[1].offset == -10 class TestArrayMapDependencyAxes: """`ArrayMap.dependency_axes` derives the input axes an array varies on from its (full-rank) shape: non-singleton axes vary, singleton axes do not.""" def test_orthogonal_single_axis(self) -> None: t = IndexTransform.from_shape((10, 20)).oindex[np.array([1, 3]), np.array([2, 4, 6])] m0, m1 = t.output[0], t.output[1] assert isinstance(m0, ArrayMap) assert isinstance(m1, ArrayMap) assert m0.dependency_axes == (0,) assert m1.dependency_axes == (1,) def test_vectorized_shares_axes(self) -> None: t = IndexTransform.from_shape((10, 20)).vindex[np.array([1, 3]), np.array([2, 4])] m0, m1 = t.output[0], t.output[1] assert isinstance(m0, ArrayMap) assert isinstance(m1, ArrayMap) assert m0.dependency_axes == (0,) assert m1.dependency_axes == (0,) def test_scalar_array_has_no_dependency(self) -> None: assert ArrayMap(np.ones((1, 1), dtype=np.intp)).dependency_axes == () def test_zero_length_axis_has_no_dependency(self) -> None: """An axis of size 0 carries no dependency either: it selects nothing, so the array does not vary along it any more than along a singleton.""" assert ArrayMap(np.zeros((0, 4), dtype=np.intp)).dependency_axes == (1,) assert ArrayMap(np.zeros((3, 0), dtype=np.intp)).dependency_axes == (0,) assert ArrayMap(np.zeros((0, 1), dtype=np.intp)).dependency_axes == () def test_zero_length_axis_does_not_make_a_map_correlated(self) -> None: """An empty orthogonal selection is legal, so it must classify as one.""" m = ArrayMap(index_array=np.zeros((0, 4), dtype=np.intp)) assert m.dependent_axis == 1 class TestIntersectArrayMapClassification: """`_intersect` must distinguish orthogonal (outer-product) ArrayMaps from correlated (vectorized) ones by their dependency axes, keep surviving arrays at full input rank, and preserve residual (slice) dimensions.""" def test_orthogonal_outer_product_keeps_full_rank(self) -> None: """Two arrays on distinct axes narrow independently and stay full rank; out_indices is a per-output-dim dict of surviving positions.""" t = IndexTransform.from_shape((10, 10)).oindex[np.array([1, 3, 8]), np.array([2, 6, 9])] # Chunk covering storage [0,5) x [0,5): rows 1,3 survive (out pos 0,1), # cols 2 survives (out pos 0). chunk = IndexDomain(inclusive_min=(0, 0), exclusive_max=(5, 5)) result = t.intersect(chunk) assert result is not None restricted, out_indices = result assert isinstance(restricted.output[0], ArrayMap) assert isinstance(restricted.output[1], ArrayMap) # Full input rank preserved (not raveled to 1-D). assert restricted.output[0].index_array.ndim == 2 assert restricted.output[1].index_array.ndim == 2 assert restricted.domain.ndim == 2 assert isinstance(out_indices, dict) np.testing.assert_array_equal(out_indices[0], np.array([0, 1])) np.testing.assert_array_equal(out_indices[1], np.array([0])) def test_correlated_with_residual_slice_preserves_slice_dim(self) -> None: """A vindex transform with two correlated arrays plus a residual slice dim intersects without a rank error and keeps the DimensionMap.""" t = IndexTransform.from_shape((4, 3, 5)).vindex[np.array([1, 3]), np.array([2, 0])] # Chunk covering storage [0,2) x [2,3) x [0,5): only point (1,2,*) is in # bounds on both array dims -> one surviving broadcast point. chunk = IndexDomain(inclusive_min=(0, 2, 0), exclusive_max=(2, 3, 5)) result = t.intersect(chunk) assert result is not None restricted, out_indices = result # A DimensionMap for the residual slice dim survives (no post-init error). assert any(isinstance(m, DimensionMap) for m in restricted.output) assert out_indices is not None def test_length1_orthogonal_collapses_to_a_constant(self) -> None: """A length-1 orthogonal array holds one coordinate: it is a ConstantMap. The length-1 axis stays in the domain, and the remaining genuine array intersects orthogonally — a single survivor vector, not a joint gather. """ t = IndexTransform.from_shape((6, 6)).oindex[np.array([2]), np.array([1, 3, 5])] assert isinstance(t.output[0], ConstantMap) assert t.domain.shape == (1, 3) chunk = IndexDomain(inclusive_min=(0, 0), exclusive_max=(6, 6)) result = t.intersect(chunk) assert result is not None _restricted, out_indices = result assert isinstance(out_indices, np.ndarray) np.testing.assert_array_equal(out_indices, [0, 1, 2]) class TestDerivedMapDependency: """Derived map dependencies follow the index array's shape. Vectorized and orthogonal composition must preserve the selected values and their placement for each partitioning.""" def test_a_vindex_over_a_fancy_view_is_marked_correlated(self) -> None: base = np.arange(6) view = LazyArray(base).oindex[np.array([0, 1])].vindex[np.array([[0, 1, 0], [1, 0, 1]])] np.testing.assert_array_equal( np.asarray(view.result()), base[[0, 1]][[[0, 1, 0], [1, 0, 1]]] ) def test_the_same_view_resolves_alike_however_it_is_partitioned(self) -> None: base = np.arange(36).reshape(6, 6) def build(array: LazyArray) -> LazyArray: return array.oindex[np.array([-3, -6, -4]), -4].vindex[np.array([[-2, -3]])] unpartitioned = np.asarray(build(LazyArray(base)).result()) partitioned = np.asarray(build(LazyArray(base).with_parts((3, 3))).result()) np.testing.assert_array_equal(partitioned, unpartitioned) np.testing.assert_array_equal(unpartitioned, np.array([[2, 20]])) def test_dependency_axes_are_read_from_the_shape(self) -> None: """What a map varies over is its non-singleton axes — nothing else. The retired `input_dimension` field could contradict the array it rode on; the shape cannot. """ t = IndexTransform( domain=IndexDomain.from_shape((2, 3)), output=( ArrayMap(index_array=np.array([[0, 1, 2]], dtype=np.intp)), ArrayMap(index_array=np.array([[0], [1]], dtype=np.intp)), ), ) assert t.index_array_structure == "orthogonal" def test_an_orthogonal_step_over_a_correlated_view_is_an_outer_product() -> None: """Orthogonal indexing after vectorized indexing selects an outer product.""" base = np.arange(14).reshape(7, 2) view = LazyArray(base).vindex[ np.array([[5, 5], [1, 2], [0, 4]]), np.array([[1, 1], [1, 0], [1, 0]]) ] result = np.asarray(view.oindex[np.array([1, 1, 0]), np.array([1, 1, 0, 1])].result()) assert result.shape == (3, 4) np.testing.assert_array_equal(result, np.array([[4, 4, 3, 4], [4, 4, 3, 4], [11, 11, 11, 11]])) @pytest.mark.parametrize( ("value", "description"), [(1, "one below the lower bound"), (10, "the exclusive upper bound itself")], ) def test_an_index_array_value_just_outside_the_domain_is_refused( value: int, description: str ) -> None: """Reject the coordinate below the lower bound and the exclusive upper bound.""" transform = IndexTransform.from_shape((12,))[2:10] with pytest.raises(BoundsCheckError, match="out of bounds"): transform.oindex[np.array([value, 3])] def test_an_index_array_value_at_each_end_of_the_domain_is_accepted() -> None: """The other side of the same boundary: the extremes themselves are in range.""" transform = IndexTransform.from_shape((12,))[2:10] array_map = transform.oindex[np.array([2, 9])].output[0] assert isinstance(array_map, ArrayMap) np.testing.assert_array_equal(array_map.index_array, np.array([2, 9])) # --------------------------------------------------------------------------- # Intersecting diagonal gathers # --------------------------------------------------------------------------- def test_intersecting_a_diagonal_gather_keeps_points_inside_the_domain() -> None: """Index arrays sharing an input axis intersect pointwise, like vindex.""" rows = np.array([4, 0, 2]) cols = np.array([1, 5, 2]) transform = IndexTransform( domain=IndexDomain.from_shape((3,)), output=( ArrayMap(index_array=rows), ArrayMap(index_array=cols), ), ) result = transform.intersect(IndexDomain(inclusive_min=(0, 0), exclusive_max=(3, 3))) assert result is not None restricted, survivors = result # Only the point (2, 2) has both coordinates inside [0, 3) x [0, 3). assert restricted.domain.shape == (1,) assert isinstance(survivors, np.ndarray) np.testing.assert_array_equal(survivors, [2]) np.testing.assert_array_equal(restricted.apply((0,)), (2, 2)) assert transform.intersect(IndexDomain(inclusive_min=(0, 0), exclusive_max=(1, 1))) is None def test_index_array_structure_classifies_the_three_shapes() -> None: base = IndexTransform.from_shape((4, 6)) assert (base[1:, ::2]).index_array_structure == "none" assert (base.oindex[np.array([0, 2]), slice(None)]).index_array_structure == "orthogonal" assert (base.vindex[np.array([0, 2]), np.array([1, 3])]).index_array_structure == "general" diagonal = IndexTransform( domain=IndexDomain.from_shape((2,)), output=( ArrayMap(index_array=np.array([0, 1])), ArrayMap(index_array=np.array([2, 3])), ), ) assert diagonal.index_array_structure == "general" @pytest.mark.parametrize("array_first", [False, True]) def test_intersect_rejects_array_and_affine_shared_input_axis(array_first: bool) -> None: """Joint filtering of affine and lookup coordinates must not return extra cells.""" affine = DimensionMap(0) lookup = ArrayMap(np.array([0, 3, 1, 2])) output = (lookup, affine) if array_first else (affine, lookup) transform = IndexTransform(IndexDomain.from_shape((4,)), output) output_domain = IndexDomain((0, 1), (2, 4)) if array_first else IndexDomain((1, 0), (4, 2)) with pytest.raises(NotImplementedError, match="also bound by a slice map"): transform.intersect(output_domain) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_wire_boundaries.py000066400000000000000000000044131527014137600317620ustar00rootroot00000000000000"""Boundary validation must preserve coordinates and canonical messages.""" from typing import Any import numpy as np import pytest from zarr_indexing._wire import lower_index_array from zarr_indexing.messages import NdselError, normalize_ndsel @pytest.mark.parametrize("raw", [[True, 2], [[0, False]], [True, False]]) def test_index_array_rejects_boolean_elements(raw: Any) -> None: with pytest.raises(NdselError, match="integers"): lower_index_array(raw, "index_array") @pytest.mark.parametrize("value", [int(np.iinfo(np.intp).max) + 1, int(np.iinfo(np.intp).min) - 1]) def test_index_array_rejects_out_of_range_coordinates(value: int) -> None: with pytest.raises(NdselError): lower_index_array([value], "index_array") @pytest.mark.parametrize( "raw", [[], [[], []], [0, -1, 2], [int(np.iinfo(np.intp).min), int(np.iinfo(np.intp).max)]] ) def test_index_array_preserves_integer_coordinates(raw: Any) -> None: result = lower_index_array(raw, "index_array") assert result.dtype == np.dtype(np.intp) assert result.tolist() == raw @pytest.mark.parametrize(("start", "stop"), [(-(2**63), -(2**63)), (-(2**63) + 1, -(2**63))]) def test_slice_rejects_unrepresentable_canonical_bounds(start: int, stop: int) -> None: with pytest.raises(NdselError): normalize_ndsel({"kind": "slice", "start": [start], "stop": [stop], "step": [-1]}) @pytest.mark.parametrize( "message", [ {"kind": "box", "shape": [1] * 33}, {"kind": "slice", "start": [0] * 33, "stop": [1] * 33}, {"kind": "transform", "input_shape": [1] * 33}, ], ) def test_normalize_rejects_excessive_inferred_rank(message: dict[str, Any]) -> None: with pytest.raises(NdselError, match="rank"): normalize_ndsel(message) @pytest.mark.parametrize( "message", [ {"kind": "box", "shape": [1] * 32}, {"kind": "transform", "input_rank": 32}, {"kind": "slice", "start": [-(2**63) + 1], "stop": [-(2**63) + 1], "step": [-1]}, {"kind": "slice", "start": [2**63 - 1], "stop": [-(2**63) + 1], "step": [-(2**63)]}, ], ) def test_normalize_boundary_results_are_idempotent(message: dict[str, Any]) -> None: canonical = normalize_ndsel(message) assert normalize_ndsel({"kind": "transform", **canonical}) == canonical zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_wire_bounds.py000066400000000000000000000121621527014137600311210ustar00rootroot00000000000000"""Index-array bounds are validated against raw coordinates during loading.""" from __future__ import annotations from typing import TYPE_CHECKING, Any import numpy as np import pytest from zarr_indexing import ConstantMap, IndexTransform, normalize_ndsel from zarr_indexing.messages import NdselError from zarr_indexing.output_map import output_index_map_from_json if TYPE_CHECKING: from zarr_indexing.json import IndexTransformJSON, OutputIndexMapJSON def load(values: Any, bounds: Any, entry: str, offset: int, stride: int) -> Any: output: OutputIndexMapJSON = {"index_array": values, "offset": offset, "stride": stride} if bounds is not None: output["index_array_bounds"] = bounds if entry == "output_map": return output_index_map_from_json(output) body: IndexTransformJSON = {"input_shape": list(np.asarray(values).shape), "output": [output]} return IndexTransform.from_json(body) @pytest.mark.parametrize("entry", ["transform", "output_map"]) @pytest.mark.parametrize(("offset", "stride"), [(0, 1), (10, -2), (7, 0)]) @pytest.mark.parametrize( ("values", "bounds"), [ ([-3, 0, 4], [-3, 4]), ([-3, 0, 4], ["-inf", 4]), ([-3, 0, 4], [-3, "+inf"]), ([-3, 0, 4], ["-inf", "+inf"]), ([-3, 0, 4], None), ([2], [2, 2]), ([], [2, 2]), ([], ["+inf", "+inf"]), ([], ["-inf", "-inf"]), ([[0, 2], [2, 0]], [0, 2]), ], ) def test_valid_bounds_preserve_values_and_roundtrip( values: Any, bounds: Any, entry: str, offset: int, stride: int ) -> None: model = np.asarray(values, dtype=np.intp) loaded = load(values, bounds, entry, offset, stride) expected = offset + stride * model if entry == "transform": points = np.array(list(np.ndindex(model.shape)), dtype=np.intp).reshape(-1, model.ndim) restored = IndexTransform.from_json(loaded.to_json()) np.testing.assert_array_equal(loaded.apply_many(points).reshape(model.shape), expected) np.testing.assert_array_equal(restored.apply_many(points).reshape(model.shape), expected) # Further selection and affine adjustment operate on validated coordinates. if model.ndim == 1 and model.size > 1: selected = loaded.oindex[[model.size - 1, 0]].translate((3,)) np.testing.assert_array_equal( selected.apply_many([[0], [1]]).ravel(), expected[[-1, 0]] + 3 ) else: restored = output_index_map_from_json(loaded.to_json()) if isinstance(restored, ConstantMap): np.testing.assert_array_equal(np.full(model.shape, restored.offset), expected) else: assert restored == loaded np.testing.assert_array_equal(loaded.offset + loaded.stride * loaded.index_array, expected) if bounds is not None: canonical = normalize_ndsel( { "kind": "transform", "input_shape": list(model.shape), "output": [{"index_array": values, "index_array_bounds": bounds}], } ) assert canonical["output"][0]["index_array_bounds"] == bounds @pytest.mark.parametrize("entry", ["transform", "output_map"]) @pytest.mark.parametrize(("offset", "stride"), [(0, 1), (10, -2), (7, 0)]) @pytest.mark.parametrize( ("values", "bounds"), [ ([-1, 0], [0, 2]), ([0, 3], [0, 2]), ([3], [2, 2]), ([-1], [0, "+inf"]), ([3], ["-inf", 2]), ([0], ["+inf", "+inf"]), ([0], ["-inf", "-inf"]), ([[0, 1], [1, 3]], [0, 2]), ], ) def test_out_of_bounds_raw_values_are_rejected( values: Any, bounds: Any, entry: str, offset: int, stride: int ) -> None: with pytest.raises(NdselError, match="index_array_bounds") as exc: load(values, bounds, entry, offset, stride) assert exc.value.reason == "invalid_json" @pytest.mark.parametrize("entry", ["transform", "output_map"]) @pytest.mark.parametrize("values", [[], [1]]) @pytest.mark.parametrize("bounds", [[], [0], [0, 1, 2], [False, 2], [0.5, 2], ["bad", 2]]) def test_malformed_bounds_are_rejected(values: Any, bounds: Any, entry: str) -> None: with pytest.raises(NdselError) as exc: load(values, bounds, entry, 0, 1) assert exc.value.reason == "invalid_json" @pytest.mark.parametrize("entry", ["transform", "output_map"]) @pytest.mark.parametrize("values", [[], [1]]) def test_reversed_bounds_are_rejected(values: Any, entry: str) -> None: with pytest.raises(NdselError) as exc: load(values, [2, 0], entry, 0, 1) assert exc.value.reason == "bounds_out_of_order" @pytest.mark.parametrize("entry", ["transform", "output_map"]) @pytest.mark.parametrize("value", [int(np.iinfo(np.intp).min), int(np.iinfo(np.intp).max)]) def test_integer_extreme_bounds_are_exact(entry: str, value: int) -> None: result = load([value], [value, value], entry, 0, 1) if entry == "transform": assert result.apply((0,)) == (value,) else: assert int(result.index_array[0]) == value bounds = [value + 1, "+inf"] if value < 0 else ["-inf", value - 1] with pytest.raises(NdselError, match="index_array_bounds"): load([value], bounds, entry, 0, 1) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/tests/test_writer.py000066400000000000000000000443031527014137600301170ustar00rootroot00000000000000from __future__ import annotations from typing import Any import numpy as np import pytest from zarr_indexing.domain import IndexDomain from zarr_indexing.output_map import ArrayMap, ConstantMap, DimensionMap from zarr_indexing.transform import IndexTransform from zarr_indexing.writer import write_into class BasicSource: """Reject source reads and advanced writes, recording basic writes.""" def __init__(self, data: np.ndarray[Any, Any]) -> None: self.data = data self.shape = data.shape self.dtype = data.dtype self.calls: list[tuple[int | slice, ...]] = [] def __setitem__(self, key: tuple[int | slice, ...], value: Any) -> None: assert all(isinstance(item, (int, slice)) for item in key) assert all( not isinstance(item, slice) or item.step is None or item.step > 0 for item in key ) self.calls.append(key) self.data[key] = value @pytest.mark.parametrize( "case", ["reverse", "scalar", "newaxis", "empty", "outer", "composed", "correlated", "transpose"], ) def test_write_coordinates(case: str) -> None: data = np.arange(24).reshape(4, 6) expected = data.copy() # Select flat source identifiers independently of the transform evaluator. identifiers = np.arange(data.size).reshape(data.shape) transform = IndexTransform.from_shape(data.shape) if case == "reverse": transform = transform[::-1, ::2] identifiers = identifiers[::-1, ::2] elif case == "scalar": transform = transform[2, 3] identifiers = identifiers[2, 3] elif case == "newaxis": transform = IndexTransform( IndexDomain.from_shape((1, 4, 6)), (DimensionMap(1), DimensionMap(2)) ) identifiers = identifiers[None] elif case == "empty": transform = transform[1:1, :] identifiers = identifiers[1:1, :] elif case == "outer": transform = transform.oindex[[3, 1, 3], [5, 0, 5]] identifiers = identifiers[np.ix_([3, 1, 3], [5, 0, 5])] elif case == "composed": transform = transform.oindex[[3, 1, 3], :].oindex[[2, 0], [5, 0]] identifiers = identifiers[[3, 1, 3]][np.ix_([2, 0], [5, 0])] elif case == "correlated": transform = IndexTransform( IndexDomain.from_shape((3, 2)), ( ArrayMap(np.array([[3], [0], [3]], dtype=np.intp)), ArrayMap(np.array([[5, 1], [0, 2], [5, 1]], dtype=np.intp)), ), ) identifiers = identifiers[np.array([[3], [0], [3]]), np.array([[5, 1], [0, 2], [5, 1]])] else: transform = IndexTransform( IndexDomain.from_shape((6, 4)), (DimensionMap(1), DimensionMap(0)) ) identifiers = identifiers.T values = np.arange(identifiers.size).reshape(identifiers.shape) + 100 for index, value in zip(np.asarray(identifiers).flat, values.flat, strict=True): expected.flat[index] = value source = BasicSource(data) assert write_into(source, transform, values) is None np.testing.assert_array_equal(data, expected) if case in ("reverse", "scalar", "newaxis", "transpose"): assert len(source.calls) == 1 elif case == "empty": assert source.calls == [] @pytest.mark.parametrize("fancy", [False, True]) def test_write_alias_and_broadcast(fancy: bool) -> None: data = np.arange(6) expected = data[::-1].copy() transform = IndexTransform.from_shape(data.shape) transform = transform.oindex[[5, 4, 3, 2, 1, 0]] if fancy else transform[::-1] write_into(BasicSource(data), transform, data) np.testing.assert_array_equal(data, expected) write_into(BasicSource(data), transform, np.array([[17]])) np.testing.assert_array_equal(data, np.full(6, 17)) def test_write_broadcast_error_before_mutation() -> None: source = BasicSource(np.zeros((4, 6))) with pytest.raises(ValueError): write_into(source, IndexTransform.from_shape(source.shape), [1, 2]) assert source.calls == [] def test_write_cast_error_before_mutation() -> None: source = BasicSource(np.zeros(2, dtype=np.int64)) with pytest.raises(ValueError): write_into(source, IndexTransform.from_shape(source.shape), ["1", "invalid"]) assert source.calls == [] def test_write_bounds_error_before_mutation() -> None: source = BasicSource(np.zeros(2)) transform = IndexTransform( IndexDomain.from_shape((2,)), (ArrayMap(np.array([0, 2], dtype=np.intp)),) ) with pytest.raises(IndexError, match="outside"): write_into(source, transform, [1, 2]) assert source.calls == [] def test_write_readonly_error() -> None: data = np.zeros(2) data.flags.writeable = False with pytest.raises(ValueError, match="read-only"): write_into(data, IndexTransform.from_shape(data.shape), 1) def test_write_constant_repeated_destination() -> None: source = BasicSource(np.zeros(3)) transform = IndexTransform(IndexDomain.from_shape((2, 3)), (ConstantMap(1),)) write_into(source, transform, [[1], [9]]) np.testing.assert_array_equal(source.data, [0, 9, 0]) assert len(source.calls) == 6 @pytest.mark.parametrize("scalar", [False, True]) def test_write_scalar_source_and_shared_dimensions(scalar: bool) -> None: if scalar: source = BasicSource(np.array(0)) transform = IndexTransform.from_shape(()) expected = np.array(7) else: source = BasicSource(np.zeros((3, 3), dtype=np.int64)) transform = IndexTransform(IndexDomain.from_shape((3,)), (DimensionMap(0), DimensionMap(0))) expected = np.diag([7, 7, 7]) write_into(source, transform, 7) np.testing.assert_array_equal(source.data, expected) def test_write_rank_error_before_mutation() -> None: source = BasicSource(np.zeros((2, 2))) with pytest.raises(ValueError, match="output rank"): write_into(source, IndexTransform.from_shape((2,)), 1) assert source.calls == [] @pytest.mark.parametrize("masked_source", [False, True]) @pytest.mark.parametrize("fancy", [False, True]) def test_write_masked_values(masked_source: bool, fancy: bool) -> None: data: Any = np.arange(12).reshape(3, 4) if masked_source: data = np.ma.array(data, mask=np.ones(data.shape, dtype=bool)) expected = data.copy() values = np.ma.array([[31, 32, 33, 34]], mask=[[False, True, False, True]]) transform = IndexTransform.from_shape(data.shape) if fancy: transform = transform.oindex[[2, 0, 2], :] expected[[2, 0, 2], :] = values else: transform = transform[::-1, :] expected[::-1, :] = values write_into(data, transform, values) np.testing.assert_array_equal(np.ma.getdata(data), np.ma.getdata(expected)) np.testing.assert_array_equal(np.ma.getmaskarray(data), np.ma.getmaskarray(expected)) def test_write_masked_lazy_alias() -> None: from zarr_indexing.lazy_array import LazyArray source = np.ma.array(np.arange(6), mask=[True, False, False, True, False, True]) expected = source[::-1].copy() array = LazyArray(source) array[::-1] = array np.testing.assert_array_equal(source.data, expected.data) np.testing.assert_array_equal(source.mask, expected.mask) @pytest.mark.parametrize("backend", ["numpy", "zarr"]) @pytest.mark.parametrize("case", ["basic", "outer", "vector", "newaxis", "empty", "alias"]) def test_lazy_write_and_assignment(backend: str, case: str) -> None: from zarr_indexing.lazy_array import LazyArray initial = np.arange(20).reshape(4, 5) source: Any = initial.copy() if backend == "zarr": zarr = pytest.importorskip("zarr") source = zarr.create_array({}, shape=initial.shape, chunks=(2, 3), dtype="int64") source[:] = initial array = LazyArray(source) identifiers = initial.copy() expected = initial.copy() if case == "basic": view = array[::-1, ::2][1:, 1:] identifiers = identifiers[::-1, ::2][1:, 1:] elif case == "outer": view = array.oindex[[3, 0, 3], [4, 1]].oindex[[2, 0, 1], [1, 0]] identifiers = identifiers[np.ix_([3, 0, 3], [4, 1])][np.ix_([2, 0, 1], [1, 0])] elif case == "vector": view = array.oindex[[3, 0, 3], :].vindex[[2, 0, 1], [4, 4, 1]] identifiers = identifiers[[3, 0, 3], :][[2, 0, 1], [4, 4, 1]] elif case == "newaxis": view = array[None, 1:3, :] identifiers = identifiers[None, 1:3, :] elif case == "empty": view = array[2:2, :] identifiers = identifiers[2:2, :] else: array[::-1, :] = array np.testing.assert_array_equal(source[:], initial[::-1]) return values = np.arange(identifiers.size).reshape(identifiers.shape) + 100 for identifier, value in zip(identifiers.flat, values.flat, strict=True): expected.flat[identifier] = value assert view.write(values) is None np.testing.assert_array_equal(source[:], expected) # All selector assignment entry points write through to the original source. array.oindex[[0, 3], [1, 4]] = 77 expected[np.ix_([0, 3], [1, 4])] = 77 array.vindex[[0, 0, 2], [2, 2, 4]] = [10, 11, 12] expected[[0, 0, 2], [2, 2, 4]] = [10, 11, 12] np.testing.assert_array_equal(source[:], expected) @pytest.mark.parametrize("backend", ["numpy", "zarr"]) def test_random_selection_chain_scatter(backend: str) -> None: """Map selected integer labels back to storage without consulting transforms.""" from zarr_indexing.lazy_array import LazyArray rng = np.random.default_rng(58123) for _ in range(24): initial = np.arange(30).reshape(5, 6) source: Any = initial.copy() if backend == "zarr": zarr = pytest.importorskip("zarr") source = zarr.create_array({}, shape=initial.shape, chunks=(2, 3), dtype="int64") source[:] = initial view = LazyArray(source) identifiers = initial for _ in range(3): if rng.integers(2): rows = rng.integers(0, view.shape[0], size=4) cols = rng.integers(0, view.shape[1], size=3) view = view.oindex[rows, cols] identifiers = identifiers[np.ix_(rows, cols)] else: step = int(rng.choice([-2, -1, 1, 2])) view = view[::step, ::-1] identifiers = identifiers[::step, ::-1] values = rng.integers(100, 200, size=identifiers.shape) expected = initial.copy() for identifier, value in zip(identifiers.flat, values.flat, strict=True): expected.flat[identifier] = value view.write(values) np.testing.assert_array_equal(source[:], expected) class _ChunkTouches: """Count the storage round trips a zarr array makes, one per chunk key.""" def __init__(self) -> None: self.gets = 0 self.sets = 0 @classmethod def wrap(cls, zarr: Any) -> tuple[Any, _ChunkTouches]: counter = cls() class CountingStore(zarr.storage.MemoryStore): async def get(self, key: str, prototype: Any, byte_range: Any = None) -> Any: counter.gets += 1 return await super().get(key, prototype, byte_range) async def set(self, key: str, value: Any) -> None: counter.sets += 1 await super().set(key, value) return CountingStore(), counter @pytest.mark.parametrize("sharded", [False, True]) @pytest.mark.parametrize("case", ["outer_rows", "vector_points", "duplicates"]) def test_fancy_writes_touch_each_write_chunk_once(sharded: bool, case: str) -> None: """Storage round trips scale with touched write chunks, not selected elements.""" zarr = pytest.importorskip("zarr") from zarr_indexing.lazy_array import LazyArray store, touches = _ChunkTouches.wrap(zarr) kwargs: dict[str, Any] = {"chunks": (2, 2), "shards": (4, 4)} if sharded else {"chunks": (4, 4)} source = zarr.create_array(store, shape=(12, 12), dtype="int64", **kwargs) source[:] = 0 expected = np.zeros((12, 12), dtype=np.int64) touches.gets = touches.sets = 0 view = LazyArray(source) if case == "outer_rows": rows = [1, 5, 11] values = np.arange(3 * 12).reshape(3, 12) view.oindex[rows, :] = values expected[rows, :] = values touched = {(r // 4, c) for r in rows for c in range(3)} elif case == "vector_points": rows, cols = [0, 7, 7, 11, 3], [0, 2, 9, 11, 5] values = np.arange(5) + 10 view.vindex[rows, cols] = values expected[rows, cols] = values touched = {(r // 4, c // 4) for r, c in zip(rows, cols, strict=True)} else: view.oindex[[5, 5, 6], 2:5] = np.array([[1] * 3, [2] * 3, [3] * 3]) expected[5, 2:5] = 2 expected[6, 2:5] = 3 touched = {(1, 0), (1, 1)} gets, sets = touches.gets, touches.sets np.testing.assert_array_equal(source[:], expected) assert sets == len(touched), (sets, len(touched)) # At most the hull read plus zarr's own partial-chunk read per cell. assert gets <= 2 * len(touched), (gets, len(touched)) def test_write_grid_prefers_write_chunk_sizes_over_chunks() -> None: """A source whose write grid is coarser than its read grid is written by the write grid.""" from zarr_indexing.lazy_array import LazyArray class ShardedSource: chunks = ((2, 2, 2), (3, 3)) write_chunk_sizes = ((6,), (3, 3)) def __init__(self) -> None: self.data = np.zeros((6, 6), dtype=np.int64) self.shape = self.data.shape self.dtype = self.data.dtype self.boxes: list[tuple[slice, ...]] = [] def __getitem__(self, key: Any) -> Any: return self.data[key] def __setitem__(self, key: Any, value: Any) -> None: self.boxes.append(key) self.data[key] = value source = ShardedSource() LazyArray(source).oindex[[0, 5], :] = 7 expected = np.zeros((6, 6), dtype=np.int64) expected[[0, 5], :] = 7 np.testing.assert_array_equal(source.data, expected) # One read-modify-write per touched write cell: two column shards, each # spanning all six rows, rather than the four inner chunks a read grid names. assert len(source.boxes) == 2 assert all(box[0] == slice(0, 6) for box in source.boxes) def test_write_only_source_falls_back_to_element_assignment() -> None: source = BasicSource(np.zeros((3, 4), dtype=np.int64)) transform = IndexTransform.from_shape(source.shape).oindex[[2, 0], [1, 3]] write_into(source, transform, np.array([[1, 2], [3, 4]])) assert len(source.calls) == 4 expected = np.zeros((3, 4), dtype=np.int64) expected[np.ix_([2, 0], [1, 3])] = [[1, 2], [3, 4]] np.testing.assert_array_equal(source.data, expected) def test_masked_zero_rank_affine_write_keeps_payload() -> None: from zarr_indexing.lazy_array import LazyArray source = np.ma.array([1, 2, 3]) LazyArray(source)[1].write(np.ma.array(7, mask=True)) np.testing.assert_array_equal(source.data, [1, 7, 3]) np.testing.assert_array_equal(np.ma.getmaskarray(source), [False, True, False]) def test_write_rank_error_precedes_value_broadcasting() -> None: source = np.zeros((3, 4), dtype=np.int64) transform = IndexTransform.from_shape((3,)) with pytest.raises(ValueError, match="rank"): write_into(source, transform, np.arange(6).reshape(2, 3)) np.testing.assert_array_equal(source, 0) def test_empty_selection_still_validates_values() -> None: source = np.zeros(5, dtype=np.int64) with pytest.raises(ValueError, match="broadcast"): write_into(source, IndexTransform.from_shape((5,))[2:2], np.arange(3)) with pytest.raises(ValueError): write_into(source, IndexTransform.from_shape((5,))[2:2], ["not a number"]) np.testing.assert_array_equal(source, 0) def test_gridless_readable_source_never_reads() -> None: """Without a write grid the hull could be the whole array, so elements are assigned.""" from zarr_indexing.lazy_array import LazyArray class ReadableSource(BasicSource): def __getitem__(self, key: Any) -> Any: raise AssertionError("gridless writes must not read the source") source = ReadableSource(np.zeros((4, 4), dtype=np.int64)) LazyArray(source).vindex[[0, 3], [3, 0]] = [1, 2] expected = np.zeros((4, 4), dtype=np.int64) expected[[0, 3], [3, 0]] = [1, 2] np.testing.assert_array_equal(source.data, expected) assert len(source.calls) == 2 def test_unfactorable_transform_falls_back_to_elements_on_a_gridded_source() -> None: """A diagonal reads one input axis twice, which the planner rejects; it is still written.""" class GriddedSource(BasicSource): chunks = ((2, 2), (2, 2)) def __getitem__(self, key: Any) -> Any: return self.data[key] source = GriddedSource(np.zeros((4, 4), dtype=np.int64)) diagonal = IndexTransform(IndexDomain.from_shape((4,)), (DimensionMap(0), DimensionMap(0))) write_into(source, diagonal, np.arange(1, 5), write_grid=None) from zarr_indexing.grid import dimension_grids_from_chunks source.data[:] = 0 source.calls.clear() write_into( source, diagonal, np.arange(1, 5), write_grid=dimension_grids_from_chunks(source.chunks, (4, 4)), ) np.testing.assert_array_equal(source.data, np.diag([1, 2, 3, 4])) assert len(source.calls) == 4 def test_writes_bypass_the_reader_so_a_caching_reader_serves_stale_values() -> None: """Pins the documented hazard: the reader is not invalidated by `write`.""" from zarr_indexing.lazy_array import LazyArray from zarr_indexing.reader import ReadContext, basic_reader class CachingReader: def __init__(self) -> None: self.cache: dict[tuple[Any, ...], Any] = {} def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: key = (context.transform.domain.shape, str(context.transform.to_json())) if key not in self.cache: basic_reader.read_into(source, context, out) self.cache[key] = out.copy() out[...] = self.cache[key] source = np.arange(6) view = LazyArray(source).with_reader(CachingReader())[1:4] np.testing.assert_array_equal(view.result(), [1, 2, 3]) view.write([7, 8, 9]) np.testing.assert_array_equal(source, [0, 7, 8, 9, 4, 5]) np.testing.assert_array_equal(view.result(), [1, 2, 3]) zarr-developers-zarr-python-a2f2415/packages/zarr-indexing/uv.lock000066400000000000000000003436451527014137600253470ustar00rootroot00000000000000version = 1 revision = 3 requires-python = ">=3.12" [[package]] name = "babel" version = "2.18.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d", size = 9959554, upload-time = "2026-02-01T12:30:56.078Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, ] [[package]] name = "backrefs" version = "8.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ec/56/4744bcd0c82184e80c52b0ac4076c261a8ffa1f1b343ff2f6e89ce0e1cef/backrefs-8.0.tar.gz", hash = "sha256:b556cd7d36c3a3a2f256b89590b176b8eddfb73bcfaee3a3ddd84ea66d21ce50", size = 7013081, upload-time = "2026-07-26T19:54:24.638Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e3/fd/9bf53b6a6f6f519ffaac765df2f2a25e5c2fc6d32cfd2b2747099e72c911/backrefs-8.0-py310-none-any.whl", hash = "sha256:4a627b817fd2dce43b79ab48da63613340509381cd8ce0897078a0bce79a2ab8", size = 380377, upload-time = "2026-07-26T19:54:17.457Z" }, { url = "https://files.pythonhosted.org/packages/e1/29/4bd7ae72a2634da00379c2b3bcc5439e7c94620235c6afea8af15229a973/backrefs-8.0-py311-none-any.whl", hash = "sha256:f0c35cf0102ba6b6070c12a492be3c1c1d3f5839529784b9a9565d6d04569a01", size = 392169, upload-time = "2026-07-26T19:54:18.782Z" }, { url = "https://files.pythonhosted.org/packages/29/13/232505664e8e2a0c7a2eb0c505cfade9d715538f89a5d62bc4c272968f62/backrefs-8.0-py312-none-any.whl", hash = "sha256:87f0fae8c5f207fe9f4b2887efc71d42f4900ac78faa1af08d675ef303692dc5", size = 398084, upload-time = "2026-07-26T19:54:19.954Z" }, { url = "https://files.pythonhosted.org/packages/8a/69/47a3dc20abc4fa5486655fde681bd55e63211b46c886d8c02223d6468431/backrefs-8.0-py313-none-any.whl", hash = "sha256:601ce68ca12385dbda06ce264406b4c4210cf5b79fd0fd627592365c92f29a88", size = 400040, upload-time = "2026-07-26T19:54:21.194Z" }, { url = "https://files.pythonhosted.org/packages/1c/cf/e5f9b68a5b0e939a2fb933a66c20180d0c9241bf8927f7a47fa48c1675e9/backrefs-8.0-py314-none-any.whl", hash = "sha256:9ec96efa080938be92323e8e730e57718c9c88eb15ad70bbef4e1766df591408", size = 411903, upload-time = "2026-07-26T19:54:23.221Z" }, ] [[package]] name = "certifi" version = "2026.7.22" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, ] [[package]] name = "charset-normalizer" version = "3.4.9" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0", size = 319300, upload-time = "2026-07-07T14:33:15.666Z" }, { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" }, { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" }, { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" }, { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" }, { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" }, { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" }, { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" }, { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" }, { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" }, { url = "https://files.pythonhosted.org/packages/0e/42/6dbc00b8cd16011691203e33570fa42ed5746599a2e878112d16eab403a3/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642", size = 151185, upload-time = "2026-07-07T14:33:30.781Z" }, { url = "https://files.pythonhosted.org/packages/80/cc/f920afd1a23c58ccd53c1d36085a71893a4737ff5e66e0371efab6809850/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0", size = 162557, upload-time = "2026-07-07T14:33:32.176Z" }, { url = "https://files.pythonhosted.org/packages/f0/e6/0386d43a261ff4e4b30c5857af7df877254b46bec7b9d1b74b6bf969a90b/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2", size = 152665, upload-time = "2026-07-07T14:33:33.711Z" }, { url = "https://files.pythonhosted.org/packages/b2/06/97ec2aeae780b31d742b6352218b43841a6871e2564578ca522dce4a45c3/charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614", size = 317688, upload-time = "2026-07-07T14:33:35.408Z" }, { url = "https://files.pythonhosted.org/packages/d0/39/8ff066c672434225f8d25f8b739f992af250944392173dcc88362681c9bf/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698", size = 214982, upload-time = "2026-07-07T14:33:36.996Z" }, { url = "https://files.pythonhosted.org/packages/92/8f/3a47a3667c83c2df9483d91644c6c107de3bf8874aa1793da9d3012eb986/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b", size = 236460, upload-time = "2026-07-07T14:33:38.536Z" }, { url = "https://files.pythonhosted.org/packages/f1/60/b22cdbee7e4013dab8b0d7647fc6181120fbbbc8f7025c226d15bd5a47fc/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9", size = 232003, upload-time = "2026-07-07T14:33:40.059Z" }, { url = "https://files.pythonhosted.org/packages/ea/f8/72eb13dcabe7257035cea8aefd922caad2f110d252bf9f67c4c2ca763aee/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33", size = 223149, upload-time = "2026-07-07T14:33:41.631Z" }, { url = "https://files.pythonhosted.org/packages/b0/3e/faee8f9de92b14ee1198e9163252bb15efee7301b31256a3b6d9ebfdd0dd/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63", size = 207901, upload-time = "2026-07-07T14:33:43.209Z" }, { url = "https://files.pythonhosted.org/packages/3a/25/45f30093ae27dd7b92a793b61882a38685f993700113ca36e0c9c14965e1/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0", size = 219176, upload-time = "2026-07-07T14:33:44.725Z" }, { url = "https://files.pythonhosted.org/packages/48/18/c8f397329c35e32f6a837e488986f4ae03bd2abebc453b48714991630c2f/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe", size = 217356, upload-time = "2026-07-07T14:33:46.192Z" }, { url = "https://files.pythonhosted.org/packages/86/7e/5ce0bba863470fd1902d5e5843968951bddf38abe4742fc97116ef4598b3/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35", size = 209614, upload-time = "2026-07-07T14:33:47.705Z" }, { url = "https://files.pythonhosted.org/packages/6c/ef/2473d3c4d869155be4af1191111d59c4d5c4e0173026f7e85b176e23bf65/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8", size = 224991, upload-time = "2026-07-07T14:33:49.238Z" }, { url = "https://files.pythonhosted.org/packages/d0/a3/53ddae3db108a088156aa8ddfafd411ebbc1340f48c5573f697b27f69a39/charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9", size = 150622, upload-time = "2026-07-07T14:33:50.711Z" }, { url = "https://files.pythonhosted.org/packages/e8/ef/6953a77c7cf2c2ff9998e6f575ab3e380119f100223381565a4f94c1f836/charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115", size = 161947, upload-time = "2026-07-07T14:33:52.197Z" }, { url = "https://files.pythonhosted.org/packages/6e/fb/d560d1d1555debbfe7849d9cac6145c1b537709d79576bf22557ed803b82/charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012", size = 152594, upload-time = "2026-07-07T14:33:53.486Z" }, { url = "https://files.pythonhosted.org/packages/7e/8d/496817fa0944239ecae662dd57ea765cfeaec6a735f9f025d4b7b72e7143/charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380", size = 317253, upload-time = "2026-07-07T14:33:54.994Z" }, { url = "https://files.pythonhosted.org/packages/2b/f9/ef4a69ea338ad3c0deceea0f5f7d2380ae8b52132b06d652cb0d2cd86706/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9", size = 215898, upload-time = "2026-07-07T14:33:56.334Z" }, { url = "https://files.pythonhosted.org/packages/8c/e7/5ddfd76fc061eb52de219658a4aa431cbacadf0a0219c8854f00da50d289/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4", size = 236718, upload-time = "2026-07-07T14:33:57.9Z" }, { url = "https://files.pythonhosted.org/packages/49/ba/768fa3f36048d81c477a0ce61f813bc1454d80917ccfe550abd9f44f5e24/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a", size = 232519, upload-time = "2026-07-07T14:33:59.811Z" }, { url = "https://files.pythonhosted.org/packages/f4/c4/b3e049d2aa3766180c78507110543d9d50894cc97f57de543f1be521dcdc/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046", size = 223143, upload-time = "2026-07-07T14:34:01.517Z" }, { url = "https://files.pythonhosted.org/packages/19/79/55c32d06d76ae4feafe053f061f3e3ab70bcf19f4007797ce8c3efda7830/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81", size = 206742, upload-time = "2026-07-07T14:34:03.04Z" }, { url = "https://files.pythonhosted.org/packages/10/e0/47c079dd82d217c807479cd59ffd30af56307ea31c108b75758970459ad3/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917", size = 219191, upload-time = "2026-07-07T14:34:04.657Z" }, { url = "https://files.pythonhosted.org/packages/42/ab/b9bc2e77d6b44a7e46ef62ec5cac1c9a6ba7b9135a5d560f002696ec9995/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41", size = 218328, upload-time = "2026-07-07T14:34:06.115Z" }, { url = "https://files.pythonhosted.org/packages/f1/78/c9c71d599f5aa2d42bcdd35cbbd46d7f535351a57e40ff7d8e5a7e219401/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1", size = 207406, upload-time = "2026-07-07T14:34:07.554Z" }, { url = "https://files.pythonhosted.org/packages/f6/39/c914445c321a845097ce4f6ac7de9a18228a77b766272125a1ce00d851eb/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf", size = 225157, upload-time = "2026-07-07T14:34:09.061Z" }, { url = "https://files.pythonhosted.org/packages/9b/f2/c0d4b8508565a36bc5c624e88ed297f5b0b1095011034d7f5b83a69908b5/charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48", size = 151095, upload-time = "2026-07-07T14:34:10.901Z" }, { url = "https://files.pythonhosted.org/packages/49/fd/a1d26144398c67486422a72bf5812cda22cb4ccfcd95a290fb41ceb4b8e2/charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b", size = 162796, upload-time = "2026-07-07T14:34:12.47Z" }, { url = "https://files.pythonhosted.org/packages/20/95/d75e82f8ce9fd323ebf059c16c9aadefb22a1ecde13b7840b35835e4886c/charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519", size = 153334, upload-time = "2026-07-07T14:34:14.044Z" }, { url = "https://files.pythonhosted.org/packages/00/5e/17398df3a139985ba9d11ed072531986f408c8fca952835ef1ab1820c02b/charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198", size = 338848, upload-time = "2026-07-07T14:34:15.688Z" }, { url = "https://files.pythonhosted.org/packages/cd/91/7253a32e86b7e1d1239b1b36ba6dd0f021a21107ab33054b53119cc083b9/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32", size = 223022, upload-time = "2026-07-07T14:34:17.248Z" }, { url = "https://files.pythonhosted.org/packages/cb/32/2e64bd2be10e89c61e57ebe6a93fd98ae88eb7ebe414b5121f22c96c69eb/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632", size = 241590, upload-time = "2026-07-07T14:34:18.813Z" }, { url = "https://files.pythonhosted.org/packages/3d/ef/d96ec496cfea0c21db43b0ad03891308b02388d054cc902cf0e5a1ad6a88/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf", size = 239584, upload-time = "2026-07-07T14:34:20.52Z" }, { url = "https://files.pythonhosted.org/packages/d4/ce/9af95f7876194bd7a14e3dfe4a4de2e0bff02666a3910d72beafd06cc297/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990", size = 230224, upload-time = "2026-07-07T14:34:22.189Z" }, { url = "https://files.pythonhosted.org/packages/52/94/af74dde74a3996bd959c350709bfe50e297823d70a8c1cbd54b838880863/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d", size = 212667, upload-time = "2026-07-07T14:34:23.857Z" }, { url = "https://files.pythonhosted.org/packages/ee/f0/f1c4fe746c395922961b5916ed1d7d6e7d4c84851d19ed43cc89980ec953/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e", size = 227179, upload-time = "2026-07-07T14:34:25.586Z" }, { url = "https://files.pythonhosted.org/packages/e4/56/6c745619ac397e8871e2bcd3cea1eec86b877488f33888b3aef5c3ed506e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c", size = 225372, upload-time = "2026-07-07T14:34:27.212Z" }, { url = "https://files.pythonhosted.org/packages/78/ad/98aae8630ac71f16711968e38a5acfecce41b778bf2f0312851020f565a8/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2", size = 215222, upload-time = "2026-07-07T14:34:28.774Z" }, { url = "https://files.pythonhosted.org/packages/f7/40/9593d54209765207a7f11073c06494c1721e4ca4a0a426c597679bf7f91e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534", size = 231958, upload-time = "2026-07-07T14:34:30.345Z" }, { url = "https://files.pythonhosted.org/packages/b1/27/693ee5e8a18191eb38647360c51cd505013e2bd3b366aa43fd5344c21e3c/charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226", size = 155580, upload-time = "2026-07-07T14:34:31.884Z" }, { url = "https://files.pythonhosted.org/packages/80/3f/bd97d3d9c613013d07cb7733d299385b41df37f0471310f5a73dc359f0b8/charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177", size = 167620, upload-time = "2026-07-07T14:34:33.438Z" }, { url = "https://files.pythonhosted.org/packages/3d/c6/eee9dca4439b1061f76373f06ea855678cc4a64c1c3c90b50e479edbb8eb/charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501", size = 158037, upload-time = "2026-07-07T14:34:35.018Z" }, { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, ] [[package]] name = "click" version = "8.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] [[package]] name = "colorama" version = "0.4.6" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] [[package]] name = "ghp-import" version = "2.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "python-dateutil" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d9/29/d40217cbe2f6b1359e00c6c307bb3fc876ba74068cbab3dde77f03ca0dc4/ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343", size = 10943, upload-time = "2022-05-02T15:47:16.11Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", size = 11034, upload-time = "2022-05-02T15:47:14.552Z" }, ] [[package]] name = "griffe-inherited-docstrings" version = "1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffelib" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cb/da/fd002dc5f215cd896bfccaebe8b4aa1cdeed8ea1d9d60633685bd61ff933/griffe_inherited_docstrings-1.1.3.tar.gz", hash = "sha256:cd1f937ec9336a790e5425e7f9b92f5a5ab17f292ba86917f1c681c0704cb64e", size = 26738, upload-time = "2026-02-21T09:38:44.312Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/16/20/4bc15f242181daad1c104e0a7d33be49e712461ea89e548152be0365b9ea/griffe_inherited_docstrings-1.1.3-py3-none-any.whl", hash = "sha256:aa7f6e624515c50d9325a5cfdf4b2acac547f1889aca89092d5da7278f739695", size = 6710, upload-time = "2026-02-20T11:06:38.75Z" }, ] [[package]] name = "griffelib" version = "2.1.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/33/e4/8d187ea29c2e30b3a09505c567513077d6117861bde1fbd997a167f262ec/griffelib-2.1.0.tar.gz", hash = "sha256:762a186d2c6fd6794d4ea20d428d597ffb857cb56b66421651cbba15bdd5e813", size = 216234, upload-time = "2026-06-19T12:05:42.278Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e4/d3/5268aeabf2ad82658c4e2ff3a060648d0f02f3926cb53247c0e4d0dab49e/griffelib-2.1.0-py3-none-any.whl", hash = "sha256:cc7b3d2d2865ad0b909fcc38086e3f554b5ea7acbaa7bbb7ecaa3f5dfb7d9f00", size = 142560, upload-time = "2026-06-19T12:05:38.742Z" }, ] [[package]] name = "hypothesis" version = "6.164.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "sortedcontainers" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/ac/7b76103bd74d8457e4de0c6a6c3a26ac6327016438bde125e0a3de83a5b8/hypothesis-6.164.0.tar.gz", hash = "sha256:5d63d263d8c71b571638c18d9591f6e34b836c60a12469e9d9105c1c785f00f1", size = 492022, upload-time = "2026-07-30T12:39:49.085Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/7c/fe/d5b75a55892b33e72945f82efc71f645d29c0bfdb9f00727f7535a52edcc/hypothesis-6.164.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:14b861ac3353f8643b82a3ba76b8a0a54d2a06160c32b9a1f64a8ab41b179089", size = 771561, upload-time = "2026-07-30T12:39:00.404Z" }, { url = "https://files.pythonhosted.org/packages/1c/b0/2f01e9efc7267446bad0e2a68f7472daa174a72553d213b16aefe44b2bda/hypothesis-6.164.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:3d8c8bb00a4b86ae90b9ad41f3e1c99d016ec3e64c0ff9d676a4bb7be4f56948", size = 767079, upload-time = "2026-07-30T12:39:23.123Z" }, { url = "https://files.pythonhosted.org/packages/c3/26/d7bcd26b58e1df2bd39116b924b2a72676215d9650e68cbff9a629c3ce30/hypothesis-6.164.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e80e3ba8eaf37664eaa0f2625cef120b330b128a7df570210cf8be4f5ae65aaa", size = 1096364, upload-time = "2026-07-30T12:38:49.972Z" }, { url = "https://files.pythonhosted.org/packages/9d/17/99fe7ea866935da83444c3ef7885a14fc7349d96ff61c6faebd37ef4edf2/hypothesis-6.164.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8cdf70f821e2d2f3a0bccaab29830aea8aefb63a77806e7e91246fb65a10c8d3", size = 1124963, upload-time = "2026-07-30T12:39:13.1Z" }, { url = "https://files.pythonhosted.org/packages/38/e8/df08be6296cbc1271d44e81f8ff9dcd6267a07552fb768e0fdc166e93d40/hypothesis-6.164.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcc3743e22b3cffa7267b4bc74d03628606e4a115495728e986a7be220987315", size = 1145886, upload-time = "2026-07-30T12:39:45.612Z" }, { url = "https://files.pythonhosted.org/packages/4e/72/d5cf6fbfac40891d4281f630e16a6eb217ff56f97e350a06e0fd9322aa6a/hypothesis-6.164.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:730f09d4afcd8a918b3d589bfb6421e3b41c057aa57652a773ef4f512cc60836", size = 1101181, upload-time = "2026-07-30T12:39:05.194Z" }, { url = "https://files.pythonhosted.org/packages/fb/ff/7ceb002329febffb678b65835ca6e9479a916325d088aadb0210d07f8252/hypothesis-6.164.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9651cb48cb5a995295b442138d15d381547b935dcb0066fca7148a7955347400", size = 1137970, upload-time = "2026-07-30T12:39:16.076Z" }, { url = "https://files.pythonhosted.org/packages/7c/8f/c12c697b73ca9ca24d8a913879e3e0a9db86479754c7221554247c701565/hypothesis-6.164.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:51d161d2655dd86143b370c577267b5b7b4c2e8fcb8a3f22c1a787572aad707c", size = 1270184, upload-time = "2026-07-30T12:38:54.436Z" }, { url = "https://files.pythonhosted.org/packages/0e/2f/93f1c850c794fc9c80f5e61b3b20652126b865e6f57b348ae530446aadc7/hypothesis-6.164.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:e8a250552390128b57e3afe55035ce2c2cb1f6f0919817657854244f071bc5be", size = 1397987, upload-time = "2026-07-30T12:38:21.113Z" }, { url = "https://files.pythonhosted.org/packages/c5/b8/bab2546325e15e87c8518dfbca263c81dbc35d566c516d66c9da98a38b77/hypothesis-6.164.0-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:570cd51944e1cc3443847d8afa3d17fcf8aac475a1f744c9e7318a5ad7ef5c9f", size = 1270755, upload-time = "2026-07-30T12:38:51.571Z" }, { url = "https://files.pythonhosted.org/packages/6a/4e/ea97dd39678a42dc5a24e3e2a64d3b950fad9fb1dcce8d7be5afb52a0335/hypothesis-6.164.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3a423e543055b3de5af7a7624c4285422541658367211fa293a3a57dd0ad01ba", size = 1312888, upload-time = "2026-07-30T12:38:30.847Z" }, { url = "https://files.pythonhosted.org/packages/44/84/a6f2d5b12b23d65f16eb398750e430065f9d1f40f4418569e3b87ef58d23/hypothesis-6.164.0-cp310-abi3-win32.whl", hash = "sha256:f5e51490b2ce64c66138f24477d83c71b6224ab0ef65700da10187c464b54e94", size = 657401, upload-time = "2026-07-30T12:39:11.581Z" }, { url = "https://files.pythonhosted.org/packages/f5/d3/c5ee410daa594cac2d3fe1fbe5473f2390e35f4369e168a817e43341ce2f/hypothesis-6.164.0-cp310-abi3-win_amd64.whl", hash = "sha256:c9059dfbb039342b6590bbce207f90e0f9a80fdf45a404c68c2d3e598be78ab3", size = 663566, upload-time = "2026-07-30T12:39:30.27Z" }, { url = "https://files.pythonhosted.org/packages/90/91/4942fe3f2f08b920368ed5a2937346259e843e382205513b4a0e70d2de9d/hypothesis-6.164.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6bc3373fe550cf4d7cadb94ceaeb91e431e1418a96b7baa330487366eaa67d3c", size = 773152, upload-time = "2026-07-30T12:38:33.328Z" }, { url = "https://files.pythonhosted.org/packages/eb/df/e66d052386a2b6c3e2f3eab32a02d7de3c9c59cd21d5dd58c08ecfa715f0/hypothesis-6.164.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2780297ca68929b153eff7effb2ebe67e9487d2fd9f49fa961007f8f2d236c9e", size = 764713, upload-time = "2026-07-30T12:38:48.59Z" }, { url = "https://files.pythonhosted.org/packages/f5/d5/5a50d14b8f04809e973c4dea884b367fef3663ff253c1205fa9e96229ef9/hypothesis-6.164.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b400bb4eb5a4a1e19cd5af3cc63817909e6b54b4603e04022bdba46860913d7", size = 1095160, upload-time = "2026-07-30T12:38:58.925Z" }, { url = "https://files.pythonhosted.org/packages/58/01/781b19ce4382ec239c4dc6ec3bd9f195e69e5570f2814bbf04b5781ecb18/hypothesis-6.164.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fca6632933fc506dd96926d9383483e4c0066c7ff62c748d059a3276da761e7", size = 1145199, upload-time = "2026-07-30T12:39:09.904Z" }, { url = "https://files.pythonhosted.org/packages/e9/64/30e016863515ca01c1c738b05dd50491353d3ccae6432362e56e0c15d0da/hypothesis-6.164.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b9e1f6e89e5ec34735b727f3ce41d12e7f3b8efc162c91c8a225e10b54b504b4", size = 1267980, upload-time = "2026-07-30T12:38:18.733Z" }, { url = "https://files.pythonhosted.org/packages/84/23/17eb8d67d59ecd3a820c905fbdf514e371dd7d01631e62a304cdd5793abe/hypothesis-6.164.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:51b0f967f608707b24ed37a298174ae6eec7899bfe3f271d1c3062c39ad66c06", size = 1312181, upload-time = "2026-07-30T12:38:36.056Z" }, { url = "https://files.pythonhosted.org/packages/42/69/cff9f3cd9524252adda7c8e0e129dfc176e72f64fdf0bf1552d1ea43d78d/hypothesis-6.164.0-cp312-cp312-win_amd64.whl", hash = "sha256:5770df7d518bf867a9379e9081abd9e44db1d15473430e26a0946438c08c5926", size = 660690, upload-time = "2026-07-30T12:38:28.107Z" }, { url = "https://files.pythonhosted.org/packages/ba/b4/729697380a22dc2ce8feae3c64b08bf3bd3c27e99c3706cb9bdac40c6fc8/hypothesis-6.164.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:29e7cb48974cb9fd87602e20625c890385793c6b56c18a957085a9c291f56ef8", size = 773046, upload-time = "2026-07-30T12:39:40.473Z" }, { url = "https://files.pythonhosted.org/packages/38/35/72374f02d90dfda198afd8aac6b1e7d1184506f97e62ebcf3d2c1e5bf761/hypothesis-6.164.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1ff8c3819345be8dd15ee6588ee9383869a54c9a3d2232cce5e26b456424135d", size = 764659, upload-time = "2026-07-30T12:38:55.896Z" }, { url = "https://files.pythonhosted.org/packages/6e/75/fb26388915d71e5949b98ccd0c9d95edcbe6b45d0370f177d43633d81ae2/hypothesis-6.164.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33e88be13fac3ff7cb789a0b4cc43d99fb297db085f529fbb363188141c7d5bf", size = 1095078, upload-time = "2026-07-30T12:38:34.677Z" }, { url = "https://files.pythonhosted.org/packages/be/63/f6da6e39667d39a1e44c5df82fbe6cff070c29aaffa9beb62a5322e7d8ae/hypothesis-6.164.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2e296d03a77355ce2e1c32e85a636b555edf0ddaaef277f98f1b84fe38a4595", size = 1145015, upload-time = "2026-07-30T12:39:26.487Z" }, { url = "https://files.pythonhosted.org/packages/88/c7/55ba09727da3d9a60628c50e31e6083a36f403cb230f5e1a7bd1749a5c39/hypothesis-6.164.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:53698a1b246714539dd0ecc2d556cde613d74e9f7385ec4109e0651ab2d382d6", size = 1268027, upload-time = "2026-07-30T12:38:25.676Z" }, { url = "https://files.pythonhosted.org/packages/ff/35/4789cade332f799b0e8f2f7ea0fe2aae6157a85e60f74497e316dd17a7e3/hypothesis-6.164.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:004c92c4b869f8e258f0641101b7743cae8420436f4465383f681c086ef95c9d", size = 1311895, upload-time = "2026-07-30T12:39:14.621Z" }, { url = "https://files.pythonhosted.org/packages/12/8a/18d85e624f8631aec42daa8a2f07c6edcedb7385b2c0f375ba8a30cbd065/hypothesis-6.164.0-cp313-cp313-win_amd64.whl", hash = "sha256:4878f81fa92a580d3e16b53e64e01a9d9fe1dca5973783558493a003138dbd36", size = 660656, upload-time = "2026-07-30T12:38:37.696Z" }, { url = "https://files.pythonhosted.org/packages/c7/06/3c144d427799c7c72befb0bb3b199d419a89b96e1002fd8f0cc94c84ffb7/hypothesis-6.164.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:9110010bdf6deb3ba9134f8ce8b683e8bb0fba108a351045c96d60c410eb6963", size = 773254, upload-time = "2026-07-30T12:38:38.919Z" }, { url = "https://files.pythonhosted.org/packages/74/2d/b61a10d9e70df04aa7e8f34efef8e4afe364e8995c59f894e1c35b428214/hypothesis-6.164.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4df103e5d32b47d574c6e857d45361e2cba5a198d6dae4e4ee1bd248b3a2cbfa", size = 764786, upload-time = "2026-07-30T12:38:24.464Z" }, { url = "https://files.pythonhosted.org/packages/99/68/7f80ac7bdffe78686135311c919534be411d4565c2a5ba38fd389880c553/hypothesis-6.164.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4abec95020960c0ed08e5be318d2bcdde79f2c6fc7785e368a9389d31d3e802a", size = 1095578, upload-time = "2026-07-30T12:38:57.422Z" }, { url = "https://files.pythonhosted.org/packages/45/f9/97dcbac776bcf33cb4241b52111527821f707b60a84d03d0ea670b09a134/hypothesis-6.164.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b106756cc9abd50ab1632541ea7b7223792d877a084726aa0304237d758181e", size = 1145207, upload-time = "2026-07-30T12:38:23.387Z" }, { url = "https://files.pythonhosted.org/packages/a4/df/68184b6f71540435c895cf35ad1d67a3634a887c597ab38d3372c0d20186/hypothesis-6.164.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:11c4aab2ae6757fc4bc3bbf009487e24fd3490365817bbf40b9ec85a7e02fabb", size = 1268357, upload-time = "2026-07-30T12:38:52.946Z" }, { url = "https://files.pythonhosted.org/packages/a0/76/6a6851dc8af89a5c0418937d38456417b2a1fc9db15c992b9cb43d53a7a3/hypothesis-6.164.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4713edecbc0969557ca135769a36d1e524c8e3b7a2b271de48d98fa29f681bf6", size = 1312183, upload-time = "2026-07-30T12:39:28.604Z" }, { url = "https://files.pythonhosted.org/packages/2f/19/83adeb1f8f045bd8a1ab9822d0c3db28b337d37fff01d809fcd6e3ea70f8/hypothesis-6.164.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:e6882d316c390d33c55ec8f1675f35ab238d7c0473ccf8d235c69eaef6c621b9", size = 604771, upload-time = "2026-07-30T12:39:33.579Z" }, { url = "https://files.pythonhosted.org/packages/0b/62/fcb48ebfbccdc5b695de175b9d1d344b3688782150f0603124bb70c0891b/hypothesis-6.164.0-cp314-cp314-win_amd64.whl", hash = "sha256:7c3357633b38bca8c927fd90d02b39a0a3f35f24cdbcfb2fb1dcf69a3f63bd85", size = 660570, upload-time = "2026-07-30T12:39:43.898Z" }, { url = "https://files.pythonhosted.org/packages/42/61/5857da7db0435fa69df658a9eafba62eb8a1319454005ce2a0d97f6f9e4d/hypothesis-6.164.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:53152cb549f52d661c47768d0d12a192ef26a7a9758a7f13b8ec41e8e63d6325", size = 771839, upload-time = "2026-07-30T12:38:26.842Z" }, { url = "https://files.pythonhosted.org/packages/6c/9c/22292a9dab1c544362d1759244132c7d71a9d9d5eda5d454ec735fba6bd3/hypothesis-6.164.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cee7898ad84b63da6506ae48483bb36f319a25ea4c2b1d2df47d021cc4080c24", size = 763363, upload-time = "2026-07-30T12:38:20.042Z" }, { url = "https://files.pythonhosted.org/packages/e8/29/cc0c6e9a065a32f93fe52dde746232f007d2cabf619d4e7b1b37bd34c424/hypothesis-6.164.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0def33f0d236e54144a5218997e4492925144d4615f25fdbb4ac8e47b7b709e6", size = 1094171, upload-time = "2026-07-30T12:39:35.158Z" }, { url = "https://files.pythonhosted.org/packages/a7/59/37040d0776a29d4bc6d0ca9a50ca2755200007e4a8ddc27b010115b69c85/hypothesis-6.164.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:471fd80d70f2df606b1320276168bc2c6007a586124a1d81628264ccb9266f68", size = 1144089, upload-time = "2026-07-30T12:38:43.024Z" }, { url = "https://files.pythonhosted.org/packages/fa/10/5235ed3c090a2f12fa15cc1d08e5a36cfa31bc0607c45199b0806e930ab4/hypothesis-6.164.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2eb285756aee62890fd08d6e97cf77651dfe7c093ceac094df52120a7a8dbe68", size = 1266595, upload-time = "2026-07-30T12:39:36.979Z" }, { url = "https://files.pythonhosted.org/packages/7f/97/ffc4cee4dfdffe658e839d5f4df72ae3fa7bfea9401550b475d9700e0ee2/hypothesis-6.164.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7c5215b5568968c35c6e124e5a4a8068f80419d6171414ddf735b49e1df1ab59", size = 1310998, upload-time = "2026-07-30T12:38:45.788Z" }, { url = "https://files.pythonhosted.org/packages/dd/08/681d4a272cd2812151581c3328e41a80a34e420d676e419a25b4b9dc2291/hypothesis-6.164.0-cp314-cp314t-win_amd64.whl", hash = "sha256:a845e59fae87bb47a6fb84e0d5adb5679b3b55042fc3f8791da91486103cfbf0", size = 660724, upload-time = "2026-07-30T12:38:40.341Z" }, ] [[package]] name = "idna" version = "3.18" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] [[package]] name = "iniconfig" version = "2.3.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] [[package]] name = "jinja2" version = "3.1.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markupsafe" }, ] sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] [[package]] name = "markdown" version = "3.10.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/29/6f/da4c6aea59b3001f2e8c0ec7497475aadaf3b021c10cab5b2858f0f32b26/markdown-3.10.3.tar.gz", hash = "sha256:3589362618f743188b4d955b874402bc814f4f83f544dc207719f4baa7d9c45f", size = 372596, upload-time = "2026-07-30T19:05:29.005Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/64/69/4a5af2bc115a9a33fefe51709749de8262be3f9ba063d1753a837cdbc49c/markdown-3.10.3-py3-none-any.whl", hash = "sha256:fa6c92a00a4a3c98b22728c64a935ae1928250ae65058a6ded814d2cc29a4cea", size = 110757, upload-time = "2026-07-30T19:05:27.883Z" }, ] [[package]] name = "markupsafe" version = "3.0.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] [[package]] name = "mergedeep" version = "1.3.4" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/3a/41/580bb4006e3ed0361b8151a01d324fb03f420815446c7def45d02f74c270/mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8", size = 4661, upload-time = "2021-02-05T18:55:30.623Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/2c/19/04f9b178c2d8a15b076c8b5140708fa6ffc5601fb6f1e975537072df5b2a/mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307", size = 6354, upload-time = "2021-02-05T18:55:29.583Z" }, ] [[package]] name = "mkdocs" version = "1.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "ghp-import" }, { name = "jinja2" }, { name = "markdown" }, { name = "markupsafe" }, { name = "mergedeep" }, { name = "mkdocs-get-deps" }, { name = "packaging" }, { name = "pathspec" }, { name = "pyyaml" }, { name = "pyyaml-env-tag" }, { name = "watchdog" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bc/c6/bbd4f061bd16b378247f12953ffcb04786a618ce5e904b8c5a01a0309061/mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2", size = 3889159, upload-time = "2024-08-30T12:24:06.899Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/22/5b/dbc6a8cddc9cfa9c4971d59fb12bb8d42e161b7e7f8cc89e49137c5b279c/mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e", size = 3864451, upload-time = "2024-08-30T12:24:05.054Z" }, ] [[package]] name = "mkdocs-autorefs" version = "1.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown" }, { name = "markupsafe" }, { name = "mkdocs" }, ] sdist = { url = "https://files.pythonhosted.org/packages/52/c0/f641843de3f612a6b48253f39244165acff36657a91cc903633d456ae1ac/mkdocs_autorefs-1.4.4.tar.gz", hash = "sha256:d54a284f27a7346b9c38f1f852177940c222da508e66edc816a0fa55fc6da197", size = 56588, upload-time = "2026-02-10T15:23:55.105Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/28/de/a3e710469772c6a89595fc52816da05c1e164b4c866a89e3cb82fb1b67c5/mkdocs_autorefs-1.4.4-py3-none-any.whl", hash = "sha256:834ef5408d827071ad1bc69e0f39704fa34c7fc05bc8e1c72b227dfdc5c76089", size = 25530, upload-time = "2026-02-10T15:23:53.817Z" }, ] [[package]] name = "mkdocs-get-deps" version = "0.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mergedeep" }, { name = "platformdirs" }, { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ce/25/b3cccb187655b9393572bde9b09261d267c3bf2f2cdabe347673be5976a6/mkdocs_get_deps-0.2.2.tar.gz", hash = "sha256:8ee8d5f316cdbbb2834bc1df6e69c08fe769a83e040060de26d3c19fad3599a1", size = 11047, upload-time = "2026-03-10T02:46:33.632Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/88/29/744136411e785c4b0b744d5413e56555265939ab3a104c6a4b719dad33fd/mkdocs_get_deps-0.2.2-py3-none-any.whl", hash = "sha256:e7878cbeac04860b8b5e0ca31d3abad3df9411a75a32cde82f8e44b6c16ff650", size = 9555, upload-time = "2026-03-10T02:46:32.256Z" }, ] [[package]] name = "mkdocs-material" version = "9.7.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "babel" }, { name = "backrefs" }, { name = "colorama" }, { name = "jinja2" }, { name = "markdown" }, { name = "mkdocs" }, { name = "mkdocs-material-extensions" }, { name = "paginate" }, { name = "pygments" }, { name = "pymdown-extensions" }, { name = "requests" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f1/cd/c05d3a530ba7934f144fb45f7203cd236adc25c7bdcc34673d202f4b0278/mkdocs_material-9.7.7.tar.gz", hash = "sha256:c0649c065b1b0512d60aad8c10f947f8e455284475239b364b610f2deb4d0855", size = 4097923, upload-time = "2026-07-17T16:21:33.156Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ad/21/17c1bc9e6f47c972ad66fb2ac2568f99f90f1207eeb6fc3b34d094dba7b5/mkdocs_material-9.7.7-py3-none-any.whl", hash = "sha256:8ea9bb1737a5b524a5f9dcf2e1b4ebda8274ae3008aa7845720a97083bef708f", size = 9305438, upload-time = "2026-07-17T16:21:30.017Z" }, ] [[package]] name = "mkdocs-material-extensions" version = "1.3.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/79/9b/9b4c96d6593b2a541e1cb8b34899a6d021d208bb357042823d4d2cabdbe7/mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443", size = 11847, upload-time = "2023-11-22T19:09:45.208Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/5b/54/662a4743aa81d9582ee9339d4ffa3c8fd40a4965e033d77b9da9774d3960/mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31", size = 8728, upload-time = "2023-11-22T19:09:43.465Z" }, ] [[package]] name = "mkdocstrings" version = "1.0.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jinja2" }, { name = "markdown" }, { name = "markupsafe" }, { name = "mkdocs" }, { name = "mkdocs-autorefs" }, { name = "pymdown-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/71/f85bdf13355073ae15a7375f09879375a830553552e58c1c4b7e0bbc5c8b/mkdocstrings-1.0.6.tar.gz", hash = "sha256:a0b8c2bdd29a6416c80d717aa369bbf7831946bd9f23c2a66db1b1dbe7693dbd", size = 100649, upload-time = "2026-07-11T19:38:05.732Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/5d/5b/4c1902e8bdd5c4db63284e9d101dece4038d4025d6d88850ffe0a1578980/mkdocstrings-1.0.6-py3-none-any.whl", hash = "sha256:2703708697487d1b6d6d7b412e176fa436edf120c1bf81dc9e126b12d00893c7", size = 35787, upload-time = "2026-07-11T19:38:04.417Z" }, ] [[package]] name = "mkdocstrings-python" version = "2.0.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffelib" }, { name = "mkdocs-autorefs" }, { name = "mkdocstrings" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c5/65/0db919b7c91bf033b42d04f15ff16a56f4edd6afb26e84d448359660ece3/mkdocstrings_python-2.0.8.tar.gz", hash = "sha256:34545b54681cf0b212aab95394d37b0f5326a93b102f6ad1df3d965c79384c9d", size = 201771, upload-time = "2026-08-31T20:16:06.698Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0b/b7/a5a854d4e44853f26decc0c5e0d68df052399f3da344a313da4178f00816/mkdocstrings_python-2.0.8-py3-none-any.whl", hash = "sha256:e555d9ef53b0febdeae4dd603144d4f86ec90fef672276645ea3bb61232c3d24", size = 105443, upload-time = "2026-08-31T20:16:05.25Z" }, ] [[package]] name = "numpy" version = "2.5.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/22/fd/89965aa4ac08c74998539fcbf24fa3540f3e15237fbeb6bcf9c908f4aade/numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3", size = 20755553, upload-time = "2026-07-04T17:08:00.933Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/62/7b/14687aa674250e5e546f616f486b0d56d3631cd5b2415739141ce40bdcea/numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277", size = 16801574, upload-time = "2026-07-04T17:06:12.423Z" }, { url = "https://files.pythonhosted.org/packages/e1/19/cc5bb2a3f2913d27d6dbb2c78d25921fabaedc6741d4a5a615a11f3c5bf3/numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1", size = 11772250, upload-time = "2026-07-04T17:06:15.726Z" }, { url = "https://files.pythonhosted.org/packages/42/77/fdf34a71dd30f54979b18603bee915e0aaf825b07afe79acd60b04b691e2/numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0", size = 5331516, upload-time = "2026-07-04T17:06:17.913Z" }, { url = "https://files.pythonhosted.org/packages/ce/e2/eb7efa015b4cce41e2517bf182a7fce0d7d5b9d9ed76a29bfa0f4fe4505c/numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e", size = 6664863, upload-time = "2026-07-04T17:06:20.02Z" }, { url = "https://files.pythonhosted.org/packages/a9/4b/a2b32dd94ee9ffbeecb28152240042a3949db33b1c834d44090b80e1b3b8/numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75", size = 15167977, upload-time = "2026-07-04T17:06:21.621Z" }, { url = "https://files.pythonhosted.org/packages/b8/a9/6e73d68500f80773f65f0654ea932019d6694329a0eb0ed0533de38df376/numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca", size = 16672469, upload-time = "2026-07-04T17:06:24.064Z" }, { url = "https://files.pythonhosted.org/packages/24/7d/ad3e59015135f5261c95fd4cafeff159c955febd83a99a1d9250c4233815/numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3", size = 16527531, upload-time = "2026-07-04T17:06:26.69Z" }, { url = "https://files.pythonhosted.org/packages/83/d0/a39b2fbcde9cb17a1dac678f254b33a6336298af9df338824c685425d5e8/numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9", size = 18431940, upload-time = "2026-07-04T17:06:29.521Z" }, { url = "https://files.pythonhosted.org/packages/04/12/cff070947791c1ed425ff76413189adbdc2fbe215eba7ce7fa454a03c7f8/numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2", size = 6066764, upload-time = "2026-07-04T17:06:32.571Z" }, { url = "https://files.pythonhosted.org/packages/65/66/53f31807a48a750f9d748da273bc3fcedd12b27ff1f3e373bfec55ef2dc0/numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2", size = 12430966, upload-time = "2026-07-04T17:06:34.926Z" }, { url = "https://files.pythonhosted.org/packages/2b/2a/d1a88066b1c14186f5d3c0d18c94f17b064511982bab0578d49ee9d43c29/numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b", size = 10350488, upload-time = "2026-07-04T17:06:37.785Z" }, { url = "https://files.pythonhosted.org/packages/eb/07/ec2a3f0c91761581d4b7104a740791800025983f9a4dc4e73f91a99aeac4/numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1", size = 16796419, upload-time = "2026-07-04T17:06:40.37Z" }, { url = "https://files.pythonhosted.org/packages/ab/ab/ddb499fc4f8780354395face5b65c7fd107bcd6e1d667a5f07d046956f6f/numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6", size = 11765832, upload-time = "2026-07-04T17:06:42.768Z" }, { url = "https://files.pythonhosted.org/packages/88/b3/3c28c558a09fc72100c646dac6d2fce8e834c471b0edca01a29996706117/numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d", size = 5325143, upload-time = "2026-07-04T17:06:45.466Z" }, { url = "https://files.pythonhosted.org/packages/5e/0e/ce19b985bb15c596f4f05954e76cccc77c845083b3b8f938a6c68e523128/numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1", size = 6659749, upload-time = "2026-07-04T17:06:47.288Z" }, { url = "https://files.pythonhosted.org/packages/2e/20/1ee6614d64332a1bba6411f38e68cb79eec1b2459e20a623777c5c5492a2/numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd", size = 15164716, upload-time = "2026-07-04T17:06:49.494Z" }, { url = "https://files.pythonhosted.org/packages/ed/a7/2bcd3fdbb87804755c35b729bf8709d62025c5f4cfd7d5b2415997097515/numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a", size = 16661440, upload-time = "2026-07-04T17:06:52.061Z" }, { url = "https://files.pythonhosted.org/packages/fc/d7/a41e3310c886fe457d36e670bbf24fae411aca8a7b6ad92a32afd924077c/numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7", size = 16526305, upload-time = "2026-07-04T17:06:54.605Z" }, { url = "https://files.pythonhosted.org/packages/53/75/4333a9a707c1edd3a4e1a0c58eca52c0f31e55089fa80db02b5565b24df7/numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6", size = 18423008, upload-time = "2026-07-04T17:06:57.54Z" }, { url = "https://files.pythonhosted.org/packages/ee/90/e314a32b1c11a2ffe818ddad3a57b50b4b6e1b6c487192eb50cdef0415d0/numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9", size = 6063885, upload-time = "2026-07-04T17:07:00.14Z" }, { url = "https://files.pythonhosted.org/packages/10/70/800b3fca480af32df9e8ea9f3d4a0c8feb4b32d7f195d174eabbda4829ad/numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74", size = 12425674, upload-time = "2026-07-04T17:07:02.387Z" }, { url = "https://files.pythonhosted.org/packages/8b/0b/196350c122f50f6ca56846f2d71efd5e0d24b7b2e07355e019b2e2c7a11e/numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107", size = 10350256, upload-time = "2026-07-04T17:07:04.878Z" }, { url = "https://files.pythonhosted.org/packages/db/f4/731b6085a83faf6ca843394cbd5e217280c214399f7e8b21b9f552af0ae2/numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8", size = 16795063, upload-time = "2026-07-04T17:07:07.374Z" }, { url = "https://files.pythonhosted.org/packages/bf/64/0e215f2048dd11a55bb989ed41b3585ef57452404e638d703a211a3e4157/numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75", size = 11776652, upload-time = "2026-07-04T17:07:09.907Z" }, { url = "https://files.pythonhosted.org/packages/b5/59/2b844c7a6e9deff69b404a66221e1542937734f65d5e6e39411876053862/numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2", size = 5335944, upload-time = "2026-07-04T17:07:12.227Z" }, { url = "https://files.pythonhosted.org/packages/86/51/9bf7cb2cabcebc9e017e4ec7e6322b378317a542c08b4cb68479c1efc716/numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b", size = 6656266, upload-time = "2026-07-04T17:07:14.368Z" }, { url = "https://files.pythonhosted.org/packages/83/3e/fb7615b211b82a32f44d5180a6d421b61f84d4fadd578b48ba4ac34e189f/numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95", size = 15179720, upload-time = "2026-07-04T17:07:16.272Z" }, { url = "https://files.pythonhosted.org/packages/41/5f/0f992cb24560673496c5d68de61913b57166ce530ffda07c1f280e0cc464/numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21", size = 16664835, upload-time = "2026-07-04T17:07:19.021Z" }, { url = "https://files.pythonhosted.org/packages/a2/2f/97d6475ee91afe2587797d09446f9d3e475ad4cb681662d824809327b75a/numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373", size = 16539135, upload-time = "2026-07-04T17:07:22.015Z" }, { url = "https://files.pythonhosted.org/packages/c4/5b/4db81e4ba0be7e2776b1de68c82aa862c7f8ec27e1b4927d4ae075e20678/numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438", size = 18426684, upload-time = "2026-07-04T17:07:24.941Z" }, { url = "https://files.pythonhosted.org/packages/1f/64/c0ba2d90724d450279a7df8f32057241070250a26a7e2b5337d77347f481/numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace", size = 6116103, upload-time = "2026-07-04T17:07:27.622Z" }, { url = "https://files.pythonhosted.org/packages/c1/1a/837f9ed7405adcd7a40538792eb169eddd8fa5630c16a1ef49dae71a30f4/numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a", size = 12562177, upload-time = "2026-07-04T17:07:29.887Z" }, { url = "https://files.pythonhosted.org/packages/22/ed/49707938b6dd0a78a9178dd93227dc89e4c11af47f5c798d70366e8d0483/numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0", size = 10627739, upload-time = "2026-07-04T17:07:32.568Z" }, { url = "https://files.pythonhosted.org/packages/a6/c7/bb4b882cfe7f299cbc8b66e42e7dd78cf9d14e40f9469fc5e3db7e15b3bd/numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22", size = 11894709, upload-time = "2026-07-04T17:07:34.941Z" }, { url = "https://files.pythonhosted.org/packages/40/3f/5af7f4a7f6224aef48017aa82bb6174c7a659d724be0c75017b7e64a55b4/numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7", size = 5453810, upload-time = "2026-07-04T17:07:37.495Z" }, { url = "https://files.pythonhosted.org/packages/20/c9/3474309bc94d634d3f9c3eddf03250ecb8c22cd948ef16fef69a77cc5d7b/numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d", size = 6761189, upload-time = "2026-07-04T17:07:39.563Z" }, { url = "https://files.pythonhosted.org/packages/90/8a/558ae39fdd55d7e7f7fef9a84a6e964ac6b23edbd2a07e52bb084500507d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09", size = 15225039, upload-time = "2026-07-04T17:07:41.682Z" }, { url = "https://files.pythonhosted.org/packages/63/27/ca7392b2d030277bdf0273e7d23255b3ee57d57a7c170a6f4fb3981e1e5d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4", size = 16701306, upload-time = "2026-07-04T17:07:44.611Z" }, { url = "https://files.pythonhosted.org/packages/02/42/03d53ae7996c44d4374a8262e9dc41671fd56cbb98f7d47ef85cf5da4c6b/numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1", size = 16589955, upload-time = "2026-07-04T17:07:47.694Z" }, { url = "https://files.pythonhosted.org/packages/7b/15/6c1784ae469640e65db111e9a34b3d0f14d91e8a38b9ce34810ced370dbb/numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077", size = 18464252, upload-time = "2026-07-04T17:07:50.684Z" }, { url = "https://files.pythonhosted.org/packages/94/a8/f98e50356cf167df656c526c2dfeec2d7dde182f2a3da4b458a5938e2776/numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf", size = 6263298, upload-time = "2026-07-04T17:07:53.445Z" }, { url = "https://files.pythonhosted.org/packages/72/ac/96ae880cdecad0b3275d9359fcec72667b49a4863c9f12942e43679dda02/numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af", size = 12748623, upload-time = "2026-07-04T17:07:55.384Z" }, { url = "https://files.pythonhosted.org/packages/a1/5a/4d2b1601df3602dba7a14f3348ba9bfe94a18adb428e693df6154c293831/numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb", size = 10697674, upload-time = "2026-07-04T17:07:58.506Z" }, ] [[package]] name = "packaging" version = "26.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] [[package]] name = "paginate" version = "0.5.7" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ec/46/68dde5b6bc00c1296ec6466ab27dddede6aec9af1b99090e1107091b3b84/paginate-0.5.7.tar.gz", hash = "sha256:22bd083ab41e1a8b4f3690544afb2c60c25e5c9a63a30fa2f483f6c60c8e5945", size = 19252, upload-time = "2024-08-25T14:17:24.139Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/90/96/04b8e52da071d28f5e21a805b19cb9390aa17a47462ac87f5e2696b9566d/paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591", size = 13746, upload-time = "2024-08-25T14:17:22.55Z" }, ] [[package]] name = "pathspec" version = "1.1.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, ] [[package]] name = "platformdirs" version = "4.11.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/78/9b/560e4be8e26f6fd133a03630a8df0c663b9e8d61b4ade152b72005aec83b/platformdirs-4.11.0.tar.gz", hash = "sha256:0555d18370482847566ffabcaa53ad7c6c1c29f195989ae1ed634a05f76ea1e0", size = 31953, upload-time = "2026-07-21T13:09:36.565Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/7d/68/d8d58938dfb1370b266a1a729e6d77a985be23689a0496498ee17b2cbf90/platformdirs-4.11.0-py3-none-any.whl", hash = "sha256:360ccded2b7fce0af0ff80cc8f5942a1c5d99b0e856033acb030bfc634709e74", size = 23247, upload-time = "2026-07-21T13:09:35.422Z" }, ] [[package]] name = "pluggy" version = "1.6.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] [[package]] name = "pygments" version = "2.20.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] [[package]] name = "pymdown-extensions" version = "11.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown" }, { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/21/a9/5f0c535ba3b08fe09270c16808e053a968868242ecbd5676d4e3a488bf28/pymdown_extensions-11.0.1.tar.gz", hash = "sha256:dd2905ae6fc5b75582fafb139a1266ffc754705efa902aa50067fa7ff4f94ec0", size = 857113, upload-time = "2026-07-02T17:59:22.955Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d6/54/da572c98c0b77626a91b5d3b89f0231d8bff5125c225420908632f8b342d/pymdown_extensions-11.0.1-py3-none-any.whl", hash = "sha256:db3943a62bab7e03af1364f0c4083e64b91fb097675a4b6cceccfbe9a77e5eb2", size = 269455, upload-time = "2026-07-02T17:59:21.271Z" }, ] [[package]] name = "pytest" version = "9.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] [[package]] name = "python-dateutil" version = "2.9.0.post0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "six" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] [[package]] name = "pyyaml" version = "6.0.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] [[package]] name = "pyyaml-env-tag" version = "1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/2e/79c822141bfd05a853236b504869ebc6b70159afc570e1d5a20641782eaa/pyyaml_env_tag-1.1.tar.gz", hash = "sha256:2eb38b75a2d21ee0475d6d97ec19c63287a7e140231e4214969d0eac923cd7ff", size = 5737, upload-time = "2025-05-13T15:24:01.64Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/04/11/432f32f8097b03e3cd5fe57e88efb685d964e2e5178a48ed61e841f7fdce/pyyaml_env_tag-1.1-py3-none-any.whl", hash = "sha256:17109e1a528561e32f026364712fee1264bc2ea6715120891174ed1b980d2e04", size = 4722, upload-time = "2025-05-13T15:23:59.629Z" }, ] [[package]] name = "requests" version = "2.34.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, { name = "charset-normalizer" }, { name = "idna" }, { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] [[package]] name = "ruff" version = "0.16.8" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ba/78/449cb84790bd5cc3823b2652ee405a4558856e5c4195aee3a16bf7b3eb5d/ruff-0.16.8.tar.gz", hash = "sha256:9247bf92b5f04d825c8639a4fe423ec2e4222acd9222e58412b0dab7e442798b", size = 4938814, upload-time = "2026-09-16T15:54:46.688Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ac/25/6071aabc530e9be7e2c195e8fe3f7aea2735405b6cf447212832d7811831/ruff-0.16.8-py3-none-linux_armv6l.whl", hash = "sha256:6ffbd6d87383c1edf5f6fa890f10200950240d7c1a16052a19a09d3a2307dd38", size = 10048966, upload-time = "2026-09-16T15:53:57.605Z" }, { url = "https://files.pythonhosted.org/packages/54/98/07f90ecbc74dd5fb5764f11f2bc774d6a7cffef92d2ff5f5b4e9e23c754e/ruff-0.16.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:42ed6b878ed61e3acca92f2730a17acff39286944ea82398544696366a6f925e", size = 10165498, upload-time = "2026-09-16T15:54:01.14Z" }, { url = "https://files.pythonhosted.org/packages/fe/1f/e6a712e3b47cad4a40600134105ed193cb773f618a42eb7ba323cb812cc0/ruff-0.16.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7ea781c7f2afba8c6a505ea0fb3f994020249e0c450635f5381286fea6b46170", size = 9830004, upload-time = "2026-09-16T15:54:03.998Z" }, { url = "https://files.pythonhosted.org/packages/23/f2/311a08776d75d81c7676e20b6b020ae63cbe881fcdc7a8dd64e6e18bdd93/ruff-0.16.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8efeae3bbe414a5efefda11a792dfb51ef90ac48d50c4830de2f644caf3e8659", size = 9986558, upload-time = "2026-09-16T15:54:06.804Z" }, { url = "https://files.pythonhosted.org/packages/f3/ed/37b6cb3d3ba8c73e68ae3eb1d502383beb5aa05a582bb7bb3a922f929f54/ruff-0.16.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a79b795469fef7fc6e908b218eed2eb17332afd85031db6480dc864560e69b2", size = 9877332, upload-time = "2026-09-16T15:54:09.552Z" }, { url = "https://files.pythonhosted.org/packages/22/cc/40873a8f36ad084cc540d55fcca7077264d5b13b24659e9180c176fb2b08/ruff-0.16.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3fdc5563cdc50555e6fba39322850860e9267c1b3d12c26a74729d8604c3c812", size = 10507125, upload-time = "2026-09-16T15:54:12.152Z" }, { url = "https://files.pythonhosted.org/packages/c3/e4/fc91a642b78ccbab6b9477720f3644ae7a10a9bcce69a934679cd64f62bc/ruff-0.16.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:34508983c70665578dab88f5223d8e6228307e1135398ca8bfc8b7e9501e282b", size = 11336694, upload-time = "2026-09-16T15:54:15.489Z" }, { url = "https://files.pythonhosted.org/packages/c2/3d/bbd2a9a600a4e73dc3e7548a249c8d1671273464b55822c6fae50f602dff/ruff-0.16.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:644bb578569e0ffc575741232bd385dacdd6fbe123f1a729e7a225f54aa3957f", size = 10774448, upload-time = "2026-09-16T15:54:18.16Z" }, { url = "https://files.pythonhosted.org/packages/1a/41/d83af9879a7b6e8bf5fe16b1da0b134049d2f5d3afac12defb0897cb84bd/ruff-0.16.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15e7d226246961db9235098333caa13063906d3851136b84c2900b82f5daa1df", size = 10323796, upload-time = "2026-09-16T15:54:20.743Z" }, { url = "https://files.pythonhosted.org/packages/f5/2c/cefd07bfe914b84943ea769ade8d607bd22750b965d3228eefd7cebd15d0/ruff-0.16.8-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:a2bf6bc3e9ebdd4449abc6f06cf64b98051a2c61cf94d2fe9596518c881f1a1e", size = 10514115, upload-time = "2026-09-16T15:54:23.497Z" }, { url = "https://files.pythonhosted.org/packages/f3/9d/76a2e26c79a23be6e6e3664c57bec9e9fc8de155cfb9e4b67ea91b64f9d7/ruff-0.16.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6ca111ba0849539165e9e59d2b442542f3c1e8060ebbdea82494f1ffbccb1e1f", size = 10072582, upload-time = "2026-09-16T15:54:26.185Z" }, { url = "https://files.pythonhosted.org/packages/2e/d4/f42edddb39668af1a559ceafa3823aedd65633a48dc9768e775485faa2c1/ruff-0.16.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:359a1e5b495448ee1e91018064382ebc86f90e8aac2fed222c7d0e4e8df85fd2", size = 9879644, upload-time = "2026-09-16T15:54:29.278Z" }, { url = "https://files.pythonhosted.org/packages/f8/d4/913e3195d95e0378786c6656945c865f534a3560e29139da4882aff630d1/ruff-0.16.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:59e8f5681349474110b24d62e93cfda6593f5fa3473446ca3705200cac1a08b9", size = 10231569, upload-time = "2026-09-16T15:54:32.036Z" }, { url = "https://files.pythonhosted.org/packages/2b/c4/8aa6ea0bdcedbd1bf87397e2fc4ed8406448ea5842f8660bc6e5f163039d/ruff-0.16.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:efa3e7a16d1baaa79957888dfdf8be9ef2e44db81cb032af06d76632ab59e773", size = 10663666, upload-time = "2026-09-16T15:54:34.838Z" }, { url = "https://files.pythonhosted.org/packages/3d/02/7f10ef4700bc223c30a3fdd10631a29830c45524b810a3c7ed947af64591/ruff-0.16.8-py3-none-win32.whl", hash = "sha256:55793ba85c69921e89be061426d91a78652d6e50317c962240922747a4eb713f", size = 10093472, upload-time = "2026-09-16T15:54:37.47Z" }, { url = "https://files.pythonhosted.org/packages/1e/5d/a509c07d714b6da88f2c518b4637cf6f1d46b074be8f0f1e5fb9ff5126fe/ruff-0.16.8-py3-none-win_amd64.whl", hash = "sha256:a6b85621fd3c81e31fc5f5add09c9c078b430db3595ca632efafdec9e64ebfaa", size = 10586899, upload-time = "2026-09-16T15:54:40.488Z" }, { url = "https://files.pythonhosted.org/packages/fe/a0/50787329e4f20bf9dc9f6230015d46ec69c51a97ace5bc202dae4755365d/ruff-0.16.8-py3-none-win_arm64.whl", hash = "sha256:d075e820af612102ce217f07cc93e69f9490b10ec13ea85fa87bd03d996cef8a", size = 10386316, upload-time = "2026-09-16T15:54:43.332Z" }, ] [[package]] name = "six" version = "1.17.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] [[package]] name = "sortedcontainers" version = "2.4.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, ] [[package]] name = "urllib3" version = "2.8.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, ] [[package]] name = "watchdog" version = "6.0.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220, upload-time = "2024-11-01T14:07:13.037Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/39/ea/3930d07dafc9e286ed356a679aa02d777c06e9bfd1164fa7c19c288a5483/watchdog-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdd4e6f14b8b18c334febb9c4425a878a2ac20efd1e0b231978e7b150f92a948", size = 96471, upload-time = "2024-11-01T14:06:37.745Z" }, { url = "https://files.pythonhosted.org/packages/12/87/48361531f70b1f87928b045df868a9fd4e253d9ae087fa4cf3f7113be363/watchdog-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c7c15dda13c4eb00d6fb6fc508b3c0ed88b9d5d374056b239c4ad1611125c860", size = 88449, upload-time = "2024-11-01T14:06:39.748Z" }, { url = "https://files.pythonhosted.org/packages/5b/7e/8f322f5e600812e6f9a31b75d242631068ca8f4ef0582dd3ae6e72daecc8/watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0", size = 89054, upload-time = "2024-11-01T14:06:41.009Z" }, { url = "https://files.pythonhosted.org/packages/68/98/b0345cabdce2041a01293ba483333582891a3bd5769b08eceb0d406056ef/watchdog-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:490ab2ef84f11129844c23fb14ecf30ef3d8a6abafd3754a6f75ca1e6654136c", size = 96480, upload-time = "2024-11-01T14:06:42.952Z" }, { url = "https://files.pythonhosted.org/packages/85/83/cdf13902c626b28eedef7ec4f10745c52aad8a8fe7eb04ed7b1f111ca20e/watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:76aae96b00ae814b181bb25b1b98076d5fc84e8a53cd8885a318b42b6d3a5134", size = 88451, upload-time = "2024-11-01T14:06:45.084Z" }, { url = "https://files.pythonhosted.org/packages/fe/c4/225c87bae08c8b9ec99030cd48ae9c4eca050a59bf5c2255853e18c87b50/watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a175f755fc2279e0b7312c0035d52e27211a5bc39719dd529625b1930917345b", size = 89057, upload-time = "2024-11-01T14:06:47.324Z" }, { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079, upload-time = "2024-11-01T14:06:59.472Z" }, { url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078, upload-time = "2024-11-01T14:07:01.431Z" }, { url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076, upload-time = "2024-11-01T14:07:02.568Z" }, { url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077, upload-time = "2024-11-01T14:07:03.893Z" }, { url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078, upload-time = "2024-11-01T14:07:05.189Z" }, { url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077, upload-time = "2024-11-01T14:07:06.376Z" }, { url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078, upload-time = "2024-11-01T14:07:07.547Z" }, { url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065, upload-time = "2024-11-01T14:07:09.525Z" }, { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070, upload-time = "2024-11-01T14:07:10.686Z" }, { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, ] [[package]] name = "zarr-indexing" source = { editable = "." } dependencies = [ { name = "numpy" }, ] [package.optional-dependencies] testing = [ { name = "hypothesis" }, ] [package.dev-dependencies] docs = [ { name = "griffe-inherited-docstrings" }, { name = "mkdocs" }, { name = "mkdocs-material" }, { name = "mkdocstrings" }, { name = "mkdocstrings-python" }, { name = "pymdown-extensions" }, { name = "ruff" }, ] test = [ { name = "hypothesis" }, { name = "pytest" }, ] [package.metadata] requires-dist = [ { name = "hypothesis", marker = "extra == 'testing'", specifier = ">=6.160.0" }, { name = "numpy", specifier = ">=2" }, ] provides-extras = ["testing"] [package.metadata.requires-dev] docs = [ { name = "griffe-inherited-docstrings", specifier = "==1.1.3" }, { name = "mkdocs", specifier = "==1.6.1" }, { name = "mkdocs-material", specifier = "==9.7.7" }, { name = "mkdocstrings", specifier = "==1.0.6" }, { name = "mkdocstrings-python", specifier = "==2.0.8" }, { name = "pymdown-extensions", specifier = "==11.0.1" }, { name = "ruff", specifier = "==0.16.8" }, ] test = [ { name = "hypothesis", specifier = ">=6.160.0" }, { name = "pytest" }, ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/000077500000000000000000000000001527014137600237775ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/.readthedocs.yaml000066400000000000000000000052471527014137600272360ustar00rootroot00000000000000# Read the Docs configuration for the zarr-metadata docs site, separate from # the zarr-python site configured by the repo-root .readthedocs.yaml. The RTD # project for zarr-metadata must set its configuration-file path to # packages/zarr-metadata/.readthedocs.yaml. version: 2 build: os: ubuntu-22.04 tools: python: "3.12" jobs: post_checkout: # Cancel pull request builds that do not touch this package. Exit code # 183 cancels the build and reports success to the Git provider. Scoped # to PR builds ("external" versions) because origin/main is only a # meaningful diff base there. The shared docs config and header script # live outside the package directory but change this site too. # The diff is taken against the merge base (origin/main...HEAD), not # against the tip of main: a two-dot diff also reports everything that # landed on main after the PR branched, which made every PR that was # behind main build every site. The checkout is shallow, so fetch the # full history first; without a merge base the diff fails and the build # runs, which is the safe default. - | if [ "$READTHEDOCS_VERSION_TYPE" = "external" ]; then git fetch --quiet --unshallow origin main pull/$READTHEDOCS_VERSION_NAME/head || true; if git diff --quiet origin/main...HEAD -- packages/zarr-metadata packages/mkdocs-base.yml packages/source-version.js; then exit 183; fi fi install: - pip install --upgrade pip - pip install ./packages/zarr-metadata --group packages/zarr-metadata/pyproject.toml:docs build: html: # Build from inside the package rather than pointing `-f` at its config # from the repo root. mkdocs resolves some settings relative to the # current working directory rather than to the config file, so building # from elsewhere looks for them in the wrong place -- and silently, since # the paths are valid, just wrong. zarr-indexing hit this: with # `pymdownx.snippets` and a relative `base_path`, its snippets were # searched for under the repo-root docs/ and the build failed with # SnippetMissingError, while `just docs-check` passed because it runs # from here. Building from the package directory makes this identical to # the local and CI invocations, so a green build there means a green # build here. # # $READTHEDOCS_OUTPUT is absolute, so the cd does not affect it. - cd packages/zarr-metadata && mkdocs build --strict --site-dir $READTHEDOCS_OUTPUT/html mkdocs: configuration: packages/zarr-metadata/mkdocs.yml zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/CHANGELOG.md000066400000000000000000001163551527014137600256230ustar00rootroot00000000000000# Release notes ## 0.6.0 (2026-10-26) ### Features - **Breaking:** `ValidationProblem` and `MetadataValidationError` check at run time what their annotations declare, and raise `TypeError` otherwise. A problem's `loc` is a tuple of strings and integers, so a list, which was accepted before, is now refused, as is a `bool`; its `message` is a string and its `kind` one of the kinds; an error holds problems. A caller that is not type-checked fails where it builds the value, not later, where a location is read. ([#4422](https://github.com/zarr-developers/zarr-python/pull/4422)) - `zarr_metadata.typed_json.check(value, SomeTypedDict)` type-checks a JSON value against any of the package's TypedDicts -- `ZarrV3ArrayMetadataJSON`, `GzipCodecConfiguration`, or one of your own -- and gives back a value of it, or None, with every problem located. A TypedDict reads as the typing spec defines it, whatever the runtime's attributes say: a key is required by its own `Required` or `NotRequired`, or else by the `total` of the class that declared it, read from the evaluated annotations, so a module written with `from __future__ import annotations` reads like any other; `closed`, `extra_items` and openness come from the class, or from its bases when it says neither; and each annotation is evaluated in the module of the class that wrote it. A key a closed TypedDict does not declare is reported, as the new problem kind `unknown_key`, and left out of the value, which still comes back; a union of TypedDicts tagged by a `Literal` key is read by the branch the tag names. ([#4432](https://github.com/zarr-developers/zarr-python/pull/4432)) - **Breaking:** `must_understand: false` is refused at every extension point of an array document, codecs and storage transformers included, not only at `data_type`, `chunk_grid` and `chunk_key_encoding`: ignoring a codec gives wrong bytes as surely as ignoring a data type gives wrong values, and the spec naming only three points is read as an oversight. A document that declares a codec or a storage transformer ignorable now has a problem at `codecs.N.must_understand` or `storage_transformers.N.must_understand`, so the readers refuse it, and no model writes one. The JSON schema `zarr_metadata.pydantic` generates for an array document says the same, and so does `resolve`, which reads one field on its own. ([#4434](https://github.com/zarr-developers/zarr-python/pull/4434)) - Each extension's metadata is a definition: a value holding the name the metadata carries, the TypedDict its configuration is -- the one declaration of that JSON -- and plain functions over that TypedDict for the rules. `zarr_metadata.definition` and `zarr_metadata.v3.definition` are the doors, and their module docstrings the guide. Every extension the package models is one: the codecs `blosc`, `bytes`, `cast_value`, `crc32c`, `gzip`, `scale_offset`, `sharding_indexed`, `transpose` and `zstd`; every data type, raw bits included (`r16` is read by the `r*` definition with `{"bits": 16}`, the size its name carries); the `regular` and `rectilinear` chunk grids; and the `default` and `v2` chunk key encodings. Each is a constant in its own module -- `GZIP_CODEC`, `STRUCT_DATA_TYPE` -- and `CORE` holds what the specification defines, `CORE_AND_EXTENSIONS` what `zarr-extensions` registers besides. A third party writes a TypedDict, a rule function and a definition, and adds it to a scope; a definition checks itself when built, and nothing happens at class creation. Reading a field is three steps, each usable on its own by a caller holding nothing but JSON. `check(value, SomeTypedDict)`, from `zarr_metadata.typed_json`, type-checks JSON against a TypedDict and needs nothing else; `definition.read_configuration(configuration)` is the check, each nested field's envelope judged, and then the rules; `resolve(field, CodecDefinition, scope)` reads a whole field in a scope -- its envelope judged, its name related to a definition, its configuration judged, each nested field read the same way -- and returns what the scope made of it, with every problem located. A name nothing in scope claims is left unjudged. A shard's pipelines, a cast's target type and a struct's field types are nested fields, annotated `ZarrV3CodecField` and `ZarrV3DataTypeField`, and read in the scope their field is read in. A codec definition also says whether the size of what it gives out is `static` or `dynamic`, and a member annotated `ZarrV3StaticCodecField` takes codecs of static size only: a shard's `index_codecs`, where the spec forbids a compressor, so a `gzip` there is a problem at its place. The TypedDict reads as the typing spec defines it -- `total`, `Required`, `NotRequired`, `closed` and `extra_items` mean what they mean to a type checker, whether or not its module postpones annotations. A key a closed TypedDict does not declare is reported, as `unknown_key`, and left out of what the rules are handed. A definition refuses a TypedDict that says nothing of the keys it does not declare, and a member typed `ZarrV3MetadataFieldJSON`, which would check as plain JSON. `canonical_of(field, problems)` gives a field without problems in its simplest equivalent spelling: each nested field in its own simplest spelling, then its definition's own `canonical`, and the envelope in the fewest words every reader takes, with no `must_understand`. A field with any problem, an unknown key included, has none, since a simpler spelling would erase what its author wrote. What `canonical` gives is judged again, and one that does not hold is a `ValueError`, a fault in the definition. ([#4434](https://github.com/zarr-developers/zarr-python/pull/4434)) - **Breaking:** the v3 readers read each extension point through its definition: `read_array_metadata_v3` judges the data type, the chunk grid, the chunk key encoding, each codec and each storage transformer against the definition that claims its name in a scope, `CORE_AND_EXTENSIONS` unless a `context` is passed, so a gzip `level` of 99, or a key a codec does not declare, is a problem located in the configuration, and a document holding one, accepted before, is refused. So do the v3 group readers, and the v3 model classes' `from_json` and `from_key_value` take the same `context`. A name nothing in scope claims is left unjudged. A problem in a field that a configuration holds, such as a codec inside a shard, is reported at that field's own location, and the field that holds it is still read. ([#4436](https://github.com/zarr-developers/zarr-python/pull/4436)) - **Breaking:** a v3 array's `fill_value` is judged against its `data_type`. Each data type's definition declares the JSON shape of its fill value, `fill_value`, and the rules for one of that shape, `fill_value_rules`: an `int8` fill value of 300, a `float32` hex string of another width, and a struct fill value missing a field are each a problem at `fill_value`, where the package accepted them before. `fill_value_problems(data_type, value)` judges a fill value against a data type field a scope read, and a field that is read keeps the fields it read inside as `nested`, a `Nested` mapping by location. A data type nothing in scope claims leaves its fill value unjudged. ([#4443](https://github.com/zarr-developers/zarr-python/pull/4443)) - **Breaking:** a v3 array's sharding codec is judged against the shard it is handed, and its two pipelines are read as the array's is. Its `chunk_shape` has an inner chunk length for each axis of the shard, dividing each length the shards take along it -- the shard being the chunk the codec is handed, so a transposed shard is divided along its transposed axes. Its inner codecs are handed the inner chunks, of the shard's data type, and its index codecs the shard index, of `uint64`, with an axis more than the shard. A shard that does not divide, or a pipeline that does not fit what it is handed -- an index `bytes` codec without an `endian` -- has a problem where it sits, where the package accepted it before. A codec definition says what the pipelines it holds are handed, `pipelines`, and each codec's `Stage` keeps the stages of those pipelines as `inner`. ([#4443](https://github.com/zarr-developers/zarr-python/pull/4443)) - **Breaking:** a v3 array's `chunk_grid` is judged against its `shape`, by the grid's definition. A regular grid whose `chunk_shape` does not have one length per dimension of the shape, and a rectilinear grid whose `chunk_shapes` does not have one entry per dimension, or whose chunk lengths fall short of their dimension, each have a problem in `chunk_grid.configuration`, where the package accepted them before. ([#4443](https://github.com/zarr-developers/zarr-python/pull/4443)) - **Breaking:** a v3 array's `codecs` are read as a pipeline: in order -- array -> array codecs, then one array -> bytes codec, then bytes -> bytes codecs -- and each codec judged against the chunk it is handed, which is the grid's chunks of the array's data type, as the array -> array codecs before it hand them on. A codec out of order, a second array -> bytes codec or none at all; a `bytes` codec without an `endian` handed numbers of several bytes, or handed values that vary in size; a `transpose` whose `order` has another number of axes than its chunk; a `cast_value` to or from a data type that models no real numbers, wrapping to one that is not integral, or mapping a scalar that is not a fill value of the data type on its side; a `scale_offset` handed values that are no numbers, or with an `offset` or `scale` that is not a value of its data type; and a struct field whose values vary in size are each a problem where they sit, where the package accepted them before. A reading records each codec with the chunk it is handed, as a `Stage`, and the lengths a grid's chunks take; `storage_of(data_type)` says how a data type's values are stored. A definition's `rules` are handed the fields its configuration holds as the scope read them, and the kinds gain `chunk_lengths` (grids), `storage` (data types), and `chunk_rules` and `transition` (codecs). ([#4443](https://github.com/zarr-developers/zarr-python/pull/4443)) - **Breaking:** metadata is now read against the definitions in a scope, a `ZarrV3Context` or `ZarrV2Context` that a reader builds from the codecs, data types and other fields it supports, and a model is the document together with the scope it was read in. The `read_*` readers report every problem at its location, `unclaimed_fields` lists each field the scope does not know, so a reader can tell whether it can open a node, repair readers undo known zarr-python writer bugs, and JSON Schemas are exported. Code written for 0.5.0 must change: models are no longer dataclasses, their field members hold `AcceptedField` or `UnclaimedField` values, `update` takes JSON and takes members the spec does not define through `extra_fields`, the `validate_*`, `is_*` and `parse_*` functions for documents and the `...Partial` TypedDicts are removed, some names moved to another module, the v2 constants and models follow the v3 names and shapes, and a v2 time dtype without a unit is refused. `to_json` now writes each field as it was read, rather than shortening a codec to a bare name that zarr-python could not open, and zarr-metadata now depends on `annotated-types`. ([#4490](https://github.com/zarr-developers/zarr-python/pull/4490)) ### Bugfixes - The v2 array validator follows the spec on two points it was stricter than: members outside the `.zarray` definition are accepted when they are JSON values with string keys, and are kept and written back by `to_json` and `to_key_value`; any other member is a problem, as in a v3 document (the spec says other keys ["SHOULD NOT be present ... and SHOULD be ignored"](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), unlike `.zgroup`'s ["MUST NOT"](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313)), and `filters: []` is accepted (["a list of JSON objects providing codec configurations, or null"](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79) sets no minimum). The JSON Schema that `zarr_metadata.pydantic` generates follows: it accepts other members, and the filter pipeline has no minimum length. ([#4365](https://github.com/zarr-developers/zarr-python/pull/4365)) - Concrete v3 entity types are now assignable to the fields they describe. Previously, none of the package's canonical codec / chunk-grid / chunk-key-encoding / data-type types (e.g. `BloscCodecMetadata`, `RegularChunkGridMetadata`) satisfied `ZarrV3MetadataFieldJSON`, so a type checker rejected putting them into the very fields they document (`codecs`, `chunk_grid`, `data_type`, ...). Three changes fix this: - `ZarrV3NamedConfigJSON.name` and `.configuration` are now `ReadOnly` (PEP 705), making them covariant so concrete `name: Literal[...]` and required-`configuration` shapes are accepted. - `ZarrV3NamedConfigJSON` is now `closed` (PEP 728): the spec's named-configuration envelope has exactly `name` / `configuration` / `must_understand`, and closing the type also makes it usable as a `JSONValue` (needed for e.g. the `sharding_indexed` inner `codecs`). - Every concrete `*Object` / `*Configuration` TypedDict is now `closed`, and object forms declare `must_understand: NotRequired[bool]` (any v3 metadata field may carry the extension member). **Soft-breaking** for type-checking consumers: dicts with keys beyond the declared shape no longer satisfy the closed types, and `name` / `configuration` can no longer be mutated through `ZarrV3NamedConfigJSON`. Both were previously accepted by type checkers but produced documents outside the spec's shapes. Generating a pydantic JSON Schema for the `zarr_metadata.pydantic` field types no longer emits warnings. ([#4378](https://github.com/zarr-developers/zarr-python/pull/4378)) - Attributes are user data, and a non-finite number in them is read, validated and written back. zarr-python writes attributes with the defaults of Python's `json` module, so an xarray `_FillValue` or a CF `missing_value` of NaN is stored as a bare `NaN`, which RFC 8259 lacks; the model refused such a document and could not read the store at all. A node's `attributes`, a v2 `.zattrs`, the `.zattrs` entries of a v2 `.zmetadata`, and the attributes of the nodes a v3 group's inline `consolidated_metadata` holds may now hold `NaN`, `Infinity` and `-Infinity`, in every reader, such as `read_array_metadata_v3`, and in each model's `from_json`, `from_key_value` and `to_key_value`. Wherever the spec interprets a value it spells those numbers as strings, so anywhere else a non-finite number is still refused, and the store reader now says where one is instead of failing to decode the document. A model whose document the reader refuses raises `MetadataValidationError` with every problem, instead of being written: - a non-finite number outside attributes, which was refused before too; - a non-string attribute key, which was written as a string; - a value that is not JSON, which raised `TypeError`; - anything else the reader refuses, such as `dimension_names` longer than `shape`, which was written and then refused on read. Any sequence is written as an array. A number too large for a float, such as `1e400`, is read as infinity, so in attributes it is now accepted and written back as `Infinity`. Through `zarr_metadata.pydantic`, pydantic writes a non-finite attribute as `null` unless the enclosing model sets `ser_json_inf_nan="constants"`. ([#4420](https://github.com/zarr-developers/zarr-python/pull/4420)) - `from_key_value` takes a mapping keyed by `str` or by any narrower key type, so reading back what `to_key_value` wrote type-checks. `to_key_value` keys its result by the store-key literals, and since a mapping's key type is invariant, a `Mapping[str, bytes]` parameter refused it. ([#4421](https://github.com/zarr-developers/zarr-python/pull/4421)) - `MetadataValidationError` pickles and copies with its message. A copy was built from the error's message, and its message came out as the original's characters, one per line. ([#4422](https://github.com/zarr-developers/zarr-python/pull/4422)) - A `bytes` or `bytearray` is not a JSON array. Where a document expects an array, an empty one passed as an array with no items: the v3 parser returned `storage_transformers=b""`, and the v2 readers took `dtype=b""`, then failed when the model was written. The readers and models now refuse one at `storage_transformers` and at a v2 `dtype` and its field records. At `codecs`, `dimension_names` and v2 `filters`, which already refused one, the problem now says the value has the wrong type. ([#4423](https://github.com/zarr-developers/zarr-python/pull/4423)) - A model read by `from_json` or `from_key_value` shares no mutable state with the document it was read from, as a document from `to_json` already shares none with its model. A frozen model held the caller's own dictionaries, so editing the input after reading it changed the model, and could make `to_key_value` fail. ([#4426](https://github.com/zarr-developers/zarr-python/pull/4426)) - `from_key_value` checks that each store value it reads is `bytes`, as its signature says: a `str` was decoded as a JSON document, and `None` or a `memoryview` raised `TypeError`. Each is now a `MetadataValidationError` with an `invalid_type` problem at the store key. ([#4428](https://github.com/zarr-developers/zarr-python/pull/4428)) - `ZarrV3ArrayMetadata.create_default(shape=...)` derives a chunk length of 1 for a dimension of length 0, where it wrote 0: the regular grid asks for chunk lengths greater than zero, and `zarr` does not open a grid with one. ([#4443](https://github.com/zarr-developers/zarr-python/pull/4443)) ### Improved Documentation - The docs now render the release notes as a page instead of linking to `CHANGELOG.md` on GitHub, so each docs version shows its own changelog. ([#4311](https://github.com/zarr-developers/zarr-python/pull/4311)) ### Misc - `ZarrV2ConsolidatedMetadataJSON.zarr_consolidated_format` is typed `Literal[1]` rather than `int`. Format 1 is the only defined `.zmetadata` format and the runtime validator already rejected anything else, so the type now carries the constraint the validator enforces instead of contradicting it. ([#4378](https://github.com/zarr-developers/zarr-python/pull/4378)) - **Breaking:** `MetadataValidationError.problems` is now a tuple instead of a list. Iteration and indexing are unchanged; code that changes a report must first copy it with `list(problems)`. `MetadataValidationError` still accepts any problem sequence. ([#4378](https://github.com/zarr-developers/zarr-python/pull/4378)) - `ZarrV2ZGroupJSON` and `ZarrV2GroupMetadataJSON` are closed (PEP 728), as the spec's "Other keys MUST NOT be present" and the v2 group validator already said, and as its pydantic schema already was. A type checker now refuses a v2 group document with a key it does not declare. ([#4430](https://github.com/zarr-developers/zarr-python/pull/4430)) ## 0.5.0 (2026-08-14) ### Bugfixes - `JSONValue`'s array arm is now the covariant `Sequence["JSONValue"]` rather than the invariant `list["JSONValue"] | tuple["JSONValue", ...]`. Values typed with a narrower element type — a `list[str]` field on a TypedDict, a `Sequence[float]` — now count as JSON values, and TypedDicts whose fields carry precise types are now assignable to `Mapping[str, JSONValue]`. Type-level cost, accepted deliberately: `Sequence` says nothing about the concrete container and admits `str`/`bytes`, so runtime code narrowing a JSON array must exclude `str`/`bytes`/`bytearray` — as it already had to, since `str` was always a union arm. ([#4264](https://github.com/zarr-developers/zarr-python/pull/4264)) ### Deprecations and Removals - Unified the naming grammar for SCREAMING_SNAKE constants with the one used for type names. A constant's name is now a purely syntactic transformation of the name of the `Literal` type it manifests, so the format version is spelled `ZARR_V2`/`ZARR_V3` and comes first, matching the `ZarrV2`/`ZarrV3` prefix on the corresponding type: - `ARRAY_METADATA_STORE_KEY_V2` → `ZARR_V2_ARRAY_METADATA_STORE_KEY` - `ARRAY_METADATA_STORE_KEY_V3` → `ZARR_V3_ARRAY_METADATA_STORE_KEY` - `ATTRIBUTES_STORE_KEY_V2` → `ZARR_V2_ATTRIBUTES_STORE_KEY` - `GROUP_METADATA_STORE_KEY_V2` → `ZARR_V2_GROUP_METADATA_STORE_KEY` - `GROUP_METADATA_STORE_KEY_V3` → `ZARR_V3_GROUP_METADATA_STORE_KEY` - `CONSOLIDATED_METADATA_STORE_KEY_V2` → `ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY` - `ARRAY_ORDER_V2` → `ZARR_V2_ARRAY_ORDER` - `ARRAY_DIMENSION_SEPARATOR_V2` → `ZARR_V2_ARRAY_DIMENSION_SEPARATOR` - `CONSOLIDATED_METADATA_KEY_V3` → `ZARR_V3_CONSOLIDATED_METADATA_KEY` The old names are removed, not aliased. This supersedes the 0.4.0 convention under which type names put the format version first while constants put it last: every constant that manifests a `Literal` type now follows the same rule as that type. The last of those is the one rename the syntactic rule does not force: `ZARR_V3_CONSOLIDATED_METADATA_KEY` manifests no `Literal` type, so it is outside the rule and was renamed for consistency with its siblings. Digit runs stay glued to the token they follow, so spec vocabulary is preserved: `Uint8DataTypeName` pairs with `UINT8_DATA_TYPE_NAME` (not `UINT_8_...`) and `Crc32cCodecName` with `CRC32C_CODEC_NAME`. No dtype, codec, chunk-grid, or chunk-key-encoding constant changed name. Constants that do not manifest a `Literal` type are outside the rule and are unchanged: the `*_METADATA_*_KEYS_V2`/`_V3` key sets, the `CANONICAL_*_HEX_FLOAT*` bit patterns, and `UNSET`. The key sets keep the version-last spelling, so `zarr_metadata.model` exports both `ARRAY_METADATA_REQUIRED_KEYS_V2` and `ZARR_V2_ARRAY_METADATA_STORE_KEY`. They name validation policy rather than a spec document, have no paired type to derive from, and renaming them would be a second breaking change buying only cosmetic consistency — so it is deliberately deferred. `tests/test_public_api.py::test_constant_names_derive_from_their_type_names` derives every constant name from the type it manifests and asserts they match, so the two grammars cannot diverge again. Store keys also moved to the modules that describe the documents they name, matching the package's layering (the `v2`/`v3` modules describe the specs; the `model` layer is built on top of them). `ZARR_V2_ATTRIBUTES_STORE_KEY` now lives in `zarr_metadata.v2.attributes` beside the `.zattrs` type it names, rather than in the array model; the other five moved likewise, and `ZarrV2AttributesStoreKey` is no longer an array-specific concept. `zarr_metadata.model` re-exports all six, so `from zarr_metadata.model import ZARR_V2_ARRAY_METADATA_STORE_KEY` is unaffected. `CONSOLIDATED_METADATA_KEY_V3` moved to `zarr_metadata.v3.consolidated` and was renamed to `ZARR_V3_CONSOLIDATED_METADATA_KEY` for consistency. It is not a store key: unlike v2's `.zmetadata` file, v3 consolidated metadata is embedded as an extension field inside the group's own `zarr.json`. All seven keys and the six store-key `Literal` aliases are now also exported from the top-level `zarr_metadata` namespace, alongside the document types and the rest of the spec vocabulary, so `from zarr_metadata import ZARR_V2_ARRAY_METADATA_STORE_KEY` works. The model layer's validators, parsers, type guards, and metadata key sets remain `zarr_metadata.model` imports. ([#4232](https://github.com/zarr-developers/zarr-python/pull/4232)) ### Misc - The source distribution now ships an explicit allowlist (`/src`, `/tests`, `/docs`, `/mkdocs.yml`, `/justfile`, `/CHANGELOG.md`) rather than whatever happens to sit in the package directory, so an sdist both tests and documents itself and cannot pick up scratch files from the tree it was built in. ([#4248](https://github.com/zarr-developers/zarr-python/pull/4248)) ## 0.4.0 (2026-07-29) ### Features - Added `zarr_metadata.model`: frozen-dataclass models (`ZarrV2ArrayMetadata`, `ZarrV3ArrayMetadata`, `ZarrV2GroupMetadata`, `ZarrV3GroupMetadata`, `ZarrV2ConsolidatedMetadata`, `ZarrV3ConsolidatedMetadata`, `ZarrV3NamedConfig`) that are canonical, semantically lossless representations of Zarr metadata documents, plus structural validators (`validate_*` / `is_*` / `parse_*`). Every v3 extension point (data type, chunk grid, chunk key encoding, codecs, storage transformers) is held as `ZarrV3NamedConfig`: a name, configuration, and `must_understand` obligation; nothing is interpreted. On the wire, an empty configuration with the default obligation uses the spec's plain-string shorthand. Model fields are annotated with the role alias `ZarrV3MetadataField` (today exactly `ZarrV3NamedConfig`), so annotations convey the logical meaning and stay put if the spec adds another field form. Validation is strict about what the types declare: v2 `dtype` / `order` / `compressor` / `filters` / `dimension_separator` shapes and the fixed `zarr_format` / `node_type` literals are all enforced. Every `ValidationProblem` carries a machine-readable `kind` (`missing_key` / `invalid_type` / `invalid_value` / `invalid_json`) so consumers can dispatch on the failure mode without matching message strings, and every ingestion failure — including missing store keys and undecodable bytes in `from_key_value` — surfaces as `MetadataValidationError`. An adversarial review added further structural checks: JSON booleans are not accepted as dimension lengths, dimensions are non-negative, `dimension_names` must have one entry per dimension of `shape`, `attributes` and `configuration` values are JSON-checked recursively (like `fill_value`), non-finite floats and non-standard JSON constants are rejected, abstract mappings and sequences normalize to encoder-safe canonical containers, v2 `shape` and `chunks` must have the same rank, non-null v2 filter pipelines contain at least one filter, document `TypeIs` guards only narrow values that already use the declared canonical containers, and the inline consolidated-metadata envelope and entries are deep-validated so the group validator's verdict always agrees with the model constructor. The v3 models expose `must_understand_fields`: the subset of `extra_fields` not explicitly waived with `must_understand: false` (fields are implicitly must-understand per the spec). Readers discharge the spec's fail-to-open duty by subtracting the extension names they recognize; the model only partitions by obligation, since recognition is reader-specific. Optional pydantic integration ships as `zarr_metadata.pydantic` (importing it requires pydantic 2.13 or newer; the core package does not depend on it): one `Annotated` field type per model, validating raw documents through `from_json`, passing core-model instances through unchanged, serializing via `to_json`, and publishing JSON Schemas derived from private constrained document types that mirror the independently expressible runtime rules. Cross-field cardinality relations still require runtime validation. The instances are the core model classes, so values interoperate freely with non-pydantic code. `create_default` keeps its output self-consistent: overriding `shape` without a chunk grid derives one regular chunk covering the array (v3 `chunk_shape == shape`; v2 `chunks == shape`) instead of silently keeping the scalar default's 0-d grid. A v2 `.zarray` that omits `dimension_separator` is interpreted with the v2 convention's default `"."` (the model previously normalized absence to `"/"`, which would misaddress the chunks of real-world default-separator arrays). The value is never null: absent, `"."`, or `"/"` are the only spellings. Optional document keys use `UNSET` — a PEP 661 sentinel (`typing_extensions.Sentinel`), usable directly in type expressions — never `None`: in a model, `None` always corresponds to a JSON `null` in the document (a v2 `compressor`, an unnamed dimension inside `dimension_names`), and `UNSET` always means the key is absent. Checker note: ty types the sentinel exactly; pyright needs `<= 1.1.404` until microsoft/pyright#11115 is fixed (this package's CI pins it); mypy users need a `cast` or `type: ignore` at narrowing sites until python/mypy#21647 merges. This keeps semantically distinct spellings distinct — an absent `dimension_names` ("there are no dimension names") and an explicit `[null, null]` ("every dimension has a name, which is null") are different documents and round-trip as such. The `consolidated_metadata: null` written by a historical zarr-python bug is the one deliberate exception to faithful round-tripping: those stores remain readable, but the bug spelling is repaired to absence on read and never written back. The v2 models treat the `.zattrs` file's presence as part of the store: `attributes` is `UNSET` when no `.zattrs` file exists (and `to_key_value` emits none), while an explicit empty `.zattrs` is `{}` and round-trips as a file. Previously `to_key_value` always emitted `.zattrs`, silently adding a file to stores that never had one. The store-key `Literal` aliases (`ZarrV2ArrayMetadataStoreKey`, `ZarrV2AttributesStoreKey`, ...) are exported from `zarr_metadata.model` alongside their constants, and each `to_key_value` return type is keyed by them, so the set of store keys a model can emit is visible in its signature. `from_key_value` deliberately keeps `Mapping[str, bytes]` input: it accepts any string-keyed store mapping and ignores unrelated keys. `to_json` returns a document that shares no mutable state with the model: every value that can hold a mutable container (attributes, configurations, extra fields, v2 codec configurations, fill values, consolidated entries) is deep-copied on the way out, so editing a serialized document can never silently mutate the frozen model that produced it. ([#4119](https://github.com/zarr-developers/zarr-python/issues/4119)) ### Improved Documentation - `zarr-metadata` now has a standalone documentation site at , with a comprehensive API reference covering every public module, versioned by this package's release tags. The package also gained a `justfile` collecting its development commands (`test`, `lint`, `typecheck`, `docs-check`, `docs-serve`, `changelog-draft`), which the package CI workflow now delegates to. ([#4208](https://github.com/zarr-developers/zarr-python/issues/4208)) ### Deprecations and Removals - The document (TypedDict) types are renamed to put the format version at the front of the name and to mark the JSON-document form with a `JSON` suffix, so a format version can never be misread as a class revision and the bare entity names are reserved for the `zarr_metadata.model` dataclasses: - `ArrayMetadataV2` → `ZarrV2ArrayMetadataJSON` (and `...Partial` accordingly) - `ArrayMetadataV3` → `ZarrV3ArrayMetadataJSON` (and `...Partial` accordingly) - `GroupMetadataV2` → `ZarrV2GroupMetadataJSON` (and `...Partial` accordingly) - `GroupMetadataV3` → `ZarrV3GroupMetadataJSON` (and `...Partial` accordingly) - `ConsolidatedMetadataV2` → `ZarrV2ConsolidatedMetadataJSON` - `ConsolidatedMetadataV3` → `ZarrV3ConsolidatedMetadataJSON` - `NamedConfigV3` → `ZarrV3NamedConfigJSON` - `MetadataV3` → `ZarrV3MetadataFieldJSON` (the union of the bare-name and named-configuration spellings of one metadata field) - `ExtensionFieldV3` → `ZarrV3ExtensionField` - `CodecMetadataV2` → `ZarrV2CodecMetadata` - `DataTypeMetadataV2` → `ZarrV2DataTypeMetadata` - `ArrayOrderV2` → `ZarrV2ArrayOrder` - `ArrayDimensionSeparatorV2` → `ZarrV2ArrayDimensionSeparator` - `ZArrayMetadata` → `ZarrV2ZArrayJSON` (the strict on-disk `.zarray` document) - `ZGroupMetadata` → `ZarrV2ZGroupJSON` (the strict on-disk `.zgroup` document) - `ZAttrsMetadata` → `ZarrV2ZAttrsJSON` (the `.zattrs` document) The old names are removed, not aliased. The `zarr_metadata.pydantic` field types take the bare entity names (`ZarrV3ArrayMetadata`, ...), matching the model classes they validate into. The conventions, stated once for future additions: CamelCase type names put the format version first (`ZarrV2ArrayMetadataJSON`, `ZarrV3ArrayMetadataStoreKey`), while SCREAMING_SNAKE constants and snake_case functions put it last (`ARRAY_METADATA_STORE_KEY_V2`, `validate_array_metadata_v3`). The `JSON` suffix marks a raw-document type whose bare name is taken by (or reserved for) a `zarr_metadata.model` dataclass; raw field-level types the models hold verbatim (`ZarrV2CodecMetadata`, `ZarrV3ExtensionField`) keep their bare names. Extension-entity types put the registered entity name first and end in exactly one role suffix (`BloscCodecMetadata`, `Uint8DataTypeName`) — the `V2` in `V2ChunkKeyEncodingMetadata` is that encoding's entity name, not a format version, which is always spelled `ZarrV2`/`ZarrV3`. Every public type name is checked against this grammar by `tests/test_public_api.py::test_public_type_names_comply_with_naming_grammar`. ([#4119](https://github.com/zarr-developers/zarr-python/issues/4119)) ## 0.3.0 (2026-06-19) ### Deprecations and Removals - Introduces a new `JSONValue` type that models python objects that serialize directly to JSON. This type is used to annotate the contents of `attributes` and `fill_value` fields, replacing the use of the overly wide `object` type. This is technically a breaking change. ([#4037](https://github.com/zarr-developers/zarr-python/pull/4037)) - Promoted a curated "front door" of names to the top-level `zarr_metadata` namespace, so consumers can write e.g. `from zarr_metadata import ArrayMetadataV3, ShardingIndexLocation, BLOSC_CNAME` instead of importing from deep submodule paths. The front door covers every metadata-document TypedDict, each codec/chunk-grid/chunk-key-encoding canonical type, the full data-type trio for every dtype, and every constant + `Literal` pair. Deep submodule paths continue to work unchanged. Several promoted names were given clearer, less ambiguous spellings than their deep-module names, since they now appear bare at the top level: `Endian`/`ENDIAN` → `Endianness`/`ENDIANNESS`, `IndexLocation`/`INDEX_LOCATION` → `ShardingIndexLocation`/`SHARDING_INDEX_LOCATION`, `RoundingMode`/`ROUNDING_MODE` → `CastRoundingMode`/`CAST_ROUNDING_MODE`, `OutOfRangeMode`/`OUT_OF_RANGE_MODE` → `CastOutOfRangeMode`/`CAST_OUT_OF_RANGE_MODE`, `DateTimeUnit` → `NumpyTimeUnit`, `NamedConfig` → `NamedConfigV3`, and `MetadataFieldV3` → `MetadataV3` (matching the name `zarrs` uses for this `name`-or-`{name, configuration}` shape). Also added the `NUMPY_TIME_UNIT` runtime constant (a `Final` tuple paired with the `NumpyTimeUnit` Literal) in `zarr_metadata.v3.data_type.numpy_timedelta64`. ([#4083](https://github.com/zarr-developers/zarr-python/pull/4083)) ## 0.2.0 (2026-05-19) ### Bugfixes - `GzipCodecConfiguration.level` is now required, and `GzipCodecMetadata` no longer accepts the bare-string `"gzip"` form. The codec's compressed output depends on `level`, so metadata that omits it cannot reproducibly identify the chunk bytes produced by a writer. **Breaking** for consumers that previously typed gzip codec metadata as the bare string or constructed a `GzipCodecConfiguration` without `level`. ([#3978](https://github.com/zarr-developers/zarr-python/pull/3978)) - `BytesCodecObject.configuration` is now `NotRequired`. The configuration has no required keys (`endian` is conditionally required at runtime based on data type), so the object form may omit it entirely — matching the bare-string short-hand. **Soft-breaking** for consumers that previously relied on `configuration` always being present. ([#3978](https://github.com/zarr-developers/zarr-python/pull/3978)) - Better modelling of Zarr v2 stored metadata. Zarr v2 splits a node's metadata across two JSON documents (`.zarray`/`.zgroup` and `.zattrs`), but `GroupMetadataV2` had no `attributes` field while `ArrayMetadataV2` did — an inconsistency. `GroupMetadataV2` now also has an optional `attributes` field, and `ArrayMetadataV2.attributes` is now `NotRequired` for symmetry. **Soft-breaking** for consumers that relied on `ArrayMetadataV2.attributes` always being present. ([#3962](https://github.com/zarr-developers/zarr-python/pull/3962)) ### Features - Added `ArrayMetadataV3Partial`, `GroupMetadataV3Partial`, `ArrayMetadataV2Partial`, and `GroupMetadataV2Partial` — sibling TypedDicts to the existing full metadata types, declared with `total=False` so every field is `NotRequired`. Use these when typing dicts that intentionally hold a subset of a complete metadata document (test fixtures, fragment templates, in-progress builders). An equivalence test pins each `Partial` to the keys and value types of its full sibling so the two cannot drift. ([#3982](https://github.com/zarr-developers/zarr-python/pull/3982)) - Added three new top-level types modelling the **strict on-disk** shape of Zarr v2 metadata documents: `ZArrayMetadata` (the `.zarray` file), `ZGroupMetadata` (the `.zgroup` file), and `ZAttrsMetadata` (the `.zattrs` file). Use these when you want a type that faithfully matches what's stored on disk; use the merged `ArrayMetadataV2`/`GroupMetadataV2` when you want the in-memory representation a Python program typically works with. ([#3962](https://github.com/zarr-developers/zarr-python/pull/3962)) - Added typed constants exposing the spec-permitted values of constrained Literal fields, importable at the per-codec module level. For example, `from zarr_metadata.v3.codec.bytes import ENDIAN` provides `("little", "big")` as a tuple, enabling runtime iteration or validator generation without re-stating the Literal values by hand. ([#3978](https://github.com/zarr-developers/zarr-python/pull/3978)) ## 0.1.1 (2026-05-06) ### Misc - First usable release on PyPI. Version 0.1.0 was uploaded then deleted to reserve the project name; this version is the first one PyPI will install. No source changes from 0.1.0. ([#3949](https://github.com/zarr-developers/zarr-python/pull/3949)) ## 0.1.0 (2026-05-01) ### Features - Initial release. Provides `TypedDict` definitions and `Literal` aliases for the JSON shapes specified by Zarr v2 and v3 metadata, plus a subset of `zarr-extensions` types and the un-specified-but-widely-used consolidated metadata documents. Pair with a runtime validator like `pydantic` to check JSON loaded from disk. ([#3919](https://github.com/zarr-developers/zarr-python/pull/3919)) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/LICENSE.txt000066400000000000000000000021441527014137600256230ustar00rootroot00000000000000The MIT License (MIT) Copyright (c) 2015-2025 Zarr Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/README.md000066400000000000000000000525051527014137600252650ustar00rootroot00000000000000# zarr-metadata Documentation: Basic tools for modelling Zarr metadata, with minimal dependencies. `zarr-metadata` is developed in the [zarr-python repository](https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata) and released independently of `zarr` itself. Install it with: ``` pip install zarr-metadata ``` ## Who needs this This library supports software that interacts with Zarr metadata documents. ## What this is This is *not* a full Zarr implementation. It's a collection of data structures and routines that closely model the content of the Zarr specifications, such as: - **Typed JSON shapes** ([`zarr_metadata.v2`](https://zarr-metadata.readthedocs.io/en/latest/api/v2/) and [`zarr_metadata.v3`](https://zarr-metadata.readthedocs.io/en/latest/api/v3/)): `TypedDict` definitions and `Literal` aliases for the JSON documents specified by the [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html) and [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) specifications, plus types for [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/) and a few widely-used-but-unspecified entities (e.g. consolidated metadata). - **Document models** ([`zarr_metadata.model`](https://zarr-metadata.readthedocs.io/en/latest/api/model/)): immutable models of whole metadata documents, each one a document and the scope it was read in, with readers that report every problem at its location, and store-key (de)serialization. A document produced by `to_json` shares no mutable state with the model that produced it. - **Definitions and scopes** ([`zarr_metadata.definition`](https://zarr-metadata.readthedocs.io/en/latest/api/definition/), with each format's kinds in `zarr_metadata.v3.definition` and `zarr_metadata.v2.definition`): each extension's configuration as a `TypedDict` with its rules, and the scope that decides which definitions read a document. `zarr_metadata.v3` and `zarr_metadata.v2` export every definition and scope of their format, such as `BYTES_CODEC` and `CORE_AND_EXTENSIONS`. - **JSON type checking** ([`zarr_metadata.typed_json`](https://zarr-metadata.readthedocs.io/en/latest/api/typed_json/)): `check` checks a JSON value against any `TypedDict` and reports every problem with its location. - **Problems** ([`zarr_metadata.problems`](https://zarr-metadata.readthedocs.io/en/latest/api/problems/)): the problem values every reader returns, and the error that carries them. - **JSON Schema** ([`zarr_metadata.json_schema`](https://zarr-metadata.readthedocs.io/en/latest/api/json_schema/)): the JSON Schema of a `TypedDict`, a v3 field, or a whole `zarr.json`. - **Optional Pydantic integration** ([`zarr_metadata.pydantic`](https://zarr-metadata.readthedocs.io/en/latest/api/pydantic/), requires Pydantic 2.13 or newer): each model as a Pydantic field type that validates raw documents through the same strict parser. ## What this is for - **Static Type Checking** The public `TypedDict` definitions describe the static JSON shape of Zarr metadata. - **Dynamic JSON validation** For strict, loc-aware validation of JSON loaded from disk, use the model parser: ```python test="true" session="readme" import json from zarr_metadata.model import ZarrV3ArrayMetadata with open("zarr.json", "rb") as f: raw = json.load(f) metadata = ZarrV3ArrayMetadata.from_json(raw) ``` The optional Pydantic integration delegates raw input to the same strict parser and returns the same normalized model class: ```python test="true" session="readme" from pydantic import TypeAdapter import zarr_metadata.pydantic as zmp adapter: TypeAdapter[zmp.ZarrV3ArrayMetadataField] = TypeAdapter(zmp.ZarrV3ArrayMetadataField) metadata = adapter.validate_python(raw) encoded = metadata.to_key_value()["zarr.json"] ``` A field annotated `Annotated[ZarrV3ArrayMetadata, zmp.InScope(CONTEXT)]` is read in `CONTEXT`, whatever the validation context holds, and a model instance given to it is read again in `CONTEXT`. Note that a bare `TypeAdapter` over a public `TypedDict` is a **coercive** shape adapter, not a Zarr conformance validator; it may coerce values or discard members that the strict model parser rejects. This is better suited for handling input provided from Python objects (e.g., user-authored dicts) than JSON. ## Validation boundary The readers and model constructors enforce the declared document structure and a small set of context-free consistency rules, e.g.: - fixed format literals, - finite JSON numbers outside `attributes` and the members nothing in scope reads - non-negative dimensions - one `dimension_names` entry per array dimension. In a v3 document, validation also checks each extension point: the data type, the chunk grid, the chunk key encoding, each codec, and each storage transformer. Each one is checked by the definition that claims its name in a scope. The scope is `CORE_AND_EXTENSIONS` unless a `context` is passed. If the definition rejects a configuration, the reader rejects it too, and a configuration key the definition does not declare is reported as `unknown_key`. If no definition in the scope claims a name, the reader does not check that extension point, and the consumer decides whether to support it. Some checks relate one part of a v3 document to another: - The fill value is checked against the data type the document names, by that data type's definition. - The chunk grid is checked against the shape, by the grid's definition. - The codecs are checked as a pipeline. In order, each codec's definition checks the codec against the chunk it receives. This includes a shard's inner codecs and index codecs. The readers do no arithmetic on values. For example, they do not check whether a fill value survives a `cast_value` round trip. Some choices that the spec text leaves open, or answers in two ways: - **`attributes` may hold `NaN`, `Infinity` and `-Infinity`.** The spec does not interpret attributes, and zarr-python and xarray write these numbers there (for example, a CF `_FillValue`). The models read them, and `to_key_value` writes them back as bare tokens, which a strict JSON parser rejects. So may a member the spec does not define when no definition in scope claims it: nothing reads it, so it is kept as written. `check`, from `zarr_metadata.typed_json`, rejects a non-finite number anywhere, including in attributes. - **A reader walks at most 256 levels of nesting.** A value nested deeper is an `invalid_value` problem at the first level past the limit, wherever it is. Every reader, writer and comparison uses one stack frame per level, and `copy.deepcopy` (and `pickle` before Python 3.12) uses two. A document at the limit therefore uses about half of the interpreter's default recursion limit, and the rest is left for the caller. - **`consolidated_metadata: null` is a problem.** A zarr-python 3.0.x bug wrote it. The spec requires an object, and the package accepts only an object. A reader of those stores removes the key before reading. - **An extension name must follow the spec's naming rule**, `^[a-z][a-z0-9-_.]+$`, or be a URI, which earlier versions of the spec required. Any other name is rejected before any definition is consulted, so `""` and `"foo/bar"` are problems, not unknown extensions. - **`must_understand: false` is rejected at every extension point**, including codecs and storage transformers, although the core spec names only the data type, chunk grid and chunk key encoding. A reader that skips a codec reads wrong bytes, just as a reader that skips a data type reads wrong values. On an unknown top-level member, `must_understand` keeps its meaning: a reader can skip that member. - **A v2 datetime or timedelta dtype must have a unit**, as in `.md`, where - `xxxx` is the pull request number associated with this entry - `` is one of: - feature - bugfix - doc - removal - misc Inside the file, please write a short description of what you have changed, and how it impacts users of `zarr-metadata`. A `zarr-metadata` release runs `towncrier build` in `packages/zarr-metadata/`, which consumes the fragments here and updates `CHANGELOG.md`. Fragments that describe parent zarr-python changes (not the metadata package) belong in the top-level `changes/` directory, not here. zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/000077500000000000000000000000001527014137600247275ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/_static/000077500000000000000000000000001527014137600263555ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/_static/favicon-96x96.png000066400000000000000000000306521527014137600313210ustar00rootroot00000000000000PNG  IHDR_`{` pHYsodtEXtSoftwarewww.inkscape.org< IDATx}y|Uι7&lU* Ep\@[kTlUYVZpׯ-l, [s}~$dAk|>瓛s3g33̼!_L;[wG4kڽ3>B~I Nj4!oPD1۔Ec.:Џ[O觎/B}J1 8]Q?~jkPp_zkg-~޽)1 Ժ\Q6e`HJܗ6s\3_|77=el0"d~z"oeOU05uम8?:ՑLs\E!A @*G`EG3nyv,Ƽy``JZ{SNJOV:sϏbgav LI++?j]-HL2 @m4Pe?  W=SRG*>s~RL+Wzh##yx(f'Ҥf3Uu(w"T<ʍ1qsR*ں|QQI]g̘Ƭxibhӭ 5ulɭOqJ[;;Ң*& g@TYU2yj,P>x{ xɩcֵʚ`?}6x )鷉Ӳ+\ [Ϙ:D$bU)+[0{qwE7pg#;HNCP@.' Y譠XKʜ/P/B8Am (9ɩ=a 8m/SJP@䌽i臦 ,ŰaN~RMI @9Tp&Ӧ4pĽPP6JP@: 5a Q2X qYsABj:X<.+cFK5*U(bjpWc?!}t'SNNnZOBL،!\*PxkO (,VhUTJAIDêKTh=]OO0j܂{Rn+f q/' ܝ:Vw *DX`ЗWǧ6i,}nS{{0* / UU (B 0ķ'VZר@AțT^ Zma ~-RYok{"|ܓr;STp8M1= N]xԩ^DJ4OǍkC~)ό,E6p nV+WT,b*5By D:Ú aJw05q1QO ;~q僸d51}b,VG0J??Dl7.2 ot@⪺ZyKlJ!U-zXBc}(A@8|FͣJKߕ:2Q [<6()2N8` sZ,)eq)缚;kȖJs|Zeו7*n2o7.7.E-lϩW%aŷ64 Hxq5 z8Фp>tQ$ d47&DLݿx4?)DZV "y6Xg( ~e<@ߙRYR "H FZwqoѲ*t|2W|Ci>F?0{vwzB7~/0*9h+0C ͡##)OhNT >w`h#% '`( DD!Pט+cP( 'xT*a_Z5e x'Zע;y6D4- ϪЮL|rOmiЂ;ҴᢶOM-/6?ߞziB9P*04S Eu[EG'T 6"ʩɇ4RBѾ@c |d!,T]CEd)W>Z/17=S.ʽl^pGڈcǛwFA `KڠȦd]?9 e< ($!.wΚy`9nP MAαl>Xt6)4(!BԪqwP0ɢʐ29R/0yPp|YhV9֬TV4)hZS FE,VGZ^ aè7rN6@S@h~"꧍JE&oP4!olʱFbTI`w礼a"bEzͅbN~ b(dTJZuN`IQ%~Im1aӞmܜ'x0[Ӯ0/9Pg:B՚MH*:٬c O|>TjoMPgO~NMy$5PGřlmo+w4gE:gPz*QxǧaQ((/pTLe!7ՎP&ݒr[ȴUWCT,#FNu8~3?EřTD<6:oTUNLFk>7W5`ѨuY[9OzaFaNC_B y<  "T=ϟ"F>2U`(+Ð!bJEWK@ ^PbaO Fv!haM1~sj]vuK[?X?2'h kB-nB4Ӓ⛩R0ECi,Fbj a: Tx9 ŹUv@nr1}N`JDOl']Ee+P@ c^\(^v*|V֌V[ ot8Qv-(1#(ڏtQTWVy^H{5,n9 _hcn"332 iDR +AѼTO+EDs?Ӳ-9EUcmeybǍV1< ࣇ]-b!D(1fsS GԀ } \!hy!^֎@?cRD!!1<:-$({“YK/]/ynq A5\c%a͑Yn44հxZ_ +jpt q`&irOF:w/~*\ 2WXz9M-^*pݣQȱy_?@zcf^N0hr86\)x[‚ugXZ_(e~ kS{Ѱ#do OCo4/{Mפ-=={>+n-I >6v̈́Nt|P : z>('9"*!  KoJHd)GE\S9fŽ YT: 9b~9;}Æ9y^N`EG']s6Wj%7匵Y;44Vϻ6/+ŏ Jc"JƊ3&K:pgm˽&9/lkTEIR\.}UCRA _5aW( RϘm]?lﶻl;ImVtd ux÷uO ax$r!jY2Hv5`DK(o—o.E_'dP!lTQ2kjw%{$6cg%W"WGp:aPdCؓsXSH a5*Q<Ab1mJ|E_0. r,Q$?P`(R_m DQbuTk0jḡř g &Lrx0P_!1jʹ&&&Ð(xD,C·X|*FQo AH>[N1Q ^-[FLO`̵ KcJAVN:q*_F~qu) [~?BcwxhI]q.B,(ixsyE9Z=[Rb|@C9(HX[l_&)]Ȁ7aGoi,:U\|dgH3Q =GV7{tP5 7~E֢WCAi ֬R;8A#Ty>MZ1s. |ǮFq}qYv qlr`с|EnRs!Q:Xf+xEx~u\1FH8(bǯ-iJIdR'D++prx8GI):.{ K__xXرaK"6 X9K^w߻ӰмE(AhXDT mOcbRQ|@8MD$ @ @b\$bt8״Pk& fOtBە~ո6-}W$_0&@1VeίJKSCŪhZJ.IJM5r3 U6F"ʭ\h`!?PDLZ_("\͸[gToP}KHKT嶫fm-zn;#6v@Ҫ̑'>r/NYrq1RUw|_Օ(P\t= H;Ϫ)Zճ1lS+Ix'ňK#]}6u6e;!du_C pX**kK! ⥊ CT.\/ f Ljoʏ+δ5u+z*H@L\xәG[gVgV铲5Y=>^1B!K|?MH^仔)iܾq 5B .O|?=}Kڛ蚤)\PĔ x8!ѫG@(3h ^!r}y@LB0 @O(e @  &nF0a]:^0bcdJ@e8u'^A+rW6볖$&R>ԛhfv^ \UZJU/۩Ԭ|%u A_cؔ r)aIiD4% ;J\[<_Ghx| aY1G@IBX,p։׋h*^A;aY̱'>䁹|7`3qmf6벖" & #2J %oj$|9Ǹq8m<}k3tBPgkάWi&~[ˡ ~`\[mȮm? 4ZE%RQyD-y+ ˡӫ19B0"==/#ȦUt4]G*e/XDZuygUP-&l=9cLwOy\y f'3I0y&;7_ ~ygϲI\T­q"T+`Z8*ēˡ;ib ]4ŠsP#ė;;ܳ'~!Z8B\2VnkCNipeE‡Y*9gJ>=8+쉷>kbg1/I,*-D\@=v;{mZ;*Uv/?g9,ļ'aFV\C :ʤL6Q1miZ dhaJ<,GYb5;_X ⨴X*VvM^(EPq|3b( @a/vgc"_fTšt= H_%m^ `uNKA?kXk7V T3_d|ܮ/hQ:6Uv[2Ƽ.)'M@FtMF?jqgАٱ <F@\<6$Y/,J ,!Ey%,σEZW><[Q"-l"X4wŚQ 5C{Q+v:g25dlX)DXv>9ּNą3/rD?P*th/9cmpVmV,PLcIb@ E),h Og2@b1w">*+iCz2h7lYxfb ,$]Nu$ V&^tGtvfΧE;%/NT^M{[pUcM=UxNW(QM(ׯM+@5>I 8Aܤ1՛UpQJ'C` g7 J#!JhL[(1 p !&7cs˹CL*NH9@nSwN,}hy92XptxP͵`PkRxl(( p˜/`uE=%bVLObBkZbU>AOg2@M>6SyQ-,z(-[Z|V8S_hKKP U`)[O򛤼&5f썟\U62 6qbU>"%kN [ZΟZ&7rZ{ZOnTR*]7N-OIO=c]sZyШ+IDATC/ez0jk&Q1Xl'7œpuPϱchx$i<#=LLbn5`F\*MdkU1ݱfHӊp6f= yֈ@?fTW=T8Y|V!6C }%mfX/]*QlX@oqB6]~~!"cU 9J6튲'9 E^''|0?x4I1K|7"'69wMy6)IۃY𼈳F;|o29 1EN y@rH845r@9MdQǻ ]y쨈l-%1nͤRxai}R R N1*qn1eNN;u<2wǤ& 3<#|#Ltb '#IK`Mvib"'q@>oS43J׭QEW $x-0 !v&LwERQ"tbα4_y[B^ IROF8eNns |3m ^~~G L}5' ؽ>|Jq\[¬`u7I0bgl&`˕>%1)n>s%?d.|hS]Q\z9|(no|a-9f+ʚK1,q՞;ɗU%*_Ew t]PV' C Sꞕp6-$ \-*;j! &8CWp=4ri6aM>Pó%c*h jΘ.|Q۵=9-YYs-o$ulC+e"¸=FĵPTx>l@a I^<I#K;2/uIjn,=&(XW)F<|xڢHkt8\wJ7w(̚-:#"먦l06"ozL]CKk!kO㼢YA7$|8^ jMLr.gԵ b8AN}pEYaszulI/;\U@JҫZf#\Xs$:f?4|`-Q,B޼Nز%6h#y}UU#ו7"hֹFQW8%"$h4 OA]㳏Յ}K@ Եp|;~=:Ʀ@5\-ƴH+Q-:N?@H:75z":Ŷش3}ug$n "5rК֯gj&.TDU\C]5o -άʲ56 @vDo8r8if4[KH*B:8`-6"\E]JH)(hSTAYu^ Z\$EꉾBy񱣱җ:ޢWu$Mgͨ1]2>V ^ojr#&K7 4II &]=}w5Zlp|C;(Dt й£a@m롚s(X `@zqf:8FL6*3[F8:KB+_$1\'|z90\}t_QK.qW* N)] JjTf, V䁮p&$h6G^/" o|yƷ3RR0oGt"sLrM| "za/xOD2IċJ0ӎ՘JFu}]4.< K57L#o B):N͖ `w](́J\aoZl} # M;9~{!}| (G+ Њ4mE00/Qrt zi3\ߤqYUfSlZ4D`rb ÜM1F V VpХh[HuG6ŤjU݋BPū$k(-(kec =\}a da"N*.2˻ɊP " "(pw?xe%Fw<:iPk.6.S0ad=•K闱G2ĹVZwK}@;7Lic<l;Q{wX%A_@ML ŀB]xc'FkTOAVSo M49ENUO1KC9f}t; /kf>wUo~MiK,"po $*ec&jTzhJ,Ö˻MIfa}6 }#qB,{F߂_&xUŴ<0@@Ǵu|{CɁ5V'Tn~/.K꿬롩l$BdgC@9@HK)>(oSG]m09_//l`ZAUC{i[3lAl`vEE3=~GĔ,":jƝ@@VO2.HLD\rNԢ#~'$Q8@|v56/Üc:'E;)籓Y='Y-O&9}yF~ $ho$J߳MZe10*b{p2NG /7G{?occ7V#Qğ퐠(3*D8GoI* G{C Ю_>V̥a@sϟs:+"Aa"U;~1AnkL~0Muwc$W}Chsw0ۧ}raCB A.@yWS?AdژJoRT>pMO'#ek"Hȫ :PRO)J!K޷Afm $xQRϢ{ߣ>Tom _WXڻhNN:Ax*)zʚ?ڷW!ekYн42E"T>?U>F~l/+ LS1:F+}KNKh:P XtQ| ?X;^A~(V/Wf%SVg7و_U y9D>wpIIk~FޟX;/[ˊ욹?[+*vZ_ %S^8P= 0+/-ӷ^Jч >i}r$S_Oi*2ފ"z%~qWbE>PP|EStEGIENDB`zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/_static/logo_bw.png000066400000000000000000001302301527014137600305120ustar00rootroot00000000000000PNG  IHDR;' pHYsodtEXtSoftwarewww.inkscape.org< IDATxweſ'*z/JE$XAPDP,"EcB'$T@swݛ[vw&a3;3LfO;Mm| i`S} p x #:~O\I@G>"lO}0&!3@҂fk-mhA$_9|\VOg;$D/V}8bʬےfOFO!\F?pgRFDaڡfm-mLhA=%\ '`U%m/nJNnCkafFDd~XAZgI Mmt IHpq&َdugr1{Jr "aljTN :5†>`;*02j;Hon* e{퇳6Npm' a n "h|mVCl-=cI>&96@4@((؜ WCIH&@5l?j{o"Ev Q5N~~ĹDk!=^3*ʮ6ʇ6 %]O [mOg֖t%Q}Av2;W$%%V"%Q{y=*Kfj-mm"#ͪ$|emDEp'HZ]߁ZnU ؾ gˀ%)ٽL'b oo!ۻ"@[ʹҖ6ʅ?֐4wh{vw%$b½;U*ؾvD@e"PbV:f2 83Q0hq@{D>eHn+(Ǡz? m_ߑ_J?Hپ6DK*%sA;rj>&V3O2ydžO؟HZS?Aw( 1nD CI%͞a?Dr$/i1= 4z=牂zPjm"zpJ9QEڶi; `" lntl?b{wo?ptw"42+ߵճr۷؜eETM3$ S/ؾ$G(AO$*r? կ/}H׏/IkBfD?{%}Նھd_9&m"A#]2T]?ARC ukcny J "69)#m"!ipJ o0gٞ i>I?&V'ja*y,~Ib:v[;+$ݚ ]}uG`:%]@^˒Z"`yg`I)'j$yo=IPҠt̵D#]@Ds"p /k$),mm" izY/CϘWTϰ$ 'V+'hc~MJTz?.bI;p*bg &TؖxOP^M KT<t0տhNopgyG%#iDWph.o V++$N;~&#|rVvtLt2MaDLpe6;F؏StES϶oYVi/b;p$mJ[=DPRHZJ9 @~JO耤Y$} s% |pyu o 2:Hŝ$4I ?Ib[mD^B- Fm@JT\+b`:rֿKo }ۯf;$C+)Ⱦ}SmN̾ W1"sm+>̹S#B[moDc}"%]/iVƴ&@b$*fWZgyߵRcV%ϲ}4%N_y8C$R;X81<.\I-%+QDTUA+mi>DP0rյ2W}nyIRbFU"N?+N+UO=Is~=O'dAV3Ոt笪*IҖ6Mv΢ !f f߰ tمh\7hi Y]7t 04/iY\Dp`%}w{,Dq&I:"rޱ}*q?!fAVgcۗ+ɃehA!iy>&WOf;$KQy=B&nm?퐴BfOw8-N9H!NeH*[ 矑SCp:jپ4.:E4 PY|&Բֵ}IHP-Ⱦ*nAu3q5?afrE~Y"|BY*ꨡ~IIJ*oG*})ؖO Dd(hv*Pwkjl͒6bfYf6{%%_TTz !OA%imD<.o"S(< 54ۯ8<*n &qDѽFJZ@[>hA 3N Oy\g{wۏdJP("fQ[logCIJ xhΘ,NE' }๞aOT֕ڣHGnKR"XI^X|xc$]*ilTM Fr  2 CII1nS6}[%ܓ&Ni$A[?7ڎО Bc%Yۇ߉zBB)7D 췉>V%JYhD (:l~ SkMmoo{tB=_̸Z@G Wтm*%%i۴ k 7\PC |pYΫm#^E:jHBkc&~B) άJ@eFD)^%T" M !.!(t"eKJ5諔 5tY!oßTu/Wo&{]^:"[.HL"X{I[iˌ6Gi?Jv9we*$,/ 'S>TN%~PJ@'!I>w:A#Jӻ7!ZRMg+K=sKWeu7qDr6;=̃B`8i:`=W#`SSvrhAp.!(#ђsmܷ%P5`:۟%rnǁnJDo%-J[g`P&1%1+#:e9~$&*yW};gY)G#. f$"iṱ7'Խ>tߊ|!;hSNJ#D $-H(OYҢt/-%rć>Q{q"B(C])TJ^7"gsٴHC]fqؾM[ vj*}o-DPI`s'$zìl?(WzxH\9,nN)ˣOaT%z.e90%eFHZ" "Chs._}:9꩒pE~S Y{nM~Wbu71* _}4B8f{Jfhfx"HѾ$\e†i|OFPzMbt<#neZuM&+TvqH좟P?]U1^F-!HE[Typ K"ݭ,%|@K ZLGei{gCxa.xT?R8%."bGS$+i4>3qJRDgÈ(eJex 8P҅` 3?N$mlPQɹ;AP:28G_ee|uv7&k]s~o)7DJw9s&ί׼8?~؏-DBND3E\ߧ{%Dc[1r+܀H F L,IS`Ԓu쾩A\~k4|D aʄAD /%GwbaG;P"ȝw!6uoRA-k5r]X!  M:"yJ6L%s(V x A5g3޹sOlݍ&1+wMO={I:G/4r_9KYryTk4r? {O:>"3/SYU/֋7r[d[Mt#PZd-ˀmSI;/gaBd-~,!iI)δ^?ƶwtgq%U( ?ۇI $!,m<{r:WRpG1Ht>BDږm_'ދ&+OL\h#>!ҫ>l$ \IyVm_6` ؾ3PjFk;X< ih"L~%:"s%[5Y~&GDҟȉKHc}oJ]_''EދU 7F%(HFҹEQBkI[H h[f\Omqu$]In`;ۛپ%PJ.%{ucI:D[\m\8{?LRJiU؅8#W9l]VW&4A%j5?$7D"<{K*~FHRҭ(`âJ`{}(icI!^M gWt9ک0z/CED ɾ}fu pOCtZ, jwf+R},WsM OHru4yG,%X,Nؤ;DNe Dζ>BvJqY~%IQnmqDKZFA(qf!tn8{};fBZ!T\߈Ⱦa´+˄2fDCgNn#'%f%𴤣LkD!"ZÍ&}O$]AUu=h`9z iDcLR fu(Akؾ̎tIK@p3>K<!\JX{.Ljb1$$͞Kt(n/iaIlߢY 8ʹ}+s7)`AW+Ժzί*cVLdD{&E%]Md{l_-2 >#-JA,nx=O$RJL ?hZ,nNIJ*itL^Kθ"~vI)ؾǡE|lWK!ym'lo)j=Jk $B^JURv,†:C;usI-" Db\L8Z/?S .$ t*TcnF[nVIrt1/?$zť&Ky#b<[%m^uDZZ] }ǒ]㊲eIJ2ɝ~'eUA@#'왻FZLda"PV;#[LdW{Zr_N IYG[#iBd:h"xh-Хg `{h85pAlе㬈:c5uL w ծHj=1pHزi8T=磓}˥:fC(Lvԋk4r2&\BhkHʒ.#L9S*/q%I3)>[cb0Ii qm_DpE |䟳klGZ%mYdI7B{~/͟*2UB.gyNI!{|^]={ A)wWqL(x8h=. KWZ 8/$-JjWqI!<9ϓl:"go]yk%_ #ϑ=i$ T4K Ot=N1ˊwslCʶ~:ܟ>W%~ؙP޳K*]v#N퓈GkRtL̾7&ۑeq{ *y({+{$"b#Enۛ1C}]!jW4&JS"f%ճY%}hA0.+'V)t=?FgCl`{ tD+m !(#IG*T;O"N DrJDs~$iUΑ6ȗ/r!TV9%2V9;꟱}-NI#"4HBuf3K2[IXYk}G@$DR@*)>'>O6t RN lL\OΚuaDjLz8I~? Srg5өr~Ba:۟#3)t4zv-(:SOu?Um i&I_"rEClf0Nˈ[0 %m-p|aM17PQ:D,_}<1PJJG[#cG ߓ4WZpH[!2&K:9V9W}} DȆڙ00m7DxbWvShy'$>Db++>" H :f5[oZhT6= z晴hڞh7}',>? +&1XpLmI~|'Jsg5 <漶fq0"+y6IwJƮDOHE`ݹyieOo_~'}@WD]'JodKysߚ56Zg!8Թ˛`3zB&C3N`ihV"?&ɇP"ȝwM\תڿ Y`>^D;`"po Rq{NuN5103dh/"ȝgh6TM8iM W\N'\RELxֶuV&*Eonp31{9q6!Te}Cۏ;J+8L~3=:Thzn3$m,?RXOT6 b87cx*]ew+AdRyλH9y%*LyDϨ$( ˕yqys\Ūp:4J-+܊cV:a74Օ$*1CCv%^Jc SWzHPBQ Yn];"M4.YP^ }9R{pZ"2ȶ=I+4 Rs&n]Lص59X0& qZxX`7,DpE'`00pTt]Ev'n&B܊ n~9T q*D68b%Qpc( Rg̻(6L+ad-DH-s/KhI;lHrnݒ=LqʥaP @{^L/Yhd!wEm>I?&ϣ e"|PZG?i(bvZ/QU#(:v71`ygvG5bBL8lEbQķ}Ʊ"=+Dͫ%DйfHHMUԂl?nv3i= Nq y@7=,L;&v-!r_&dų+ON :jM=r.^",?1X ݥhx>O_爕ڜ=vB't|&1AP,Nm9{4p^/128QO"s,O#]> gMwVMlε7]L'^%bKk7s bD!{ N3`f[ N3E`:캿h6Nvď0krߝ*HN&~'RKQh԰gX7Ib;[zY~DPu?q]]++,Wڅ{mH:^}^È`<1뙩EFg[3f|105A}Mgs YDP1nԵFsGtAdX>%zve.@ -vcܱ?=ǵFUe:fӯQ)MB'8f#edb%17L!zg\ubQG[#ۆ\4h~k$NU͆\DDo4Ca p"ȝkA |؟!QǞ@ gc{Ufn TV9 'ܹf%T͒/t̿DPtnLO lߋܗ` Um_`#I+zМԇs׃ eGPM 6l!n)s*kW_Q_},`."qay2^ j>ù4VAl%V;9̒ftBcۗ_%R鉖y(s = ¾+"Gv_xr:ۛ^dA瞠7DQ9WRthZVO>q% 'f?iel!yQ  > C:^s" /0ۥ?kFS`I$_z$a,TI :d@dJ+JXRf&'[IK%B;^IT(/uTմ!tI&=9".olWwt,A'K߲-b2I߂qC /)H^xך=Ol/ ʊw኶ t.}$>]B\GdVђ" `NIG+B DI?V`{Qkq _km7UgIZQ@x)Qu.|PHIe{'Idժi(FLntvwz>W~*l_:kBY~Mmb\fI߭oBY~M Sg[DTM>T;T(3|L ٶoT(3 a Sz/Qʍp '#M69aBCTbf<jԴA:_]a{q4J>;[8+w7Mss2NdqQzu]Q.KdAS?U)5@?u9Y"d QP]Ki-M@ɛteљޢB1^G#yrˆ wשLa2B7A9k<%sQDl IOV$Ot!\{x"#D;ih"EBT҃kvJkfp?,P]}D.yt&7FhOqy#e%[BƎZ J@c]1 {FJ}tB wluMt& WFjYQ!oP ʔ^ FD1yp][{z"f^9rT/QJ"CZHxM" "}gc#jU1JB`UޔsTִ 2?Ϫ +?nk%Gq{:z&jCOg[Dpף~nzw[XUOde:_`bN`/Oo z"'>/rͷy1Ι۟/T=矦fz?ـê_|%[_VAe5ZI\@=Wo$ eMKwUm bUѝy@ Ltܶ͏ "ȝcUBѩp4D;Dzw̞?o{3T=_7]>ӥEfoV5:& zkᏤmEQĤHVV 8b `abV]-x<F w]Od0y9=~WY"ȝkyCGD5~oQ]Jo"ȝkq"e!uκ5jk|BH>,"YNg\5qC7:$T1.:} `Qn`~^'O"ȝs}B*SJ7w.1~ȕkιדI~pyA>\aD;T@~d&uM4nΏxE "ةqzm5'J5[?Z-,I 0:BBk8Ν|jy>\D;jD|u~gM!{xA܇Cz mdê*2@snc{ wϫNmE969T ө}IKX-[ݗ́ԖHJxgJߦ5jsw4 7d!J'>p5BO5G~,7,imI"~!=.=PNy[v>ˁ+3Ǹ!١,$i sّ _eNIKH:Xz,[e%w_.6t qw_"N$eR[tm&P"᭪G( =D5&,#Fc$!IqoTWY<$ T m2TfϷe*4\Bvܳ燉fP8Uz =O$3VWW9tf" 2ysE3 /$af0 "$":[!&bf}-0YDP tlQ ;pG>*i?Ie2L< ih"'m:(PIݖq91fDǿy qZxN(z6N !A\!y!8 IDAT `DEVt1ޗ?loݠ0"0 . o"c1+cms{آ쭅1ߙ(+rKnj=Qk<熢D+Ξt 1{>(R)+'f$?{87 $ϿI9:-_4S."fx \H}C4^@}@t %DBI1#[UnVO娵ؚHIoM ~l!cdBY_1F/[XU0 B{BY_s|{I,JFmD3S2E8 Y*_o5Gޑ#eF bJS sܫJ^5sݺ e}ݺO7zof֙a6B# aof;(?O/h6LJ )]+k؝DDO=#VKec#OKI*]FÄi좣UTTmA~BZ7TB`53b?w+y?o@5.Oa6rhdT?3=ښA)isfk|@F#_k{pIQi\Wk{H\({y?y"x&}}z FʈX]^ :DE8Sh$UY9YzRtQo"x`ߺM(^~Z[)iK9맵M~MNj&,K6üt/iր؃Ӧt6X(x_] /&Hy\hT%]gJ>&ӆM,N*Mf?gI! {zaзs2xw#xxʢ);#ʻW#x2gd=}A4l27f@ۓ>&:~nPY7ǁTCJHRM|d2m -5p+in>I4R$~pt@%$i~G|D|йh8һ lIK8RkB FKR_[>)R~Zov{@'y'"TxLܾ<j˰&XH/X i*Yq^:E;yߟ2(v :k*U6:STabB,%F;'ǷI)\"xG;qsz-zswc4`*) z"e" j_0@TKgD0 eDK Bx2#!bpz a~&0"4%I}rNFgEm`"s/o,t!>مXd(+ؚh"ڳӾi Dp `U"]`7 9,A)t#UILt'r$\,SeR"ڲϚ??p21~p,J | NԎRks>AaSB7(C Y8Dp7}{XwsLEsL7<;.L2s?/*v=,&N=Jϋӿ@$7L/꼫Sk*g #˃ҾB|$W9ྡFu $D+KD:5sm_j" آ=FD{+n_ϋ:+OGׄ,0g6^%ēzyaU.'ܗYm`bJ‰ Ust],JGɈs0ߤVpK$ZIԸu込'[+KӞ=Okk%lFR{g$Q4DW Ù{q/˧OyA#.IzLuƔ/T4D;ZTzFZ,IrGS!.>AsSSAyl"K]ql"حt[F5aIyjBwZ/o36"@op(U}ʶٞDYqԮ 5Ν2lWHnݏ{7ΕuYc6o"ȝ+Br{AşרYDpU[+}=ښE}˳,pgϽݚI 2&Y5m I+{q/u>F?ݪ:6zAg"]cGt. t~(Џ5r׼Z@%*}w^z #:y< $L`:7?{5R{]^h"k칯[3` UAt;uW4dySIcmѽETdךsJW?_7dY*j:% ?Շk5rޜ jW.M^"ȝ{з^CCYeJ+h-HW2aL]H'u*I Ve mI`(Z`U@'Ś/ΡT$q>l= $]At_ݝRwMzYl^%/9&lY6s/wƃv)-TzL 2ۤ?$&ƏJb{סia18c gSi$=PzlKYg>_y=sDG@Ҽk2yJ)a0{R^D*qAlV HXҵ|(1F?q?.C""Ë;HF|#%]5LstmK̾ Hڱ$"i3=Ah <"iIKaDB\s}RB~|;۟LQ#h&nmON򜓉xh"C"Ck 9%};+"h]q;j" M-$HvGٞ؏kOW]߯;Ɂ$Mjo9E!p;w ,pK"OůIڭUN _-n8Q"]VdXC칖Z=pTLevwT%ixLjk/#k߱A&n WFzف HY# 9Ջńnm$NloDI[fmmlAL>C~Q "")kt_0ř8MdxuWqc{, B^X1-Cd$qCO}uqtvt7QXXk@{ppr#N%G4Y+/<!/#Dђ)it@ڊhY1K6"x=dyÞHIY͞>"xL9ED̯:î^A `8 ,\1ll`$Jخ$jIt^ ؙhi]4lHmeb2&D;u56x>:x;RK5+B6ğԓ_FDd܍I@1[=*I!VOa*fu]L`FM[ݷc+󋘚f/R2._:70ϡk.l5=Ap" N U){6 P sx@58 o)0 ~l=#Y}d*dFR'*'J\>Ͽ cVagc4E<4 :3`0D5?c &`6< 1 < )y .;PyK œ,`s@0ػK ˍ;Z ؤ뀍n;œxX`t5ɘ | $a>󆑰t X$1 F=s,my[cj/ONtṖ޽Ԓ 6sA"`YnJ/"7vJ/92>[XKD~h94~;ƹAֶzY0UʁԯQNu}#nKkf}Մ,]Ǒp,_31Kle^=j:smD`הv}#:N}M@45ihVe/2?^=EkWyj] O`klJQv`8aJ-hb;gǩ:5!'y)&(C ߺotsA=,TOP< /y. :i @$fǥzkT4OW`-&u%vR ?vExڤ>y'mf}t;m\xAAY|3{pNF193asDuw Sɳne"ۙ<&%o;0۸hV:ҙs5 - g]C+gx:y}@(J>w6LM{ֹWT\G61zbS*9oM^]=gL/8B3:  O1&/#4#? ZNYscҙ|ys]֍F;0Vݻ)ST6?7-4`j0aXI?갎4 Ob!m3ԬS/^#=G]7:( sݎZr}o^1X6 trfeq.|=:@pSDѨHhU`zQ0ؤ>o ;+>^ 1 @:]Jjd1;Q"j,-Ldqձt~ ̍#R{QI7Kq&@ib!&mOk`+fe;Sa%*I[Ž3hBE[?DFaoKuQO6>JR#mܗǗ 8gu9a69 I1cA9T{G W`v>F`D=A٪>`˔{hܾ@uNب~/1}7fg-O~\zc]V!,k㫐LcowJ!f7|)y? _i;$R΢nE=gi@0:à"9{U0wKv^~T*QGs F`;bQ\`#2T5&_FD.G9Xؚ1hζM{Rnrvt[\¾Cqrgk`:PW2ԹfeOUy{*jrƃ-UD-"sBrэk+NTuOq"rl?a^{Ն>{3|‹Yѭe[D#KAۯs&Qh-=ϰҾXBxDd]}Z(|9܋l*_ZU/P眘A8/S+38@SS|3hHYrܩDn*%"wzMXUw8KU.U<.66?!"r)zn**ZU$P~rm"%1-RED6aδ$lt݉7cYdLwE1)\Tum/"W#`"rl6U@w[q܄ƞ#!ߦ(.XTVU=;X*`5Uc?ueEp5~sY;rgv" ni1/ߡ\ȳ?i=owwb~uebϻ`ܤFN%VDn t$nW̑n&8^Un?_D\DDUoP 1­˼xN]YUx\DB2S~rFg)쎥- xՐ|ADnż?6M9R/bdI-`goϩOs`7"2n>_cQ;aEDVMSUט ,0Yȝ/̠5E&^D6(uv9`}DD>)"sPa}|wM%(*G!zݬ, "ce70eU4M\e@#܅%ZrۘzNcc.L $MKw*g`n^&'߉ )]Tu *PDd-8p,{f(c?J0cm\)HWs AD1[}6]FUG"2 S̕ee#pu,՘671ak~EGUC}ܔI_]T u~  evl?^DVO%"lmP< "r|"RDlVYA2yLD~#"PkUj"" @S_d[Dʧ> =L׶{3l"f8K\63JOv"2DD+6=װ~|XW1<9/~o\K)qvEhWU0я1{S"r,EO-`!"GF睱\[IQ~`m0PZ >z~3'beHeT("+O4e)X׃JYEd{؄˘}foL[lE-sgU}3b襃v&Y3!pt3TVeb_YeǕU|9H\-V|cq$b*Tɍdmjwϐ}^#矩\C>vI*bgEd]U)e"" w.qҹ)\*qy}XҒMV%ލx P;NúvMly4WI1gȍT`nآy'$aoߐˠy4r0xύ \qQlBI@-9FݱI:Wi1sN uR?Qad{L}r!v^VN^QUwW`XIQE= dłnR]"6KAv"2W`':tH n6뤝@NTջ6U[OSTS2Tf|TTuoˆW]DdV]}jrv^E)j#10+[z`p3EUvrz}ʹy`ێ}|$n;:X,=D9_Ի2 m ^.Ŏ آ2 p"o\i00[]9}p.R똼UC̰HNa+4ͨR'̻ظ`/lXjձNVP,yMj\s;E}iדL#yAFHXEΙSx9sxV`;˝u+C۫{؂Pl15$o܀3O`zsh#,0I|!IE^W@p-ٔ{X [=y@mn༅&܁}\+ar}Nd)kD̹ KFuٯp;k 4Բt- MTu];C+>,y&$oPU{> 9Ey. BF.%o XA?у+MF 0Y=7*uX Mx0ZتzNKDm|Té7OQO9vC:uWot#MQUb;ϰ8rswCU͘3N͂ǿ~8a݌:"Ra~2 6<xπ hV`? , r?c`m(Ẹ(Z(6^t5XTu[U}b~G_&h>wEdu+F?WakO~JTm(p<:FUl wajDdj9L|CDDU_R=1OIb-ȡEdg .vD(⤥'N 8Kڬ펠9E^I'N5d'oU^YUN1P``gJ-x+Q^:ק^MtN?vioo!ro4 MO;Q׻ uѹESzr1A ɓ1!hsGp=nEd-ڽ2N{iz~8UTې'w93|SUM7vqW/Wv [zW:7qK\P\0j(eiK64;9{K}}|@UǪ6,3"qM1 Q# q$)Fv&?[h/Fn T0z^O."b.rSnvgU3m "Ww0X-f;ޫSռs`;c/lFap(ˋHsyj0[곆'@UP/Ӏ47XGx6sǟR`1/'Ed)Xr%zX=9lHDf @p'nEd%6nC 9iyd|U V8]jUU&ȩy#TXt7{<^y ˅dn軬s jIrԵ1p$"u[0oTjx4-"2^FI}('@USo`;'.X)4 s~skw|&7`jرW 0o Tz_y>y `ҟJVkD珹X \=)~%UDvEDl3>&/aY䗘q<9DdI5 [ٕFx.ƀOl+sF ە螳{5<z9T:ф~Kla1IU6[?hX:)Zg|NyROƂ[^{ =Lu1Ot0p{6:^5F\07УgI:}~ |,8l6{ kȍ1GD +߂5vQl]L0Kñzt&S khAws܉cvu/K5fo+h {Sa4KNPLB6?\?l\ Surl)ob.g<+#'[@1[INԄi՗$FQ=/nK-~1u[QףwFZf>]ds),eXm^(II{ Q3˯BoM {~FybapM\6h'@?.t] |6z0u#>-> e,z 3FzTsyUGU_la6>-LR0{8\Px6Deչ$"si-i11]Tml?D$YHGQqXD|VEI*FN,kGLm7;9}+)G"UT]*"ct?V 0X%q6A{K>--duy|LVS3T#j5':?P|؊Y_DZX9]D84DX_lr/a RG2 SȢjRrIDV.P x8#U-."u.T}71/ae^EUwVXqrYr>I9CUn.+"{0U}GUD"" n^r5f=#DdUg`p8.LY׮=]Gz!s>*"8HߤH nd˫mEԧ4. {Kl% ],( 9:K/~#LQlS^#rN;܆IԅᣱLyDs fG#Ryck ωߝTuo#Hujap4?ʢ1x[US!9/v9`ivOȟEl@`YU=Z={ß\Vnc\DV.m֪QD61~M\ӛ&"},/d?Ezy9oZnݱ03"X7Tl2= MTLPvϊNhvAgbk8=slLLe~[UDd# PPڕU˪zQD>-/qY6U1CmY,$7q=bǼ\t9 S IVՋQՇׯ9^\馷st˥4J!~$aڙ<n}fC8zQWTul¿|VDK1j@oL?"r'N7>dk6:MD>+`zמCYV~0ض9O(716w6똧G9$uIT>`x&"ϩ.)Ɋ:U}BU=H0RDfqc;;T^xex+"?tиmr.3MdѠ51pR^JPD=svR¤sQ>aYLwbmq SsCuPNC_3SUbovtX;׀2WgfJjɧ'_׾Lm U%c !; mӷnem)cWx+܄ֈ!"TY[S;U}-眡RUB0]jn""C05f^4INS6j-tMIU?ݕcDj,] 'I_d8Xs, 9cމr v9ޙC,-7鐜er(0F51yWj4̟<<2feU㨝ȸΉ|SYpۤ 3֟V{=<1,dHm/? 4`]w`A;P}:u~$uӁ"  pNH9gL߶i㯍r0qNJir. ލǗvS4ߩ :ԟI5Mu[zo,lg|Pl UBH15)=g^UO)* nƶƹ V\UO:B?TZ5_ñ08*lz[t/sB+~4йn7")!/}AYdXqwMBé^ $cgqr/ȭ"RLwIZ_Hܷz r1Ꝫ%$d :YD@/GWxڀ^D vo^,:'uCOE%\l4>>4էM?|!A Ys<: e_U}.ώ-hבX,w:8F;@ΡC )p\."hy*ޯK&`1"7u">[rH,.'0yZON+,A͌&ǰTuFgYʭuj)瓱ޔ5ZrE`lpɄ1X`ap^9uZ?EUPʩzx( F(ux#1qsbtʐ\mOlGX~TzUY.`y[U g鿈ccٶDp,DYr5Xn!͈8fHBΠxHc SSLI^X<r>XY{,S0='ՠ«U=cmU'ۖ*P19oay޾'4i7ƿ77XkJLy`q 7zj,vu$q&yu~c 0IJu8Z/EA; kV A\=̉yDq) >|OEm}MlK\>SfsbiI?ckn6np$`; +^Ԓzr 'Q|R0Db'l0 s_%:$^H>W)gs5)@,F1kt"X NWUr Qm( x3Xvv SiFQ }04l낼>1sh:SǶz0{a/:gg6s"S!vf.@?ph4&n~w ͎Ss C7Lt/.54i=c%o Ixɩ_)6ű'0% IDAT#ð߷9KDFtoo<*m2N\QC0/'iJ}bf2"Iw}n@1lE=x8 :>'5wΙSo\ 7BHZϩՕ z~~} &aa 1ف`{ÛCvcx$n zFEmu.Y'OֹSN^vaʶv4ZM+1>w $fd{X`nI/1a_ѹ6z\@!]"_ ႣR8j2Ap:pF/-dHiܤGyyfǢ:̡~7mȦF<rG@$rǜXv\ 5 5 c-ژ` oC4,J)|sMQ/CY;@Jd=$U P{Y^_%\S:'!;z3ECbF= C15 9fR@1\sc7HMQ$@V֘,<@0ծ З75c5k>Π%I|L.FotYww5s}c1Liiw^?B-m"+(`a [?A,n|65L~ҹ/ >2Z)>$횢]8t3Z DɬVIiCRELHzy3x 7gӤ_+`oܗrcc 8cI$,]qZ`ie1q .'1:LWY {ݢmD|b>SkحY$b`6IrxqB .)=4X5vPQ>8rфB_\tB^aWz&$j+qG$Ĥ6{~puh|)F#Ra .V`I,tfkyXXӀM3ֹ%mQY$]{ S;7*Apl]ȭÝ2UJP j;"b~0H. ^baEmL&۸@5Z$DL8vyqژ sZ:>b65y Koy\Ӄ?P  B?@=9rXG$;X@\YƦcIj % ?6Hi|-L=MIYaܾo@d,%m"e;Sj<U5 9$zOw:9H7|AV}M6uj`Tu)ouHlG! ,/px@jn>iT u4Lw1f^wm8:>L.-,~8l܃vVvJN:=hɓm=cN'"{ xG=dԍ- RWMQ?ek{'M`.|a5Lډ]ʩS--!y@P }KMJJw\A$;Sˆ19HP5-Ap|V,CxX%^w-8m>8υ:otð#(v`y,l[خlF`WUB:b❽@ϳo ys`>-{*(?!a+lhuX1/i)5$wn׻ygLp/&M}raicrmh6ha,[Е^gWy[`m:סqAC,Emw_ G_m9^:ݍ:w  rP IR/,V (ՂI7Y8Y1?nM^M̨=_/? 5ޝ]ڣvA:5KD ߐѣ@6~l!ݮVGiIxpDM-9} z}"8>.T|0lgZ,+@zp7},Km &c9Buq\Fo`hK s Em#NX[ F) `.yjMٝ׷)yR}G;O`ޯr/ ׇc|,&^# nG6}- <|Z z3{%9v?IIkaPj>>R@пL@3f OM;? 1 <oos<\D~Zs!!q`nH1^,ݜl( hVƬ~BzSו/zN県̓y:3:h&Lr}g_H/6vBR[+C<isb>!ߙdd#g2j126y AҞ_f Y_ǼVe-BJs XYg~dm@e!5?;QU!$S>`*Ȯ>U%@azFP4x> n$6>4NmJC>rަ_Z=J8bǀp-s@01c19O]JNF] Jfyh a!Do>{H_:s!jc4S9(,Z ز< 4I=(|!gGa`~rg&0u,#V6`TZ/6&$/zޙUO$[a_Ƒn̍vRAb>B do8KӔ>G6>SMcr"`zL*azLͲ,ALAbAEa;es u,.MkŔTf\ A!HVvCj3!!OD,@pNJ/ 7a{|nCo2^V=a.|jAp4̮/OsChH2w yv1,khHNS1oq Kz4+azC?``M,㢟 _P}<^TY fd ?̉ޚ'R,Z5|<灎KSf !# q=H>΋r]Ӏ;jwoApFM27va`);%xka^eJ֠aA|\zţk8|Fie6*Qv>T[ Ga\tt}p;nL-92@;6DEXyTxhStY99jxtX.Z_CByBi˧LQbI`B"p0{%mPql}F?AÈ}% --,|6vgs`ѫ{NMoppZ@i ~>t ^< qtިu09xNT^d2ps6Ki'F@qz/3yM'ȟC;sQ-vu2o >8i"ӱE:E3=z..qYf@ Gb}xՑFN?kM>ϲkbrLnp*nhev`>1=qoQ;gE\Q7b0}z:D<`ǐFFB & eJ;S$ ZtNHN ]f%' xK6A#e 4gt}O_B}@`.҄:F?97~dVxT*RH8c'28=1w@lW+fk(ރ@")cev`]}e*KVb =PSDlQ;!Xj a@N, ac@1 WƖ@9gǂT[:~2 hLbhL=jҨ27;< \rkdBw2,12Ņg:0ƂI&%ȨQyѹ濍@@&!& PeX,Ѧ/{1Cw6jΞ#c">lr ziF*[r֜|g*6~,UtkjIb-^vϔl2Sk._ |AmO4_tlԇp|lkq >hp?fy5`v0%-t6i!@O (cpR=@:25c+Iks7_@sWB8JI2c0OxB!Ā:-݇Es  N>WjaG[߁ 8/$X880nA }pw_3Z<C~@ijjwyo$jv>N@1vEu~tWƂEOƈ3E`W/{9@?NM#rOGG%@pf<UQ09m)s\:q^-y1;X8]@p׬mv؏PSK6[78>;>wtaMLv,6rtk$ح*3۷g?뻡0+M>-aғw'ΦN [q*G,ƍgC;+'=Rv`tpͳ5@>NmꁏβԦ3V*2NR,Vt?g@ [ITs <BV Y3} iP(yD7l׌v(LG5yK,nX3p^2]F:'{hU-Bmd76= /|)&$ ?vAOꝌsd]"Va}7!}.{OTgKm +;^K4! IM 6vyAP$Aot\@#7V@KF3c}"ى@~#N,U>R)91a 3 \}ׇ(l}1JP{d1,˽-ޙ//:P( `/I7f 6 ˛Xg?U;^ ޻+5C؝Z`h6z/j =yn ~_ s+ *wk[!d Pl 3v6aX^wϩATOa:a0AKqS,yxUji@$p6qF?$벯0Vh&`.|Ϫt ,mz$Ir=GCXm0/,A=Ob*@+x޷q1_Un H*`T! 4eHLI!Vtd˫!z|ј`L;NU3NVꙪ111K;eTOٺY~ EjleyUN{+)LTUK2/𜈜'"KDOQ1ϜK#hY/\fT%JI|szVRQg),a/39QU uIh<!;}rKOva"܉YVSߧ]SI$T|U; C8]D5S^@0DtY "܅0|X^?TPtjS2\gCy3/gw#!Q pS},)hG!ZDҗ@/>!he(RzVq|+ y Vt; *i[T5U=R3b@p;`po@Iq*(?/ (zJ!)8$`vB@)Kt bK2Cy-þwA+ ,ͷ8@X<: NN=$cTPvvxȻ6^+)KΨ6B]@ N3T0@0l"2|::۵*/f1) 4 niPy_fm{.ʮUIi$ J @/ fxm&"bju#҃:K#YEd$XP([WI+a솒NC@_GD2&IDAT~R X5X[U.L%v"ҿ3ƅ6RտVc2>{b w; 1*X#C| u^x*K hPv"2qY\mCKf5+h/; u~pB.!zT obpv?Fp^U},B@DȞl =<:.`KU]WUt%YT9Uй8 /"NjȂSU"`a ~UX"p vc1PJ*UFc*aO3mG]2vn1PU{+lb6oC8@X;?=8A-F "sXʇ0Pn6QT2I`gsc0NDYP0k܉`B%=#p/f9F Tu3U_̅9Dl~R,╪Q͊PUoWխ1k01&+@t2 @Dv0Gr;u*-NuTuku<UZxsœJz. Dd9GAznU?*S 7Cpۋo2 l ,r=nw~% 9;/mMu+DdVU鸾<DdX{XD(U}+sL`?3ډ+[,%w"ӱ-Ѫd+d0 RJ ,?cT3^LDvf%S|FKDdj=ŗ\7uLmM4TvJ fߋaY!x@\)$7KDdML=}'h0ފKUu|vK?:qlT QI%^RK<`F; !"ñ~wL>p!pڳ+EDd>,oQ,ӁX;EERHWP3|ʇW(U}]#.}nc * _DrQYs6aLtǩޡJ:@m/LwSU})UR`pIT޾,Ӂ׈7V6 HW.J*) ORXU}ȾUҾ 1o1ATVI%e~h,2NRY%K534UPp*PIl>lhR* "" cʷJ*pR@,/".MTէ[%TRI#i1SnKSJJ(ITYZ2 ̃%G;M_%$J ^x g<C%TRI^2ˀ/m,Z-#_%TRI4#"]/ epl/sTRI%386 hY/\$:ZTR ^xiuPJ*?""CP c6 X2(K/ TT*rKGaia40܎!._TRIdf/3x |K>ZUo(cTRI2 /H۱ WֱJ*T Xe/ 8X\B% rVfj UX|'p){%T26`/qJ燀+@)4 R$c'PB% i` 4[aQT2U^T*R@gRA""|.\3w:URI%TНT@Ёpq*@¤`mH]l?~zET7iUhRAFqpN_EJz%K* (Xpׁ_;EJ )LDd5 ,Yv}nrK%Ys;.pTPɀfG* (*۩0(U}ȾURI#iW@I"",CNQ7[%x``JLDdla"pZd*y- P0}B̆Jq=dfppbG* ""KPT"V T@0HDDvf>.VЧ p`,w'?aǿA̴y!MN lH1/;knc bah ;*14mj166hC?{S:~哪[om7s {OK+`58$k}u 5pLw.ҤFJ>A @3( WM>D'.}smR] @(-af7w l".}smR @;)-cf7 Zw,$Bƀ3Xh r-A.E`o˹633b$3ES0\OI> vSZF!lz}FAx8Cؒsm; LG!I AX,w7d Lc@@J~5 Qw'd @fdfo<=nkr ̅B Rr4Vl IJx7 RdNl/*`8:ǁ!w+Z`I+2+ ̋ \ Ǥqk6 B R*4 Cx5׺N^R'R.'K .l k 3XG7Rd^)38C8>'ӴqF B }QpBN쩬 AIv]&H_]o6l5@E!(| ա.m(2h T!ci'^?wgG=(g dQV>Zv!ϸ&HV%w.4@HU(R ӽ6 UHLsxɶRt\ F_AxNAH9i  HR9+  uHm FQwߙk]S)R7 N (pI}ls?3KZRRwl3 L]'ĥ+@jA!3 =;AxUi 3;xbـ3z@F!1D.crtK\F 4B ef+G`6ϙ O( 4F Owuߛ Hk٩2J`wg!`RT6Ruqx0=`p~mD6 @L!2SN;gxH(zfv @?v[()[ IENDB`zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/_static/source-version.js000077700000000000000000000000001527014137600360532../../../source-version.jsustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/api/000077500000000000000000000000001527014137600255005ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/api/definition.md000066400000000000000000000002061527014137600301500ustar00rootroot00000000000000--- title: definition --- ::: zarr_metadata.definition options: inherited_members: false show_if_no_docstring: false zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/api/index.md000066400000000000000000000052721527014137600271370ustar00rootroot00000000000000--- title: API reference --- # API reference The package is organized to mirror the structure of the Zarr specifications: - [`zarr_metadata.json_schema`](json_schema.md) — JSON Schema output for any `TypedDict`, one v3 field, or a whole v3 `zarr.json` - [`zarr_metadata.model`](model.md) — document models (each one a document and the scope it was read in), the readers that return problems instead of raising them, and the `UNSET` sentinel - [`zarr_metadata.problems`](problems.md) — `ValidationProblem`, `ProblemKind`, `Problems` and `MetadataValidationError`, which every reader reports, and `shown`, which formats a value for a message - [`zarr_metadata.pydantic`](pydantic.md) — optional Pydantic field types over the models - [`zarr_metadata.typed_json`](typed_json.md) — `check`, which type-checks a JSON value against any of the package's `TypedDict`s, read as the typing spec defines them, with every problem located - [`zarr_metadata.v2`](v2.md) — `TypedDict` shapes for Zarr v2 documents (`.zarray`, `.zgroup`, `.zattrs`, `.zmetadata`), and `zarr_metadata.v2.definition`: the v2 kinds of metadata, the scope `CORE_V2`, and the readers of one `dtype` or codec. Each dtype family's definition is in `zarr_metadata.v2.data_type`, and each codec's in `zarr_metadata.v2.codec` - [`zarr_metadata.v3`](v3/index.md) — `TypedDict` shapes for Zarr v3 documents, with subpackages for [chunk grids](v3/chunk_grid.md), [chunk key encodings](v3/chunk_key_encoding.md), [codecs](v3/codec.md), and [data types](v3/data_type.md) - [`zarr_metadata.definition`](definition.md) — what both formats share for reading a field: `Definition`, `resolve`, the three results of a reading (`AcceptedField`, `UnclaimedField`, `RefusedField`), and the scope type `Context`. Its module docstring is the guide - [`zarr_metadata.v3.definition`](v3/definition.md) — the five v3 kinds of metadata (`CodecDefinition`, `DataTypeDefinition`, ...), the codec pipeline types, and the v3 scopes `CORE` and `CORE_AND_EXTENSIONS` The document types, models, and spec vocabulary — including the store keys — are re-exported at the top level, so `from zarr_metadata import ZarrV3ArrayMetadataJSON` and `from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON` are equivalent. The model layer's readers are imported from [`zarr_metadata.model`](model.md) directly, and the problem types every reader reports from [`zarr_metadata.problems`](problems.md). ## Common types Two cross-cutting types are also exported from the top-level `zarr_metadata` namespace: `JSONValue`, documented in [`zarr_metadata.typed_json`](typed_json.md), and `ZarrV3NamedConfigJSON`, documented in [`zarr_metadata.v3`](v3/index.md). zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/api/json_schema.md000066400000000000000000000000721527014137600303120ustar00rootroot00000000000000--- title: json_schema --- ::: zarr_metadata.json_schema zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/api/model.md000066400000000000000000000000561527014137600271230ustar00rootroot00000000000000--- title: model --- ::: zarr_metadata.model zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/api/problems.md000066400000000000000000000000641527014137600276450ustar00rootroot00000000000000--- title: problems --- ::: zarr_metadata.problems zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/api/pydantic.md000066400000000000000000000000641527014137600276350ustar00rootroot00000000000000--- title: pydantic --- ::: zarr_metadata.pydantic zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/api/typed_json.md000066400000000000000000000000701527014137600301750ustar00rootroot00000000000000--- title: typed_json --- ::: zarr_metadata.typed_json zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/api/v2.md000066400000000000000000000013121527014137600263460ustar00rootroot00000000000000--- title: v2 --- ::: zarr_metadata.v2 options: members: false ::: zarr_metadata.v2.array ::: zarr_metadata.v2.group ::: zarr_metadata.v2.attributes ::: zarr_metadata.v2.codec ::: zarr_metadata.v2.codec.checksum ::: zarr_metadata.v2.codec.compression ::: zarr_metadata.v2.codec.filters ::: zarr_metadata.v2.codec.vlen ::: zarr_metadata.v2.consolidated ::: zarr_metadata.v2.definition options: inherited_members: false show_if_no_docstring: false ::: zarr_metadata.v2.data_type ::: zarr_metadata.v2.data_type.fixed_width ::: zarr_metadata.v2.data_type.object ::: zarr_metadata.v2.data_type.scalar ::: zarr_metadata.v2.data_type.struct ::: zarr_metadata.v2.data_type.time zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/api/v3/000077500000000000000000000000001527014137600260305ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/api/v3/chunk_grid.md000066400000000000000000000002631527014137600304700ustar00rootroot00000000000000--- title: chunk_grid --- ::: zarr_metadata.v3.chunk_grid options: members: false ::: zarr_metadata.v3.chunk_grid.regular ::: zarr_metadata.v3.chunk_grid.rectilinear zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/api/v3/chunk_key_encoding.md000066400000000000000000000003121527014137600321740ustar00rootroot00000000000000--- title: chunk_key_encoding --- ::: zarr_metadata.v3.chunk_key_encoding options: members: false ::: zarr_metadata.v3.chunk_key_encoding.default ::: zarr_metadata.v3.chunk_key_encoding.v2 zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/api/v3/codec.md000066400000000000000000000006371527014137600274350ustar00rootroot00000000000000--- title: codec --- ::: zarr_metadata.v3.codec options: members: false ::: zarr_metadata.v3.codec.blosc ::: zarr_metadata.v3.codec.bytes ::: zarr_metadata.v3.codec.cast_value ::: zarr_metadata.v3.codec.crc32c ::: zarr_metadata.v3.codec.gzip ::: zarr_metadata.v3.codec.scale_offset ::: zarr_metadata.v3.codec.sharding_indexed ::: zarr_metadata.v3.codec.transpose ::: zarr_metadata.v3.codec.zstd zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/api/v3/data_type.md000066400000000000000000000015721527014137600303310ustar00rootroot00000000000000--- title: data_type --- ::: zarr_metadata.v3.data_type options: members: false ::: zarr_metadata.v3.data_type.bool ::: zarr_metadata.v3.data_type.int8 ::: zarr_metadata.v3.data_type.int16 ::: zarr_metadata.v3.data_type.int32 ::: zarr_metadata.v3.data_type.int64 ::: zarr_metadata.v3.data_type.uint8 ::: zarr_metadata.v3.data_type.uint16 ::: zarr_metadata.v3.data_type.uint32 ::: zarr_metadata.v3.data_type.uint64 ::: zarr_metadata.v3.data_type.float16 ::: zarr_metadata.v3.data_type.float32 ::: zarr_metadata.v3.data_type.float64 ::: zarr_metadata.v3.data_type.complex64 ::: zarr_metadata.v3.data_type.complex128 ::: zarr_metadata.v3.data_type.raw ::: zarr_metadata.v3.data_type.bytes ::: zarr_metadata.v3.data_type.string ::: zarr_metadata.v3.data_type.numpy_datetime64 ::: zarr_metadata.v3.data_type.numpy_timedelta64 ::: zarr_metadata.v3.data_type.struct zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/api/v3/definition.md000066400000000000000000000002111527014137600304740ustar00rootroot00000000000000--- title: definition --- ::: zarr_metadata.v3.definition options: inherited_members: false show_if_no_docstring: false zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/api/v3/index.md000066400000000000000000000005501527014137600274610ustar00rootroot00000000000000--- title: v3 --- ::: zarr_metadata.v3 options: members: false ::: zarr_metadata.v3.ZarrV3MetadataFieldJSON ::: zarr_metadata.v3.ZarrV3NamedConfigJSON ::: zarr_metadata.v3.ZarrV3ExtensionField ::: zarr_metadata.v3.NodeName ::: zarr_metadata.v3.NodePath ::: zarr_metadata.v3.array ::: zarr_metadata.v3.group ::: zarr_metadata.v3.consolidated zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/context.md000066400000000000000000000157771527014137600267560ustar00rootroot00000000000000# Context Resolving metadata documents requires a context, a `Context`, that defines which metadata definitions apply to a JSON metadata document. This library models resolved metadata as a `(Document, Context)` pair, which might be a bit unintuitive. We explain these design decisions here. ## Why do we need a context The Zarr V3 spec defines the JSON structure of `zarr.json` for arrays and groups exactly, by defining the fields that must be present and the JSON types they must have. But the __interpretation__ of those fields as actual data types, codecs, chunk grids, etc is not strictly defined by the spec. Relating a JSON value like `"bytes"` to the `bytes` codec requires a mapping from names to codec types. `zarr-metadata` makes this explicit by validating JSON documents with a __context object__. That context object contains definitions of codecs, data types, etc, which can be matched to fields in the JSON document. ### Context example We will walk through an example that shows how `zarr-metadata` uses context to parse the same metadata in different ways. ```python test="true" session="context" from zarr_metadata import ( MetadataValidationError, ZarrV3ArrayMetadata, ZarrV3ArrayMetadataJSON, ZarrV3MetadataFieldJSON, ) from zarr_metadata.model import read_array_metadata_v3 from zarr_metadata.v3 import ( BYTES_CODEC, GZIP_CODEC, DEFAULT_CHUNK_KEY_ENCODING, FLOAT64_DATA_TYPE, REGULAR_CHUNK_GRID, ZarrV3Context, ) codecs: tuple[ZarrV3MetadataFieldJSON, ...] = ( {"name": "bytes", "configuration": {"endian": "little"}}, {"name": "gzip", "configuration": {"level": 99}}, {"name": "unknown"}, ) document: ZarrV3ArrayMetadataJSON = { "zarr_format": 3, "node_type": "array", "shape": (100,), "data_type": "float64", "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (10,)}}, "chunk_key_encoding": {"name": "default"}, "fill_value": 0, "codecs": codecs, } ``` In the above code snippet we define a `zarr.json` document with three codec declarations (we will just call them codecs for brevity): one named `bytes`, another named `gzip`, and a third named `unknown`. The `bytes` codec here is correctly configured according to the `bytes` codec spec. The `gzip` declaration here violates the `gzip` codec spec because the `level` configuration field is too high. And the `unknown` codec is unknown, it has no specification and cannot be resolved. These three codecs represent three different scenarios that this library handles, two with distinct failure modes. We demonstrate how this works in the next code snippet. ```python test="true" session="context" # A context with one data type, one chunk grid, one chunk key encoding # and one codec. There's no gzip definition, and no unknown codec definition. context = ZarrV3Context( data_types=[FLOAT64_DATA_TYPE], chunk_grids=[REGULAR_CHUNK_GRID], chunk_key_encodings=[DEFAULT_CHUNK_KEY_ENCODING], codecs=[BYTES_CODEC], ) # The gzip codec is unclaimed, and its configuration is not checked. The # document is valid in this context. small = ZarrV3ArrayMetadata(document, context=context) print(small.codecs[1]) # UnclaimedField(json=mappingproxy({'name': 'gzip', 'configuration': mappingproxy({'level': 99})}), name='gzip', read_as=, configuration=mappingproxy({'level': 99})) ``` In this example, our context object only contained one codec definition, for the `bytes` codec. But the array metadata document contained three codecs, named `"bytes"`, `"gzip"` and `"unknown"`. After resolution the `gzip` codec was explicitly flagged as unrecognized. But we can actually check that codec declaration against a real codec definition by expanding our context to include the `GZIP_CODEC` definition: ```python test="true" session="context" new_context = context.update(codecs={**context.codecs, "gzip": GZIP_CODEC}) ``` `update` returns a new context with the `codecs` slot replaced. We pass the codecs the old context already has, plus `gzip`, so the new context claims both `bytes` and `gzip`. The old context is unchanged. Now read the same document in the new context. `read_array_metadata_v3` returns everything the read found, instead of raising, so we can look at each codec: ```python test="true" session="context" reading = read_array_metadata_v3(document, context=new_context) for loc, field in reading.fields(): if loc[0] == "codecs": print(loc, type(field).__name__) # ('codecs', 0) AcceptedField # ('codecs', 1) RefusedField # ('codecs', 2) UnclaimedField for problem in reading.problems: print(problem.loc, problem.kind, problem.message) # ('codecs', 1, 'configuration', 'level') invalid_value expected an integer in [0, 9], got 99 print(reading.metadata) # None ``` These are the three scenarios: - `bytes` is parsed to an `AcceptedField`. A definition in the context claims the name `bytes`, and the configuration passes its checks. - The malformed `gzip` is parsed to a `RefusedField`. A definition claims the name `gzip`, but the configuration fails one of its checks: a `level` of 99 is outside the range gzip allows. The read reports the problem, with its location in the document, and builds no model, so `reading.metadata` is `None`. Building the model directly raises the same problem as a `MetadataValidationError`. - The `unknown` codec is parsed to an `UnclaimedField`. Nothing in the context claims the name, so it is kept as written and not checked. This is not a problem: the spec lets a document name codecs a reader does not know. So one JSON document can be valid in one context and invalid in another. This is why a model is a document _and_ the context it was read in: the context defines the interpretation of the document. A model keeps its context, and moving it to another context reads the document again. Our `small` model was valid in its original context, but it is not valid in the new one: ```python test="true" session="context" try: small.refined_in(new_context) except MetadataValidationError as error: print([problem.loc for problem in error.problems]) # [('codecs', 1, 'configuration', 'level')] ``` An unclaimed field is never an error, but a reader that does not understand a codec cannot decode the array's chunks. `unclaimed_fields` lists every field that nothing in the context claims, so a reader can check that it understands everything before it opens the array. With a valid `gzip` level, the document reads in the new context, and only `unknown` is left: ```python test="true" session="context" fixed_codecs = (codecs[0], {"name": "gzip", "configuration": {"level": 1}}, codecs[2]) fixed = ZarrV3ArrayMetadata({**document, "codecs": fixed_codecs}, context=new_context) print(list(fixed.unclaimed_fields)) # [('codecs', 2)] # In the small context, gzip is unclaimed too. print(list(small.unclaimed_fields)) # [('codecs', 1), ('codecs', 2)] ``` A reader that can decode `bytes` and `gzip` but not `unknown` should refuse to open this array, even though its metadata is valid. Or at least, it's not __invalid__ in its context. zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/docs/index.md000066400000000000000000000524001527014137600263610ustar00rootroot00000000000000# zarr-metadata Basic tools for modelling Zarr metadata, with minimal dependencies. `zarr-metadata` is developed in the [zarr-python repository](https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata) and released independently of `zarr` itself. Install it with: ``` pip install zarr-metadata ``` ## Who needs this This library supports software that interacts with Zarr metadata documents. ## What this is This is *not* a full Zarr implementation. It's a collection of data structures and routines that closely model the content of the Zarr specifications, such as: - **Typed JSON shapes** ([`zarr_metadata.v2`](api/v2.md) and [`zarr_metadata.v3`](api/v3/index.md)): `TypedDict` definitions and `Literal` aliases for the JSON documents specified by the [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html) and [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) specifications, plus types for [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/) and a few widely-used-but-unspecified entities (e.g. consolidated metadata). - **Document models** ([`zarr_metadata.model`](api/model.md)): immutable models of whole metadata documents, each one a document and the scope it was read in, with readers that report every problem at its location, and store-key (de)serialization. A document produced by `to_json` shares no mutable state with the model that produced it. - **Definitions and scopes** ([`zarr_metadata.definition`](api/definition.md), with each format's kinds in `zarr_metadata.v3.definition` and `zarr_metadata.v2.definition`): each extension's configuration as a `TypedDict` with its rules, and the scope that decides which definitions read a document. `zarr_metadata.v3` and `zarr_metadata.v2` export every definition and scope of their format, such as `BYTES_CODEC` and `CORE_AND_EXTENSIONS`. - **JSON type checking** ([`zarr_metadata.typed_json`](api/typed_json.md)): `check` checks a JSON value against any `TypedDict` and reports every problem with its location. - **Problems** ([`zarr_metadata.problems`](api/problems.md)): the problem values every reader returns, and the error that carries them. - **JSON Schema** ([`zarr_metadata.json_schema`](api/json_schema.md)): the JSON Schema of a `TypedDict`, a v3 field, or a whole `zarr.json`. - **Optional Pydantic integration** ([`zarr_metadata.pydantic`](api/pydantic.md), requires Pydantic 2.13 or newer): each model as a Pydantic field type that validates raw documents through the same strict parser. ## What this is for - **Static Type Checking** The public `TypedDict` definitions describe the static JSON shape of Zarr metadata. - **Dynamic JSON validation** For strict, location-aware validation of JSON loaded from disk, use the model parser: ```python test="true" session="index" import json from zarr_metadata.model import ZarrV3ArrayMetadata with open("zarr.json", "rb") as f: raw = json.load(f) metadata = ZarrV3ArrayMetadata.from_json(raw) ``` The optional Pydantic integration delegates raw input to the same strict parser and returns the same normalized model class: ```python test="true" session="index" from pydantic import TypeAdapter import zarr_metadata.pydantic as zmp adapter: TypeAdapter[zmp.ZarrV3ArrayMetadataField] = TypeAdapter(zmp.ZarrV3ArrayMetadataField) metadata = adapter.validate_python(raw) encoded = metadata.to_key_value()["zarr.json"] ``` A field annotated `Annotated[ZarrV3ArrayMetadata, zmp.InScope(CONTEXT)]` is read in `CONTEXT`, whatever the validation context holds, and a model instance given to it is read again in `CONTEXT`. Note that a bare `TypeAdapter` over a public `TypedDict` is a **coercive** shape adapter, not a Zarr conformance validator; it may coerce values or discard members that the strict model parser rejects. This is better suited for handling input provided from Python objects (e.g., user-authored dicts) than JSON. ## Validation boundary The readers and model constructors enforce the declared document structure and a small set of context-free consistency rules, e.g.: - fixed format literals, - finite JSON numbers outside `attributes` and the members nothing in scope reads - non-negative dimensions - one `dimension_names` entry per array dimension. In a v3 document, validation also checks each extension point: the data type, the chunk grid, the chunk key encoding, each codec, and each storage transformer. Each one is checked by the definition that claims its name in a scope. The scope is `CORE_AND_EXTENSIONS` unless a `context` is passed. If the definition rejects a configuration, the reader rejects it too, and a configuration key the definition does not declare is reported as `unknown_key`. If no definition in the scope claims a name, the reader does not check that extension point, and the consumer decides whether to support it. Making the context and explicit part of metadata parsing allows different libraries, which might support different sets of Zarr features, to formally represent exactly the codecs, data types, etc they support when parsing metadata. Some checks relate one part of a v3 document to another: - The fill value is checked against the data type the document names, by that data type's definition. - The chunk grid is checked against the shape, by the grid's definition. - The codecs are checked as a pipeline. In order, each codec's definition checks the codec against the chunk it receives. This includes a shard's inner codecs and index codecs. The readers do no arithmetic on values. For example, they do not check whether a fill value survives a `cast_value` round trip. Some choices that the spec text leaves open, or answers in two ways: - **`attributes` may hold `NaN`, `Infinity` and `-Infinity`.** The spec does not interpret attributes, and zarr-python and xarray write these numbers there (for example, a CF `_FillValue`). The models read them, and `to_key_value` writes them back as bare tokens, which a strict JSON parser rejects. So may a member the spec does not define when no definition in scope claims it: nothing reads it, so it is kept as written. `check`, from `zarr_metadata.typed_json`, rejects a non-finite number anywhere, including in attributes. - **A reader walks at most 256 levels of nesting.** A value nested deeper is an `invalid_value` problem at the first level past the limit, wherever it is. Every reader, writer and comparison uses one stack frame per level, and `copy.deepcopy` (and `pickle` before Python 3.12) uses two. A document at the limit therefore uses about half of the interpreter's default recursion limit, and the rest is left for the caller. - **`consolidated_metadata: null` is a problem.** A zarr-python 3.0.x bug wrote it. The spec requires an object, and the package accepts only an object. A reader of those stores removes the key before reading. - **An extension name must follow the spec's naming rule**, `^[a-z][a-z0-9-_.]+$`, or be a URI, which earlier versions of the spec required. Any other name is rejected before any definition is consulted, so `""` and `"foo/bar"` are problems, not unknown extensions. - **`must_understand: false` is rejected at every extension point**, including codecs and storage transformers, although the core spec names only the data type, chunk grid and chunk key encoding. A reader that skips a codec reads wrong bytes, just as a reader that skips a data type reads wrong values. On an unknown top-level member, `must_understand` keeps its meaning: a reader can skip that member. - **A v2 datetime or timedelta dtype must have a unit**, as in ` --8<-- "CHANGELOG.md" zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/justfile000066400000000000000000000061771527014137600255620ustar00rootroot00000000000000# Development verbs for the zarr-metadata package. Recipes run with this # directory as the working directory regardless of where `just` is invoked. # List available recipes # Quoted arguments must survive delegation from the root Justfile. set positional-arguments default: @just --list # Run the test suite on the project interpreter; extra args are passed to pytest test *args: uv run --group test pytest tests "$@" # The oldest and newest interpreters the package is declared for: the floor is # `requires-python`, the ceiling the newest classifier, both read from # pyproject.toml. CI's matrix is written out in its workflow, so a change to # the declared range is made there too. # uv fetches an interpreter it does not have. A difference in how a version # materialises annotations or resolves a stub shows up here, not only in CI. # Run the test suite on the oldest and newest supported interpreters test-versions *args: #!/usr/bin/env bash set -euo pipefail read -r floor ceiling < <(uv run python - <<'EOF' import re import tomllib project = tomllib.load(open("pyproject.toml", "rb"))["project"] floor = re.fullmatch(r">=\s*(\d+\.\d+)", project["requires-python"]).group(1) declared = [ classifier.rsplit(" ", 1)[1] for classifier in project["classifiers"] if classifier.startswith("Programming Language :: Python :: 3.") ] print(floor, max(declared, key=lambda version: tuple(map(int, version.split("."))))) EOF ) for version in "$floor" "$ceiling"; do echo "== python $version ==" uv run --python "$version" --group test pytest tests "$@" done # Lint with the same invocation CI uses lint: uvx ruff check . # Run under the interpreter CI runs pyright under, so a version-dependent # stdlib or stub difference shows up here rather than only in CI. The latest # pyright: the sentinel typing it once lacked is fixed, so nothing holds it back. # Type-check the package, sources and tests alike typecheck: uv run --python 3.11 --group test --with pyright pyright # Run everything CI runs for this package, on both ends of the version range check: lint typecheck test-versions docs-check # Preview the changelog that the next release would generate changelog-draft: uvx towncrier build --draft --version Unreleased # Build this package's documentation site, warnings as errors docs-check: env DISABLE_MKDOCS_2_WARNING=true uv run --group docs mkdocs build --strict # With no argument, uses port 8000 if free, otherwise an ephemeral free port; # an explicitly requested port is used as-is so a conflict fails loudly. # Serve this package's documentation site docs-serve port="": #!/usr/bin/env bash set -euo pipefail port="{{ port }}" if [ -z "$port" ]; then port=$(uv run --group docs python -c ' import socket s = socket.socket() try: s.bind(("127.0.0.1", 8000)) except OSError: s.close() s = socket.socket() s.bind(("127.0.0.1", 0)) print(s.getsockname()[1]) s.close() ') fi exec env DISABLE_MKDOCS_2_WARNING=true uv run --group docs mkdocs serve -a "localhost:$port" zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/mkdocs-base.yml000077700000000000000000000000001527014137600320332../mkdocs-base.ymlustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/mkdocs.yml000066400000000000000000000057501527014137600260110ustar00rootroot00000000000000INHERIT: mkdocs-base.yml site_name: zarr-metadata # The package lives in the zarr-python monorepo; point the header source # widget at the package directory rather than the repository root. repo_name: zarr-python/packages/zarr-metadata repo_url: https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata # Absolute because mkdocs would otherwise append this to repo_url's subpath. edit_uri: https://github.com/zarr-developers/zarr-python/edit/main/packages/zarr-metadata/docs/ site_description: Spec-defined metadata types, models, and validators for Zarr v2 and v3. site_url: !ENV [READTHEDOCS_CANONICAL_URL, 'https://zarr-metadata.readthedocs.io/'] # The docs tests run each python fence marked `test="true"`, and superfences # refuses a fence with options it does not know. zarr-python's hook, shared # through a symlink as mkdocs-base.yml is, renders such a fence as a plain # code block. hooks: - mkdocs_hooks.py nav: - index.md - Parsing / Validation: parsing-validation.md - Context: context.md - API Reference: - api/index.md - ' zarr_metadata.definition': api/definition.md - ' zarr_metadata.json_schema': api/json_schema.md - ' zarr_metadata.model': api/model.md - ' zarr_metadata.problems': api/problems.md - ' zarr_metadata.pydantic': api/pydantic.md - ' zarr_metadata.typed_json': api/typed_json.md - ' zarr_metadata.v2': api/v2.md - ' zarr_metadata.v3': - api/v3/index.md - ' zarr_metadata.v3.chunk_grid': api/v3/chunk_grid.md - ' zarr_metadata.v3.chunk_key_encoding': api/v3/chunk_key_encoding.md - ' zarr_metadata.v3.codec': api/v3/codec.md - ' zarr_metadata.v3.data_type': api/v3/data_type.md - ' zarr_metadata.v3.definition': api/v3/definition.md - Release notes: release-notes.md # This site is a Read the Docs subproject of zarr-python; give readers a way # back to the parent docs, which list every companion package. - 'zarr-python ↪': https://zarr.readthedocs.io/ zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/mkdocs_hooks.py000077700000000000000000000000001527014137600325122../../mkdocs_hooks.pyustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/pyproject.toml000066400000000000000000000232601527014137600267160ustar00rootroot00000000000000[build-system] requires = ["hatchling>=1.29.0", "hatch-vcs", "setuptools-scm>=9"] build-backend = "hatchling.build" [project] name = "zarr-metadata" dynamic = ["version"] description = "Spec-defined metadata types, models, and validators for Zarr v2 and v3." readme = "README.md" requires-python = ">=3.11" license = "MIT" license-files = ["LICENSE.txt"] authors = [ { name = "Davis Bennett", email = "davis.v.bennett@gmail.com" }, ] classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Intended Audience :: Information Technology", "Intended Audience :: Science/Research", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Topic :: Scientific/Engineering", "Topic :: Software Development :: Libraries :: Python Modules", "Typing :: Typed", ] keywords = ["zarr"] dependencies = [ # The bounds a TypedDict's members carry -- `Gt`, `Ge`, `Lt`, `Le`, # `Interval` -- which the checker enforces, in the vocabulary pydantic # and Hypothesis read as well. "annotated-types>=0.6", # >=4.16: first release where `Sentinel` pickles by reference # (`__reduce__` returns the sentinel's name), so `UNSET` — and any model # holding it — can cross process boundaries and be deep-copied with its # singleton identity intact. 4.15 and earlier refuse to pickle sentinels. "typing_extensions>=4.16", ] [project.urls] Homepage = "https://github.com/zarr-developers/zarr-python" Source = "https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata" Issues = "https://github.com/zarr-developers/zarr-python/issues" Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-metadata/CHANGELOG.md" Documentation = "https://zarr-metadata.readthedocs.io/" [dependency-groups] test = [ "pytest", "pydantic>=2.13", "jsonschema", "hypothesis", "pytest-examples>=0.0.18", ] docs = [ # Pins match the zarr-python docs environment in the repo-root # pyproject.toml so every site renders with the same toolchain, and the # shared packages/mkdocs-base.yml sees one set of plugin versions. # Bump them together with the root. "mkdocs-material==9.7.7", "mkdocs==1.6.1", "mkdocstrings==1.0.6", # pymdown-extensions 12.3 changed the Highlight extension API; mkdocstrings 1.0.6 # cannot drive it (Highlight.__init__() missing argument "md"). Match the # version in the repo-root uv.lock until mkdocstrings is bumped everywhere. "pymdown-extensions==11.0.1", "mkdocstrings-python==2.0.8", "griffe-inherited-docstrings==1.1.3", # mkdocstrings uses ruff to format rendered signatures "ruff==0.16.8", ] [tool.hatch.version] source = "vcs" tag-pattern = '^zarr_metadata-v(?P.+)$' # `git_describe_command` ensures we get the zarr_metadata tags instead of latest. # `local_scheme` strips the git commit info so the appending info is just a counter from latest tag. # test-pypi doesn't accept git commit info in tags, and the count should be enough to distinguish unique runs. raw-options = { root = "../..", scm = { git = { describe_command = "git describe --dirty --tags --long --match zarr_metadata-v*" } }, local_scheme = "no-local-version" } [tool.hatch.build.targets.wheel] packages = ["src/zarr_metadata"] # An allowlist, so nothing that merely happens to sit in the package directory # — a scratch script, a stray notebook — can ride along in a release. The list # keeps an sdist self-testing and self-documenting: every fixture this suite # reads is a JSON file sitting next to the test module that loads it, so # `/tests` is the whole test dependency, and `/docs` plus `/mkdocs.yml` are a # self-contained site (mkdocstrings reads `src`; the shared config and header # script are copied in below) so `just docs-check` runs from an unpacked sdist too. `changes/` # and `.readthedocs.yaml` are deliberately absent: towncrier fragments are # repo bookkeeping, and the RTD config addresses paths from the repo root. # `pyproject.toml`, `README.md` and `LICENSE.txt` are added by hatchling itself. [tool.hatch.build.targets.sdist] include = [ "/src", "/tests", "/docs", "/mkdocs.yml", "/justfile", "/CHANGELOG.md", ] # The shared docs config, header script and docs hook are symlinks out of the # package directory in the repository; ship the files they point at, since a # link out of the sdist dangles and pip refuses to unpack it. exclude = ["/mkdocs-base.yml", "/docs/_static/source-version.js", "/mkdocs_hooks.py"] [tool.hatch.build.targets.sdist.force-include] "../mkdocs-base.yml" = "mkdocs-base.yml" "../../mkdocs_hooks.py" = "mkdocs_hooks.py" "../source-version.js" = "docs/_static/source-version.js" [tool.ruff] extend = "../../pyproject.toml" target-version = "py311" # zarr-python's docs hook, symlinked here for this package's site; the # repository root's settings lint it, without this package's stricter rules. extend-exclude = ["mkdocs_hooks.py"] [tool.ruff.lint] # `Any` defeats the point of a package whose product is precise types, so it # is banned in annotations here rather than merely discouraged. Use `object` # for "any value" (the caller must narrow) and the document TypedDicts where # the shape is known; a genuinely dynamic annotation needs an explicit noqa # saying why. extend-select = ["ANN401"] [tool.pytest.ini_options] minversion = "7" testpaths = ["tests"] xfail_strict = true addopts = ["-ra", "--strict-config", "--strict-markers"] filterwarnings = [ "error", # Pydantic validates these public immutable-shape TypedDicts correctly but # cannot enforce the type checker's ReadOnly mutation restriction. "ignore:Items? .* using the `ReadOnly` qualifier.*:UserWarning:pydantic._internal._generate_schema", ] [tool.numpydoc_validation] checks = [ "GL10", "SS04", "PR02", "PR03", "PR05", "PR06", ] # Pyright runs unpinned, from the justfile, which is what CI runs. [tool.pyright] include = ["src", "tests"] # Pyright is the only checker that runs on this package (the root mypy # config covers the root package alone), and it honours `# type: ignore` # as a blanket suppression of every diagnostic on the line. So that # spelling is switched off: a suppression is `# pyright: ignore[rule]`, # names the rule, and is an error once it stops suppressing anything. enableTypeIgnoreComments = false reportUnnecessaryTypeIgnoreComment = "error" # `tests` is a package imported as `tests.*`, which pytest resolves from the # rootdir; pyright needs the same root on its search path. extraPaths = ["."] enableExperimentalFeatures = true typeCheckingMode = "strict" pythonVersion = "3.11" # Tests are checked, but not for the inference strictness the sources are held # to: test bodies are full of untyped JSON literals whose inferred types are of # no interest, and strict mode reports about ninety of those. What is kept is # everything a rename or a signature change breaks -- a dangling import, a call # that no longer type-checks, an argument of the wrong type -- which is the # reason for checking tests at all. Tests also reach into private modules and # define functions purely to be refused at registration, both deliberately. [[tool.pyright.executionEnvironments]] root = "tests" reportMissingTypeArgument = false reportPrivateUsage = false reportUnknownArgumentType = false reportUnknownLambdaType = false reportUnknownMemberType = false reportUnknownParameterType = false reportUnknownVariableType = false reportUnusedFunction = false [tool.towncrier] # Fragments for this package live alongside the package source, separate # from the parent zarr-python `changes/` directory, so a PR touching only # `packages/zarr-metadata/` produces a release note for this package only. directory = "changes" filename = "CHANGELOG.md" package = "zarr_metadata" underlines = ["", "", ""] title_format = "## {version} ({project_date})" issue_format = "[#{issue}](https://github.com/zarr-developers/zarr-python/pull/{issue})" start_string = "\n" # Indents entries by four spaces, so the docs render multi-paragraph entries # as one list item. It lives outside `changes/`, which holds only fragments, # and its name ends in `.md`, which is how towncrier knows it is Markdown. template = "towncrier-template.md" # Declaring any type replaces towncrier's built-in set, so all five the # `changes/README.md` menu offers are restated here. They are the defaults # verbatim except for `misc`, whose `showcontent` towncrier defaults to false: # a `misc` entry would render as a bare PR link, which tells a reader nothing. # A change worth a release note is worth a sentence, whatever its category. [[tool.towncrier.type]] directory = "feature" name = "Features" showcontent = true [[tool.towncrier.type]] directory = "bugfix" name = "Bugfixes" showcontent = true [[tool.towncrier.type]] directory = "doc" name = "Improved Documentation" showcontent = true [[tool.towncrier.type]] directory = "removal" name = "Deprecations and Removals" showcontent = true [[tool.towncrier.type]] directory = "misc" name = "Misc" showcontent = true [tool.ruff.lint.flake8-type-checking] # A TypedDict's annotations are evaluated at run time: the checker reads them. # A definition's annotations are too: `Definition` finds its function members # with `get_type_hints`. runtime-evaluated-base-classes = [ "typing_extensions.TypedDict", "typing.TypedDict", "zarr_metadata.definition._definition.Definition", "zarr_metadata.definition._definition.WithFillValue", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/000077500000000000000000000000001527014137600245665ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/000077500000000000000000000000001527014137600274045ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/__init__.py000066400000000000000000000275741527014137600315340ustar00rootroot00000000000000"""Types, models and readers for Zarr v2 and v3 metadata documents. This namespace is a convenience: it re-exports the names most code needs, with Zarr v3 first. Each name also has one home module, where its documentation lives: - `zarr_metadata.v3` and `zarr_metadata.v2`: the JSON types of each document, extension and data type; - `zarr_metadata.model`: the document models and their readers; - `zarr_metadata.definition`: definitions, scopes and the field reader that both formats share; - `zarr_metadata.problems`: the problems every reader reports; - `zarr_metadata.typed_json`: `check`, the type checker for JSON; - `zarr_metadata.json_schema`: JSON Schema output; - `zarr_metadata.pydantic`: the optional pydantic field types. """ from importlib.metadata import version from zarr_metadata.model import ( UNSET, ZarrV2ArrayMetadata, ZarrV2ArrayMetadataUpdate, ZarrV2ConsolidatedMetadata, ZarrV2GroupMetadata, ZarrV2GroupMetadataUpdate, ZarrV3ArrayMetadata, ZarrV3ArrayMetadataUpdate, ZarrV3ConsolidatedMetadata, ZarrV3ConsolidatedMetadataInput, ZarrV3GroupMetadata, ZarrV3GroupMetadataUpdate, ZarrV3NodeMetadataInput, ) from zarr_metadata.problems import MetadataValidationError, ProblemKind, ValidationProblem from zarr_metadata.typed_json import JSONValue from zarr_metadata.v2.array import ( ZARR_V2_ARRAY_DIMENSION_SEPARATOR, ZARR_V2_ARRAY_METADATA_STORE_KEY, ZARR_V2_ARRAY_ORDER, ZarrV2ArrayDimensionSeparator, ZarrV2ArrayMetadataJSON, ZarrV2ArrayMetadataStoreKey, ZarrV2ArrayOrder, ZarrV2ZArrayJSON, ) from zarr_metadata.v2.attributes import ( ZARR_V2_ATTRIBUTES_STORE_KEY, ZarrV2AttributesStoreKey, ZarrV2ZAttrsJSON, ) from zarr_metadata.v2.codec import ZarrV2CodecMetadata from zarr_metadata.v2.consolidated import ( ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY, ZarrV2ConsolidatedMetadataJSON, ZarrV2ConsolidatedMetadataStoreKey, ) from zarr_metadata.v2.data_type import ZarrV2DataTypeMetadata from zarr_metadata.v2.group import ( ZARR_V2_GROUP_METADATA_STORE_KEY, ZarrV2GroupMetadataJSON, ZarrV2GroupMetadataStoreKey, ZarrV2ZGroupJSON, ) from zarr_metadata.v3 import ZarrV3ExtensionField, ZarrV3MetadataFieldJSON, ZarrV3NamedConfigJSON from zarr_metadata.v3.array import ( ZARR_V3_ARRAY_METADATA_STORE_KEY, ZarrV3ArrayMetadataJSON, ZarrV3ArrayMetadataStoreKey, ) from zarr_metadata.v3.chunk_grid.rectilinear import ( RECTILINEAR_CHUNK_GRID_NAME, RectilinearChunkGridMetadata, RectilinearChunkGridName, ) from zarr_metadata.v3.chunk_grid.regular import ( REGULAR_CHUNK_GRID_NAME, RegularChunkGridMetadata, RegularChunkGridName, ) from zarr_metadata.v3.chunk_key_encoding.default import ( DEFAULT_CHUNK_KEY_ENCODING_NAME, DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR, DefaultChunkKeyEncodingMetadata, DefaultChunkKeyEncodingName, DefaultChunkKeyEncodingSeparator, ) from zarr_metadata.v3.chunk_key_encoding.v2 import ( V2_CHUNK_KEY_ENCODING_NAME, V2_CHUNK_KEY_ENCODING_SEPARATOR, V2ChunkKeyEncodingMetadata, V2ChunkKeyEncodingName, V2ChunkKeyEncodingSeparator, ) from zarr_metadata.v3.codec.blosc import ( BLOSC_CNAME, BLOSC_CODEC_NAME, BLOSC_SHUFFLE, BloscCName, BloscCodecMetadata, BloscCodecName, BloscShuffle, ) from zarr_metadata.v3.codec.bytes import ( BYTES_CODEC_NAME, ENDIANNESS, BytesCodecMetadata, BytesCodecName, Endianness, ) from zarr_metadata.v3.codec.cast_value import ( CAST_OUT_OF_RANGE_MODE, CAST_ROUNDING_MODE, CAST_VALUE_CODEC_NAME, CastOutOfRangeMode, CastRoundingMode, CastValueCodecMetadata, CastValueCodecName, ) from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC_NAME, Crc32cCodecMetadata, Crc32cCodecName from zarr_metadata.v3.codec.gzip import GZIP_CODEC_NAME, GzipCodecMetadata, GzipCodecName from zarr_metadata.v3.codec.scale_offset import ( SCALE_OFFSET_CODEC_NAME, ScaleOffsetCodecMetadata, ScaleOffsetCodecName, ) from zarr_metadata.v3.codec.sharding_indexed import ( SHARDING_INDEX_LOCATION, SHARDING_INDEXED_CODEC_NAME, ShardingIndexedCodecMetadata, ShardingIndexedCodecName, ShardingIndexLocation, ) from zarr_metadata.v3.codec.transpose import ( TRANSPOSE_CODEC_NAME, TransposeCodecMetadata, TransposeCodecName, ) from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME, ZstdCodecMetadata, ZstdCodecName from zarr_metadata.v3.consolidated import ( ZARR_V3_CONSOLIDATED_METADATA_KEY, ZarrV3ConsolidatedMetadataEnvelopeJSON, ZarrV3ConsolidatedMetadataJSON, ) from zarr_metadata.v3.data_type.bool import ( BOOL_DATA_TYPE_NAME, BoolDataTypeName, BoolFillValue, ) from zarr_metadata.v3.data_type.bytes import ( BYTES_DATA_TYPE_NAME, BytesDataTypeName, BytesFillValue, ) from zarr_metadata.v3.data_type.complex64 import ( COMPLEX64_DATA_TYPE_NAME, Complex64DataTypeName, Complex64FillValue, ) from zarr_metadata.v3.data_type.complex128 import ( COMPLEX128_DATA_TYPE_NAME, Complex128DataTypeName, Complex128FillValue, ) from zarr_metadata.v3.data_type.float16 import ( FLOAT16_DATA_TYPE_NAME, Float16DataTypeName, Float16FillValue, ) from zarr_metadata.v3.data_type.float32 import ( FLOAT32_DATA_TYPE_NAME, Float32DataTypeName, Float32FillValue, ) from zarr_metadata.v3.data_type.float64 import ( FLOAT64_DATA_TYPE_NAME, Float64DataTypeName, Float64FillValue, ) from zarr_metadata.v3.data_type.int8 import ( INT8_DATA_TYPE_NAME, Int8DataTypeName, Int8FillValue, ) from zarr_metadata.v3.data_type.int16 import ( INT16_DATA_TYPE_NAME, Int16DataTypeName, Int16FillValue, ) from zarr_metadata.v3.data_type.int32 import ( INT32_DATA_TYPE_NAME, Int32DataTypeName, Int32FillValue, ) from zarr_metadata.v3.data_type.int64 import ( INT64_DATA_TYPE_NAME, Int64DataTypeName, Int64FillValue, ) from zarr_metadata.v3.data_type.numpy_datetime64 import ( NUMPY_DATETIME64_DATA_TYPE_NAME, NUMPY_TIME_UNIT, NumpyDatetime64DataTypeName, NumpyDatetime64FillValue, NumpyTimeUnit, ) from zarr_metadata.v3.data_type.numpy_timedelta64 import ( NUMPY_TIMEDELTA64_DATA_TYPE_NAME, NumpyTimedelta64DataTypeName, NumpyTimedelta64FillValue, ) from zarr_metadata.v3.data_type.raw import RawBytesDataTypeName, RawBytesFillValue from zarr_metadata.v3.data_type.string import ( STRING_DATA_TYPE_NAME, StringDataTypeName, StringFillValue, ) from zarr_metadata.v3.data_type.struct import ( STRUCT_DATA_TYPE_NAME, StructDataTypeName, StructFillValue, ) from zarr_metadata.v3.data_type.uint8 import ( UINT8_DATA_TYPE_NAME, Uint8DataTypeName, Uint8FillValue, ) from zarr_metadata.v3.data_type.uint16 import ( UINT16_DATA_TYPE_NAME, Uint16DataTypeName, Uint16FillValue, ) from zarr_metadata.v3.data_type.uint32 import ( UINT32_DATA_TYPE_NAME, Uint32DataTypeName, Uint32FillValue, ) from zarr_metadata.v3.data_type.uint64 import ( UINT64_DATA_TYPE_NAME, Uint64DataTypeName, Uint64FillValue, ) from zarr_metadata.v3.group import ( ZARR_V3_GROUP_METADATA_STORE_KEY, ZarrV3GroupMetadataJSON, ZarrV3GroupMetadataStoreKey, ) __version__ = version("zarr-metadata") __all__ = [ "BLOSC_CNAME", "BLOSC_CODEC_NAME", "BLOSC_SHUFFLE", "BOOL_DATA_TYPE_NAME", "BYTES_CODEC_NAME", "BYTES_DATA_TYPE_NAME", "CAST_OUT_OF_RANGE_MODE", "CAST_ROUNDING_MODE", "CAST_VALUE_CODEC_NAME", "COMPLEX64_DATA_TYPE_NAME", "COMPLEX128_DATA_TYPE_NAME", "CRC32C_CODEC_NAME", "DEFAULT_CHUNK_KEY_ENCODING_NAME", "DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR", "ENDIANNESS", "FLOAT16_DATA_TYPE_NAME", "FLOAT32_DATA_TYPE_NAME", "FLOAT64_DATA_TYPE_NAME", "GZIP_CODEC_NAME", "INT8_DATA_TYPE_NAME", "INT16_DATA_TYPE_NAME", "INT32_DATA_TYPE_NAME", "INT64_DATA_TYPE_NAME", "NUMPY_DATETIME64_DATA_TYPE_NAME", "NUMPY_TIMEDELTA64_DATA_TYPE_NAME", "NUMPY_TIME_UNIT", "RECTILINEAR_CHUNK_GRID_NAME", "REGULAR_CHUNK_GRID_NAME", "SCALE_OFFSET_CODEC_NAME", "SHARDING_INDEXED_CODEC_NAME", "SHARDING_INDEX_LOCATION", "STRING_DATA_TYPE_NAME", "STRUCT_DATA_TYPE_NAME", "TRANSPOSE_CODEC_NAME", "UINT8_DATA_TYPE_NAME", "UINT16_DATA_TYPE_NAME", "UINT32_DATA_TYPE_NAME", "UINT64_DATA_TYPE_NAME", "UNSET", "V2_CHUNK_KEY_ENCODING_NAME", "V2_CHUNK_KEY_ENCODING_SEPARATOR", "ZARR_V2_ARRAY_DIMENSION_SEPARATOR", "ZARR_V2_ARRAY_METADATA_STORE_KEY", "ZARR_V2_ARRAY_ORDER", "ZARR_V2_ATTRIBUTES_STORE_KEY", "ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY", "ZARR_V2_GROUP_METADATA_STORE_KEY", "ZARR_V3_ARRAY_METADATA_STORE_KEY", "ZARR_V3_CONSOLIDATED_METADATA_KEY", "ZARR_V3_GROUP_METADATA_STORE_KEY", "ZSTD_CODEC_NAME", "BloscCName", "BloscCodecMetadata", "BloscCodecName", "BloscShuffle", "BoolDataTypeName", "BoolFillValue", "BytesCodecMetadata", "BytesCodecName", "BytesDataTypeName", "BytesFillValue", "CastOutOfRangeMode", "CastRoundingMode", "CastValueCodecMetadata", "CastValueCodecName", "Complex64DataTypeName", "Complex64FillValue", "Complex128DataTypeName", "Complex128FillValue", "Crc32cCodecMetadata", "Crc32cCodecName", "DefaultChunkKeyEncodingMetadata", "DefaultChunkKeyEncodingName", "DefaultChunkKeyEncodingSeparator", "Endianness", "Float16DataTypeName", "Float16FillValue", "Float32DataTypeName", "Float32FillValue", "Float64DataTypeName", "Float64FillValue", "GzipCodecMetadata", "GzipCodecName", "Int8DataTypeName", "Int8FillValue", "Int16DataTypeName", "Int16FillValue", "Int32DataTypeName", "Int32FillValue", "Int64DataTypeName", "Int64FillValue", "JSONValue", "MetadataValidationError", "NumpyDatetime64DataTypeName", "NumpyDatetime64FillValue", "NumpyTimeUnit", "NumpyTimedelta64DataTypeName", "NumpyTimedelta64FillValue", "ProblemKind", "RawBytesDataTypeName", "RawBytesFillValue", "RectilinearChunkGridMetadata", "RectilinearChunkGridName", "RegularChunkGridMetadata", "RegularChunkGridName", "ScaleOffsetCodecMetadata", "ScaleOffsetCodecName", "ShardingIndexLocation", "ShardingIndexedCodecMetadata", "ShardingIndexedCodecName", "StringDataTypeName", "StringFillValue", "StructDataTypeName", "StructFillValue", "TransposeCodecMetadata", "TransposeCodecName", "Uint8DataTypeName", "Uint8FillValue", "Uint16DataTypeName", "Uint16FillValue", "Uint32DataTypeName", "Uint32FillValue", "Uint64DataTypeName", "Uint64FillValue", "V2ChunkKeyEncodingMetadata", "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingSeparator", "ValidationProblem", "ZarrV2ArrayDimensionSeparator", "ZarrV2ArrayMetadata", "ZarrV2ArrayMetadataJSON", "ZarrV2ArrayMetadataStoreKey", "ZarrV2ArrayMetadataUpdate", "ZarrV2ArrayOrder", "ZarrV2AttributesStoreKey", "ZarrV2CodecMetadata", "ZarrV2ConsolidatedMetadata", "ZarrV2ConsolidatedMetadataJSON", "ZarrV2ConsolidatedMetadataStoreKey", "ZarrV2DataTypeMetadata", "ZarrV2GroupMetadata", "ZarrV2GroupMetadataJSON", "ZarrV2GroupMetadataStoreKey", "ZarrV2GroupMetadataUpdate", "ZarrV2ZArrayJSON", "ZarrV2ZAttrsJSON", "ZarrV2ZGroupJSON", "ZarrV3ArrayMetadata", "ZarrV3ArrayMetadataJSON", "ZarrV3ArrayMetadataStoreKey", "ZarrV3ArrayMetadataUpdate", "ZarrV3ConsolidatedMetadata", "ZarrV3ConsolidatedMetadataEnvelopeJSON", "ZarrV3ConsolidatedMetadataInput", "ZarrV3ConsolidatedMetadataJSON", "ZarrV3ExtensionField", "ZarrV3GroupMetadata", "ZarrV3GroupMetadataJSON", "ZarrV3GroupMetadataStoreKey", "ZarrV3GroupMetadataUpdate", "ZarrV3MetadataFieldJSON", "ZarrV3NamedConfigJSON", "ZarrV3NodeMetadataInput", "ZstdCodecMetadata", "ZstdCodecName", "__version__", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/_common.py000066400000000000000000000061611527014137600314110ustar00rootroot00000000000000""" JSON types that the shared reading code needs. `JSONValue` is the type of any JSON value. `ZarrV3NamedConfigJSON` is the `{name, configuration, must_understand}` envelope of a v3 field. It is v3's, but it lives here because the shared field reader in `zarr_metadata.definition` uses the envelope as its default. Public homes: `zarr_metadata.typed_json` for `JSONValue`, `zarr_metadata.v3` for the envelope. """ from collections.abc import Mapping from typing import NotRequired from typing_extensions import ReadOnly, TypeAliasType, TypedDict JSONValue = TypeAliasType( "JSONValue", int | float | bool | str | list["JSONValue"] | tuple["JSONValue", ...] | Mapping[str, "JSONValue"] | None, ) """A recursive type alias for JSON-encodable values. Defined via `TypeAliasType` (rather than a plain `TypeAlias`) so the self-reference is a named recursion point that pydantic can resolve when building a `TypeAdapter`; a bare recursive `TypeAlias` raises `PydanticUserError`/`RecursionError` at validation time. """ UserJSONValue = TypeAliasType("UserJSONValue", JSONValue) """User data in a document, such as a node's attributes: a `JSONValue` in which a number may also be `NaN`, `Infinity` or `-Infinity`. The spec only requires attributes to be JSON values, and zarr-python writes these numbers as bare tokens, so a reader accepts them where a document holds user data. A configuration member annotated with this alias is checked that way; a member annotated `JSONValue` must hold finite numbers, as RFC 8259 requires. """ class ZarrV3NamedConfigJSON(TypedDict, closed=True): """ Externally-tagged union member for a metadata field. The optional `configuration` mapping holds arbitrary JSON-encodable values. `must_understand` is implicitly true when absent. `name` and `configuration` are `ReadOnly` (PEP 705) so that concrete entity types — `BloscCodecObject`, `RegularChunkGridObject`, and the rest — are assignable to this type, and therefore to `ZarrV3MetadataFieldJSON`. Without `ReadOnly` both items are invariant, so a concrete `name: Literal["blosc"]` does not satisfy `name: str`, and a required `configuration` does not satisfy a `NotRequired` one. That made the package's own codec types unusable in the very fields they describe (`codecs`, `data_type`, `chunk_grid`, ...), and made `TypeIs`-based codec classification impossible to declare, since `TypeIs` requires the narrowed type to be assignable to the input type. `must_understand` stays writable: nothing needs to narrow it, and keeping it mutable lets writers set it on an already-constructed field. The type is `closed` (PEP 728): the spec's named-configuration envelope has exactly these three members, and closing it is also what makes this type — and every concrete entity type embedding it, e.g. the `sharding_indexed` configuration's inner `codecs` list — assignable to `Mapping[str, JSONValue]` (i.e. usable as a `JSONValue`). """ name: ReadOnly[str] configuration: NotRequired[ReadOnly[Mapping[str, JSONValue]]] must_understand: NotRequired[bool] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/_hierarchy.py000066400000000000000000000131771527014137600321040ustar00rootroot00000000000000"""The tree that the nodes of a Zarr hierarchy form, in either format. "A Zarr hierarchy is a tree structure, where each node in the tree is either a group or an array. Group nodes may have children but array nodes may not" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L177-L181). Zarr v2 hierarchies have the same structure. The rules for node names and paths differ between the formats, so each format checks its paths before it checks the tree. """ from __future__ import annotations from typing import TYPE_CHECKING, Literal from zarr_metadata._json import ValidationProblem, shown if TYPE_CHECKING: from collections.abc import Mapping NodeType = Literal["array", "group"] """The two kinds of node in a hierarchy.""" def tree_problems(nodes: Mapping[str, NodeType | None]) -> tuple[ValidationProblem, ...]: """The reasons `nodes` (the node type of each node, keyed by a valid node path) do not form a tree. Reports one problem per affected node, at that node's path. Each node except the root must have its parent in `nodes`, and that parent must be a group. A node below an array is an `invalid_value` at the node's path. If groups are missing above a node, the nearest missing group is reported once as a `missing_key` at that group's path. The message says how many more groups are missing above it, including the root. A node whose type is unknown (None) is treated as a group. The paths are not checked; each format checks its own path rules first. """ problems: list[ValidationProblem] = [] tree = _Tree() for path, node_type in nodes.items(): tree.hold(path, node_type) reported: set[str] = set() for path in nodes: if path == "/": continue holder, missing = tree.above(path) if holder == "array": # No node may be below an array, no matter how many groups are # missing between them. message = ( f"expected a node below a group, got {shown(path)}, below the array " f"{shown(tree.holder_of(path))}" ) problems.append(ValidationProblem((path,), message, "invalid_value")) elif missing != 0: nearest = path.rpartition("/")[0] or "/" if nearest not in reported: reported.add(nearest) above = missing - 1 more = ( "" if above == 0 else f", and {above} group{'s' if above != 1 else ''} above it" ) message = f"missing the group holding {shown(path)}{more}" problems.append(ValidationProblem((nearest,), message, "missing_key")) return tuple(problems) class _Branch: """A name in the tree of present paths: whether a node is present at it, the node's type, and the names below it.""" __slots__ = ("children", "held", "node_type") def __init__(self) -> None: self.held = False self.node_type: NodeType | None = None self.children: dict[str, _Branch] = {} class _Tree: """The present paths as a tree of their names, so each path is walked once, in time proportional to its length. Walking up a path by building each ancestor's string costs the path's length for each ancestor. A hostile key of a million characters would make that quadratic. Here a path is split once and walked name by name. """ __slots__ = ("root",) def __init__(self) -> None: self.root = _Branch() def hold(self, path: str, node_type: NodeType | None) -> None: """Mark the node at `path` (a node path) as present, with type `node_type`.""" branch = self.root for name in _names(path): branch = branch.children.setdefault(name, _Branch()) branch.held = True branch.node_type = node_type def above(self, path: str) -> tuple[NodeType | None, int]: """For the node at `path` (a node path below the root), return the type of the nearest present node above it and the number of groups missing between them. The type is `"group"` for a node of unknown type, and None if no node above is present.""" names = _names(path) branch: _Branch | None = self.root holder: NodeType | None = None held_depth = -1 for depth, name in enumerate(names[:-1], start=0): if branch is not None and branch.held: holder, held_depth = branch.node_type or "group", depth branch = None if branch is None else branch.children.get(name) # The parent, at the last depth walked, may also be present. if branch is not None and branch.held: holder, held_depth = branch.node_type or "group", len(names) - 1 return holder, len(names) - 1 - held_depth def holder_of(self, path: str) -> str: """The path of the nearest present node above the node at `path`. `path` must be a node path below the root, with a present node above it.""" names = _names(path) branch: _Branch | None = self.root holder = "/" for depth, name in enumerate(names[:-1]): if branch is not None and branch.held: holder = "/" + "/".join(names[:depth]) if depth != 0 else "/" branch = None if branch is None else branch.children.get(name) if branch is not None and branch.held: holder = "/" + "/".join(names[:-1]) return holder def _names(path: str) -> list[str]: """The names between a node path's `/` characters. Empty for the root's path.""" return [] if path == "/" else path[1:].split("/") __all__ = ["NodeType", "tree_problems"] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/_json.py000066400000000000000000001045221527014137600310720ustar00rootroot00000000000000"""JSON values and the problems found in them. This is the lowest layer of the reader. `refine_json` is the single walk that both normalizes and checks a value. `refine_user_data` is the same walk over user data, where a non-finite number is kept as a float. Every layer (the checker, the definitions and the model) reports with `ValidationProblem` and `MetadataValidationError`. """ from __future__ import annotations import dataclasses import json import math from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import Final, Literal, TypeAlias, TypeGuard, TypeVar, cast, get_args, overload from zarr_metadata._common import JSONValue from zarr_metadata._sentinel import UNSET ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json", "unknown_key"] """Machine-readable classification of a `ValidationProblem`. - `missing_key`: a required key (document key or store key) is absent. - `invalid_type`: a value has the wrong structural type (e.g. a string where a mapping is required, a non-JSON-serializable object). - `invalid_value`: a value has an acceptable type but an invalid content (e.g. `zarr_format: 2` in a v3 document, `order: "Q"`). - `invalid_json`: bytes that do not decode as JSON. - `unknown_key`: a key that an object's type does not declare, where the type is closed (as a closed TypedDict is). Zarr configurations rarely state whether they are closed (zarr-developers/zarr-specs#270 has been open since 2023), and many readers reject such a key. It has its own kind so that a caller who tolerates it can distinguish it from a wrong value, and so that it never hides other problems in the same object. """ PROBLEM_KINDS: Final[tuple[ProblemKind, ...]] = get_args(ProblemKind) """The valid problem kinds. `ValidationProblem` checks its `kind` against these.""" JSON_DEPTH: Final = 256 """The maximum number of nesting levels a reader walks. A value nested deeper is a problem at the first level past the limit. This means no document, however deep, makes a reader exceed the interpreter's recursion limit: the package refines every value before reading it, and refining stops at this depth. Every reader, writer and comparison uses one stack frame per level, and `copy.deepcopy` (and `pickle` before Python 3.12) uses two. A document at the limit therefore uses about half of the interpreter's default limit of a thousand frames, and the rest is left for the caller. """ _PAST_THE_LEVELS: Final = f"nested deeper than the {JSON_DEPTH} levels a reader walks" """The message for a container nested past `JSON_DEPTH`.""" class _Ctx(Mapping[str, JSONValue]): """The type of a problem's `ctx`: a private copy, with arrays as tuples, checked to be JSON when created and never modified after. It is a separate type so that a problem built from another problem's `ctx` (as `replace` builds one) can tell the value was already checked and skip the check. A mapping of any other type is checked and copied. """ __slots__ = ("_held",) def __init__(self, held: dict[str, JSONValue]) -> None: self._held = held def __getitem__(self, key: str) -> JSONValue: return self._held[key] def __iter__(self) -> Iterator[str]: return iter(self._held) def __len__(self) -> int: return len(self._held) def __repr__(self) -> str: return repr(self._held) def __reduce__(self) -> tuple[type[_Ctx], tuple[dict[str, JSONValue]]]: return _Ctx, (self._held,) _NO_CTX: Final[Mapping[str, JSONValue]] = _Ctx({}) def _no_ctx() -> Mapping[str, JSONValue]: """The empty `ctx`, for a problem that states no more than the expected type.""" return _NO_CTX @dataclass(frozen=True, slots=True) class ValidationProblem: """A single problem found in a value: its location, what is wrong, the kind of problem, and the data used in the message. `loc` is the path from the root of the checked value to the bad value, e.g. `("codecs", 0, "name")` in a document. An empty `loc` refers to the root. `kind` classifies the failure for programmatic dispatch; `message` is the human-readable description. `input` and `ctx` hold the data in the message, in the same way that pydantic's `ErrorDetails` and zod's issues do. `input` is the JSON found at `loc`, e.g. `12` for a gzip `level` of 12. It is `UNSET` when nothing is there (zod does the same for a missing key; pydantic gives the object that lacks the key), or when the value there is not JSON, in which case the message shows it. `input` is the object the caller passed in, not a copy, as in pydantic: if the caller later changes its document, `input` shows the change. `ctx` holds what was expected, when that is more than a type: - `gt`, `ge`, `lt` and `le`: the bounds from the value's type or from a rule, with the names pydantic uses. For example `{"ge": 0, "le": 9}` for a gzip `level`, whose type is `Annotated[int, Interval(ge=0, le=9)]`. - `expected`: the values of a closed set, like zod's `values`, in the order the message lists them. For example the values of a `Literal`, of `node_type`, or of `zarr_format`. Neither field takes part in equality or the repr: two problems are equal when they have the same location, message and kind. Every function that returns or raises problems fills `input` from the value its caller passed, so a rule only needs to give the location. """ loc: tuple[str | int, ...] message: str kind: ProblemKind input: JSONValue | UNSET = dataclasses.field( default=UNSET, kw_only=True, compare=False, repr=False ) ctx: Mapping[str, JSONValue] = dataclasses.field( default_factory=_no_ctx, kw_only=True, compare=False, repr=False ) def __post_init__(self) -> None: # Enforce the annotations at runtime. A rule written without a type # checker (an extension's rule may be) then fails where it builds # the problem, instead of reporting a problem at an invalid location. loc = cast("object", self.loc) if not isinstance(loc, tuple) or not all( isinstance(part, str) or (isinstance(part, int) and not isinstance(part, bool)) for part in cast("tuple[object, ...]", loc) ): msg = f"a ValidationProblem's loc is a tuple of keys and indices, got {loc!r}" raise TypeError(msg) message = cast("object", self.message) if not isinstance(message, str): msg = f"a ValidationProblem's message is a string, got {message!r}" raise TypeError(msg) kind = cast("object", self.kind) if not isinstance(kind, str) or kind not in PROBLEM_KINDS: msg = f"a ValidationProblem's kind is one of {PROBLEM_KINDS!r}, got {kind!r}" raise TypeError(msg) ctx = cast("object", self.ctx) if isinstance(ctx, _Ctx): # Another problem's `ctx`, already checked when that problem was # created. `replace` passes it to the copy. return if ( not isinstance(ctx, Mapping) or not all(isinstance(key, str) for key in cast("Mapping[object, object]", ctx)) or not is_json(dict(cast("Mapping[str, object]", ctx))) ): msg = f"a ValidationProblem's ctx is an object of JSON values, got {ctx!r}" raise TypeError(msg) # Store a view of a private deep copy, with arrays as tuples. Then # neither this view nor the caller's original mapping can modify a # raised error, which is a finished report. held = cast( "dict[str, JSONValue]", copied(cast("JSONValue", arrays_to_tuples(dict(cast("Mapping[str, object]", ctx))))), ) object.__setattr__(self, "ctx", _Ctx(held)) def __str__(self) -> str: location = ".".join(str(part) for part in self.loc) if self.loc else "" return f"{location}: {self.message}" def __reduce__( self, ) -> tuple[Callable[..., ValidationProblem], tuple[object, ...]]: # Pickle and copy by calling the constructor again. The view that # holds `ctx` does not pickle, but the dict it wraps does. return (_problem, (self.loc, self.message, self.kind, self.input, dict(self.ctx))) Problems: TypeAlias = tuple[ValidationProblem, ...] """Every problem a reader found, in the order it found them.""" def _problem( loc: tuple[str | int, ...], message: str, kind: ProblemKind, found: JSONValue | UNSET, ctx: Mapping[str, JSONValue], ) -> ValidationProblem: """Rebuild a problem from the values `ValidationProblem.__reduce__` returns.""" return ValidationProblem(loc, message, kind, input=found, ctx=ctx) class MetadataValidationError(ValueError): """Raised when a value fails validation, by the entry points that raise instead of returning problems. Carries every problem found (not just the first) in `.problems`, as an immutable tuple: a raised error is a finished report, and a caller inspecting it must not be able to edit the record. """ problems: tuple[ValidationProblem, ...] def __init__(self, problems: Sequence[ValidationProblem]) -> None: self.problems = tuple(problems) for entry in cast("tuple[object, ...]", self.problems): # Enforce the annotation at runtime. A caller that is not # type-checked and passes something else fails here, instead of # later, at the code that reads a `loc` from it. if not isinstance(entry, ValidationProblem): msg = ( "MetadataValidationError takes ValidationProblem values, " f"got {type(entry).__name__}" ) raise TypeError(msg) super().__init__("\n".join(str(problem) for problem in self.problems)) def __reduce__( self, ) -> tuple[ type[MetadataValidationError], tuple[tuple[ValidationProblem, ...]], dict[str, object] ]: # By default an exception pickles and copies by calling its class # with its `args`, which here hold the message. This one is rebuilt # from its problems instead, and the rest of its state (including # its notes) is restored after. return (type(self), (self.problems,), self.__dict__) def prefixed( loc_head: str | int, problems: Sequence[ValidationProblem] ) -> tuple[ValidationProblem, ...]: """Prepend `loc_head` to the `loc` of every problem (for nested validators).""" return tuple(dataclasses.replace(p, loc=(loc_head, *p.loc)) for p in problems) def value_at(value: object, loc: tuple[str | int, ...]) -> object: """The value inside `value` at `loc`, where each key names an object member and each index an array element; `UNSET` if nothing is there.""" for part in loc: if isinstance(part, str) and is_object(value): members = value if part not in members: return UNSET value = members[part] elif isinstance(part, int) and is_array(value) and 0 <= part < len(value): value = value[part] else: return UNSET return value def with_input( problems: Sequence[ValidationProblem], value: object, loc: tuple[str | int, ...] = () ) -> tuple[ValidationProblem, ...]: """Set each problem's `input` to the JSON in `value` at the problem's `loc`. `value` itself sits at `loc`. Every function that checks a value calls this on the problems it found. This way a rule only gives the location of a problem, and the problem carries the value found there. The function the caller called is the last to apply it, so a problem's `input` comes from the value the caller passed in, not from a copy made by an inner reader. If a problem's `loc` points to nothing in `value` (a missing key), or to something that is not JSON, the problem keeps its current `input`: `UNSET`, unless an inner reader found JSON there. This ensures no problem holds a value that might not pickle or copy. """ if len(problems) == 0: return () filled: list[ValidationProblem] = [] for found in problems: if found.loc[: len(loc)] == loc: there = value_at(value, found.loc[len(loc) :]) if there is not found.input and _holdable(there): found = dataclasses.replace(found, input=cast("JSONValue", there)) filled.append(found) return tuple(filled) def _holdable(value: object) -> bool: """Whether a problem can hold `value` as its input: JSON, nested no deeper than `JSON_DEPTH`. A value the validator did not walk (a member it only reports) may be nested deeper than that. Such a value is not stored, because `is_canonical_json` rejects it. """ return is_canonical_json(value, finite=False) def not_an_object(value: object) -> tuple[ValidationProblem, ...]: """The single problem, at the root, for a document that is not an object.""" return with_input((ValidationProblem((), "expected an object", "invalid_type"),), value) def validate_json(value: object, loc: tuple[str | int, ...] = ()) -> tuple[ValidationProblem, ...]: """Return every reason that `value`, located at `loc`, is not JSON, each at its own location: a non-finite float, a non-string key, a value of no JSON type, or nesting deeper than `JSON_DEPTH`, counted from the document root above `loc`.""" return with_input(refine_json(value, loc)[1], value, loc) def is_object(value: object) -> TypeGuard[Mapping[object, object]]: """Whether `value` is a JSON object as Python represents one: a mapping with any keys. The same test as `isinstance(value, Mapping)`, but it narrows the type to a mapping of `object`. The bare check leaves the key and value types unknown to a type checker. """ return type(value) is dict or isinstance(value, Mapping) def refined_object(value: object) -> dict[str, JSONValue]: """Refine `value`, an object that a read found no problems in, as user data. The result is the document a model holds. Raises `TypeError` if `value` is not an object. This cannot happen after a read that found no problems. """ refined, _ = refine_user_data(value) if not isinstance(refined, dict): msg = f"expected an object a read found nothing wrong with, got {shown(value)}" raise TypeError(msg) return refined def object_at(document: Mapping[str, JSONValue], key: str) -> dict[str, JSONValue]: """The object at `key` in the refined `document`, where a read found one; `TypeError` if the value is not an object, which that read rules out.""" value = document[key] if not isinstance(value, dict): msg = f"expected an object at {key!r}, got {shown(value)}" raise TypeError(msg) return value def is_json_object(value: object) -> TypeGuard[Mapping[str, object]]: """Whether `value` is a JSON object with string keys, as JSON requires: a mapping whose keys are all strings.""" return isinstance(value, Mapping) and all( isinstance(key, str) for key in cast("Mapping[object, object]", value) ) def is_list_or_tuple(value: object) -> TypeGuard[list[object] | tuple[object, ...]]: """Whether `value` is a list or a tuple: the two containers canonical JSON uses for an array.""" return isinstance(value, (list, tuple)) def is_tuple(value: object) -> TypeGuard[tuple[object, ...]]: """Whether `value` is a tuple, narrowed to a tuple of `object`.""" return isinstance(value, tuple) def is_array(value: object) -> TypeGuard[Sequence[object]]: """Whether `value` is a JSON array as Python represents one: a sequence that is not text or bytes.""" return isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)) def refine_json( value: object, loc: tuple[str | int, ...] = () ) -> tuple[JSONValue | None, tuple[ValidationProblem, ...]]: """`value` as JSON with arrays as tuples, or None with every reason it is not JSON. A single walk both normalizes and checks: a mapping becomes a `dict` with string keys, a sequence becomes a tuple, and a float must be finite. For a value that is not JSON, the result is None, with the problems located at the leaves that are not JSON. JSON's `null` is also None, but with no problems, so `value` is JSON exactly when there are no problems. """ return _refine(value, loc, finite=True) def refine_user_data( value: object, loc: tuple[str | int, ...] = () ) -> tuple[JSONValue | None, tuple[ValidationProblem, ...]]: """Refine user data the same way `refine_json` refines JSON, but keep a non-finite number as a float. A node's attributes are user data. The spec only requires each to be a JSON value, and does not interpret any of them. zarr-python writes them with the defaults of Python's `json` module, so an attribute can hold `NaN`, `Infinity` or `-Infinity` (for example xarray's `_FillValue` or a CF `missing_value`), which RFC 8259 does not allow. Wherever the spec interprets a value, it writes those numbers as strings. So a validator uses `refine_json` for values the spec interprets, and this function only for user data. """ return _refine(value, loc, finite=False) _Refined = tuple[JSONValue | None, tuple[ValidationProblem, ...]] def nested_past_the_levels(value: object, loc: tuple[str | int, ...]) -> ValidationProblem | None: """The problem for `value` if it is a container (an object or an array) at `loc` deeper than `JSON_DEPTH`; None for a scalar, or for a container within the limit. `_refine` applies this rule to each container it reaches, through `_past_the_levels`, which handles a value known to be a container. A reader also calls this for each container it walks without refining (a document's members, or the consolidated metadata it descends into). This bounds a chain of documents in the same way as any other nesting, and checks every container at its location, with the same result as `refine_json` of the whole document. """ if len(loc) < JSON_DEPTH or isinstance(value, (str, int, float, bool)) or value is None: return None if not isinstance(value, (Mapping, Sequence)) or isinstance(value, (bytes, bytearray)): return None return _past_the_levels(loc) def _past_the_levels(loc: tuple[str | int, ...]) -> ValidationProblem: """The problem for a container at `loc` deeper than `JSON_DEPTH`: what `nested_past_the_levels` returns for a value known to be a container.""" return ValidationProblem(loc, _PAST_THE_LEVELS, "invalid_value") def within( problems: Sequence[ValidationProblem], at: tuple[str | int, ...] ) -> tuple[ValidationProblem, ...]: """Make the locations of `problems`, found in a value at `at`, relative to that value. This is the reverse of `prefixed`. It is for a reader that counts nesting levels from the document it was given, but reports locations within the document it reads. A problem that is not below `at` raises `TypeError`. Otherwise a reader that located a problem from the wrong root would report it in the wrong place. """ if len(problems) == 0: return () for problem in problems: if problem.loc[: len(at)] != at: msg = f"a problem at {problem.loc!r} does not sit below {at!r}" raise TypeError(msg) return tuple(dataclasses.replace(p, loc=p.loc[len(at) :]) for p in problems) def _refine(value: object, loc: tuple[str | int, ...], *, finite: bool) -> _Refined: """`refine_json`, but a non-finite number counts as JSON unless `finite` is true.""" # Check the exact types `json.loads` builds first, by type alone. Nearly # every value is one of these, so only the remaining values reach the # subclass checks below (a `StrEnum` member, an `IntEnum` member, a # mapping proxy). if type(value) is str or type(value) is bool or value is None: return value, () if type(value) is int: if _writable(value): return value, () return None, (ValidationProblem(loc, _too_many_digits(value), "invalid_value"),) if type(value) is float: if not finite or math.isfinite(value): return value, () return None, (_not_finite(value, loc),) cls = type(value) if cls is dict: mapping = True elif cls is list or cls is tuple: mapping = False else: if isinstance(value, float): if not finite or math.isfinite(value): return float(value), () return None, (_not_finite(value, loc),) if isinstance(value, int) and not isinstance(value, bool) and not _writable(value): return None, (ValidationProblem(loc, _too_many_digits(value), "invalid_value"),) if isinstance(value, bool): return value, () if isinstance(value, str): # A subclass (such as a `StrEnum` member) becomes the string JSON writes for it. return str.__str__(value), () if isinstance(value, int): return int(value), () mapping = is_object(value) if not mapping and not is_array(value): return None, ( ValidationProblem( loc, f"not a JSON-serializable value: {shown_by_python(value)}", "invalid_type", ), ) # The value is a container. Check its depth at its location with the # rule from `nested_past_the_levels`; only containers reach this point. if len(loc) >= JSON_DEPTH: return None, (_past_the_levels(loc),) if mapping: # Walk the members here rather than in a separate function, so that # each level of nesting costs one frame, `JSON_DEPTH` at most. members: dict[str, JSONValue] = {} found_in_members: list[ValidationProblem] = [] for key, item in cast("Mapping[object, object]", value).items(): if not isinstance(key, str): found_in_members.append( ValidationProblem( loc, f"non-string key {shown_key(key)} in JSON object", "invalid_type" ) ) continue member, found = _refine(item, (*loc, key), finite=finite) found_in_members.extend(found) if len(found) == 0: members[key] = member return (members if len(found_in_members) == 0 else None), tuple(found_in_members) entries: list[JSONValue] = [] found_in_entries: list[ValidationProblem] = [] for index, item in enumerate(cast("Sequence[object]", value)): entry, found = _refine(item, (*loc, index), finite=finite) found_in_entries.extend(found) if len(found) == 0: entries.append(entry) return (tuple(entries) if len(found_in_entries) == 0 else None), tuple(found_in_entries) def _not_finite(value: float, loc: tuple[str | int, ...]) -> ValidationProblem: """The problem for a non-finite number at `loc`, since JSON has no such numbers.""" return ValidationProblem(loc, f"non-finite float {value!r} is not JSON", "invalid_value") def _too_many_digits(value: int) -> str: """The message for an integer the interpreter will not convert to text, stating its size.""" return ( f"an integer of {value.bit_length()} bits has more digits than JSON text holds " "here, as sys.get_int_max_str_digits bounds it" ) def _writable(value: int) -> bool: """Whether the interpreter can convert `value` to text, as `json.dumps` and `repr` do. `sys.set_int_max_str_digits` limits this, and an integer past the limit could never be written back.""" if value.bit_length() <= 64: return True try: str(value) except ValueError: return False return True def json_text(value: JSONValue) -> str: """`value` as the JSON text `json.dumps` writes for it, with object keys sorted. `==` compares this text for a JSON value the package does not interpret. This is not Python's `==` on the value, which treats `true` as equal to `1`, `-0.0` as equal to `0.0`, and `NaN` as unequal to everything. It compares what a document contains: two values written the same way are the same value to every reader. """ return _TEXT_ENCODER.encode(value) def _as_object(value: object) -> dict[object, object]: """Convert a mapping that `json.dumps` does not handle (such as a read-only view) to a dict, so it is written as an object. Anything else raises the `TypeError` that `json.dumps` raises.""" if is_object(value): return dict(value) msg = f"{value!r} is not JSON" raise TypeError(msg) _TEXT_ENCODER: Final = json.JSONEncoder(sort_keys=True, ensure_ascii=False, default=_as_object) """The encoder `json_text` uses. `json.dumps` builds the same encoder on every call; this one is built once.""" def shown(value: object) -> str: """`value` as a problem's message shows it: as JSON (`null`, `[1, 2]`), or as its repr when it is not JSON. A value the interpreter will not write (an integer with too many digits, or a value nested too deep) is described instead.""" if isinstance(value, int) and not isinstance(value, bool) and not _writable(value): return f"an integer of {value.bit_length()} bits" refined, problems = _refine(value, (), finite=False) if any(problem.message == _PAST_THE_LEVELS for problem in problems): # Nested deeper than `JSON_DEPTH`. Say so instead of using the repr, # which overflows at a depth set by the interpreter and platform. return "a value nested too deep to show" if len(problems) != 0: # Not JSON. return shown_by_python(value) try: return json.dumps(refined, ensure_ascii=False) except ValueError: # An integer with more digits than the interpreter converts to text. # If it is the value itself, show its size. If a container holds it, # Python cannot write the container either. if isinstance(value, int): return f"an integer of {value.bit_length()} bits" return shown_by_python(value) def shown_key(key: object) -> str: """A non-string key as a message shows it: as Python shows it, since JSON has no such key to write. An integer with more digits than the interpreter writes is shown by its size.""" if isinstance(key, int) and not isinstance(key, bool): try: return repr(key) except ValueError: return f"an integer of {key.bit_length()} bits" return shown_by_python(key) def shown_by_python(value: object) -> str: """`value` as Python shows it (its repr), for a non-JSON value a reader finds. If the interpreter cannot write it (because it holds an integer with too many digits), the result says so.""" try: return repr(value) except ValueError: return f"a value of type {type(value).__name__} the interpreter will not write" V = TypeVar("V") def choices(allowed: Sequence[object]) -> str: """A closed set of values as a message names it: `"C"` alone, or `one of ["C", "F"]`.""" values = [shown(value) for value in listed(allowed)] return values[0] if len(values) == 1 else f"one of [{', '.join(values)}]" def listed(allowed: Sequence[V]) -> tuple[V, ...]: """A closed set of values without duplicates, in the order a message lists them: sorted by their JSON text.""" written = {shown(value): value for value in allowed} return tuple(written[json_text] for json_text in sorted(written)) def outside_of( loc: tuple[str | int, ...], value: object, allowed: Sequence[object] ) -> ValidationProblem: """The problem for `value`, found at `loc`, when it is not in the closed set `allowed`. The message lists the set. The kind tells whether the value has the wrong JSON type, or the right type and the wrong value. The `ctx` holds the set under `expected`. """ message = f"expected {choices(allowed)}, got {shown(value)}" expected = cast("tuple[JSONValue, ...]", listed(allowed)) return ValidationProblem(loc, message, refused_kind(value, allowed), ctx={"expected": expected}) def json_type(value: object) -> str: """The JSON type of `value`, as a message names it: "a string", "a number", "null".""" if value is None: return "null" if isinstance(value, bool): return "a boolean" if isinstance(value, (int, float)): return "a number" if isinstance(value, str): return "a string" if isinstance(value, Mapping): return "an object" if isinstance(value, (list, tuple)): return "an array" return "a value" def refused_kind(value: object, allowed: Sequence[object]) -> ProblemKind: """The kind of problem for `value` outside a closed set: `invalid_type` when no member of the set has its JSON type, else `invalid_value`.""" same_type = json_type(value) in {json_type(entry) for entry in allowed} return "invalid_value" if same_type else "invalid_type" def is_canonical_json(value: object, *, finite: bool = True) -> TypeGuard[JSONValue]: """Whether `value` already uses the concrete containers in `JSONValue`, nested no deeper than `JSON_DEPTH`. A non-finite number counts only when `finite` is false, as in user data such as attributes: the document's reader decides where such a number is allowed. The walk uses one frame per level of nesting, `JSON_DEPTH` at most, as `refine_json` does. A reader does not accept a container deeper than that as JSON, so this function does not either, and the walk stops there. """ return _is_canonical(value, 0, finite=finite) def _is_canonical(value: object, depth: int, *, finite: bool) -> bool: """`is_canonical_json` of `value`, which is `depth` levels deep.""" if isinstance(value, float): return not finite or math.isfinite(value) if isinstance(value, (str, int, bool)) or value is None: return True if depth >= JSON_DEPTH: return False if is_list_or_tuple(value): for item in value: # noqa: SIM110 - a loop, not a generator, is one frame per level if not _is_canonical(item, depth + 1, finite=finite): return False return True if is_object(value): for key, item in value.items(): if not isinstance(key, str) or not _is_canonical(item, depth + 1, finite=finite): return False return True return False def is_json(value: object) -> TypeGuard[JSONValue]: """Whether `value` is a canonical JSON structure (recursively).""" return is_canonical_json(value) def parse_json(value: object) -> JSONValue: """Return a canonical `JSONValue`, or raise `MetadataValidationError`.""" refined, problems = refine_json(value) if len(problems) != 0: raise MetadataValidationError(with_input(problems, value)) return refined @overload def frozen(value: Mapping[str, JSONValue]) -> Mapping[str, JSONValue]: ... @overload def frozen(value: JSONValue) -> JSONValue: ... def frozen(value: JSONValue) -> JSONValue: """`value` as a read-only view at every level: each object a mapping proxy, each array a tuple, so the result cannot be changed in place. The result shares the scalars with `value`, and copies only the containers the view needs. A model uses this to expose its document. """ cls = type(value) if cls in _SCALARS: return value if cls is dict or isinstance(value, Mapping): members = cast("Mapping[str, JSONValue]", value) return cast( "JSONValue", MappingProxyType( { key: item if type(item) in _SCALARS else frozen(item) for key, item in members.items() } ), ) if cls is tuple or cls is list or isinstance(value, (tuple, list)): return tuple( item if type(item) in _SCALARS else frozen(item) for item in cast("Sequence[JSONValue]", value) ) return value _SCALARS: Final[frozenset[type]] = frozenset({str, int, float, bool, type(None)}) """The Python types of JSON scalars. A read-only view shares these values without copying.""" def copied(value: JSONValue) -> JSONValue: """A deep copy of `value` that shares no containers with it: each object a new `dict`, each array a new array of the same type. The copy uses one frame for each level of nesting, as `refine_json` does, so any refined value can be copied, however deep it is. """ if isinstance(value, Mapping): members: dict[str, JSONValue] = {} for key, item in value.items(): members[key] = copied(item) return members if isinstance(value, (tuple, list)): # Use `map`, not a comprehension. Before Python 3.12 a comprehension # uses its own frame; this keeps it to one frame per level. entries = list(map(copied, value)) return entries if isinstance(value, list) else tuple(entries) return value def arrays_to_tuples(obj: object) -> object: """Recursively convert array-like values to tuples and mappings to dicts. A tuple or dict whose contents need no conversion is returned as is, not copied. """ if is_array(obj): sequence = obj # Use `map` and loops, not comprehensions. Before Python 3.12 a # comprehension uses its own frame; this keeps it to one frame per # level, as in `copied`. converted_sequence = tuple(map(arrays_to_tuples, sequence)) if isinstance(obj, tuple) and all( converted is original for converted, original in zip(converted_sequence, sequence, strict=True) ): return sequence return converted_sequence if is_object(obj): mapping = obj converted: dict[object, object] = {} for key, value in mapping.items(): converted[key] = arrays_to_tuples(value) if isinstance(obj, dict) and all(converted[key] is value for key, value in mapping.items()): return mapping return converted return obj __all__ = [ "MetadataValidationError", "ProblemKind", "Problems", "ValidationProblem", "arrays_to_tuples", "choices", "copied", "frozen", "is_canonical_json", "is_json", "json_type", "listed", "nested_past_the_levels", "not_an_object", "outside_of", "parse_json", "prefixed", "refine_json", "refine_user_data", "refused_kind", "shown", "shown_key", "validate_json", "value_at", "with_input", "within", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/_numpy.py000066400000000000000000000060771527014137600312770ustar00rootroot00000000000000"""NumPy encodings that both Zarr formats use in fill values: numpy time ticks and `NaT`, and base64 byte strings. A v3 data type and a v2 dtype of the same NumPy family write their fill values the same way, so each encoding is defined once here. """ import re from typing import Annotated, Final, Literal, NewType from annotated_types import Interval NUMPY_TIME_UNITS: Final = ( "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", ) """The units NumPy accepts for a datetime64 or timedelta64 (https://numpy.org/doc/stable/reference/arrays.datetime.html#datetime-units). The v3 types also accept `"generic"`.""" NUMPY_TIME_MAX_SCALE_FACTOR: Final = 2**31 - 1 """The largest `scale_factor` that numpy stores, because the field is a signed int32.""" NumpyTimeScaleFactor = Annotated[int, Interval(ge=1, le=NUMPY_TIME_MAX_SCALE_FACTOR)] """The number of units in one tick: a positive int32.""" NumpyTimeTicks = Annotated[int, Interval(ge=-(2**63), le=2**63 - 1)] """A count of ticks, as a fill value writes it: a signed 64-bit integer. https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L109-L112 """ NOT_A_TIME_TICKS: Final = -(2**63) """The tick count that stores `NaT`. A fill value may write this count for `NaT`. "`"fill_value": "NaT"` and `"fill_value": -9223372036854775808` should be treated as equivalent representations of the same scalar value" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L114-L116). The `numpy.timedelta64` spec says the same (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.timedelta64/README.md?plain=1#L117-L119). """ def numpy_time_fill_value_canonical( configuration: object, nested: object, value: NumpyTimeTicks | Literal["NaT"] ) -> NumpyTimeTicks | Literal["NaT"]: """Return the canonical spelling of a numpy time fill value: `"NaT"` for `NaT` however it is written, and any other count of ticks as written.""" return "NaT" if value in ("NaT", NOT_A_TIME_TICKS) else value Base64Bytes = NewType("Base64Bytes", str) """A standard-alphabet base64-encoded byte sequence.""" _BASE64_RE: Final = re.compile(r"^[A-Za-z0-9+/]*={0,2}$") def base64_bytes(value: str) -> Base64Bytes: """Validate `value` as a Base64Bytes and brand it. Raises ValueError if `value` is not standard-alphabet base64 (length must be a multiple of 4 once padded; only `A-Z`, `a-z`, `0-9`, `+`, `/`, and trailing `=` padding are permitted). """ if len(value) % 4 != 0 or not _BASE64_RE.fullmatch(value): raise ValueError(f"Expected standard-alphabet base64, got {value!r}") return Base64Bytes(value) __all__ = [ "NOT_A_TIME_TICKS", "NUMPY_TIME_MAX_SCALE_FACTOR", "NUMPY_TIME_UNITS", "Base64Bytes", "NumpyTimeScaleFactor", "NumpyTimeTicks", "base64_bytes", "numpy_time_fill_value_canonical", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py000066400000000000000000000134221527014137600332520ustar00rootroot00000000000000"""Private input types used only to generate accurate Pydantic JSON schemas. These repeat the document TypedDicts in a stricter form: extension names match the spec's pattern, `must_understand` is `true` at an extension point, and `codecs` is not empty. They exist because pydantic builds a schema from Python types and cannot embed a JSON Schema that has its own `$defs`, such as the ones `zarr_metadata.json_schema` writes; the v3 document schemas are recursive, so they cannot be inlined either. `test_the_pydantic_schema_agrees_with_the_package_schema` checks that the two schemas accept the same documents. """ from __future__ import annotations import re from collections.abc import Mapping # resolved by Pydantic at runtime from typing import Annotated, Literal, NotRequired from pydantic import Field from typing_extensions import TypedDict from zarr_metadata._common import JSONValue from zarr_metadata.definition._envelope import EXTENSION_NAME_SCHEMA_PATTERN from zarr_metadata.v2.codec import ( # resolved by Pydantic at runtime ZarrV2CodecMetadata, ) from zarr_metadata.v2.data_type import ( # resolved by Pydantic at runtime ZarrV2DataTypeMetadata, ) NonNegativeInt = Annotated[int, Field(ge=0)] ExtensionName = Annotated[str, Field(pattern=re.compile(EXTENSION_NAME_SCHEMA_PATTERN))] """An extension name as the spec defines it: the pattern `well_named` accepts. With it, the schema pydantic generates rejects the same names the reader rejects. The pattern is compiled so that pydantic matches it with Python's `re`. The pattern ends in a look-ahead, which `re` supports and pydantic's own regex engine does not. pydantic writes the pattern into the schema unchanged. """ class ZarrV3NamedConfigJSON(TypedDict, closed=True): """Closed v3 named configuration read on its own, outside a document, where `must_understand` may be `false`.""" name: ExtensionName configuration: NotRequired[Mapping[str, JSONValue]] must_understand: NotRequired[bool] class ZarrV3MandatoryNamedConfigJSON(TypedDict, closed=True): """Closed named configuration at an extension point of a document, where understanding is mandatory.""" name: ExtensionName configuration: NotRequired[Mapping[str, JSONValue]] must_understand: NotRequired[Literal[True]] ZarrV3MetadataFieldJSON = ExtensionName | ZarrV3NamedConfigJSON ZarrV3MandatoryMetadataFieldJSON = ExtensionName | ZarrV3MandatoryNamedConfigJSON ZarrV3CodecPipelineJSON = Annotated[ tuple[ZarrV3MandatoryMetadataFieldJSON, ...], Field(min_length=1) ] ZarrV2FilterPipelineJSON = tuple[ZarrV2CodecMetadata, ...] class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=JSONValue): """Schema input for a v3 array document, including arbitrary extensions.""" zarr_format: Literal[3] node_type: Literal["array"] data_type: ZarrV3MandatoryMetadataFieldJSON shape: tuple[NonNegativeInt, ...] chunk_grid: ZarrV3MandatoryMetadataFieldJSON chunk_key_encoding: ZarrV3MandatoryMetadataFieldJSON fill_value: JSONValue codecs: ZarrV3CodecPipelineJSON attributes: NotRequired[Mapping[str, JSONValue]] storage_transformers: NotRequired[tuple[ZarrV3MandatoryMetadataFieldJSON, ...]] dimension_names: NotRequired[tuple[str | None, ...]] class ZarrV3ConsolidatedMetadataEnvelopeJSON(TypedDict, closed=True): """Schema input for a group's consolidated-metadata envelope, whose entries a group does not read.""" kind: Literal["inline"] must_understand: Literal[False] metadata: Mapping[str, JSONValue] class ZarrV3ConsolidatedMetadataJSON(TypedDict, closed=True): """Schema input for the closed inline consolidated-metadata envelope, with each entry read.""" kind: Literal["inline"] must_understand: Literal[False] metadata: Mapping[str, ZarrV3ArrayMetadataJSON | ZarrV3GroupMetadataJSON] class ZarrV3GroupMetadataJSON(TypedDict, extra_items=JSONValue): """Schema input for a v3 group document, including arbitrary extensions.""" zarr_format: Literal[3] node_type: Literal["group"] attributes: NotRequired[Mapping[str, JSONValue]] consolidated_metadata: NotRequired[ZarrV3ConsolidatedMetadataEnvelopeJSON] class ZarrV2ArrayMetadataJSON(TypedDict, extra_items=JSONValue): """Schema input for the merged v2 array representation. Open, like the runtime validator: the v2 spec says other keys "SHOULD NOT be present ... and SHOULD be ignored by implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92); the group document's "MUST NOT" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313) keeps `ZarrV2GroupMetadataJSON` closed. """ zarr_format: Literal[2] shape: tuple[NonNegativeInt, ...] chunks: tuple[NonNegativeInt, ...] dtype: ZarrV2DataTypeMetadata compressor: ZarrV2CodecMetadata | None fill_value: JSONValue order: Literal["C", "F"] filters: ZarrV2FilterPipelineJSON | None dimension_separator: NotRequired[Literal[".", "/"]] attributes: NotRequired[Mapping[str, JSONValue]] class ZarrV2GroupMetadataJSON(TypedDict, closed=True): """Schema input for the closed, merged v2 group representation.""" zarr_format: Literal[2] attributes: NotRequired[Mapping[str, JSONValue]] class ZarrV2ConsolidatedMetadataJSON(TypedDict, closed=True): """Schema input matching the v2 consolidated model's structural parser.""" zarr_consolidated_format: Literal[1] metadata: Mapping[str, JSONValue] __all__ = [ "ZarrV2ArrayMetadataJSON", "ZarrV2ConsolidatedMetadataJSON", "ZarrV2GroupMetadataJSON", "ZarrV3ArrayMetadataJSON", "ZarrV3ConsolidatedMetadataEnvelopeJSON", "ZarrV3ConsolidatedMetadataJSON", "ZarrV3GroupMetadataJSON", "ZarrV3MetadataFieldJSON", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/_sentinel.py000066400000000000000000000040111527014137600317320ustar00rootroot00000000000000"""The absence sentinel for optional metadata-document keys. The models observe one invariant: `None` in a model always corresponds to a JSON `null` in the document (a v2 `compressor`/`filters` value, an unnamed dimension inside `dimension_names`), and `UNSET` always means the document key is absent. The two are never interchangeable, so a model value can never leak into a document as a spelling the writer did not intend. A problem's `input` keeps the same invariant: it is `UNSET` where nothing was found at the problem's `loc`, and `None` where a `null` was. Check with identity: `if model.dimension_names is UNSET: ...`. Because the contract is identity, the sentinel must never be reconstructed from state: pickling and copying work by *reference* (typing_extensions >= 4.16 implements `Sentinel.__reduce__` as a lookup of the sentinel's name on its defining module), so `pickle.loads(pickle.dumps(UNSET)) is UNSET` holds across process boundaries, and models holding `UNSET` pickle and deep-copy freely. Earlier typing_extensions releases could not pickle sentinels at all, which is why this package requires `>=4.16`. Checker support (PEP 661 is Final; stdlib `sentinel` arrives in Python 3.15): ty types this spelling exactly, including `is`/`is not` narrowing. Pyright supports it but a regression (1.1.405+, tracked as https://github.com/microsoft/pyright/issues/11115) degrades class-attribute reads to `Unknown`, so this package pins pyright to the last good version until the fix lands. Mypy support is in review (https://github.com/python/mypy/pull/21647); until it merges, mypy-checked consumers of these fields need a `cast` or `type: ignore` at narrowing sites. This is a deliberate short-term cost: the sentinel is the standard, and the checkers are converging on it. """ from __future__ import annotations from typing_extensions import Sentinel UNSET = Sentinel("UNSET") """Marks a metadata-document key as absent (PEP 661 sentinel; usable directly in type expressions, e.g. `tuple[str, ...] | UNSET`). Test with `is UNSET`.""" zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/_typed_json.py000066400000000000000000002041561527014137600323030ustar00rootroot00000000000000"""JSON values checked against type annotations. `parser_for` turns an annotation into a parser. A parser is a function that takes a JSON value and its location, and returns the typed value and every problem found with it. `parser_for` supports only the annotations that describe JSON shapes: `int`, `float` for any number, `bool`, `str`, `None` for null, `JSONValue`, a `Literal` of JSON scalars, `tuple[T, ...]` and `tuple[T1, T2]`, a union of those, an object described by a TypedDict, `Mapping[str, V]`, a `NewType` (parsed as its supertype), and a type alias (parsed as the type it stands for). This short list keeps the module small. Any other annotation has no parser. A number type may carry bounds, written with annotated-types as pydantic reads them, for example `Annotated[int, Interval(ge=0, le=9)]`. A value outside the bounds is a problem. `constraints_of` lists the bounds the checker supports. A TypedDict is read as the typing spec defines it. This is not always what its runtime attributes say. `typeddict_keys` reads which keys the TypedDict declares, the type of each, whether each is required, and what any other key may hold. A parser returns a value of its annotation: an array comes back as a tuple, and an object comes back as a new dict of the keys its type allows. A key that a closed TypedDict does not declare is reported as `unknown_key` and left out. `check` is the full entry point for a caller that has a JSON value and a TypedDict. The public module `zarr_metadata.typed_json` exports it. This module does not know what a metadata field is. A caller with its own shapes passes a `leaf`. The parser asks the leaf first for every annotation at every depth, and uses the parser the leaf returns without changes. Parsers are compiled once per annotation and are pure functions of the value, so a union branch that did not match leaves no state behind. `json_schema` writes the same rules as a JSON Schema. `Schemas` writes each shape as the checker reads it. It asks a caller's `SchemaLeaf` first, the same way a parser asks a `Leaf`. """ from __future__ import annotations import functools import math import operator import sys import types import typing import urllib.parse from collections.abc import Callable, Iterator, Mapping, Sequence from collections.abc import Set as AbstractSet from dataclasses import dataclass from typing import ( TYPE_CHECKING, Annotated, Final, ForwardRef, Literal, Never, NewType, NoReturn, TypeAlias, TypeGuard, TypeVar, cast, get_args, get_origin, get_type_hints, ) import annotated_types import typing_extensions from typing_extensions import NoExtraItems, TypeIs, is_typeddict from zarr_metadata._common import JSONValue, UserJSONValue from zarr_metadata._json import ( ValidationProblem, choices, copied, is_canonical_json, is_json, is_list_or_tuple, is_object, listed, outside_of, refine_json, shown, with_input, ) if TYPE_CHECKING: from zarr_metadata._json import ProblemKind T = TypeVar("T") Loc: TypeAlias = tuple[str | int, ...] """The location of a value in a document: the keys and indices that lead to it.""" Parsed: TypeAlias = tuple[object, tuple[ValidationProblem, ...]] """The result of a parser: the typed value, and every problem found with it.""" class UnreadConstraintError(TypeError): """Raised for `Annotated` metadata the checker does not read, such as `MultipleOf`.""" Parser: TypeAlias = Callable[[object, Loc], Parsed] """Checks one value at `loc` against one annotation. Compiled once, run many times.""" Leaf: TypeAlias = Callable[[object], "Parser | None"] """A caller's own shapes. Asked first for every annotation; returns None to decline.""" # The qualifiers a TypedDict key may carry. `typing` and `typing_extensions` # may each define one, and a TypedDict may be written with either. _REQUIRED: Final[tuple[object, ...]] = (typing.Required, typing_extensions.Required) _NOT_REQUIRED: Final[tuple[object, ...]] = (typing.NotRequired, typing_extensions.NotRequired) _READ_ONLY: Final[tuple[object, ...]] = ( typing_extensions.ReadOnly, getattr(typing, "ReadOnly", typing_extensions.ReadOnly), ) _QUALIFIERS: Final[tuple[object, ...]] = (*_REQUIRED, *_NOT_REQUIRED, *_READ_ONLY) # A `type` statement makes a `typing.TypeAliasType`, which is not the # `typing_extensions` one on every version that has both. _ALIASES: Final[tuple[type[typing_extensions.TypeAliasType], ...]] = ( typing_extensions.TypeAliasType, cast( "type[typing_extensions.TypeAliasType]", getattr(typing, "TypeAliasType", typing_extensions.TypeAliasType), ), ) def problem( loc: Loc, message: str, kind: ProblemKind = "invalid_type" ) -> tuple[ValidationProblem, ...]: """One problem, as the one-element tuple every parser returns. It is a tuple so that a parser can return it directly, and a caller that collects problems can call `found.extend(problem(...))`. The default `kind` means a type mismatch. A value rule passes `"invalid_value"`. """ return (ValidationProblem(loc, message, kind),) def is_integer(value: object) -> TypeIs[int]: """A JSON integer: an `int`, and not a `bool`. `True` is an `int` in Python, but `true` is not a number in JSON, so the code must distinguish the two everywhere a number is expected. """ return not isinstance(value, bool) and isinstance(value, int) # --- annotations --------------------------------------------------------- def strip_annotation(annotation: object) -> tuple[object, tuple[object, ...]]: """An annotation's type, and the metadata `Annotated` wrapped it in. `Required`, `NotRequired` and `ReadOnly` are qualifiers on a TypedDict key, not part of the value's type. This function removes them together with the `Annotated` layers, in any order they were written. """ metadata: list[object] = [] while True: origin = get_origin(annotation) if origin is Annotated: inner, *extras = get_args(annotation) # Put an inner layer's metadata first, the same way `Annotated` # flattens nested layers. metadata[:0] = extras annotation = inner elif origin in _QUALIFIERS: (annotation,) = get_args(annotation) else: return annotation, tuple(metadata) def unqualified(annotation: object) -> object: """A TypedDict key's annotation with its qualifiers removed and its `Annotated` metadata kept, since that metadata is part of the value's type.""" inner, metadata = strip_annotation(annotation) if len(metadata) == 0: return inner return Annotated[(inner, *metadata)] Qualifier: TypeAlias = Literal["Required", "NotRequired", "ReadOnly"] """The name of a qualifier on a TypedDict key. `typing` and `typing_extensions` may each define one.""" def qualifiers(annotation: object) -> frozenset[Qualifier]: """The qualifiers on a TypedDict key, under any `Annotated` layers, in any order.""" found: set[Qualifier] = set() while True: origin = get_origin(annotation) if origin is Annotated: annotation = get_args(annotation)[0] continue if origin in _REQUIRED: found.add("Required") elif origin in _NOT_REQUIRED: found.add("NotRequired") elif origin in _READ_ONLY: found.add("ReadOnly") else: return frozenset(found) (annotation,) = get_args(annotation) def is_union(annotation: object) -> bool: return get_origin(annotation) in (typing.Union, types.UnionType) def is_alias(annotation: object) -> TypeGuard[typing_extensions.TypeAliasType]: """Whether `annotation` is a type alias that takes no type parameters: `type Level = int`.""" return isinstance(annotation, _ALIASES) and len(annotation.__type_params__) == 0 @functools.cache def alias_value(alias: typing_extensions.TypeAliasType) -> object: """The type `alias` stands for, evaluated in the module that defined it; `TypeError` if a name in it does not resolve. Python evaluates the value of a `type` statement on access. An alias made with `TypeAliasType` holds whatever value it was given. An alias that refers to itself, such as `TypeAliasType("Dtype", str | tuple[tuple[str, "Dtype"], ...])`, holds that reference as a string. This function resolves the string in the alias's module. """ try: # A `type` statement evaluates its value here, so a name in it that # does not resolve raises here too. holder = type( "_AliasValue", (), {"__annotations__": {"value": alias.__value__}, "__module__": alias.__module__}, ) return get_type_hints(holder, include_extras=True)["value"] except NameError as error: msg = f"{alias.__name__}: {error}; its value must resolve in its module" raise TypeError(msg) from error # --- TypedDicts ---------------------------------------------------------- @dataclass(frozen=True, slots=True) class TypedDictKeys: """The keys a TypedDict defines for an object, read as the typing spec defines them. `members` holds every key the TypedDict declares, including keys from its bases. For each key it holds the type of the value and whether the key is required. The type has its qualifiers removed and keeps any `Annotated` metadata, since a constraint there is part of the type. `extra_items` is the type any other key may hold: `Never` when the TypedDict is closed, `object` when it is open, and the `extra_items` type otherwise. `declared` is whether the TypedDict or a base set this explicitly, rather than by default. A TypedDict that sets nothing is open. """ members: Mapping[str, tuple[object, bool]] extra_items: object declared: bool @property def required(self) -> frozenset[str]: """The keys an object of this TypedDict must have.""" return frozenset(key for key, (_, required) in self.members.items() if required) @property def closed(self) -> bool: """Whether undeclared keys are forbidden: `closed=True`, or `extra_items=Never`.""" extra_items = strip_annotation(self.extra_items)[0] return extra_items is Never or extra_items is NoReturn @property def open(self) -> bool: """Whether an undeclared key may hold any value: the default, or `closed=False`.""" return strip_annotation(self.extra_items)[0] is object @functools.cache def typeddict_keys(typeddict: type) -> TypedDictKeys: """The keys `typeddict` defines for an object; `TypeError` if its annotations do not resolve. Whether a key is required follows the spec. A `Required` or `NotRequired` qualifier decides it. Otherwise the `total` setting of the class that declared the key decides it. This function reads the qualifiers from the evaluated annotations, each evaluated in the module where its class was defined. It cannot use the runtime's `__required_keys__` for this, because the runtime computes it before the annotations are evaluated: an annotation written as a string (and `from __future__ import annotations` makes every annotation a string) hides its qualifier. For a key with no qualifier, `__required_keys__` is correct and reflects the `total` setting that applied. Openness also follows the spec. `closed=True` closes the TypedDict, `extra_items=` sets the type of every other key, and `closed=False` opens it. A TypedDict that sets none of these inherits the setting of its bases (the runtime does not record this), or is open when no base sets either. """ hints = dict(_hints(typeddict)) required_at_runtime = cast("frozenset[str]", getattr(typeddict, "__required_keys__", ())) optional_at_runtime = cast("frozenset[str]", getattr(typeddict, "__optional_keys__", ())) if frozenset(hints) != required_at_runtime | optional_at_runtime: msg = ( f"{typeddict.__name__}: its annotations declare {sorted(hints)!r}, and the runtime " f"{sorted(required_at_runtime | optional_at_runtime)!r}" ) raise TypeError(msg) members: dict[str, tuple[object, bool]] = {} for key, hint in hints.items(): said = qualifiers(hint) if "Required" in said and "NotRequired" in said: msg = f"{typeddict.__name__}.{key}: Required and NotRequired both; the spec allows one" raise TypeError(msg) if "Required" in said: required = True elif "NotRequired" in said: required = False else: required = key in required_at_runtime members[key] = (unqualified(hint), required) extra_items, declared = _openness(typeddict) return TypedDictKeys(types.MappingProxyType(members), extra_items, declared) @functools.cache def _hints(typeddict: type) -> Mapping[str, object]: """Every key's annotation, evaluated in the module of the class that declared the key. This is what the typing spec requires, and `get_type_hints` does not fully do it. `get_type_hints` evaluates every inherited annotation in the subclass's own module. If a base in another module wrote a string inside an annotation, such as `tuple["Local", ...]`, that string is evaluated where `Local` may mean something else, or nothing. This happens because the runtime ties a module only to a string at the top level of an annotation. `get_type_hints` also looks up names in the class's module before the module a string is tied to, so an inherited `x: "Foo"` resolves to the subclass's `Foo` when both modules define one. To avoid both problems, this function takes a base's keys from the base, and evaluates the class's own keys (those no base provided) in its module with nothing else in scope. On Python 3.11, `typing.TypedDict` does not record a class's bases, so every key is evaluated as the class's own. That is correct unless a string nested in a base's annotation names something that only the base's module has. """ bases = _typeddict_bases(typeddict) hints: dict[str, object] = {} for base in bases: hints.update(_hints(base)) written = _written(typeddict) inherited = [_written(base) for base in bases] own = { key: annotation for key, annotation in written.items() if not any(key in given and given[key] == annotation for given in inherited) } holder = type("_Own", (), {"__annotations__": own, "__module__": typeddict.__module__}) try: hints.update(get_type_hints(holder, localns={}, include_extras=True)) except NameError as error: msg = f"{typeddict.__name__}: {error}; its annotations must resolve in its module" raise TypeError(msg) from error return types.MappingProxyType(hints) def _written(typeddict: type) -> dict[str, object]: """A TypedDict's annotations, including its bases', evaluated only as far as they evaluate without errors.""" if sys.version_info >= (3, 14): import annotationlib return dict( annotationlib.get_annotations(typeddict, format=annotationlib.Format.FORWARDREF) ) return dict(vars(typeddict).get("__annotations__", {})) def _openness(typeddict: type) -> tuple[object, bool]: """The type an undeclared key of `typeddict` may hold, and whether it was set explicitly rather than by default.""" extra_items = getattr(typeddict, "__extra_items__", NoExtraItems) if extra_items is not NoExtraItems: return _extra_items_type(typeddict, extra_items), True closed = getattr(typeddict, "__closed__", None) if closed is not None: return (Never if closed else object), True inherited = [_openness(base) for base in _typeddict_bases(typeddict)] restricted = {extra for extra, _ in inherited if strip_annotation(extra)[0] is not object} if len(restricted) > 1: msg = ( f"{typeddict.__name__}: its bases disagree on what a key they do not declare may " "hold; say it on the class, with closed= or extra_items=" ) raise TypeError(msg) if len(restricted) == 1: return restricted.pop(), True return object, any(declared for _, declared in inherited) def _extra_items_type(typeddict: type, extra_items: object) -> object: """The `extra_items` type, evaluated in the module of `typeddict`, with `ReadOnly` removed and any `Annotated` metadata kept.""" if isinstance(extra_items, (str, ForwardRef)): holder = type( "_ExtraItems", (), {"__annotations__": {"extra": extra_items}, "__module__": typeddict.__module__}, ) try: extra_items = get_type_hints(holder, include_extras=True)["extra"] except NameError as error: msg = f"{typeddict.__name__}: {error}; its extra_items must resolve in its module" raise TypeError(msg) from error return unqualified(extra_items) def _typeddict_bases(typeddict: type) -> tuple[type, ...]: """The TypedDict bases `typeddict` was declared with, without type arguments.""" bases: list[type] = [] for base in getattr(typeddict, "__orig_bases__", ()): origin = get_origin(base) or base if isinstance(origin, type) and is_typeddict(origin): bases.append(origin) return tuple(bases) # --- names of shapes in messages, and the shape of a value ----------------- def describe(annotation: object, seen: frozenset[object] = frozenset()) -> str: """The name of the annotation in an error message: "an integer", "an object".""" return _named(annotation, seen)[0] def _named(annotation: object, seen: frozenset[object]) -> tuple[str, str]: """The singular and plural names of the annotation in an error message: "an integer", "integers".""" inner = strip_annotation(annotation)[0] if inner is int: return "an integer", "integers" if inner is float: return "a number", "numbers" if inner is bool: return "a boolean", "booleans" if inner is str: return "a string", "strings" if inner is None or inner is types.NoneType: return "null", "nulls" if inner is JSONValue: return "a JSON value", "JSON values" origin = get_origin(inner) if origin is Literal: values = get_args(inner) listed = ", ".join(sorted(dict.fromkeys(shown(value) for value in values))) return choices(values), f"values in [{listed}]" if is_union(inner): # Name each shape once, since two TypedDicts are both "an object". # A broader shape absorbs a narrower one: "a number" absorbs "an # integer", and "a string" absorbs a `Literal` of strings. branches = get_args(inner) named = dict.fromkeys(_named(branch, seen) for branch in branches) if ("a number", "numbers") in named: named.pop(("an integer", "integers"), None) if ("a string", "strings") in named: for branch in filter(_strings_only, branches): named.pop(_named(branch, seen), None) return " or ".join(one for one, _ in named), " or ".join(many for _, many in named) if origin is tuple: arguments = get_args(inner) if len(arguments) == 2 and arguments[1] is Ellipsis: each = _named(arguments[0], seen)[1] return f"an array of {each}", f"arrays of {each}" if len(arguments) == 2: pair = f"[{describe(arguments[0], seen)}, {describe(arguments[1], seen)}] pair" return f"a {pair}", f"{pair}s" return f"an array of {len(arguments)} elements", f"arrays of {len(arguments)} elements" if origin in (Mapping, dict) or (isinstance(inner, type) and is_typeddict(inner)): return "an object", "objects" if isinstance(inner, NewType): return _named(inner.__supertype__, seen) if is_alias(inner): alias = inner if alias in seen or _holds(alias_value(alias), alias, frozenset()): return f"a {alias.__name__}", f"{alias.__name__} values" return _named(alias_value(alias), seen | {alias}) return "a value", "values" def _strings_only(annotation: object) -> bool: """Whether `annotation` is a `Literal` of strings, which the name "a string" covers.""" inner = strip_annotation(annotation)[0] return get_origin(inner) is Literal and all(isinstance(value, str) for value in get_args(inner)) def _holds(annotation: object, alias: object, seen: frozenset[object]) -> bool: """Whether `alias` occurs in `annotation`, following any aliases in it. Used to tell whether an alias refers to itself.""" inner = strip_annotation(annotation)[0] if inner is alias: return True if is_alias(inner): if inner in seen: return False value = alias_value(inner) return _holds(value, alias, seen | {inner}) return any(_holds(argument, alias, seen) for argument in get_args(inner)) def shape_of(annotation: object) -> str | None: """The top-level JSON shape an annotation allows, used to choose a union branch. None means any shape: a JSON value, a union that mixes shapes, or a shape that the caller's `leaf` parses. """ inner = strip_annotation(annotation)[0] seen: set[object] = set() while (isinstance(inner, NewType) or is_alias(inner)) and inner not in seen: seen.add(inner) if isinstance(inner, NewType): inner = strip_annotation(inner.__supertype__)[0] else: inner = strip_annotation(alias_value(inner))[0] if inner is int: return "int" if inner is float: return "number" if inner is bool: return "bool" if inner is str: return "str" if inner is None or inner is types.NoneType: return "null" origin = get_origin(inner) if origin is Literal: # `True` counts as a bool, not an int, as in `is_integer`. values: tuple[object, ...] = get_args(inner) shapes = {shape_of(type(value)) for value in values} return shapes.pop() if len(shapes) == 1 else None if origin is tuple: return "tuple" if origin in (Mapping, dict): return "mapping" if isinstance(inner, type) and is_typeddict(inner): return "mapping" return None def has_shape(shape: str | None, value: object) -> bool: if shape is None: return True if shape == "int": return is_integer(value) if shape == "number": return not isinstance(value, bool) and isinstance(value, (int, float)) if shape == "bool": return isinstance(value, bool) if shape == "str": return isinstance(value, str) if shape == "null": return value is None if shape == "tuple": return isinstance(value, (list, tuple)) return isinstance(value, Mapping) # "mapping" # --- the parsers --------------------------------------------------------- def _scalar(description: str, admits: Callable[[object], bool]) -> Parser: def parse(value: object, loc: Loc) -> Parsed: if admits(value): return value, () return value, problem(loc, f"expected {description}, got {shown(value)}") return parse _INTEGER: Final = _scalar("an integer", is_integer) _NUMBER: Final = _scalar( "a number", lambda value: not isinstance(value, bool) and isinstance(value, (int, float)) ) _BOOLEAN: Final = _scalar("a boolean", lambda value: isinstance(value, bool)) _STRING: Final = _scalar("a string", lambda value: isinstance(value, str)) _NULL: Final = _scalar("null", lambda value: value is None) _JSON: Final = _scalar("a JSON value", is_json) _USER_JSON: Final = _scalar("a JSON value", lambda value: is_canonical_json(value, finite=False)) def one_of(allowed: tuple[object, ...]) -> Parser: """A member whose type is a closed set of values. A value matches an entry when it is equal and of the same type. JSON `true` is not the integer 1, though in Python `True == 1`. If the value's JSON type matches none of the entries (for example a number, where every entry is a string), the problem is a wrong type. If the type matches but the value does not, the problem is a wrong value. """ def parse(value: object, loc: Loc) -> Parsed: if not any(value == entry and type(value) is type(entry) for entry in allowed): return value, (outside_of(loc, value, allowed),) return value, () return parse def sequence_of(element: Parser) -> Parser: """A member whose type is an array of one element type, parsed element by element.""" def parse(value: object, loc: Loc) -> Parsed: if not is_list_or_tuple(value): return value, problem(loc, f"expected an array, got {shown(value)}") entries = value parsed: list[object] = [] found: list[ValidationProblem] = [] for index, entry in enumerate(entries): item, problems = element(entry, (*loc, index)) parsed.append(item) found.extend(problems) return tuple(parsed), tuple(found) return parse def fixed_tuple(elements: Sequence[Parser], description: str) -> Parser: """A member whose type is an array of a fixed length, parsed position by position.""" def parse(value: object, loc: Loc) -> Parsed: if not is_list_or_tuple(value): return value, problem(loc, f"expected {description}, got {shown(value)}") entries = tuple(value) if len(entries) != len(elements): return entries, problem(loc, f"expected {description}, got {shown(entries)}") parsed: list[object] = [] found: list[ValidationProblem] = [] for position, (element, entry) in enumerate(zip(elements, entries, strict=True)): item, problems = element(entry, (*loc, position)) parsed.append(item) found.extend(problems) return tuple(parsed), tuple(found) return parse Branch: TypeAlias = tuple[str | None, Parser, frozenset[str] | None] """A branch of a union: its top-level shape (from `shape_of`), its parser, and its keys if it is a TypedDict.""" Tag: TypeAlias = tuple[str, Mapping[tuple[type, object], int]] """A key that every branch requires as a `Literal`, and the branch that each of its values selects.""" def any_of( branches: Sequence[Branch], description: str, tag: Tag | None = None, literals: frozenset[int] = frozenset(), allowed: tuple[JSONValue, ...] = (), ) -> Parser: """A member whose type is a union of shapes, parsed by the matching branch. The parser tries only the branches whose top-level shape matches the value. This way a bad element inside an array is reported at its position inside the array, rather than the whole array being reported as wrong. A value that matches no branch's shape is reported once, with the expected shapes. When every branch is a TypedDict that requires a key as a `Literal` of values unique to that branch, the value of that key selects the branch, and the problems are that branch's problems. Otherwise the value is parsed by the first branch that reports no problem (among TypedDicts, the one that declares the most of the value's keys). If every branch reports problems, the parser returns the result of the branch with the fewest problems. `literals` holds the index of each branch that is a `Literal`. When only such branches tried the value, and it is none of their values, it is reported once with the expected shapes, since naming one `Literal` would leave out the other branches. `allowed` holds the values of every `Literal` branch, in the order the message names them. Such a problem carries them under `expected` in its `ctx`, as a single `Literal`'s problem does. """ ctx: dict[str, JSONValue] = {"expected": allowed} def parse(value: object, loc: Loc) -> Parsed: present = _keys_of(value) if tag is not None and present is not None: return _by_tag(branches, tag, value, loc) clean: list[tuple[int, int, Parsed]] = [] failed: list[tuple[int, int, Parsed]] = [] for index, (shape, branch, keys) in enumerate(branches): if not has_shape(shape, value): continue result = branch(value, loc) if len(result[1]) != 0: failed.append((len(result[1]), index, result)) elif keys is None or present is None: return result else: clean.append((-len(keys & present), index, result)) if len(clean) != 0: return min(clean, key=lambda found: found[:2])[2] message = f"expected {description}, got {shown(value)}" if len(failed) != 0 and all(index in literals for _, index, _ in failed): # A `Literal` reports one problem, of the kind its own check decided. kind = min(failed, key=lambda found: found[:2])[2][1][0].kind return value, (ValidationProblem(loc, message, kind, ctx=ctx),) if len(failed) != 0: return min(failed, key=lambda found: found[:2])[2] return value, problem(loc, message) return parse def _by_tag(branches: Sequence[Branch], tag: Tag, value: object, loc: Loc) -> Parsed: """Parse `value` with the branch its tag selects. A missing tag, or a tag value no branch has, is reported at the tag's location.""" key, picks = tag if not is_object(value) or key not in value: return value, problem((*loc, key), f"missing required key {key!r}", "missing_key") said = value[key] index = picks.get((type(said), said)) if _hashable(said) else None if index is None: return value, (outside_of((*loc, key), said, tuple(entry for _, entry in picks)),) return branches[index][1](value, loc) def _keys_of(value: object) -> AbstractSet[str] | None: """The keys of `value` if it is a JSON object, else None.""" if is_object(value): return {key for key in value if isinstance(key, str)} return None def _hashable(value: object) -> bool: return value is None or isinstance(value, (str, int, float)) def object_of(members: Mapping[str, tuple[Parser, bool]], extra: Parser | None) -> Parser: """An object with declared keys, each parsed at its own key, and `extra` for every other key. `members` maps each declared key to its parser and whether it is required. A missing required key is a `missing_key` problem. `extra` parses a key the object does not declare. With no `extra` (a closed TypedDict), such a key is an `unknown_key` problem and is left out of the result. The result is a new dict, with keys in their input order. """ def parse(value: object, loc: Loc) -> Parsed: if not is_object(value): return value, problem(loc, f"expected an object, got {shown(value)}") entries = value parsed: dict[str, object] = {} found: list[ValidationProblem] = [] for key, entry in entries.items(): if not isinstance(key, str): found.extend(problem(loc, f"non-string key {shown(key)}")) continue if key in members: continue if extra is None: found.extend(problem((*loc, key), f"unexpected key {key!r}", "unknown_key")) continue parsed[key], problems = extra(entry, (*loc, key)) found.extend(problems) for key, (member, required) in members.items(): if key in entries: parsed[key], problems = member(entries[key], (*loc, key)) found.extend(problems) elif required: found.extend(problem((*loc, key), f"missing required key {key!r}", "missing_key")) return { key: parsed[key] for key in entries if isinstance(key, str) and key in parsed }, tuple(found) return parse def mapping_of(value: Parser) -> Parser: """A member whose type is an object with any keys, parsed value by value. The open counterpart of `object_of`: a `Mapping[str, V]` does not restrict the keys, only the type of each value. """ def parse(candidate: object, loc: Loc) -> Parsed: if not is_object(candidate): return candidate, problem(loc, f"expected an object, got {shown(candidate)}") entries = candidate parsed: dict[str, object] = {} found: list[ValidationProblem] = [] for key, entry in entries.items(): if not isinstance(key, str): found.extend(problem(loc, f"non-string key {shown(key)}")) continue item, problems = value(entry, (*loc, key)) parsed[key] = item found.extend(problems) return parsed, tuple(found) return parse # --- constraints --------------------------------------------------------- Constraints: TypeAlias = Mapping[str, int | float] """The bounds a type carries, keyed by the names pydantic uses: `{"ge": 0, "le": 9}`.""" _BOUNDS: Final[tuple[tuple[type, str], ...]] = ( (annotated_types.Gt, "gt"), (annotated_types.Ge, "ge"), (annotated_types.Lt, "lt"), (annotated_types.Le, "le"), ) """The annotated-types constraints the checker supports, each with its name.""" _FROM_BELOW: Final = frozenset({"gt", "ge"}) """The lower bounds.""" _FROM_ABOVE: Final = frozenset({"lt", "le"}) """The upper bounds.""" _EXCLUSIVE: Final = frozenset({"gt", "lt"}) """The exclusive bounds: a value may not equal them.""" _HOLDS: Final[Mapping[str, Callable[[float, float], bool]]] = { "gt": operator.gt, "ge": operator.ge, "lt": operator.lt, "le": operator.le, } """For each bound name, a test of whether a number satisfies that bound.""" def constraints_of(metadata: Sequence[object]) -> dict[str, int | float]: """The bounds in an `Annotated` type's metadata, by name: at most one lower and one upper bound. The supported bounds are annotated-types' `Gt`, `Ge`, `Lt` and `Le`, and an `Interval`, which unpacks to them, as pydantic reads them. A note (a string or a `Doc`) does not constrain the value and is skipped. Anything else raises `TypeError`: - a constraint the checker does not support, such as `MinLen`, `Predicate`, or pydantic's `Field`. Otherwise the type would declare a rule that the checker does not enforce. - a second bound on the same side. pydantic uses the last one. - a bound that is not a finite number. A bound is stored as the number it equals, as an integer when it is one. `Ge(True)`, `Ge(1.0)` and `Ge(1)` are the same bound, because Python's `Annotated` cache treats equal metadata as identical and may return any of them in place of another. """ said: dict[str, int | float] = {} for item in _unpacked(metadata): if isinstance(item, (str, typing_extensions.Doc)): continue name = next((name for kind, name in _BOUNDS if isinstance(item, kind)), None) if name is None: msg = f"{item!r} is not a constraint the checker reads, which are Gt, Ge, Lt and Le" raise UnreadConstraintError(msg) side = _FROM_BELOW if name in _FROM_BELOW else _FROM_ABOVE if not side.isdisjoint(said): where = "below" if side is _FROM_BELOW else "above" msg = f"{item!r} is a second bound from {where}; a type takes one from each side" raise TypeError(msg) said[name] = _bound(item, name) return said def _unpacked(metadata: Sequence[object]) -> Iterator[object]: """`metadata`, with each group of constraints in it (such as an `Interval`) unpacked.""" for item in metadata: if isinstance(item, annotated_types.GroupedMetadata): yield from _unpacked(tuple(cast("Sequence[object]", item))) else: yield item def _bound(item: object, name: str) -> int | float: """The number of bound `item` named `name`, as an integer when it is one; `TypeError` if it is not a finite number.""" bound = cast("object", getattr(item, name)) if isinstance(bound, int): return int(bound) # a `bool` becomes its integer, as in the `Annotated` cache if isinstance(bound, float) and math.isfinite(bound): return int(bound) if bound.is_integer() else bound msg = f"{item!r}: a bound is a finite number" raise TypeError(msg) def _constrainable(inner: object, said: Constraints) -> None: """Raise `TypeError` for bounds on a type that is not a number: a string, an array, a value of any type.""" if len(said) != 0 and shape_of(inner) not in ("int", "number"): msg = f"{', '.join(said)}: a bound is on a number, and {describe(inner)} is not one" raise TypeError(msg) def constrained(parse: Parser, description: str, said: Constraints) -> Parser: """Wrap `parse` to also check the bounds its type carries: one `invalid_value` problem if any bound fails. The message states what the type allows, for example "expected an integer in [0, 9], got 12", and the problem's `ctx` holds the bounds. The bounds are checked only when `parse` reports no problem, in the same way that rules run only on a value of their type. """ expected = f"expected {description} {_admitted(said)}" ctx: dict[str, JSONValue] = dict(said) holds = tuple((_HOLDS[name], bound) for name, bound in said.items()) def parse_constrained(value: object, loc: Loc) -> Parsed: typed, found = parse(value, loc) if len(found) != 0: return typed, found number = cast("float", typed) for keeps, bound in holds: if not keeps(number, bound): message = f"{expected}, got {shown(value)}" return typed, (ValidationProblem(loc, message, "invalid_value", ctx=ctx),) return typed, () return parse_constrained def _admitted(said: Constraints) -> str: """The range the bounds allow, as written after the type in a message: "in [0, 9]", "in (0, 1)", ">= 1".""" low = next(((name, bound) for name, bound in said.items() if name in _FROM_BELOW), None) high = next(((name, bound) for name, bound in said.items() if name in _FROM_ABOVE), None) if low is not None and high is not None: opening = "(" if low[0] in _EXCLUSIVE else "[" closing = ")" if high[0] in _EXCLUSIVE else "]" return f"in {opening}{shown(low[1])}, {shown(high[1])}{closing}" if low is not None: return f"{'>' if low[0] in _EXCLUSIVE else '>='} {shown(low[1])}" name, bound = cast("tuple[str, int | float]", high) return f"{'<' if name in _EXCLUSIVE else '<='} {shown(bound)}" # --- the compiler -------------------------------------------------------- _Building: TypeAlias = dict[object, Parser] """The parsers of one compilation, keyed by the TypedDict or alias each one parses. A TypedDict or alias is added before its compilation finishes, as a parser that forwards to the one being built. This way a type that refers to itself compiles to a parser that calls itself. """ def _union(inner: object, leaf: Leaf, building: _Building) -> Parser | None: arguments = get_args(inner) branches: list[Branch] = [] for branch in arguments: member = _compile(branch, leaf, building) if member is None: return None typeddict = strip_annotation(branch)[0] keys = ( frozenset(typeddict_keys(typeddict).members) if isinstance(typeddict, type) and is_typeddict(typeddict) else None ) branches.append((shape_of(branch), member, keys)) literals = frozenset( index for index, branch in enumerate(arguments) if get_origin(_unaliased(branch)) is Literal ) # Each `Literal` names its values in the order `listed` gives, and the # union names its branches in order. A value is kept once, by its JSON # text, so `false` and `0` stay two values, as in `listed`. allowed: tuple[JSONValue, ...] = tuple( { shown(value): value for index in sorted(literals) for value in listed(get_args(_unaliased(arguments[index]))) }.values() ) return any_of(branches, describe(inner), _tag(arguments), literals, allowed) def _unaliased(annotation: object) -> object: """`annotation` without its `Annotated` metadata, and with each type alias replaced by its value, until what remains is not an alias.""" inner = strip_annotation(annotation)[0] seen: set[object] = set() while is_alias(inner) and inner not in seen: seen.add(inner) inner = strip_annotation(alias_value(inner))[0] return inner def _tag(arguments: Sequence[object]) -> Tag | None: """The key that identifies which of these TypedDicts a value is, if there is one. Every branch must require the key, as a `Literal` whose values no other branch has. This is the discriminator that a union of configurations writes as `name`. """ typeddicts = [strip_annotation(argument)[0] for argument in arguments] if not all(isinstance(typed, type) and is_typeddict(typed) for typed in typeddicts): return None keys = [typeddict_keys(cast("type", typed)) for typed in typeddicts] shared = set(keys[0].required) for each in keys[1:]: shared &= each.required for key in sorted(shared): picks: dict[tuple[type, object], int] = {} for index, each in enumerate(keys): annotation = strip_annotation(each.members[key][0])[0] if get_origin(annotation) is not Literal: break values: tuple[object, ...] = get_args(annotation) if any((type(entry), entry) in picks for entry in values): break picks.update({(type(entry), entry): index for entry in values}) else: return key, types.MappingProxyType(picks) return None def _tuple(inner: object, leaf: Leaf, building: _Building) -> Parser | None: arguments = get_args(inner) if len(arguments) == 2 and arguments[1] is Ellipsis: element = _compile(arguments[0], leaf, building) return None if element is None else sequence_of(element) elements: list[Parser] = [] for argument in arguments: element = _compile(argument, leaf, building) if element is None: return None elements.append(element) return fixed_tuple(elements, describe(inner)) def _mapping(inner: object, leaf: Leaf, building: _Building) -> Parser | None: arguments = get_args(inner) if len(arguments) != 2 or arguments[0] is not str: return None value = _compile(arguments[1], leaf, building) return None if value is None else mapping_of(value) def _recursive( key: object, building: _Building, compile_: Callable[[], Parser | None] ) -> Parser | None: """The parser `compile_` builds for `key`, a TypedDict or alias that may refer to itself.""" if key in building: return building[key] built: list[Parser] = [] building[key] = lambda value, loc: built[0](value, loc) parser = compile_() if parser is None: return None built.append(parser) building[key] = parser return parser def _object(typeddict: type, leaf: Leaf, building: _Building) -> Parser | None: def compile_() -> Parser | None: keys = typeddict_keys(typeddict) members: dict[str, tuple[Parser, bool]] = {} for key, (annotation, required) in keys.members.items(): member = _compile(annotation, leaf, building) if member is None: return None members[key] = (member, required) if keys.closed: return object_of(members, None) if keys.open: # Any JSON value is allowed. A note does not change that, and a # bound cannot apply, but the metadata is still checked. _constrainable(object, constraints_of(strip_annotation(keys.extra_items)[1])) return object_of(members, _JSON) extra = _compile(keys.extra_items, leaf, building) return None if extra is None else object_of(members, extra) return _recursive(typeddict, building, compile_) def _alias( alias: typing_extensions.TypeAliasType, leaf: Leaf, building: _Building ) -> Parser | None: return _recursive(alias, building, lambda: _compile(alias_value(alias), leaf, building)) def _literal(inner: object) -> Parser | None: values: tuple[object, ...] = get_args(inner) if not all(value is None or isinstance(value, (str, int)) for value in values): return None # an enum member, or bytes: a JSON document cannot hold these # Sort the values, because `get_args` may not report them in the order # this `Literal` was written in. Two `Literal`s over the same values # compare and hash equal, so every later one resolves to the first one # built anywhere in the process. The parse is a membership test either # way. Sorting keeps the message that lists the values independent of # import order. return one_of(tuple(sorted(values, key=repr))) def _compile(annotation: object, leaf: Leaf, building: _Building) -> Parser | None: inner, metadata = strip_annotation(annotation) parse = _compile_type(inner, leaf, building) if parse is None or len(metadata) == 0: return parse said = constraints_of(metadata) if len(said) == 0: return parse _constrainable(inner, said) return constrained(parse, describe(inner), said) def _compile_type(inner: object, leaf: Leaf, building: _Building) -> Parser | None: """The parser of a type with `Annotated` already removed.""" found = leaf(inner) if found is not None: return found if inner is int: return _INTEGER if inner is float: return _NUMBER if inner is bool: return _BOOLEAN if inner is str: return _STRING if inner is None or inner is types.NoneType: return _NULL if inner is JSONValue: return _JSON if inner is UserJSONValue: return _USER_JSON origin = get_origin(inner) if origin is Literal: return _literal(inner) if is_union(inner): return _union(inner, leaf, building) if origin is tuple: return _tuple(inner, leaf, building) if origin in (Mapping, dict): return _mapping(inner, leaf, building) if isinstance(inner, type) and is_typeddict(inner): return _object(inner, leaf, building) if isinstance(inner, NewType): # In a document, a `NewType` is the same as its supertype. The # distinction exists only in code, to mark a value the code has # already checked. return _compile(inner.__supertype__, leaf, building) if is_alias(inner): return _alias(inner, leaf, building) return None def parser_for(annotation: object, leaf: Leaf) -> Parser | None: """The parser for an annotation, or None if the annotation has none. This is a small compiler that supports only the JSON shapes listed in the module docstring. It asks `leaf` first, here and at every depth (inside a union, an array, or an object), and uses what `leaf` returns without changes. The list is closed: any other annotation has no parser. A TypedDict whose annotations do not resolve, or whose bases disagree on what its other keys hold, raises a `TypeError` that names it. """ return _compile(annotation, leaf, {}) def no_leaf(annotation: object) -> Parser | None: """The leaf for a caller with no shapes of its own.""" return None def parser(annotation: object, leaf: Leaf) -> Parser: """The parser for an annotation; `TypeError` naming the part of it that has no parser.""" found = parser_for(annotation, leaf) if found is None: inner = strip_annotation(annotation)[0] unread = unread_in(inner, leaf) if isinstance(inner, type) and is_typeddict(inner) else None if unread is None: msg = f"{annotation!r} is not a shape JSON takes" raise TypeError(msg) if unread.constraint: raise UnreadConstraintError(unread.message) raise TypeError(unread.message) return found @dataclass(frozen=True, slots=True) class Unread: """The part of a TypedDict that has no parser, as `unread_in` finds it.""" message: str """The part, named down to the TypedDict that contains it, and why it has no parser.""" constraint: bool """Whether the part is a constraint the checker does not read, which a rule can check instead.""" def unread_in(typeddict: type, leaf: Leaf) -> Unread | None: """The part of `typeddict` that has no parser, named down to the TypedDict that contains it; None if every part has one.""" return _unread_in(typeddict, leaf, frozenset({typeddict}))[0] def _unread_in( typeddict: type, leaf: Leaf, visiting: frozenset[type] ) -> tuple[Unread | None, bool]: """The unread part of `typeddict`, looked inside from each TypedDict in `visiting`, itself included, and whether it is surely the part that is unread. Each TypedDict a member holds, directly, in a union or a container, or through an alias or a `NewType`, is looked inside first, as the checker meets it first. Then a member that holds a TypedDict is named when the checker does not read its own metadata, such as a constraint. A member that leads back to one of `visiting` may be unread only because of what is unread there, which is found at that TypedDict's own members. Looking inside it again would not end, so such a member is named only when nothing else is unread. """ keys = typeddict_keys(typeddict) others: list[tuple[str, object]] = [] if not (keys.closed or keys.open): others.append(("extra_items", keys.extra_items)) fallback: Unread | None = None for key, annotation in [*((key, member[0]) for key, member in keys.members.items()), *others]: failed: TypeError | None = None try: member = parser_for(annotation, leaf) except TypeError as error: failed, member = error, None if member is not None: continue inner, metadata = strip_annotation(annotation) held = _typeddicts_in(inner, frozenset()) leads_back = not visiting.isdisjoint(held) for nested in held: if nested in visiting: continue # Name the part inside a TypedDict the member holds, whether # the member's parser failed or there is none. try: deeper, sure = _unread_in(nested, leaf, visiting | {nested}) except TypeError as error: # The nested TypedDict cannot be read at all, which this # error says. return _unread_member(typeddict, key, error), True if deeper is not None: named = Unread(f"{typeddict.__name__}.{key}: {deeper.message}", deeper.constraint) if sure: return named, True if fallback is None: fallback = named leads_back = True if len(held) != 0: own = _metadata_problem(inner, metadata) if own is not None: return _unread_member(typeddict, key, own), True if leads_back: if fallback is None: fallback = _unread_member(typeddict, key, failed) continue return _unread_member(typeddict, key, failed), True return fallback, False def _typeddicts_in(annotation: object, seen: frozenset[object]) -> tuple[type, ...]: """Each TypedDict `annotation` is or holds in its type arguments, such as `Node | None` or `tuple[Node, ...]`, through aliases and `NewType`s, once each, in the order they are written. `seen` holds the aliases and `NewType`s being followed, so one that refers to itself ends.""" if isinstance(annotation, type) and is_typeddict(annotation): return (annotation,) if isinstance(annotation, NewType) or is_alias(annotation): if annotation in seen: return () try: value = ( annotation.__supertype__ if isinstance(annotation, NewType) else alias_value(annotation) ) except TypeError: # An alias that names what is not defined holds no TypedDict # this can look inside; the parser's error names it. return () return _typeddicts_in(strip_annotation(value)[0], seen | {annotation}) found: dict[type, None] = {} for argument in get_args(annotation): found.update(dict.fromkeys(_typeddicts_in(strip_annotation(argument)[0], seen))) return tuple(found) def _metadata_problem(inner: object, metadata: tuple[object, ...]) -> TypeError | None: """The error the checker raises for `metadata` on a member of type `inner`: a constraint it does not read, or a bound on a type that is not a number; None when it reads the metadata, as it does a note.""" try: _constrainable(inner, constraints_of(metadata)) except TypeError as error: return error return None def _unread_member(typeddict: type, key: str, failed: TypeError | None) -> Unread: """The member `key` of `typeddict` as unread: the error its parser raised, or, when it has no parser, that it is not a shape JSON takes.""" if failed is not None: constraint = isinstance(failed, UnreadConstraintError) return Unread(f"{typeddict.__name__}.{key}: {failed}", constraint) message = ( f"{typeddict.__name__}: {key} is not a shape JSON takes; write it as one -- a " "number, string, boolean, null, array, object, or an alias of one" ) return Unread(message, constraint=False) @functools.cache def _checker(typeddict: type) -> Parser: return parser(typeddict, no_leaf) def check( value: object, shape: type[T], loc: Loc = () ) -> tuple[T | None, tuple[ValidationProblem, ...]]: """Type-check `value` as `shape`, a TypedDict. Returns the typed value or None, and every problem. This function first refines `value` to JSON (arrays as tuples, string keys, finite floats), then checks it member by member. Each problem is located under `loc`. The returned value holds only what `shape` allows. A key that a closed TypedDict does not declare is reported as `unknown_key` and left out, and the value is still returned. Any other problem means the value is not returned (None). The nesting-depth limit is counted from the root of the document, where `loc` gives the position of `value`, so a `loc` of 255 levels leaves one more level. Raises `TypeError` if `shape` is not a TypedDict, or contains a type that has no parser. """ if not is_typeddict(shape): msg = f"{shape!r} is not a TypedDict" raise TypeError(msg) refined, problems = refine_json(value, loc) if len(problems) != 0: return None, with_input(problems, value, loc) typed, found = _checker(shape)(refined, loc) readable = all(problem.kind == "unknown_key" for problem in found) return (cast("T", typed) if readable else None), with_input(found, value, loc) # --- JSON Schema --------------------------------------------------------- JSONSchema: TypeAlias = dict[str, JSONValue] """A JSON Schema as a JSON object, with arrays as lists, which is the form validators accept.""" SchemaLeaf: TypeAlias = Callable[[object, "Schemas"], "JSONSchema | None"] """Writes a caller's own shapes into a schema. Asked first for every annotation, like a `Leaf`; returns None to decline. It receives the schema being written. This lets a caller's shape contain other shapes, written with `of`, or be written once in `$defs` with `defined`. """ DIALECT: Final = "https://json-schema.org/draft/2020-12/schema" """The dialect of every schema: JSON Schema draft 2020-12, the same dialect pydantic and zod use.""" _KEYWORDS: Final[Mapping[str, str]] = { "gt": "exclusiveMinimum", "ge": "minimum", "lt": "exclusiveMaximum", "le": "maximum", } """JSON Schema's keyword for each bound.""" _STRICTER: Final[Mapping[str, Callable[[float, float], float]]] = { "exclusiveMinimum": max, "minimum": max, "exclusiveMaximum": min, "maximum": min, } """For each keyword, the function that picks the stricter of two bounds.""" def no_schema_leaf(annotation: object, schemas: Schemas) -> JSONSchema | None: """The schema leaf for a caller with no shapes of its own.""" return None class Schemas: """One JSON Schema being written, and the `$defs` it holds so far. `of` writes an annotation as the checker reads it. It asks the leaf first at every depth, as `parser_for` asks a `Leaf`. Each TypedDict and type alias is written once in `$defs`, under its name (or its name plus a number, when another entry already has that name), and referenced wherever it occurs. A type that refers to itself becomes a schema that refers to itself. `document` returns the whole schema. """ __slots__ = ("_defs", "_leaf", "_names", "_uses") def __init__(self, leaf: SchemaLeaf = no_schema_leaf) -> None: self._leaf = leaf self._defs: dict[str, JSONSchema] = {} self._names: dict[object, str] = {} self._uses: dict[str, int] = {} def of(self, annotation: object) -> JSONSchema: """The schema of `annotation` as the checker reads it: its type, with the bounds and notes from `Annotated`. Each bound becomes the matching JSON Schema keyword (`Ge(0)` becomes `minimum`), and a `Doc` becomes the `description`. The schema keeps both the type's bounds and the bounds of its `NewType`, because the checker applies both. When both set the same keyword, the stricter bound is kept. An annotation the checker supports is written. Any other raises `TypeError`, which cannot happen if the caller already checked the annotation with `parser`. """ inner, metadata = strip_annotation(annotation) schema = self._type(inner) if len(metadata) == 0: return schema notes = [ item.documentation for item in _unpacked(metadata) if isinstance(item, typing_extensions.Doc) ] if len(notes) != 0: schema = {**schema, "description": "\n\n".join(notes)} for name, bound in constraints_of(metadata).items(): keyword = _KEYWORDS[name] held = cast("int | float | None", schema.get(keyword)) schema = {**schema, keyword: bound if held is None else _STRICTER[keyword](held, bound)} return schema def object_of(self, typeddict: type) -> JSONSchema: """The inline schema of `typeddict`: its keys, its required keys, and what any other key may hold.""" keys = typeddict_keys(typeddict) schema: JSONSchema = {"type": "object"} if len(keys.members) != 0: schema["properties"] = { key: self.of(annotation) for key, (annotation, _) in keys.members.items() } required: list[JSONValue] = [key for key in keys.members if key in keys.required] if len(required) != 0: schema["required"] = required if keys.closed: schema["additionalProperties"] = False elif not keys.open: extra = self.of(keys.extra_items) if len(extra) != 0: schema["additionalProperties"] = extra return schema def defined(self, key: object, name: str, write: Callable[[], JSONSchema]) -> JSONSchema: """A reference to the entry in `$defs` for `key`. `write` writes the entry the first time `key` is requested. The entry is named `name`, or `name` plus a number when another key already has that name. The entry is reserved before it is written, so a schema that refers to itself produces a reference to itself. If `write` fails, the reservation is removed, because a later reference to it would point to an empty schema, which accepts anything. Any entries or use counts that `write` added before it failed are also removed. """ name_held = self._names.get(key) if name_held is None: name_held, number = name, 1 while name_held in self._defs: number += 1 name_held = f"{name}{number}" # If the write fails, restore the previous state: the entries it # wrote for the types it contains, the uses it counted, and its # own name. before = (dict(self._defs), dict(self._names), dict(self._uses)) self._names[key] = name_held self._defs[name_held] = {} try: self._defs[name_held] = write() except BaseException: self._defs, self._names, self._uses = before raise self._uses[name_held] = self._uses.get(name_held, 0) + 1 return {"$ref": _pointer(name_held)} def document(self, root: JSONSchema) -> JSONSchema: """The whole schema: its dialect, `root`, and the `$defs`, sorted by name. When `root` refers to an entry that nothing else refers to, the entry is written inline instead. pydantic does the same for a model that does not refer to itself. The result shares no objects with the internal state, and no part of it shares objects with another part, so a caller may modify it freely. """ defs = dict(self._defs) target = next((name for name in defs if root == {"$ref": _pointer(name)}), None) if target is not None and self._uses[target] == 1: root = defs.pop(target) whole: JSONSchema = {"$schema": DIALECT, **root} if len(defs) != 0: whole["$defs"] = {name: defs[name] for name in sorted(defs)} return cast("JSONSchema", copied(whole)) def _type(self, inner: object) -> JSONSchema: """The schema of a type with `Annotated` already removed.""" found = self._leaf(inner, self) if found is not None: return found if inner is int: return {"type": "integer"} if inner is float: return {"type": "number"} if inner is bool: return {"type": "boolean"} if inner is str: return {"type": "string"} if inner is None or inner is types.NoneType: return {"type": "null"} if inner is JSONValue: return {} origin = get_origin(inner) if origin is Literal: # Sort the values, as `_literal` does: `get_args` reports a # `Literal`'s values in the order of the first equal `Literal` # built in the process. values: list[JSONValue] = sorted(get_args(inner), key=repr) return {"const": values[0]} if len(values) == 1 else {"enum": values} if is_union(inner): return {"anyOf": [self.of(branch) for branch in get_args(inner)]} if origin is tuple: return self._tuple(get_args(inner)) if origin in (Mapping, dict): value = self.of(get_args(inner)[1]) if len(value) == 0: return {"type": "object"} return {"type": "object", "additionalProperties": value} if isinstance(inner, type) and is_typeddict(inner): typeddict = inner return self.defined(typeddict, typeddict.__name__, lambda: self.object_of(typeddict)) if isinstance(inner, NewType): return self.of(inner.__supertype__) if is_alias(inner): alias = inner return self.defined(alias, alias.__name__, lambda: self.of(alias_value(alias))) msg = f"{inner!r} is not a shape JSON takes" raise TypeError(msg) def _tuple(self, arguments: tuple[object, ...]) -> JSONSchema: if len(arguments) == 2 and arguments[1] is Ellipsis: items = self.of(arguments[0]) return {"type": "array"} if len(items) == 0 else {"type": "array", "items": items} if len(arguments) == 0: return {"type": "array", "maxItems": 0} return { "type": "array", "prefixItems": [self.of(argument) for argument in arguments], "items": False, "minItems": len(arguments), } def _pointer(name: str) -> str: """The reference to the entry in `$defs` named `name`: a JSON pointer, escaped as a URI fragment.""" escaped = name.replace("~", "~0").replace("/", "~1") return f"#/$defs/{urllib.parse.quote(escaped, safe='')}" def json_schema(shape: type) -> JSONSchema: """The JSON Schema for the JSON that `check` accepts as `shape`, a TypedDict. The schema uses draft 2020-12 and is a JSON object, with arrays as lists and `$schema` first. The schema is built as follows: - A TypedDict becomes an object with its keys, its required keys, and what any other key may hold (nothing, for a closed TypedDict). - A bound becomes the matching JSON Schema keyword: `Ge(0)` becomes `minimum`. - A `Doc` becomes the `description`. Only a `Doc` produces a description; docstrings are for Python readers and are not used. This matches zod, which writes only what `.describe()` sets. - A `Literal` becomes its values. - A union becomes `anyOf` its branches. Each TypedDict or type alias is written once in `$defs`, under its name, and referenced wherever it occurs. The exception is `shape` itself, which is written inline unless it refers to itself. One difference comes from JSON Schema itself: it accepts a number with no fractional part, such as `1.0`, as an integer, while `check` requires `1`. Raises `TypeError` if `shape` is not a TypedDict, or contains a type that has no parser, in the same cases as `check`. """ if not is_typeddict(shape): msg = f"{shape!r} is not a TypedDict" raise TypeError(msg) _checker(shape) schemas = Schemas() return schemas.document(schemas.of(shape)) __all__ = [ "DIALECT", "Branch", "Constraints", "JSONSchema", "Leaf", "Loc", "Parsed", "Parser", "Qualifier", "SchemaLeaf", "Schemas", "Tag", "TypedDictKeys", "Unread", "UnreadConstraintError", "alias_value", "any_of", "check", "constrained", "constraints_of", "describe", "fixed_tuple", "has_shape", "is_alias", "is_integer", "is_union", "json_schema", "mapping_of", "no_leaf", "no_schema_leaf", "object_of", "one_of", "parser", "parser_for", "problem", "qualifiers", "sequence_of", "shape_of", "strip_annotation", "typeddict_keys", "unqualified", "unread_in", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/definition/000077500000000000000000000000001527014137600315345ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py000066400000000000000000000371761527014137600336630ustar00rootroot00000000000000"""Reading metadata fields against their definitions, in either Zarr format. Every extension point of a Zarr document (a v3 codec, data type, chunk grid, chunk key encoding or storage transformer; a v2 dtype or numcodecs codec) is a metadata field: a name, and a configuration whose JSON the extension defines. A **definition** declares what that JSON is and what it may contain. A definition is a value, not a class to subclass. It has: - `name`, the name the metadata uses; - `configuration`, the TypedDict for the configuration's JSON. This is the single declaration of the JSON: the type checker is compiled from it, and a checked configuration has it as its static type. It is interpreted as the typing spec defines a TypedDict: `total`, `Required`, `NotRequired`, `closed` and `extra_items` mean what they mean to a type checker, whether or not its module postpones annotations. A member's type carries its bounds, in the same way pydantic reads them: a gzip `level` is `Annotated[int, Interval(ge=0, le=9)]`; - `rules`, a function over that TypedDict that yields the spec violations a type cannot express (such as constraints between members), located in the configuration. The kind of metadata a definition defines is its type. The v3 kinds are in `zarr_metadata.v3.definition`, and the v2 kinds are in `zarr_metadata.v2.definition`. This module holds what both formats share: `Definition` and `WithFillValue`, which the kinds derive from; `resolve`, which reads a field in a scope; the three results of a reading; and `Context`, the protocol of a scope. **Reading JSON.** There are three steps. Each feeds the next, and each can be used on its own by a caller that has only JSON: 1. `check(value, SomeTypedDict)`, from `zarr_metadata.typed_json`, type-checks JSON against a TypedDict and needs nothing else. It returns a value of the TypedDict or None, and every problem, each with a location. The value contains only what the TypedDict admits. A member typed with a field alias is checked as metadata field JSON. 2. `definition.read_configuration(configuration)` returns the configuration as that definition reads it. It type-checks the configuration, checks the envelope of each nested field (an extra member, a `must_understand` of `false`), and then runs the rules, for one configuration: `GZIP_CODEC.read_configuration({"level": 12})`. 3. `resolve(field, CodecDefinition, CORE_AND_EXTENSIONS)` reads a whole field in a scope. It checks the envelope, maps the name to a definition, checks the configuration, and reads each nested field the same way. It returns the result of the reading and every problem. The result is one of three types. `AcceptedField` is a field read by the definition that claims its name, with the configuration it checked and allowed, and its nested fields, each read the same way. `UnclaimedField` is a name that nothing in scope claims. It is not checked, which is what keeps the format open. `RefusedField` is a field with problems that explain why it was rejected. Each type has the field's JSON, the `name` it is written with, the kind it was read as (`read_as`), the `definition` that claims its name (None for `UnclaimedField`), and its nested fields as the scope read them (`nested`). The two types a model can hold, `AcceptedField` and `UnclaimedField`, have a `configuration` and `to_json()`, which returns the field as a document writes it. Two of them are equal when they read the same, however each was spelled. `ResolvedField` is the union of the three, for use with `match`. `configuration_of(field, GZIP_CODEC)` returns the configuration of a field that `GZIP_CODEC` accepted, typed as its TypedDict (`GzipCodecConfiguration`), and None for any other field, including one that `GZIP_CODEC` claimed and refused. A field is read as one of the kinds of its format, with or without type arguments. `resolve(field, Definition, context)` raises `TypeError`, because nothing is filed under `Definition`. A whole v3 array document is read by `read_array_metadata_v3`, in `zarr_metadata.model`. Its reading returns each field with its location in the document. from zarr_metadata.definition import resolve from zarr_metadata.v3.codec.gzip import GZIP_CODEC from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition resolved, problems = resolve({"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE_AND_EXTENSIONS) resolved # RefusedField(..., definition=CodecDefinition(name='gzip'), ...) problems[0].loc # ('configuration', 'level') problems[0].input # 12 dict(problems[0].ctx) # {'ge': 0, 'le': 9} resolved, problems = resolve({"name": "gzip", "configuration": {"level": 5}}, CodecDefinition, CORE_AND_EXTENSIONS) resolved.definition is GZIP_CODEC # True resolved.configuration # {'level': 5} Problems are values, not exceptions: `ValidationProblem(loc, message, kind)`, with `kind` one of `invalid_type`, `invalid_value`, `missing_key`, `unknown_key` and `invalid_json`. A field with an unknown key is still read: the key is reported, and the configuration is checked without it. So a consumer that tolerates unknown keys can filter problems by kind and use the result. The field is valid only when there are no problems at all. Each problem stores the content of its message as data, as pydantic's errors and zod's issues do: `input` is the value found at `loc` (the `12` above), and `ctx` is what was expected, when that is more than a type: the bounds `{"ge": 0, "le": 9}`, or the values of a closed set. **Writing an extension.** Write a TypedDict that declares the configuration's JSON, including bounds; a function for the rules a type cannot express; and a definition. Then build a scope that contains the definition. The TypedDict must be a `typing_extensions.TypedDict`, because `closed` and `extra_items` come from PEP 728, and `typing.TypedDict` does not accept them on the Python versions this package supports. The rules receive the configuration and its nested fields as the scope read them. A field that is read stores its nested fields as `AcceptedField.nested`, a `Nested` mapping keyed by location, so a struct's rules can access its field types. `read_configuration` reads in no scope, so it passes no nested fields. A rule's message should show a value as JSON, as the package's own messages do, with `zarr_metadata.problems.shown`: `null`, `[1, 2]`, `"C"`. A rule reports the location of a problem. The value at that location becomes the problem's `input` automatically. Define each function at module top level. A model holds the definitions that read its fields, so the model can be pickled, and compares equal after loading, only if its definitions can. A lambda or a closure cannot be pickled, and a `functools.partial` can be pickled but does not compare equal to its loaded copy. from collections.abc import Iterator from typing import Annotated, NotRequired from annotated_types import Ge from typing_extensions import TypedDict from zarr_metadata import ValidationProblem from zarr_metadata.definition import Nested from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition class AcmeLz4Configuration(TypedDict, closed=True): acceleration: Annotated[int, Ge(1)] dictionary: NotRequired[str] dictionary_size: NotRequired[Annotated[int, Ge(1)]] def acme_lz4_rules( configuration: AcmeLz4Configuration, nested: Nested ) -> Iterator[ValidationProblem]: if "dictionary" in configuration and "dictionary_size" not in configuration: yield ValidationProblem( ("dictionary_size",), "a dictionary needs its size", "missing_key" ) ACME_LZ4 = CodecDefinition( name="acme.lz4", configuration=AcmeLz4Configuration, kind="bytes_bytes", size="dynamic", rules=acme_lz4_rules, ) CONTEXT = CORE_AND_EXTENSIONS.update(codecs=[*CORE_AND_EXTENSIONS.codecs.values(), ACME_LZ4]) The TypedDict declares how keys it does not declare are handled. With `closed=True`, such a key is a problem, as above. With `extra_items=`, such a key must hold that type. With `closed=False`, anything is allowed. A TypedDict that declares none of these is open by default and would accept a misspelled key silently, so a definition rejects it. Its members must be JSON shapes: `int`, `float`, `bool`, `str`, `None`, `JSONValue`, a `Literal`, `tuple[T, ...]` and `tuple[T1, T2]`, a union, a TypedDict, `Mapping[str, V]`, a `NewType` and a type alias. A number's type may carry bounds, written with annotated-types and read the way pydantic reads them: `Gt`, `Ge`, `Lt`, `Le` and `Interval`, at most one from each side, at any depth. For example, `tuple[Annotated[int, Ge(1)], ...]` bounds each element. A value out of bounds is an `invalid_value` problem. Its message says what the type admits ("expected an integer >= 1, got 0"), and its `ctx` holds the bounds. The rules run only on a configuration within its bounds, so a rule can rely on them, as pydantic's after-validators and zod's refinements do. Until a value out of bounds is fixed, it is the only problem reported for the configuration. `Annotated` may also carry a note, as a string or a `Doc`. Any other metadata (a `MinLen`, a `Predicate`, pydantic's `Field`) is rejected when the definition is built, because a type whose constraint the checker does not enforce would give false information. A data type also declares its fill value. `fill_value` is the JSON shape of a fill value, as an annotation the checker reads (for example `Int8FillValue`, whose type carries the range). `fill_value_rules` is a function that yields the spec violations in a fill value of that shape that the type cannot express, such as a hex string of the wrong width, or a number of byte values that does not match the size. The rules receive the configuration, the nested fields as the scope read them, and the typed fill value, so a struct checks each field's fill value against that field's own type. `fill_value_problems(data_type, value)` checks a fill value against a data type field the scope read. If nothing in scope claims the data type, the fill value is not checked. A data type that does not declare its fill value accepts any JSON. A fill value can have more than one spelling (`"NaN"` and `"0x7fc00000"` are the same `float32`), so a data type declares the canonical spelling of each value with `fill_value_canonical`. It receives the same arguments as the rules, plus a fill value the rules allow. Two fill values are the same value of the type exactly when their canonical spellings are written the same way: the same JSON, as `json.dumps` writes it. `==` is not the same test, because it treats `-0.0` (a distinct `float32`) as equal to `0.0`. `canonical_fill_value(data_type, value)` returns the canonical spelling, or `UNSET` for a fill value with a problem. A data type that does not declare `fill_value_canonical` keeps each value as written. **The simplest spelling.** A field without problems has a simplest equivalent spelling, and fields are compared by it. Each nested field is written in its own simplest spelling, then the definition's `canonical` is applied (blosc drops a `typesize` that `noshuffle` ignores, and a rectilinear grid run-length encodes its chunk shapes). The envelope uses the shortest form every reader accepts: a data type with nothing to configure is its bare name, and any other field is an object, `{"name": ...}`, because a Zarr v3.0 reader does not accept a short-hand name in `codecs`. Raw bits write their size back into the name, in decimal, so `r008` becomes `r8`. A field with any problem, including an unknown key, has no simplest spelling, because a simpler spelling would erase what its author wrote. The result of `canonical` is checked again, and an invalid result raises `ValueError`, which indicates a bug in the definition. `canonical_fill_value` writes a fill value in the same way, using the data type that read it. **Scopes as values.** A scope reads documents of one Zarr format. `ZarrV3Context`, in `zarr_metadata.v3.definition`, and `ZarrV2Context`, in `zarr_metadata.v2.definition`, are the scopes of each format. They are separate classes, and `Context` is the protocol both implement: the Zarr format a scope reads, and `claimant`, which says which definition of a kind reads a name. That is all `resolve` needs, so any object that provides these two can be a scope. A scope has one slot for each kind of field its format has, named like the document member that holds such fields: `ZarrV3Context(codecs=[...], data_types=[...])` builds one, and `context.codecs["gzip"]` reads one slot. A type checker rejects a definition in the wrong slot, and a scope of one format given to a reader of the other. The constructor and the reader also reject them at run time, with `TypeError`. Two scopes are equal when they are of one format and file the same definitions, and equal scopes have equal hashes. `context.union(*others)` returns the smallest scope that contains each of them, or raises a `ScopeConflictError` naming each name filed with two different definitions. `update` replaces the definitions in the slots it is given and keeps the others, and takes a slot as definitions or as a mapping by name, so `context.update(codecs={**context.codecs, "gzip": MY_GZIP})` replaces one definition on purpose. A model's `refined_in` moves it to a scope that claims more names and contradicts nothing. `refines` orders two models by how much they know. A name that was unclaimed and is now read by a definition is a gain. The reverse is a loss. One name read by two different definitions is a conflict. A definition checks itself when it is built. Each of these raises a `TypeError` that says what is wrong: a `configuration` that is not a TypedDict, that does not declare how undeclared keys are handled, or that has a member no checker reads (the error names the TypedDict that contains it); a `name` that is not a string; a member declared as a function (`rules`, `canonical`, `fill_value_rules`, `storage`, `shape_rules`, `chunk_lengths`, `chunk_rules`, `transition`, `pipelines`) that is not a function; a data type's `fill_value` that no checker reads, or that contains a metadata field, which a value of the data type never does; a codec `kind` that is not one of the three, or a `size` that is not `"static"` or `"dynamic"`; a function that is never called for a codec of its kind (chunk rules or pipelines of a bytes -> bytes codec, which receives bytes, or a `transition` of a codec that outputs bytes); a data type named the way raw bits of one size are written; an extra field named after a member the spec defines, whose `finite` is not a boolean, or whose configuration does not declare `must_understand`. Building a definition files it nowhere: only a scope built from it does. """ from zarr_metadata.definition._definition import ( AcceptedField, Context, Definition, EmptyConfiguration, Nested, RefusedField, ResolvedField, UnclaimedField, WithFillValue, canonical_fill_value, canonical_of, configuration_of, fill_value_problems, resolve, with_problems, ) from zarr_metadata.definition._scope import ( ClaimKey, Conflict, ScopeConflictError, ) __all__ = [ "AcceptedField", "ClaimKey", "Conflict", "Context", "Definition", "EmptyConfiguration", "Nested", "RefusedField", "ResolvedField", "ScopeConflictError", "UnclaimedField", "WithFillValue", "canonical_fill_value", "canonical_of", "configuration_of", "fill_value_problems", "resolve", "with_problems", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/definition/_context.py000066400000000000000000000302071527014137600337330ustar00rootroot00000000000000"""How a format's scope files its definitions. No global registry files definitions. Reading a field means deciding which definition its name refers to. That decision depends on the *scope* the field is read in, not on the definitions themselves. A scope is a value built from definitions. `Context` is the protocol a reader needs from one. `ZarrV3Context` and `ZarrV2Context`, in each format's `definition` module, are the scopes of each format. They are separate classes. Each holds its definitions as tables: one table per kind, keyed by the name each definition is filed under. The functions here file, look up, update and join such tables. """ from __future__ import annotations from collections.abc import Callable, Iterable, Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Any, TypeAlias, cast, get_args, get_origin, get_type_hints from typing_extensions import TypeIs, TypeVar from zarr_metadata.definition._definition import ( Context, Definition, as_kind, described, filed_under, format_of, kind_of, ) from zarr_metadata.definition._scope import ( Conflict, ScopeConflictError, disagreements_of, kind_name, ) if TYPE_CHECKING: from zarr_metadata.definition._definition import D from zarr_metadata.definition._scope import Claims, Disagreements from zarr_metadata.v2._context import ZarrV2Context from zarr_metadata.v3._context import ZarrV3Context Tables: TypeAlias = Mapping[type[Definition[Any]], Mapping[str, Definition[Any]]] """A scope's definitions: one table for each kind its format has, keyed by the name each definition is filed under. A table holds definitions of one kind only.""" Given: TypeAlias = Iterable[Definition[Any]] | Mapping[str, Definition[Any]] """The definitions given for one slot of a scope: an iterable of them, or a mapping from the name each is filed under to it, as a scope's slot is.""" K = TypeVar("K", bound=Definition[Any]) def slot_kinds( slots: type, accepts: Callable[[type[Definition[Any]]], TypeIs[type[K]]] ) -> Mapping[str, type[K]]: """The kind each slot of a scope holds, read from `slots`: the TypedDict a scope's constructor takes, whose keys are the slots, each annotated `Iterable[K] | Mapping[str, K]`, in that order and with `Mapping` from `collections.abc`, for a kind `K` that `accepts`, the check for a kind of the scope's format. A scope class derives its slots from it, so they are declared once. Raises `TypeError`, when the scope class is defined, for a member annotated any other way, or with a kind that is not of the format. """ kinds: dict[str, type[K]] = {} for slot, annotation in get_type_hints(slots).items(): held = _held(annotation) if held is None: msg = f"{slots.__name__}.{slot}: a slot is annotated Iterable[K] | Mapping[str, K] for a kind K, got {annotation!r}" raise TypeError(msg) kind = as_kind(held) if not accepts(kind): msg = f"{slots.__name__}.{slot}: {kind.__name__} is not a kind of the scope's format" raise TypeError(msg) kinds[slot] = kind return MappingProxyType(kinds) def _held(annotation: object) -> object: """The kind `K` that a slot annotated `Iterable[K] | Mapping[str, K]` holds, or None for any other annotation.""" arms = get_args(annotation) if len(arms) != 2 or get_origin(arms[0]) is not Iterable or len(get_args(arms[0])) != 1: return None (held,) = get_args(arms[0]) return held if arms[1] == Mapping[str, held] else None def filed( kinds: Mapping[str, type[Definition[Any]]], given: Mapping[str, Given], zarr_format: int, ) -> Tables: """The tables of the definitions `given` for each slot of a scope whose slots hold `kinds`: one table per kind, each kind of the format present. Raises `TypeError` for a slot the scope does not have, for a definition of another kind than its slot holds, and for a slot given as a mapping with a key that is not the name of its definition, and `ScopeConflictError` for two different definitions given under one name in a slot. """ unknown = sorted(set(given) - set(kinds)) if len(unknown) != 0: msg = f"a v{zarr_format} scope has no slot {', '.join(map(repr, unknown))}" raise TypeError(msg) tables: dict[type[Definition[Any]], Mapping[str, Definition[Any]]] = {} conflicts: list[Conflict] = [] for slot, kind in kinds.items(): table: dict[str, Definition[Any]] = {} definitions = given.get(slot, ()) if _is_keyed(definitions): _refuse_misnamed(slot, definitions) definitions = definitions.values() for definition in definitions: misfiled = _misfiled(slot, kind, definition) if misfiled is not None: raise TypeError(misfiled) held = table.get(definition.name) if held is not None and held != definition: conflicts.append(Conflict((kind, definition.name), held, definition)) continue table[definition.name] = definition tables[kind] = MappingProxyType(table) if len(conflicts) != 0: raise ScopeConflictError(conflicts) return MappingProxyType(tables) def updated( tables: Tables, kinds: Mapping[str, type[Definition[Any]]], given: Mapping[str, Given], zarr_format: int, ) -> Tables: """`tables`, with the table of each slot `given` replaced by the definitions given for it. The tables of the other slots are kept. Raises as `filed` does.""" replacing = filed(kinds, given, zarr_format) replaced = {kinds[slot] for slot in given} return MappingProxyType( {kind: replacing[kind] if kind in replaced else table for kind, table in tables.items()} ) def joined_tables(kinds: Mapping[str, type[Definition[Any]]], many: Iterable[Tables]) -> Tables: """The smallest tables that file everything each of `many`, tables of the format whose slots hold `kinds`, files: their join. Raises `ScopeConflictError` when two of them file different definitions under one name of a kind. """ merged: dict[type[Definition[Any]], dict[str, Definition[Any]]] = { kind: {} for kind in kinds.values() } conflicts: list[Conflict] = [] for tables in many: for kind, table in tables.items(): for name, definition in table.items(): held = merged[kind].get(name) if held is not None and held != definition: conflicts.append(Conflict((kind, name), held, definition)) continue merged[kind][name] = definition if len(conflicts) != 0: raise ScopeConflictError(conflicts) return MappingProxyType({kind: MappingProxyType(table) for kind, table in merged.items()}) def definitions_in(tables: Tables, kind: type[D]) -> Mapping[str, D]: """The table of `kind` in `tables`, by the name each definition is filed under. Tables hold every kind of their format, so a kind they do not hold is a kind of the other format, and raises `TypeError`, as anything that is not a kind does. """ asked = as_kind(kind) if asked not in tables: msg = ( f"a v{format_of(next(iter(tables)))} scope holds the kinds of its format, " f"got {kind_name(asked)} definitions, a kind of v{format_of(asked)}" ) raise TypeError(msg) return cast("Mapping[str, D]", tables[asked]) def claimant_in(tables: Tables, kind: type[D], name: str) -> D | None: """The definition of `kind` in `tables` that reads `name` (a name as a document uses it), or None: the one filed under the name `filed_under` maps `name` to.""" table = definitions_in(tables, kind) filed_name, _ = filed_under(as_kind(kind), name) return None if filed_name is None else table.get(filed_name) def disagreements(context: ZarrV3Context | ZarrV2Context, claims: Claims) -> Disagreements: """Where `context` reads `claims`, a reading's claims, differently: the names it would gain and the names it conflicts with, as `Disagreements` describes them. A claim is keyed by the name its definition is filed under, so it is looked up by that name. Raises `TypeError` for a claim of the other format's kind, as `definitions_in` does. A model calls this to check a scope it is moved to. It is not a method of the scope classes, whose public API is the definitions they hold. """ # This module files, looks up and compares the tables a scope holds, # so it reads them here. tables = context._tables # pyright: ignore[reportPrivateUsage] return disagreements_of(lambda kind, name: definitions_in(tables, kind).get(name), claims) def slot_contents( tables: Tables, kinds: Mapping[str, type[Definition[Any]]] ) -> dict[str, tuple[Definition[Any], ...]]: """The definitions in `tables`, by the slot each kind is given in: a scope's constructor arguments, which it pickles as.""" return {slot: tuple(tables[kind].values()) for slot, kind in kinds.items()} def tables_key( tables: Tables, ) -> frozenset[tuple[type[Definition[Any]], frozenset[tuple[str, Definition[Any]]]]]: """What `tables` file, as a value to hash: each kind, with its names and definitions.""" return frozenset((kind, frozenset(table.items())) for kind, table in tables.items()) def _is_keyed(given: Given) -> TypeIs[Mapping[str, Definition[Any]]]: """Whether the definitions `given` for a slot are a mapping by name rather than an iterable of definitions. A mapping keyed some other way is caught by `_refuse_misnamed`.""" return isinstance(given, Mapping) def _refuse_misnamed(slot: str, definitions: Mapping[str, object]) -> None: """Raise `TypeError` for a key of `definitions`, a slot given as a mapping, that is not the name of the definition it holds. A value that is not a definition is left for `_misfiled` to refuse.""" for key, value in definitions.items(): if isinstance(value, Definition) and key != value.name: msg = f"{slot} files each definition under its name, got {value.name!r} under {key!r}" raise TypeError(msg) def _misfiled(slot: str, kind: type[Definition[Any]], value: object) -> str | None: """Why `value` does not belong in `slot`, which holds definitions of `kind`; None when it does. `value` is checked as a value of any type, for a caller without a type checker.""" found = kind_of(value) if found is kind: return None if not isinstance(value, Definition): return f"{slot} holds {kind_name(kind)} definitions, got {value!r}" of = "a definition of no kind" if found is None else f"a {kind_name(found)} definition" return f"{slot} holds {kind_name(kind)} definitions, got {value.name!r}, {of}" def refuse_others(contexts: Iterable[object], scope: type[ZarrV2Context | ZarrV3Context]) -> None: """Raise `TypeError` unless each of `contexts` is a `scope`, the class of one format's scopes, so a scope's `union` takes only scopes of its own format.""" for context in contexts: if not isinstance(context, scope): zarr_format = format_of(next(iter(scope.kinds.values()))) msg = ( f"a scope reads documents of one Zarr format; a union of v{zarr_format} scopes " f"takes v{zarr_format} scopes, got {described(context)}" ) raise TypeError(msg) C = TypeVar("C", bound=Context) def scoped(context: object, default: C) -> C: """The scope for a reader of `default`'s format: `context`, or `default` when `context` is None. Raises `TypeError` for anything else, such as a scope of the other format, or a value that is not one of this package's scopes.""" if context is None: return default if not isinstance(context, type(default)): msg = ( f"a v{default.zarr_format} document is read in a scope of that format, a " f"{type(default).__name__}, got {described(context)}" ) raise TypeError(msg) return context __all__ = [ "Given", "Tables", "claimant_in", "definitions_in", "disagreements", "filed", "joined_tables", "refuse_others", "scoped", "slot_contents", "slot_kinds", "tables_key", "updated", ] _definition.py000066400000000000000000002321001527014137600343140ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/definition"""Reading one metadata field against its definition. There are three steps. Each step feeds the next and needs more input than the step before: 1. The type check needs the value and a TypedDict. `check` runs the checker compiled from the TypedDict's annotations over the JSON. It checks the shape member by member, locates every problem, and returns a value of the TypedDict. A key the TypedDict does not declare is reported and left out. A member that holds another metadata field (a shard's codecs, a cast's data type) is annotated with a field alias such as `ZarrV3CodecField`, and is checked as an envelope: a name, or a named configuration. Which definition the name refers to is a question for a scope, so the check does not need one. 2. The rules need the definition. They are plain functions over the checked TypedDict, for constraints that a type cannot express (a bound, or members that depend on each other). `Definition.read_configuration` runs the check and then the rules, for a caller that has one configuration. 3. The reading needs a scope. `resolve` maps the field's name to a definition through a `Context`, checks the configuration, and reads each nested field found by the check, in the same scope. A definition is a value, not a class to subclass. It holds the name the metadata uses, the TypedDict that is the single declaration of the configuration's JSON, and its rules. It checks itself when it is built. The kind of metadata it defines is its type (`CodecDefinition`, `ChunkGridDefinition`), and a scope files it by that type. Building a definition files it nowhere: only a scope built from it does. """ # The hooks of `Definition` (`_named_configuration`, `_envelope_problems` and # the rest) are private: a kind overrides them, and no user calls them. The # functions in this module that read a field call them, from outside the # class body, which pyright reports as private use. This module is where # reading is implemented, so it is allowed here, and nowhere else. # pyright: reportPrivateUsage=false from __future__ import annotations import dataclasses import functools from collections.abc import Callable, Iterable, Iterator, Mapping from dataclasses import dataclass from types import MappingProxyType from typing import ( TYPE_CHECKING, Any, ClassVar, Final, Generic, Literal, Protocol, TypeAlias, TypeGuard, cast, get_origin, get_type_hints, ) from typing_extensions import TypeAliasType, TypedDict, TypeVar, is_typeddict from zarr_metadata._common import JSONValue, ZarrV3NamedConfigJSON from zarr_metadata._json import ( Problems, ValidationProblem, copied, frozen, is_object, is_tuple, json_text, refine_json, refine_user_data, shown, with_input, ) from zarr_metadata._sentinel import UNSET from zarr_metadata._typed_json import ( Loc, Parsed, Parser, is_alias, parser, problem, typeddict_keys, unread_in, ) from zarr_metadata.definition._envelope import ENVELOPE_KEYS, envelope_problems, name_problem if TYPE_CHECKING: from collections.abc import Sequence C = TypeVar("C", default=Mapping[str, JSONValue]) """A configuration: the TypedDict that a definition uses to declare its JSON. Any configuration is a `Mapping[str, JSONValue]`. That is the default when a kind is written without a type argument: `resolve(field, CodecDefinition, context)` is typed as `CodecDefinition[Mapping[str, JSONValue]]`, with no unknown types. """ T = TypeVar("T") D = TypeVar("D", bound="Definition[Any]") F = TypeVar("F", bound="WithFillValue[Any]") def no_rules(*_: object) -> Iterator[ValidationProblem]: """The rules of a definition (of any kind) that has no rules. Every well-typed configuration is allowed.""" yield from () def unchanged(configuration: T) -> T: """The canonical form of a configuration that has no simpler spelling: the configuration itself.""" return configuration def fill_value_as_written(configuration: object, nested: object, value: T) -> T: """The canonical spelling of a fill value for a data type that has only one spelling per value: the fill value as written.""" return value class EmptyConfiguration(TypedDict, closed=True): """The configuration of a definition with nothing to configure. Its field is written with its name only.""" _FIELD_KINDS: Final[dict[TypeAliasType, type[Definition[Any]]]] = {} """Maps each field alias to the kind that a member annotated with it is read as. Each kind adds its aliases when its class is built.""" @dataclass(frozen=True, kw_only=True, slots=True) class Definition(Generic[C]): """One extension's metadata as JSON: its name, its configuration TypedDict, and its rules. `configuration` is the TypedDict, and so it is the single declaration of the JSON. The checker is compiled from it, it is the static type of a checked configuration, and a document's author writes to it. It is interpreted as the typing spec defines a TypedDict: `total`, `Required`, `NotRequired`, `closed` and `extra_items` mean what they mean to a type checker. It also declares how keys it does not declare are handled. With `closed=True`, such a key is a problem, reported and left out. With `extra_items=`, such a key must have that type. With `closed=False`, anything is allowed. `rules` yields each violation of the spec it finds in a configuration of that type. It receives only a configuration that has passed the check: it contains only what the TypedDict admits, each member is within the bounds its type declares, and the nested fields are passed as the scope read them (for example, a struct's field types). The nested fields are empty when no scope read the configuration. `read_configuration` runs both steps, for a caller that has JSON. Each function receives the configuration as a read-only view, not a `dict`. `copy.deepcopy` and `json.dumps` reject this view. A function that simplifies a spelling must build a new mapping, such as `{**configuration}` without the member, instead of editing its input. `canonical` is where two spellings of the configuration that mean the same thing are reduced to one. A definition built by hand rejects inputs it could not read with: a `configuration` that is not a TypedDict, that does not declare how undeclared keys are handled, or that has a member no checker reads (the error names the member); and a `name` or rules of the wrong type. """ _is_kind: ClassVar[bool] = False """Whether this class is a kind, which is what a scope files definitions by. Set by `class Kind(Definition, kind=True, format=3)`. Read from the class's own namespace and never inherited: a subclass of a kind is a definition of that kind, and does not declare `kind`. """ _format: ClassVar[Literal[2, 3] | None] = None """The Zarr format of the documents that contain fields of this kind. It is declared with the kind and inherited by its definitions, and it sets the format a scope reads.""" label: ClassVar[str] = "definition" """The name of the kind in error messages, for example "codec".""" field_aliases: ClassVar[tuple[TypeAliasType, ...]] = () """The field aliases used to annotate a configuration member that holds a field of this kind. For a codec: `ZarrV3CodecField` and `ZarrV3StaticCodecField`.""" name: str """The name the metadata uses. A scope files the definition under this name.""" configuration: type[C] """The configuration TypedDict.""" rules: Callable[[C, Nested], Iterable[ValidationProblem]] = no_rules """The spec violations in a well-typed configuration and its nested fields, located in the configuration.""" canonical: Callable[[C], C] = unchanged """A well-typed, allowed configuration in its simplest equivalent spelling. This handles only the definition's own members. `canonicalize` puts each nested field in its own canonical form, because it knows the location of each one. """ @classmethod def _name_problem(cls, name: str, at: Loc) -> ValidationProblem | None: """The problem to report at `at` when no document of the format may use `name` for a field of this kind; None when it may. For Zarr v3, a name may be used when it is a valid extension name under the spec.""" return name_problem(name, at) @classmethod def _well_named(cls, name: str) -> bool: """Whether a document of the format may use `name` for a field of this kind, as decided by `_name_problem`.""" return cls._name_problem(name, ()) is None @classmethod def _filed_under(cls, name: str) -> tuple[str | None, dict[str, JSONValue] | None]: """Interpret a name used in a document: return the name its definition is filed under, and the configuration encoded in the name. By default a name is filed as itself and encodes nothing: `(name, None)`. A kind whose names encode configuration overrides this. For example, the v3 data type `r16` is filed under `r*` with `{"bits": 16}`. A name that no document uses, and that exists only to file a definition, returns `(None, None)`. """ return name, None def _carrying_name(self, configuration: Mapping[str, JSONValue]) -> str | None: """The name that encodes `configuration` for this definition. This is the inverse of `filed_under`: `r16` for `r*` with `{"bits": 16}`. None when this definition's names encode nothing.""" return None @classmethod def _named_configuration( cls, value: object ) -> tuple[str | None, Mapping[str, object] | None, Problems]: """Split `value` (a field as a document of the format writes it) into `(name, configuration, problems)`, as the module-level `named_configuration` does for a v3 field.""" return named_configuration(value) @classmethod def _envelope_problems(cls, value: object) -> Problems: """Every reason `value` is not a valid envelope for a field of this kind in this format. The contents of the configuration are not checked.""" return envelope_problems(value, allow_must_understand_false=False) @classmethod def _envelope_json(cls, name: str, configuration: Mapping[str, JSONValue]) -> JSONValue: """A field with `name` and `configuration`, as a document of the format writes it, in the shortest form every reader accepts. For v3 this is an object.""" if len(configuration) != 0: return {"name": name, "configuration": configuration} return {"name": name} @classmethod def _unclaimed_json( cls, json: JSONValue, name: str, configuration: Mapping[str, JSONValue] ) -> JSONValue: """A field of this kind that no definition claims, as a document writes it: `json` is the field as written. By default the name and configuration go in the envelope every reader accepts, as for a field a definition read.""" return cls._envelope_json(name, configuration) @classmethod def _refined( cls, data: object, loc: Loc, definition: Definition[Any] | None ) -> tuple[JSONValue | None, Problems]: """`data`, a field of this kind at `loc`, refined to JSON before it is read, with every reason it is not JSON. `definition` is the definition that claims the field, if any. By default every number must be finite, as RFC 8259 requires.""" return refine_json(data, loc) @classmethod def _configuration_loc(cls, loc: Loc) -> Loc: """The location of the configuration of a field at `loc`: under `configuration` for v3, or at the field itself for a format that writes the parameters next to the name.""" return (*loc, "configuration") @classmethod def _name_loc(cls, loc: Loc) -> Loc: """The location of the name of a field at `loc` that is written as an object: under `name` for v3.""" return (*loc, "name") @classmethod def _alias_problems( cls, alias: TypeAliasType, definition: Definition[Any], loc: Loc ) -> Problems: """The problems with using `definition` in a member annotated with `alias`, one of this kind's `field_aliases`. Empty by default. A kind with an alias that admits only some of its definitions overrides this. For example, `ZarrV3StaticCodecField` admits only codecs of static size. """ return () def __init_subclass__( cls, *, kind: bool = False, format: Literal[2, 3] | None = None, **kwargs: object ) -> None: """Register a subclass. `kind=True` declares a kind, with the Zarr `format` of the documents that contain its fields. This marks the class but not its subclasses, as `typing.Protocol` and SQLAlchemy's `__abstract__` do.""" # Use the explicit form, not bare `super()`. A dataclass with slots # is rebuilt, and the cell that bare `super()` reads refers to the # discarded class. super(Definition, cls).__init_subclass__(**kwargs) # A dataclass with slots is built twice. The second build has no # class keywords but reuses the first class's namespace, and the # last class built is the one used to read documents. So the mark is # kept in the namespace, and the aliases are registered last. if kind: if format not in (2, 3): msg = ( f"{cls.__name__}: a kind declares the Zarr format its fields belong to, " f"format=2 or format=3, got {format!r}" ) raise TypeError(msg) cls._is_kind = True cls._format = format elif format is not None: msg = f"{cls.__name__}: only a kind declares a format; a definition of a kind has its kind's" raise TypeError(msg) for alias in cls.__dict__.get("field_aliases", ()): _FIELD_KINDS[alias] = cls def __post_init__(self) -> None: refusal = _malformed(self) or self._refusal() if refusal is not None: raise TypeError(refusal) kind = kind_of(self) try: _vet(self.configuration, None if kind is None else format_of(kind)) except TypeError as error: msg = f"{self.name!r}: {error}" raise TypeError(msg) from error def __repr__(self) -> str: # Keep this short, because readings that contain definitions show # them. A full repr would show every function the definition holds, # with its address. return f"{type(self).__name__}(name={self.name!r})" def _refusal(self) -> str | None: """A description of what is wrong with the members a kind adds; None if nothing is wrong, or if the kind adds no members.""" return None def _finite_numbers(self) -> bool: """Whether every number in a configuration this definition reads must be finite, as RFC 8259 requires. A kind whose configuration may hold user data overrides this.""" return True @property def requires_configuration(self) -> bool: """Whether a document must include a configuration, meaning the TypedDict has a required key.""" return len(typeddict_keys(self.configuration).required) != 0 def _check_configuration(self, value: object, loc: Loc = ()) -> tuple[C | None, Problems]: """Type-check `value` as this definition's configuration, and check the envelope of each nested field. `zarr_metadata.typed_json.check` does only the type check. This method also checks the envelope of each metadata field that a member holds. """ configuration, problems = _configuration_checked( value, self.configuration, loc, finite=self._finite_numbers() ) return configuration, with_input(problems, value, loc) def read_configuration(self, value: object, loc: Loc = ()) -> tuple[C | None, Problems]: """Type-check `value`, then check it with the rules. Return the configuration if it is valid, and every problem. The rules run only on a configuration that passed the type check, including its bounds, and whose nested fields are well formed. Such a configuration contains only what its TypedDict admits. So a rule never receives a member of the wrong type, a member out of bounds, or a member the type does not allow. No scope reads the nested fields here, so the rules see none of them as read. A rule about a nested field (for example, a struct field whose type has values of varying size) finds nothing to check. `resolve` reads the field in a scope and runs every rule. """ configuration, problems = self._check_configuration(value, loc) if configuration is None: return None, problems refused = ruled(self, lambda: self.rules(read_only(configuration), _nothing_nested()), loc) return (configuration if len(refused) == 0 else None), ( *problems, *with_input(refused, value, loc), ) def _malformed(definition: Definition[Any]) -> str | None: """A description of what makes a hand-built definition unusable, checked before its configuration is examined; None if it is usable.""" name = cast("object", definition.name) if not isinstance(name, str): return f"a definition's name is a string, got {name!r}" for member in _function_members(type(definition)): value = getattr(definition, member) if not callable(value): return f"{name!r}: {member} is a function, got {value!r}" kind = type(definition) filed, _ = kind._filed_under(name) bad = None if filed is None else kind._name_problem(name, ()) if bad is not None: return ( f"{name!r}: {bad.message}, so no document names it, and nothing would ever read " "with this definition" ) return None @functools.cache def _function_members(kind: type[Definition[Any]]) -> tuple[str, ...]: """The members `kind` declares as functions: each member whose annotation is a `Callable`.""" return tuple( member for member, annotation in get_type_hints(kind).items() if get_origin(annotation) is Callable ) def filed_under( kind: type[Definition[Any]], name: str ) -> tuple[str | None, dict[str, JSONValue] | None]: """Interpret a name used in a document: return the name its definition is filed under, and the configuration encoded in the name. A name is filed as itself and encodes nothing, except for raw bits: the data type `r16` is filed under `r*` and encodes `{"bits": 16}`. `r*` itself is filed under nothing, `(None, None)`, because no document uses it. """ return kind._filed_under(name) @dataclass(frozen=True, kw_only=True, slots=True, repr=False) class WithFillValue(Definition[C]): """A definition of a type whose arrays have a fill value: the fill value's JSON shape, the rules for it, and its canonical spelling. This is not a kind. The data type kind of each format derives from it, so `fill_value_problems` can check a fill value for either format. `fill_value` is the JSON shape of a fill value, as an annotation. The checker reads it the same way it reads a configuration's members, including any range. `fill_value_rules` yields the spec violations in a fill value of that shape that the type cannot express. It receives the configuration, the nested fields as the scope read them, and the typed fill value. `fill_value_canonical` writes a fill value without problems in the single canonical spelling of its value. So two fill values are the same value of the type exactly when their canonical spellings are written the same way. """ fill_value: object = JSONValue """The JSON shape of a fill value, as an annotation, such as `Int8FillValue`.""" fill_value_rules: Callable[[C, Nested, Any], Iterable[ValidationProblem]] = no_rules """The spec violations in a fill value of that shape, located in the fill value.""" fill_value_canonical: Callable[[C, Nested, Any], JSONValue] = fill_value_as_written """A fill value without problems, in the single canonical spelling of its value.""" def _refusal(self) -> str | None: try: _fill_value_parser(self.fill_value) except TypeError as error: return f"{self.name!r}: fill_value: {error}" return None @functools.cache def _fill_value_parser(annotation: object) -> Parser: """The checker for a fill value's JSON shape, compiled once. Raises `TypeError` naming any part that no checker reads, or any metadata field in it.""" return parser(annotation, _no_field) def kind_of(definition: object) -> type[Definition[Any]] | None: """The kind of `definition`: the nearest class in its MRO declared with `kind=True`. None for a definition with no kind, which no scope can file, and for a value that is not a definition.""" return _kind_of_class(type(definition)) @functools.lru_cache(maxsize=256) def _kind_of_class(cls: type[object]) -> type[Definition[Any]] | None: """`kind_of` for a definition of class `cls`. It walks the MRO once per class. The cache is bounded, so a class created on each call is not kept forever.""" return next((base for base in cls.__mro__ if _declares_kind(base)), None) def format_of(kind: type[Definition[Any]]) -> Literal[2, 3]: """The Zarr format of the documents that contain fields of `kind`, as declared by the kind. Raises `TypeError` for a class that is not a kind.""" found = as_kind(kind)._format if found is None: # a kind cannot be declared without one msg = f"{kind!r} declares no format" raise TypeError(msg) return found def _declares_kind(cls: type[object]) -> TypeGuard[type[Definition[Any]]]: """Whether `cls` itself was declared with `kind=True`. The mark is read from the class's own namespace, which a subclass does not share.""" return vars(cls).get("_is_kind") is True and issubclass(cls, Definition) @functools.lru_cache(maxsize=256) def _kind_named(kind: object) -> type[Definition[Any]] | None: """The kind that `kind` refers to, with type arguments removed, or None. This is the cached lookup behind `as_kind`, run once per form of a kind (`CodecDefinition`, `CodecDefinition[Any]`), since a class is a kind by its declaration. The cache is bounded, so a class created on each call is not kept forever.""" origin = get_origin(kind) or kind if isinstance(origin, type) and _declares_kind(origin): return origin return None def as_kind(kind: object) -> type[Definition[Any]]: """The kind of metadata that `kind` refers to, with type arguments removed. Raises `TypeError` if it refers to no kind. A scope files definitions by kind, so a field is read as a kind (`CodecDefinition`, or `CodecDefinition[Any]`). It is never read as the base `Definition` or as a class that declares no kind, because nothing is filed under those, and a field read as one would never be checked. """ try: found = _kind_named(kind) except TypeError: # The value is not hashable. No kind is unhashable. found = None if found is not None: return found msg = ( f"{kind!r} is not a kind of metadata; read a field as one of the kinds of a format, " "such as CodecDefinition" ) raise TypeError(msg) def field_kind(annotation: object) -> type[Definition[Any]] | None: """The kind of metadata field held by a member annotated with `annotation` (`CodecDefinition` for `ZarrV3CodecField`), or None if the member holds no field.""" if not is_alias(annotation): return None return _FIELD_KINDS.get(annotation) class Context(Protocol): """A scope: the definitions in scope while metadata of one Zarr format is read. This is what reading a field needs from a scope: the Zarr format of the documents it reads, and which definition of a kind claims a name. `ZarrV3Context` and `ZarrV2Context` implement it, each with one slot per kind of field its format has, and so does any other object that provides these two. `resolve` reads a field in any `Context`. """ @property def zarr_format(self) -> Literal[2, 3]: ... def claimant(self, kind: type[D], name: str) -> D | None: ... @dataclass(frozen=True, slots=True) class _NestedField: """A metadata field that the check found inside a configuration: its location, its kind, and its JSON. The checker returns this in place of the JSON wherever a field alias appears. `_checked` collects each one and puts its JSON back. It is carried in the value instead of being recorded separately, so a union branch that did not match leaves nothing behind. """ loc: Loc kind: type[Definition[Any]] json: JSONValue alias: TypeAliasType """Whether the member that holds it accepts only codecs of static size.""" def _field(annotation: object) -> Parser | None: """The one checker shape that this module defines: a member that holds a metadata field. The value is checked only as a bare name or an object, and passed on. Whoever reads the field (`check` without a scope, `resolve` with one) checks the envelope and maps the name to a definition. """ kind = field_kind(annotation) if kind is None or not is_alias(annotation): return None alias = annotation def parse(value: object, loc: Loc) -> Parsed: # The value is refined JSON, which the checker types only as # `object`. If it is not a field at all, the kind's envelope check # reports that. return _NestedField(loc, kind, cast("JSONValue", value), alias), () return parse def _vetting(annotation: object) -> Parser | None: """`_field`, but rejecting annotations that a definition's configuration cannot contain and still be read. A member typed with the envelope's own TypedDict (`ZarrV3MetadataFieldJSON`) is checked as plain JSON, so its name would never be mapped to a definition and its configuration would never be checked. A TypedDict that does not declare how undeclared keys are handled is open, so an undeclared key would never be reported. """ if annotation is ZarrV3NamedConfigJSON: msg = ( "a member typed ZarrV3MetadataFieldJSON checks as plain JSON, and is never read as " "a field; annotate it with the field alias of its kind: " + ", ".join(alias.__name__ for alias in _FIELD_KINDS) ) raise TypeError(msg) if ( isinstance(annotation, type) and is_typeddict(annotation) and not typeddict_keys(annotation).declared ): msg = ( f"{annotation.__name__} says nothing of the keys it does not declare, so it is " "open, and such a key would go unreported; declare it closed=True, or " "extra_items= for what such a key holds, or closed=False to take any, " "on a typing_extensions.TypedDict" ) raise TypeError(msg) return _field(annotation) def _no_field(annotation: object) -> Parser | None: """A leaf checker that rejects a field alias. A fill value is a value of its data type and never contains a metadata field.""" if is_alias(annotation) and field_kind(annotation) is not None: name = annotation.__name__ msg = f"{name} holds a metadata field, and a fill value is a value of its data type" raise TypeError(msg) return None @functools.cache def _vet(configuration: type, zarr_format: int | None) -> None: """Reject a configuration that no definition of a kind of Zarr format `zarr_format` could read with, and say what is wrong with it.""" if not is_typeddict(configuration): msg = ( f"configuration is {configuration!r}; give the TypedDict the configuration's JSON takes" ) raise TypeError(msg) def vetting(annotation: object) -> Parser | None: # A document of one format never holds a field of the other, so a # definition never reads one. kind = field_kind(annotation) if kind is not None and zarr_format is not None and format_of(kind) != zarr_format: msg = ( f"a member typed {getattr(annotation, '__name__', annotation)} holds a field of " f"a Zarr v{format_of(kind)} document, which a definition of a v{zarr_format} " "field never reads" ) raise TypeError(msg) return _vetting(annotation) vetting(configuration) unread = unread_in(configuration, vetting) if unread is not None: # A constraint the checker does not read is one a rule checks. advice = "; check it in the definition's `rules`" if unread.constraint else "" msg = f"{unread.message}{advice}" raise TypeError(msg) @dataclass(frozen=True, slots=True) class _Checker: """A TypedDict's checker, compiled once, and whether a value of the TypedDict can contain a nested field.""" parse: Parser nests: bool @functools.cache def _checker(shape: type) -> _Checker: """The checker for a TypedDict, compiled once. Raises `TypeError` naming any part that no checker reads.""" met: list[object] = [] def leaf(annotation: object) -> Parser | None: found = _field(annotation) if found is not None: met.append(annotation) return found return _Checker(parser(shape, leaf), len(met) != 0) def _checked( shape: type, value: object, loc: Loc ) -> tuple[object, Problems, tuple[_NestedField, ...]]: """Check `value` as `shape`. Return the typed value (with each nested field restored as written), every problem, and the nested fields in order.""" typed, found, nested = _typed(shape, value, loc) return _put_back(typed, _as_written, nested), found, nested def _typed( shape: type, value: object, loc: Loc ) -> tuple[object, Problems, tuple[_NestedField, ...]]: """Check `value` as `shape`, leaving each nested field where the checker found it. Return the typed value, every problem, and the nested fields in order.""" checker = _checker(shape) typed, found = checker.parse(value, loc) if not checker.nests: return typed, found, () nested: list[_NestedField] = [] _collect(typed, nested) return typed, found, tuple(nested) def _collect(value: object, nested: list[_NestedField]) -> None: """Append to `nested` each nested field that the checker returned inside `value`, in order.""" if isinstance(value, _NestedField): nested.append(value) elif is_tuple(value): for entry in value: _collect(entry, nested) elif is_object(value): for entry in value.values(): _collect(entry, nested) def _as_written(field: _NestedField) -> JSONValue: """A nested field restored as it was written.""" return field.json def _declared(field: _NestedField) -> JSONValue: """A nested field restored as it was written, except for envelope members that are not declared. Those are reported and left out, in the same way the checker leaves out a key that a closed TypedDict does not declare.""" if not isinstance(field.json, Mapping): return field.json envelope = cast("Mapping[str, JSONValue]", field.json) return {key: value for key, value in envelope.items() if key in ENVELOPE_KEYS} def _put_back( value: object, put: Callable[[_NestedField], JSONValue], nested: Sequence[_NestedField] ) -> object: """Replace each of `nested` (the fields the checker returned inside `value`) with the result of `put`. Return `value` unchanged when there are none.""" if len(nested) == 0: return value return _putting_back(value, put) def _putting_back(value: object, put: Callable[[_NestedField], JSONValue]) -> object: """`_put_back` for a value that contains nested fields: a recursive walk.""" if isinstance(value, _NestedField): return put(value) if is_tuple(value): return tuple(_putting_back(entry, put) for entry in value) if is_object(value): return {key: _putting_back(entry, put) for key, entry in value.items()} return value def _usable(problems: Sequence[ValidationProblem]) -> bool: """Whether a value with these problems can still be read. An unknown key is allowed. Any other problem is not.""" return all(found.kind == "unknown_key" for found in problems) def read_only(configuration: C) -> C: """`configuration` in the form a definition's functions receive it: a read-only view of the field's own configuration, so a function that assigns a member fails. This function makes only the top-level members read-only. The value a member holds (for example, an object in a struct's `fields`) is passed as is. While a configuration is being read, that value can be mutable, and a function that writes into it changes the configuration. An accepted field's configuration is already read-only at every level. """ return cast("C", MappingProxyType(cast("Mapping[str, JSONValue]", configuration))) def asked(definition: Definition[Any], what: str, ask: Callable[[], T], at: Loc | None = None) -> T: """Return the result of `ask`, which calls the function `what` of `definition`. A definition's functions are the extension author's code. If one raises an error, a note is added that says which definition's function raised it and, when known, where the reader was reading. """ try: return ask() except Exception as error: error.add_note(_raised_by(definition, what, at)) raise def _raised_by(definition: Definition[Any], what: str, at: Loc | None) -> str: """The note added to an error raised by the function `what` of `definition`: which definition's function raised it and, when known, where the reader was reading.""" where = "" if at is None else f", reading {at!r}" return f"raised by the {what} of {definition.name!r}{where}" def ruled( definition: Definition[Any], ask: Callable[[], Iterable[ValidationProblem]], at: Loc ) -> Problems: """The problems found by `ask`, which calls `definition`'s rules, located under `at`. Rules are the extension author's code. Their output is checked to have the declared type. If they raise an error, a note is added that says which definition's rules raised it and where they were reading. """ try: found = tuple(cast("Iterable[object]", ask())) except Exception as error: error.add_note(_raised_by(definition, "rules", at)) raise if len(found) == 0: return () for item in found: if not isinstance(item, ValidationProblem): msg = f"{definition.name!r}: its rules yield ValidationProblem values, got {item!r}" raise TypeError(msg) return _located(at, cast("tuple[ValidationProblem, ...]", found)) def _located(prefix: Loc, problems: Iterable[ValidationProblem]) -> Problems: """`problems`, each located under `prefix`. The input may be lazy, because a kind's `_envelope_problems` may return an iterator.""" found = tuple(problems) if len(found) == 0: return () return tuple(dataclasses.replace(each, loc=(*prefix, *each.loc)) for each in found) def _envelope(field: _NestedField) -> Problems: """The problems with a nested field's envelope, located at the field.""" return _located(field.loc, field.kind._envelope_problems(field.json)) def _configuration_checked( value: object, shape: type[T], loc: Loc = (), *, finite: bool = True ) -> tuple[T | None, Problems]: """Check `value` as `shape`, as `typed_json.check` does, and check the envelope of each nested field. A definition's `read_configuration` starts with this step. A member typed with a field alias holds a metadata field, and its envelope is checked the same way as in a document. A `must_understand` of `false` is a problem of the configuration, and no value is returned. An extra member is an unknown key: it is reported and left out, and the value is still returned. This matches how the checker reports and leaves out a key that a closed TypedDict does not declare. A number that is not finite is a problem unless `finite` is false, for a configuration that holds user data; the TypedDict still decides where such a number may be. """ refined, problems = (refine_json if finite else refine_user_data)(value, loc) if len(problems) != 0: return None, problems typed, found, nested = _typed(shape, refined, loc) problems = (*found, *(problem for field in nested for problem in _envelope(field))) if not _usable(problems): return None, problems return cast("T", _put_back(typed, _declared, nested)), problems def named_configuration( value: object, ) -> tuple[str | None, Mapping[str, object] | None, Problems]: """Split a metadata field into `(name, configuration, problems)`. A field is a bare name, or an object that contains a name. A `None` name means the value is not a metadata field at all. A `None` configuration means the bare spelling was used, or the key was left out. The only problem reported is a configuration that is present but not an object, at `("configuration",)`. """ if isinstance(value, str): return value, None, () if not is_object(value): return None, None, () entry = value name = entry.get("name") if not isinstance(name, str): return None, None, () if "configuration" not in entry: return name, None, () configuration = entry["configuration"] if not isinstance(configuration, Mapping): return ( name, None, problem(("configuration",), f"expected an object, got {shown(configuration)}"), ) return name, cast("Mapping[str, object]", configuration), () def _nothing_nested() -> Nested: """The nested fields of a field that contains no read fields: an empty mapping.""" return {} @dataclass(frozen=True, slots=True, kw_only=True) class AcceptedField(Generic[D]): """A field that a definition in scope read: the name it is written with, the definition, and the configuration the definition allowed. A problem with the envelope (an extra member, a `must_understand` of `false`) is reported with the field, and the field is still read. The same applies to a problem in a nested field, which belongs to that nested field, as `nested` shows. Two fields are equal when they read the same, however each was spelled, as compared by `field_key`. For example, `"bytes"` and `{"name": "bytes"}` are one field. So are a blosc with and without the `typesize` that `noshuffle` ignores, because the definition's `canonical` removes it. Equal fields have equal hashes. """ json: JSONValue """The field as written, refined (arrays as tuples). It is not used for equality.""" name: str """The name the field is written with, for example `"r16"`, even though its definition is filed under `r*`.""" definition: D """The definition that read the field.""" configuration: Mapping[str, JSONValue] """The configuration, type-checked and allowed by the rules. For raw bits, this is the configuration encoded in the name. Each nested field is written as that field's `to_json` writes it, so the configuration shows what was read, however it was spelled. For example, a shard's `"crc32c"` and `{"name": "crc32c"}` are the same index codec. """ nested: Nested = dataclasses.field(default_factory=_nothing_nested) """The fields in the configuration, each as the scope read it, keyed by location in the configuration. For example, a struct's field types are at `("fields", 0, "data_type")` and a shard's codecs are at `("codecs", 0)`. A definition's functions use this to inspect the fields inside its own configuration. """ read_as: type[Definition[Any]] = dataclasses.field(init=False, repr=False) """The kind of metadata the field was read as: the kind of its definition.""" def __eq__(self, other: object) -> bool: if not is_field(other): return NotImplemented return field_key(self) == field_key(other) def __hash__(self) -> int: return hash(field_key(self)) def __post_init__(self) -> None: # Check the annotations at runtime. A field built by hand (an # extension may do this) fails here, not later in a function that # trusts it. definition = cast("object", self.definition) kind = kind_of(definition) refusal = ( f"a field read is read by a definition of a kind, got {definition!r}" if kind is None else _misread(definition, kind, self.name) ) if refusal is not None: raise TypeError(refusal) object.__setattr__(self, "read_as", kind) # Everything the field returns is read-only at every level, so a # field cannot reach a state where its key, `==` and `refines` # disagree. object.__setattr__(self, "json", frozen(self.json)) object.__setattr__(self, "configuration", frozen(self.configuration)) object.__setattr__(self, "nested", MappingProxyType(dict(self.nested))) def __reduce__(self) -> tuple[Callable[..., AcceptedField[Any]], tuple[object, ...]]: # Read-only views cannot be pickled, so pickle the field from its constructor arguments. return _accepted_field, ( copied(self.json), self.name, self.definition, copied(self.configuration), dict(self.nested), ) def to_json(self) -> JSONValue: """The field as a document writes it, in a form every reader accepts. The configuration is as read, and the result shares no objects with the field. The envelope uses the shortest form every reader accepts. A data type with nothing to configure is written as its bare name, as core data types have been written since Zarr v3.0. Any other field is an object, `{"name": ...}`, because a Zarr v3.0 reader does not accept a bare name in `codecs` (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L585-L592). A name that encodes its configuration, as the raw bits name does, is written alone. """ return copied(written_json(self)) @dataclass(frozen=True, slots=True, kw_only=True) class UnclaimedField: """A field that nothing in scope claims: an extension that the scope does not check. Accepting such fields is what keeps the format open. It is equal to another unclaimed field when both are of the same kind, have the same name, and a document writes the same JSON for both. Nothing in scope interprets the field, so it is compared as JSON text, as `field_key` defines. """ json: JSONValue """The field as written, refined (arrays as tuples). It is not used for equality.""" name: str """The name that nothing in scope claims.""" read_as: type[Definition[Any]] """The kind of metadata the field was read as: the kind a claiming definition would have.""" configuration: Mapping[str, JSONValue] = dataclasses.field(init=False) """The configuration as written, not checked; empty when no configuration is written.""" def __eq__(self, other: object) -> bool: if not is_field(other): return NotImplemented return field_key(self) == field_key(other) def __hash__(self) -> int: return hash(field_key(self)) def __post_init__(self) -> None: # Check the annotations at runtime. Remove type arguments from # `read_as`, as `resolve` does. object.__setattr__(self, "read_as", as_kind(self.read_as)) name = cast("object", self.name) if not isinstance(name, str) or not self.read_as._well_named(name): msg = ( "a field nothing in scope claims is named as a document names a " f"{self.read_as.label}, got {name!r}" ) raise TypeError(msg) _, written, _ = self.read_as._named_configuration(self.json) configuration: Mapping[str, object] = {} if written is None else written object.__setattr__(self, "json", frozen(self.json)) object.__setattr__( self, "configuration", frozen(cast("Mapping[str, JSONValue]", configuration)) ) def __reduce__(self) -> tuple[Callable[..., UnclaimedField], tuple[object, ...]]: # Read-only views cannot be pickled, so pickle the field from its constructor arguments. return _unclaimed_field, (copied(self.json), self.name, self.read_as) @property def definition(self) -> None: """The definition that read the field. Always None, because no definition read it.""" return None @property def nested(self) -> Nested: """The nested fields as the scope read them. Always empty, because nothing read the configuration.""" return _nothing_nested() def to_json(self) -> JSONValue: """The field as a document writes it, sharing no objects with the field. The configuration is as written, in the envelope form every reader accepts, as `AcceptedField.to_json` writes it.""" return copied(written_json(self)) @dataclass(frozen=True, slots=True, kw_only=True) class RefusedField(Generic[D]): """A field that could not be read: it is not a field at all, not JSON, or rejected by the definition that claims its name. Its problems explain why.""" json: JSONValue | UNSET """The field as written, refined (arrays as tuples). `UNSET` when it is not JSON, which no document can contain.""" name: str | None """The name the field is written with; None when it has no name.""" read_as: type[Definition[Any]] """The kind of metadata the field was read as.""" definition: D | None = None """The definition that claims the name and rejected the field; None when nothing in scope claims it, or when the field has no name.""" nested: Nested = dataclasses.field(default_factory=_nothing_nested) """The nested fields, each as the scope read it; empty when the configuration was not checked against its TypedDict.""" def __eq__(self, other: object) -> bool: if not is_field(other): return NotImplemented return field_key(self) == field_key(other) def __hash__(self) -> int: return hash(field_key(self)) def __post_init__(self) -> None: # Check the annotations at runtime. Remove type arguments from # `read_as`, as `resolve` does. kind = as_kind(self.read_as) object.__setattr__(self, "read_as", kind) definition = cast("object", self.definition) refusal = None if definition is None else _misread(definition, kind, self.name) if refusal is not None: raise TypeError(refusal) if self.json is not UNSET: object.__setattr__(self, "json", frozen(self.json)) object.__setattr__(self, "nested", MappingProxyType(dict(self.nested))) def __reduce__(self) -> tuple[Callable[..., RefusedField[Any]], tuple[object, ...]]: # Read-only views cannot be pickled, so pickle the field from its constructor arguments. return _refused_field, ( UNSET if self.json is UNSET else copied(self.json), self.name, self.read_as, self.definition, dict(self.nested), ) ResolvedField = TypeAliasType( "ResolvedField", "AcceptedField[D] | UnclaimedField | RefusedField[D]", type_params=(D,) ) """One metadata field as a scope read it: accepted by the definition that claims its name, unclaimed, or refused.""" def _accepted_field( json: JSONValue, name: str, definition: Definition[Any], configuration: Mapping[str, JSONValue], nested: Nested, ) -> AcceptedField[Any]: """Rebuild an accepted field from its pickled arguments.""" return AcceptedField( json=json, name=name, definition=definition, configuration=configuration, nested=nested ) def _unclaimed_field(json: JSONValue, name: str, read_as: type[Definition[Any]]) -> UnclaimedField: """Rebuild an unclaimed field from its pickled arguments.""" return UnclaimedField(json=json, name=name, read_as=read_as) def _refused_field( json: JSONValue | UNSET, name: str | None, read_as: type[Definition[Any]], definition: Definition[Any] | None, nested: Nested, ) -> RefusedField[Any]: """Rebuild a refused field from its pickled arguments.""" return RefusedField(json=json, name=name, read_as=read_as, definition=definition, nested=nested) _FIELDS: Final = (AcceptedField, UnclaimedField, RefusedField) """The three possible results of reading a field in a scope.""" def _misread(definition: object, kind: type[Definition[Any]], name: object) -> str | None: """A description of why `definition` cannot be the definition that read a field named `name` as a `kind`; None when it can.""" if not isinstance(definition, kind): return f"a field read as a {kind.__name__} is read by one, got {definition!r}" filed = definition.name if not isinstance(name, str) or filed_under(kind, name)[0] != filed: return f"a field named {name!r} is read by the definition filed under it, got {filed!r}" return None def field_key(field: ResolvedField[Any]) -> tuple[object, ...]: """The key that `==` and `hash` compare for a field. It represents what the field means, not how it was spelled. An accepted field is compared by the definition that read it, by its configuration in the definition's canonical spelling (as `canonical_of` writes it) as JSON text, and by the key of each nested field. An unclaimed field is compared by its kind, its name and the JSON a document writes for it, as JSON text, because nothing interprets it. A refused field is compared by what was written and by the definition that rejected it. So two fields are equal when the parts a reader understands read the same, and the parts no reader interprets are written the same way. """ if isinstance(field, AcceptedField): definition = cast("Definition[Any]", field.definition) configuration: JSONValue = dict(field.configuration) # Each nested field is compared by its own key, so its location is # cleared before the definition's `canonical` sees the rest, in the # same order as `_canonical_field`. `canonical` simplifies only the # definition's own members. for loc in field.nested: configuration = _replaced(configuration, loc, None) view = read_only(cast("Mapping[str, JSONValue]", configuration)) spelled = asked( definition, "canonical", lambda: json_text(cast("JSONValue", dict(definition.canonical(view)))), ) return ("read", definition, spelled, _nested_key(field.nested)) if isinstance(field, UnclaimedField): return ("unclaimed", field.read_as, field.name, json_text(written_json(field))) return ( "refused", field.read_as, field.name, field.definition, UNSET if field.json is UNSET else json_text(field.json), _nested_key(field.nested), ) def own_key(field: AcceptedField[Any]) -> tuple[object, ...]: """The part of `field_key` for an accepted field that excludes nested fields: the definition and the canonical spelling of its own members.""" return field_key(field)[:3] def _nested_key(nested: Nested) -> tuple[tuple[Loc, tuple[object, ...]], ...]: """The key of each nested field, paired with its location.""" return tuple((loc, field_key(inner)) for loc, inner in nested.items()) def document_json(field: ResolvedField[Any]) -> JSONValue | UNSET: """A field as a document writes it, using the field's own values: as `to_json` writes it, or as originally written for a refused field. `UNSET` for a refused field that was not JSON. A writer serializes this without changing it, so it is not copied. `to_json` is a copy of this. """ if isinstance(field, RefusedField): return field.json return written_json(field) def written_json(field: AcceptedField[Any] | UnclaimedField) -> JSONValue: """A field that a scope accepted or left unclaimed, as a document writes it. This is `document_json` for a field that is JSON.""" kind = field.read_as if isinstance(field, UnclaimedField): return kind._unclaimed_json(field.json, field.name, field.configuration) if kind._filed_under(field.name)[1] is not None: return kind._envelope_json(field.name, {}) return kind._envelope_json(field.name, field.configuration) Nested: TypeAlias = Mapping[Loc, ResolvedField[Any]] """The fields in a configuration, each as the scope read it, keyed by location in the configuration.""" def is_field(value: object) -> TypeGuard[ResolvedField[Any]]: """Whether `value` is a field that a scope read: `AcceptedField`, `UnclaimedField` or `RefusedField`.""" return isinstance(value, _FIELDS) def held(field: ResolvedField[D] | UNSET) -> AcceptedField[D] | UnclaimedField: """Return `field`, which must come from a read that found no problems, as `AcceptedField` or `UnclaimedField`. Raises `TypeError` for a refused field or one never read, which such a read cannot produce.""" if isinstance(field, (AcceptedField, UnclaimedField)): return field msg = f"expected a field a read found nothing wrong with, got {field!r}" raise TypeError(msg) def fields_of( resolved: ResolvedField[Any], loc: Loc = () ) -> Iterator[tuple[Loc, ResolvedField[Any]]]: """Yield `resolved` (a field a scope read) with its location, then each nested field recursively, each with its location. `loc` is the location of `resolved`. A nested field is located inside the configuration, at the kind's `_configuration_loc` followed by the field's place, the same way `resolve` locates problems. Each field's nested fields are its `nested`. A field whose configuration was not checked has none. """ yield loc, resolved for place, inner in resolved.nested.items(): yield from fields_of(inner, (*resolved.read_as._configuration_loc(loc), *place)) def with_problems( fields: Iterable[tuple[Loc, ResolvedField[Any]]], problems: Sequence[ValidationProblem] ) -> Iterator[tuple[Loc, ResolvedField[Any], Problems]]: """Yield each of `fields` with its location and the problems in `problems` located inside it, including inside its nested fields. `fields` and `problems` must come from the same read: a reading's `fields()` and `problems`, or `fields_of` a field and the problems that `resolve` returned with it. A field's problems include those found when it was read, and those the document found with it in its position (its place in the pipeline, the chunk it receives, the array's shape). So a field with no problems is valid in that position, and `canonical_of` can write it. This groups the problems at every depth, as zod's `treeifyError` groups issues: a problem with a shard's inner codec belongs to the inner codec and to the shard. Each field comes before its nested fields, in the order of `fields_of`, so the last field that lists a problem is the innermost field containing it. A problem outside every field (with the fill value or the shape) is not listed under any field. """ located = list(fields) held: dict[Loc, list[ValidationProblem]] = {loc: [] for loc, _ in located} for found in problems: for depth in range(len(found.loc) + 1): holder = held.get(found.loc[:depth]) if holder is not None: holder.append(found) for loc, field in located: yield loc, field, tuple(held[loc]) def configuration_of(resolved: ResolvedField[Any], definition: Definition[C]) -> C | None: """The configuration of `resolved`, typed as `definition` declares it, if `definition` accepted it. `AcceptedField` stores its configuration as a read-only mapping. When called with the definition that read the field, or one equal to it (as with a pickled field), this returns a copy of it, a plain `dict` typed as the TypedDict, which shares nothing with the field. Returns None for any other field: one that another definition read, one that no definition claims, and one that `definition` claimed and refused. """ if not isinstance(resolved, AcceptedField) or resolved.definition != definition: return None return cast("C", copied(resolved.configuration)) def fill_value_problems(data_type: ResolvedField[F], value: object, loc: Loc = ()) -> Problems: """The problems with `value` as a fill value of `data_type` (a data type field a scope read). `value` is refined to JSON first. If it is not JSON, that is reported first, whatever the data type. It is then checked against the JSON shape that the data type's definition declares, and by the fill value rules, in the same way `read_configuration` reads a configuration: a key the shape does not declare is reported and left out, and the rules still check the rest. The rules see the nested fields of the configuration as the scope read them, so a struct checks each field's fill value against that field's own type. If the scope did not read the data type (it is out of scope or invalid), a JSON fill value is not checked. `loc` prefixes every problem. """ refined, problems = refine_json(value, loc) if len(problems) != 0 or not isinstance(data_type, AcceptedField): return with_input(problems, value, loc) definition, configuration = data_type.definition, data_type.configuration typed, problems = _fill_value_parser(definition.fill_value)(refined, loc) if not _usable(problems): return with_input(problems, value, loc) refused = ruled( definition, lambda: definition.fill_value_rules(read_only(configuration), data_type.nested, typed), loc, ) return with_input((*problems, *refused), value, loc) def canonical_fill_value(data_type: ResolvedField[F], value: object) -> JSONValue | UNSET: """`value` (a fill value of `data_type`, a data type field a scope read) in the single canonical spelling of its value; `UNSET` when it has a problem. The spelling comes from the data type's `fill_value_canonical`. Two fill values of a data type are the same value exactly when their canonical spellings are written the same way: the same JSON as `json.dumps` writes it. `==` is not the same test, because it treats `-0.0` as equal to `0.0`. A fill value with a problem found by `fill_value_problems` has no canonical spelling, just as `canonical_of` returns none for a field with a problem. The result is `UNSET` and not `None`, because `None` is the JSON `null`, which is a valid fill value for a data type the scope did not read (such a data type keeps a fill value as written). """ if len(fill_value_problems(data_type, value)) != 0: return UNSET refined, _ = refine_json(value, ()) return spelled_canonically(data_type, refined) def spelled_canonically(data_type: ResolvedField[F], value: JSONValue) -> JSONValue: """`value` (a fill value of `data_type` with no problems) in its canonical spelling, as `canonical_fill_value` returns it, without checking it again. The data type's `fill_value_canonical` is the extension author's code. Its result is checked to be JSON. If it raises an error, a note is added that says which data type's canonical spelling raised it. """ if not isinstance(data_type, AcceptedField): return value definition, configuration = data_type.definition, data_type.configuration spelled = asked( definition, "fill_value_canonical", lambda: cast( "object", definition.fill_value_canonical(read_only(configuration), data_type.nested, value), ), ) refined, problems = refine_json(spelled, ()) if len(problems) != 0: msg = ( f"{definition.name!r}: its fill_value_canonical gives JSON, got {spelled!r}: " f"{problems[0].message}" ) raise TypeError(msg) return refined def resolve( data: object, kind: type[D], context: Context, loc: Loc = () ) -> tuple[ResolvedField[D], Problems]: """Read `data` (one metadata field) as a `kind` in `context`. Return the result of the reading, and every problem. This runs all three steps for one field. `data` is refined to JSON and its envelope is checked. An extra member, a `configuration` that is not an object, and a `must_understand` that is not a boolean or is `false` are each a problem. The name is mapped to a definition in `context`. The configuration is checked against its TypedDict and by its rules. Each nested field found by the check is read the same way, in the same scope. A problem in a nested field belongs to that field and is reported at its location, as with a document's fields. The result is an `AcceptedField` read by the definition that claims the name; an `UnclaimedField` when nothing in scope claims the name (an extension with no model, not checked, which is what keeps the format open); or a `RefusedField`, with the problems that explain why. `loc` prefixes every problem. `kind` must be one of the kinds of a format (for v3, those `ZarrV3Kind` names; for v2, those `ZarrV2Kind` names), with or without type arguments, and `context` a scope of that format. Anything else raises `TypeError`. """ if not _is_scope(context): msg = f"a field is read in a scope, such as CORE_AND_EXTENSIONS, got {described(context)}" raise TypeError(msg) asked = as_kind(kind) _check_format(asked, context) name = asked._named_configuration(data)[0] definition = ( None if name is None or not asked._well_named(name) else context.claimant(asked, name) ) refined, problems = asked._refined(data, loc, definition) return resolve_refined(data, refined, problems, kind, context, loc) def _check_format(kind: type[Definition[Any]], context: Context) -> None: """Raise `TypeError` when `context` is a scope of a format other than `kind`'s.""" scope_format = context.zarr_format if scope_format != format_of(kind): msg = ( f"a {kind.label} is a field of a Zarr v{format_of(kind)} document, read in a scope " f"of that format, got a v{scope_format} scope" ) raise TypeError(msg) def _is_scope(value: object) -> bool: """Whether `value` has what `resolve` needs of a `Context`: a `zarr_format` of 2 or 3, and a `claimant` it can call. A class does not, though it may declare both.""" if isinstance(value, type): return False try: zarr_format = getattr(value, "zarr_format", None) claimant = getattr(value, "claimant", None) except Exception: # noqa: BLE001 - any value may be given, and only its shape is asked return False return type(zarr_format) is int and zarr_format in (2, 3) and callable(claimant) def described(context: object) -> str: """`context`, a value given as a scope, as an error message names it: its class and the format it reads, or its repr when it reads none, or its class when even that fails.""" zarr_format: object = None if not isinstance(context, type): try: zarr_format = getattr(context, "zarr_format", None) except Exception: # noqa: BLE001 - any value may be given, and the message must still be built zarr_format = None if type(zarr_format) is int and zarr_format in (2, 3): return f"a {type(context).__name__}, a v{zarr_format} scope" try: return repr(context) except Exception: # noqa: BLE001 - as above return f"a {type(context).__name__}" def resolve_refined( data: object, refined: JSONValue | None, problems: Problems, kind: type[D], context: Context, loc: Loc = (), ) -> tuple[ResolvedField[D], Problems]: """`resolve` for `data` that the caller has already refined. `refined` and `problems` are the result of `refine_json(data, loc)`. This is for a reader that refines a document's members once and keeps the result (the document a model holds is built from it), instead of refining each field again here. `problems` shows whether `data` is JSON. If it contains any problems, `refined` is not used. """ asked = as_kind(kind) _check_format(asked, context) if len(problems) != 0: # The value is not JSON, so it is not read. If it has a name, the # name still determines which definition claims it. A name that is # not a valid extension name under the spec is a problem here, as on # the other path, and no definition is looked up for it. name = asked._named_configuration(data)[0] bad = None if name is None else asked._name_problem(name, asked._name_loc(loc)) claimant = None if name is None or bad is not None else context.claimant(asked, name) refused = RefusedField(json=UNSET, name=name, read_as=asked, definition=claimant) found = problems if bad is None else (bad, *problems) return cast("ResolvedField[D]", refused), with_input(found, data, loc) resolved, found = _resolve_field(refined, asked, context, loc) return cast("ResolvedField[D]", resolved), with_input(found, data, loc) def _resolve_field( data: JSONValue, kind: type[Definition[Any]], context: Context, loc: Loc ) -> tuple[ResolvedField[Definition[Any]], Problems]: """Check the envelope of a refined field, then read the field. The result of the reading depends only on the configuration. An extra member or a `must_understand` of `false` does not affect the configuration, so it is reported alongside the field that was read, and later layers can still check that field. """ envelope = _located(loc, kind._envelope_problems(data)) resolved, found = _read(data, kind, context, loc) return resolved, (*envelope, *found) def _read( data: JSONValue, kind: type[Definition[Any]], context: Context, loc: Loc ) -> tuple[ResolvedField[Definition[Any]], Problems]: name, given, malformed = kind._named_configuration(data) if name is None: return RefusedField(json=data, name=None, read_as=kind), () if not kind._well_named(name): # The envelope check that every reader runs first reports this. No # definition is looked up for the name. return RefusedField(json=data, name=name, read_as=kind), () definition = context.claimant(kind, name) if len(malformed) != 0: # The configuration is not an object, which the envelope problems # report. The name still determines which definition claims the # field. return RefusedField(json=data, name=name, read_as=kind, definition=definition), () if definition is None: return UnclaimedField(json=data, name=name, read_as=kind), () _, carried = kind._filed_under(name) if carried is not None: return _read_carried(data, name, kind, definition, given, carried, loc) at = kind._configuration_loc(loc) if given is None and definition.requires_configuration: missing = problem(at, f"{name!r} requires a configuration", "missing_key") return RefusedField(json=data, name=name, read_as=kind, definition=definition), missing typed, found, nested = _typed(definition.configuration, {} if given is None else given, at) # Read the nested fields first, each one stack frame deeper, so the # rules see them as the scope read them. Each is put back as a document # writes it, so the configuration shows what was read, however each was # spelled. Their problems are reported after the problems from the # rules. within: dict[Loc, ResolvedField[Any]] = {} written: dict[Loc, JSONValue] = {} inside: list[ValidationProblem] = [] for field in nested: inside.extend(_envelope(field)) inner, found_inside = _read(field.json, field.kind, context, field.loc) within[field.loc[len(at) :]] = inner # The input was refined JSON, so the result is JSON, or a field # refused for its contents, never for not being JSON. written[field.loc] = cast("JSONValue", document_json(inner)) inside.extend(found_inside) inside.extend(_sized(field, inner)) # The rules may look up a nested field by its name, so they run only # when every nested field has a name. Any other problem with a nested # field belongs to that field and is reported at its location, as with # a document's fields. sound = _usable(found) and all(_named(field) for field in nested) configuration = ( cast( "Mapping[str, JSONValue]", _put_back(typed, lambda field: written[field.loc], nested), ) if sound else None ) own = list(found) if configuration is not None: own.extend( ruled(definition, lambda: definition.rules(read_only(configuration), within), at) ) if configuration is None or not _usable(own): refused = RefusedField( json=data, name=name, read_as=kind, definition=definition, nested=within ) return refused, (*own, *inside) read = AcceptedField( json=data, name=name, definition=definition, configuration=configuration, nested=within ) return read, (*own, *inside) def _named(field: _NestedField) -> bool: """Whether a nested field has a name, and its configuration (if any) is an object.""" name, _, malformed = field.kind._named_configuration(field.json) return name is not None and len(malformed) == 0 def _read_carried( data: JSONValue, name: str, kind: type[Definition[Any]], definition: Definition[Any], given: Mapping[str, object] | None, carried: Mapping[str, JSONValue], loc: Loc, ) -> tuple[ResolvedField[Definition[Any]], Problems]: """Read a field whose name encodes its configuration (raw bits, such as `r16`) with the definition its name is filed under. The document wrote a name, so a problem with the configuration encoded in the name (such as a size that is not a positive multiple of 8) is a problem of the field. A configuration written next to the name must be empty, so each member in it is an undeclared key. """ _, beside, _ = _checked( EmptyConfiguration, {} if given is None else given, kind._configuration_loc(loc), ) configuration, judged = definition.read_configuration(carried) # A problem with the configuration encoded in the name belongs to the # field. It is located at the field, where the name is, and the # `input` and `ctx` that described a configuration member do not apply # to the name, so they are cleared. problems = ( *beside, *(dataclasses.replace(found, loc=loc, input=UNSET, ctx={}) for found in judged), ) # A name is a single token. An undeclared part of the encoded # configuration cannot be left out, as an extra key next to the name # can, so any problem with the encoded configuration refuses the field. if configuration is None or not _usable(beside) or len(judged) != 0: refused = RefusedField(json=data, name=name, read_as=kind, definition=definition) return refused, problems return AcceptedField( json=data, name=name, definition=definition, configuration=configuration ), problems def _sized(field: _NestedField, inner: ResolvedField[Any]) -> Problems: """The problems with the definition that read `inner` in the member that holds `field`, as the kind of `field` reports them for its alias. A name that nothing in scope claims is not checked. Nothing is known about it. """ definition = inner.definition if definition is None: return () return field.kind._alias_problems(field.alias, definition, field.loc) def canonicalize( data: object, kind: type[D], context: Context, loc: Loc = () ) -> tuple[JSONValue | None, Problems]: """`data` (one metadata field) in its simplest equivalent spelling, and every problem. Only a field without problems has a simplest spelling. A simplest spelling expresses what the author wrote in fewer words. A key the TypedDict does not declare, an extra envelope member, or a `must_understand` of `false` is something the author wrote that the simplest spelling would erase. So a field with any problem returns None, with its problems. Otherwise each nested field is written in its own simplest spelling, and then the definition's `canonical` handles the rest. Its result is checked again, so a `canonical` that returns an invalid configuration raises `ValueError`, which indicates a bug in the definition, not in the field. The envelope uses the shortest form every reader accepts. A data type with nothing to configure is written as its bare name, as core data types have been written since Zarr v3.0. Any other field is an object, `{"name": ...}`, because a Zarr v3.0 reader does not accept a short-hand name in `codecs` (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L585-L592). There is no `must_understand`, because leaving it out means `true`. A name that nothing in scope claims keeps the configuration it was written with, because only its own definition could decide how to simplify it. For a field that a scope has already read, use `canonical_of` with its problems. """ resolved, problems = resolve(data, kind, context, loc) return canonical_of(resolved, problems), problems def canonical_of( resolved: ResolvedField[Any], problems: Sequence[ValidationProblem] ) -> JSONValue | None: """`resolved` (a field a scope read) in its simplest equivalent spelling, as `canonicalize` writes it; None when it has a problem. `problems` are the field's problems, as returned by `resolve`, or as `with_problems` returns them for each field of a reading. Only a field with no problems has a simplest spelling. A simpler spelling of a field with a problem would erase what its author wrote (such as a key its TypedDict does not declare), or would write something invalid. The result is built from what the scope read, without reading the field again, and is plain JSON that shares nothing with the field. """ if len(problems) != 0: return None simplest = _simplest(resolved) return None if simplest is None else copied(simplest) def _simplest(field: ResolvedField[Any]) -> JSONValue | None: """A field in its simplest spelling; None when the field or one of its nested fields was refused, because a refused field has no simplest spelling.""" if isinstance(field, AcceptedField): return _canonical_field(field) if isinstance(field, UnclaimedField): return field.to_json() return None def _canonical_field(resolved: AcceptedField[Any]) -> JSONValue | None: """An accepted field in its simplest equivalent spelling: nested fields first, then its own members. None when a nested field was refused.""" definition, name = resolved.definition, resolved.name configuration: JSONValue = dict(resolved.configuration) for loc, inner in resolved.nested.items(): simplest = _simplest(inner) if simplest is None: return None configuration = _replaced(configuration, loc, simplest) # Every function of a definition receives this read-only view. The # result of `canonical` (the view itself when nothing is simplified) is # converted to a dict. view = read_only(cast("Mapping[str, JSONValue]", configuration)) simplified = asked( definition, "canonical", lambda: dict(cast("Mapping[str, JSONValue]", definition.canonical(view))), ) _, refused = definition.read_configuration(simplified) if len(refused) != 0: msg = ( f"{definition.name!r}: its canonical gave {simplified!r}, which does not hold: " f"{list(refused)!r}" ) raise ValueError(msg) carrying = definition._carrying_name(simplified) if carrying is not None: return carrying return resolved.read_as._envelope_json(name, simplified) def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: """`value` with the item at `path` replaced by `new`. `path` must come from a check of `value`.""" if len(path) == 0: return new step, rest = path[0], path[1:] if isinstance(step, str) and isinstance(value, Mapping): return {**value, step: _replaced(value[step], rest, new)} if isinstance(step, int) and isinstance(value, tuple): return (*value[:step], _replaced(value[step], rest, new), *value[step + 1 :]) msg = f"{path!r} does not lead into {value!r}" raise TypeError(msg) __all__ = [ "AcceptedField", "C", "Context", "Definition", "EmptyConfiguration", "Nested", "Problems", "RefusedField", "ResolvedField", "UnclaimedField", "WithFillValue", "as_kind", "asked", "canonical_fill_value", "canonical_of", "canonicalize", "configuration_of", "document_json", "field_key", "field_kind", "fields_of", "filed_under", "fill_value_as_written", "fill_value_problems", "format_of", "held", "is_field", "kind_of", "named_configuration", "no_rules", "own_key", "read_only", "resolve", "resolve_refined", "ruled", "spelled_canonically", "unchanged", "with_problems", "written_json", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/definition/_envelope.py000066400000000000000000000142301527014137600340620ustar00rootroot00000000000000"""The default field envelope: how a field is written as a name or as an object with `name`, `configuration` and `must_understand`, and which names are valid extension names. This is the envelope Zarr v3 uses. `Definition` uses it by default, and a kind of another format (the v2 kinds) overrides each method that uses it. """ import functools import re from typing import Final from zarr_metadata._json import ValidationProblem, is_object, shown, shown_key ENVELOPE_KEYS: frozenset[str] = frozenset({"name", "configuration", "must_understand"}) """The keys a metadata field's envelope declares. Any other key is an unknown key.""" _EXTENSION_NAME: Final = re.compile(r"[a-z][a-z0-9_.-]+") """A registered extension name: the spec's `^[a-z][a-z0-9-_.]+$`, with the hyphen moved last so no regex engine reads it as a range.""" _URI: Final = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*:[A-Za-z0-9._~:/?#\[\]@!$&'()*+,;=%-]+") """A URI, to the extent this module checks names. It matches an RFC 3986 scheme, a colon, and one or more characters from the URI unreserved and reserved sets plus the `%` of percent-encoding (https://www.rfc-editor.org/rfc/rfc3986#section-2). Earlier versions of the spec required names of this form. The characters are listed explicitly instead of written as `\\S`. Python's `re` and ECMA-262 (the regex dialect of a JSON Schema `pattern`) disagree on what `\\S` matches: `\\x1c`-`\\x1f` and `\\x85` are whitespace in one, and `\\ufeff` is whitespace in the other. With `\\S`, a schema validator could accept a name that this reader rejects, or the reverse. """ EXTENSION_NAME_SCHEMA_PATTERN: Final = rf"^({_EXTENSION_NAME.pattern}|{_URI.pattern})(?![\s\S])" r"""The names `well_named` accepts, as a JSON Schema `pattern`. It matches either pattern against the whole string. It uses `(?![\s\S])` instead of `$`, because `$` also matches before a final newline. `_RAW_BYTES_SCHEMA_PATTERN` does the same.""" @functools.lru_cache(maxsize=4096) def well_named(name: str) -> bool: """Whether `name` is a valid extension name under the spec. Results are cached per name, because a document uses few names and a reader checks each one many times. A registered name "MUST start with one lower case letter a-z and then be followed by only lower case letters a-z, numerals 0-9, underscores, dots and dashes", regex `^[a-z][a-z0-9-_.]+$` (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1603-L1606); a URI, which earlier versions of the spec required, is "still permitted" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1614-L1618). """ return _EXTENSION_NAME.fullmatch(name) is not None or _URI.fullmatch(name) is not None def name_problem(name: str, at: tuple[str | int, ...]) -> ValidationProblem | None: """The problem to report at `at` when `name` is not a valid extension name under the spec; None when it is valid, as decided by `well_named`.""" if well_named(name): return None message = ( "expected an extension name -- lower-case letters, digits, '-', '_' and '.', " f"starting with a letter -- or a URI, got {shown(name)}" ) return ValidationProblem(at, message, "invalid_value") def envelope_problems( value: object, *, allow_must_understand_false: bool ) -> tuple[ValidationProblem, ...]: """Every reason `value` is not a valid v3 metadata field envelope. The contents of the configuration are not checked. The envelope is the part around the configuration: a `name` that is a valid extension name under the spec, a `configuration` that is an object with string keys, a boolean `must_understand`, and no other keys. `resolve` calls this on a field it has already narrowed to JSON, so the configuration is walked only once. """ if isinstance(value, str): bad = name_problem(value, ()) return () if bad is None else (bad,) if not is_object(value): return ( ValidationProblem( (), f"expected a metadata field (string or extension object), got {shown(value)}", "invalid_type", ), ) field = value problems: list[ValidationProblem] = [] for key in field: if not isinstance(key, str): problems.append( ValidationProblem( (), f"non-string metadata field key {shown_key(key)}", "invalid_type" ) ) elif key not in ENVELOPE_KEYS: problems.append(ValidationProblem((key,), f"unexpected key {key!r}", "unknown_key")) if "name" not in field: problems.append(ValidationProblem(("name",), "missing required key", "missing_key")) elif not isinstance(field["name"], str): problems.append(ValidationProblem(("name",), "expected a string name", "invalid_type")) elif (bad := name_problem(field["name"], ("name",))) is not None: problems.append(bad) if "configuration" in field: configuration = field["configuration"] if not is_object(configuration): problems.append( ValidationProblem(("configuration",), "expected an object", "invalid_type") ) elif not all(isinstance(k, str) for k in configuration): problems.append( ValidationProblem(("configuration",), "expected string keys", "invalid_type") ) if "must_understand" in field: must_understand = field["must_understand"] if not isinstance(must_understand, bool): problems.append( ValidationProblem(("must_understand",), "expected a boolean", "invalid_type") ) elif not allow_must_understand_false and not must_understand: problems.append( ValidationProblem( ("must_understand",), "false is not supported at this extension point", "invalid_value", ) ) return tuple(problems) __all__ = [ "ENVELOPE_KEYS", "EXTENSION_NAME_SCHEMA_PATTERN", "envelope_problems", "name_problem", "well_named", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py000066400000000000000000000237001527014137600333600ustar00rootroot00000000000000"""Operations on scopes: which definition a reading records for each name, how one reading refines another, and where a scope disagrees with a reading. A model is a document read in a scope. The meaning of the document depends only on what the scope returned for the names the document uses (its *claims*), not on everything the scope contains. Two readings of one document are ordered by how much they know. If a name was unclaimed and a definition later reads it, the name gains meaning and loses none. If one definition reads a name and then a different definition reads it, that is a conflict. The internal `disagreements` function in `zarr_metadata.definition._context`, which a model uses to compare a scope with its claims, is built on these operations. A scope's `union` raises the `ScopeConflictError` defined here when two scopes file different definitions under one name. """ from __future__ import annotations from collections.abc import Mapping from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal, TypeAlias, cast from zarr_metadata.definition._definition import ( AcceptedField, Definition, RefusedField, UnclaimedField, as_kind, field_key, fields_of, filed_under, format_of, own_key, resolve, ) if TYPE_CHECKING: from collections.abc import Callable, Iterable, Sequence from zarr_metadata._typed_json import Loc from zarr_metadata.definition._definition import D, ResolvedField ClaimKey: TypeAlias = tuple[type[Definition[Any]], str] """A kind and the name a definition is filed under. A scope's `claimant` looks up definitions by this key.""" Claims: TypeAlias = Mapping[ClaimKey, Definition[Any] | None] """The claims a reading records for each name a document uses: the definition that read the name, or None if the name was unclaimed.""" @dataclass(frozen=True, slots=True) class Conflict: """One place where two readings of a name disagree: the key, the definition one reading claimed, the definition the other found, and the location in the document when known.""" key: ClaimKey claimed: Definition[Any] | None found: Definition[Any] | None loc: Loc | None = None written: str | None = None """The name the document uses at the conflict location, for example `r16` for a key of `r*`; None when no document is available.""" def __str__(self) -> str: kind, filed = self.key name = filed if self.written is None else self.written where = "" if self.loc is None else f" at {self.loc!r}" claimed, found = definition_said(self.claimed), definition_said(self.found) if claimed == found: # Nothing a description shows tells them apart, such as their # rules, so say that they differ. return f"{kind_name(kind)} {name!r}{where}: two different definitions, each {claimed}" return f"{kind_name(kind)} {name!r}{where}: claimed {claimed}, found {found}" def definition_said(definition: Definition[Any] | None) -> str: """Describe a definition for an error message. The description includes the configuration TypedDict, which distinguishes it from other definitions with the same name. Returns "no definition" for None.""" if definition is None: return "no definition" return f"{definition!r} of {definition.configuration.__qualname__}" def kind_name(kind: type[Definition[Any]]) -> str: """The name of a definition kind for an error message: its label, for example "codec" for `CodecDefinition`.""" return kind.label class ScopeConflictError(ValueError): """Raised when two scopes, or a scope and a reading, give one name two different meanings, or when one would lose a meaning the other has. Every conflict is stored in `.conflicts`, in the same way that `MetadataValidationError` stores every problem. """ def __init__(self, conflicts: Sequence[Conflict]) -> None: self.conflicts: tuple[Conflict, ...] = tuple(conflicts) super().__init__("; ".join(str(conflict) for conflict in self.conflicts)) def __reduce__( self, ) -> tuple[type[ScopeConflictError], tuple[tuple[Conflict, ...]], dict[str, object]]: # Pickle and copy the exception as it was raised, keeping its notes # and attributes. The default exception reduce calls the constructor # with the message, not the conflicts. return type(self), (self.conflicts,), dict(self.__dict__) def claim_key(field: ResolvedField[Any]) -> ClaimKey | None: """The key `field` is claimed under: its kind and the name its definition is filed under (for example `r*` for `r16`). Returns None for a field with no name.""" if field.name is None: return None filed, _ = filed_under(field.read_as, field.name) return None if filed is None else (field.read_as, filed) def claims_of( fields: Iterable[tuple[Loc, ResolvedField[Any]]], ) -> dict[ClaimKey, Definition[Any] | None]: """The claims that `fields` (each paired with its location) record for each name: the definition that read the name, or None if the name was unclaimed. `fields` must come from one reading, as returned by `fields_of` or a reading's `fields()`. If two fields claim the same name with different definitions, this raises `ScopeConflictError`, because no single scope could have read them both. """ claims: dict[ClaimKey, Definition[Any] | None] = {} conflicts: list[Conflict] = [] for loc, field in fields: key = claim_key(field) if key is None: continue definition = field.definition if key in claims and claims[key] != definition: conflicts.append(Conflict(key, claims[key], definition, loc)) continue claims[key] = definition if len(conflicts) != 0: raise ScopeConflictError(conflicts) return claims def refines(field: ResolvedField[Any], other: ResolvedField[Any]) -> bool: """Whether `field` contains everything `other` contains: it reads the same where both read, and it may also read what `other` left unclaimed. This is the order in which one reading of a document refines another. Suppose `other` is unclaimed and `field` is accepted. This counts as a gain when re-reading the unclaimed field's JSON, with `field`'s definition and the definitions of the fields nested in `field`, produces `field`. Two spellings of one configuration count as one gain, because `==` treats them as one field, so the order is transitive through equality. The reverse direction (accepted to unclaimed) is a loss. One name read by two different definitions is a conflict. A refused field refines only itself. Two fields that refine each other are equal. """ if isinstance(field, RefusedField) or isinstance(other, RefusedField): return field == other if isinstance(other, UnclaimedField): if isinstance(field, UnclaimedField): return field_key(field) == field_key(other) return claim_key(field) == claim_key(other) and _gained(field, other) if isinstance(field, UnclaimedField): return False if field.definition != other.definition or own_key(field) != own_key(other): return False if set(field.nested) != set(other.nested): return False return all(refines(field.nested[loc], other.nested[loc]) for loc in field.nested) def _gained(field: AcceptedField[Any], other: UnclaimedField) -> bool: """Whether reading `other` in the scope formed by `field`'s own claims produces `field`. This is the test for a gain.""" again, _ = resolve(other.json, field.read_as, _Claimed.of(field)) return again == field @dataclass(frozen=True, slots=True) class _Claimed: """The scope formed by a field's own claims: its definition and the definitions of its nested fields, keyed by kind and filed name. `_gained` re-reads a field in this scope.""" filed: Mapping[ClaimKey, Definition[Any]] zarr_format: Literal[2, 3] @classmethod def of(cls, field: AcceptedField[Any]) -> _Claimed: claimed = claims_of(fields_of(field)) return cls( {key: definition for key, definition in claimed.items() if definition is not None}, format_of(field.read_as), ) def claimant(self, kind: type[D], name: str) -> D | None: """The definition of `kind` among the claims that reads `name`, or None if there is none. This matches a scope's `claimant`.""" asked = as_kind(kind) filed, _ = filed_under(asked, name) if filed is None: return None return cast("D | None", self.filed.get((asked, filed))) @dataclass(frozen=True, slots=True) class Disagreements: """The places where a scope reads a reading's claims differently: the names that would gain a meaning, and the names that conflict (including names that would lose a meaning).""" gains: tuple[ClaimKey, ...] conflicts: tuple[Conflict, ...] @property def agrees(self) -> bool: """Whether the scope reads every claim identically.""" return len(self.gains) == 0 and len(self.conflicts) == 0 def disagreements_of( claimant: Callable[[type[Definition[Any]], str], Definition[Any] | None], claims: Claims ) -> Disagreements: """The `Disagreements` between `claimant` and `claims`. For each key, compare the definition `claimant` returns (looked up by kind and filed name, like a scope's tables) with the definition `claims` records.""" gains: list[ClaimKey] = [] conflicts: list[Conflict] = [] for key, claimed in claims.items(): kind, name = key found = claimant(kind, name) if found == claimed: continue if claimed is None: gains.append(key) else: conflicts.append(Conflict(key, claimed, found)) return Disagreements(tuple(gains), tuple(conflicts)) __all__ = [ "ClaimKey", "Claims", "Conflict", "Disagreements", "ScopeConflictError", "claim_key", "claims_of", "disagreements_of", "kind_name", "refines", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/json_schema.py000066400000000000000000000053241527014137600322530ustar00rootroot00000000000000"""JSON Schema output: a TypedDict, one Zarr v3 field, or a whole v3 `zarr.json`. Each function writes JSON Schema draft 2020-12, for a validator in another language or for an editor that checks a document while it is written. A schema describes each member on its own, so it does not contain the rules that relate members to each other, and a document it accepts may still have a problem. **A TypedDict.** `json_schema(typed_dict)` writes the rules that `zarr_metadata.typed_json.check` applies. The schema accepts the values that `check` finds no problems with. A TypedDict becomes an object with its keys, closed or open as the TypedDict declares. A bound becomes the matching JSON Schema keyword: `Interval(ge=0, le=9)` becomes `minimum` and `maximum`. Each TypedDict or type alias is written once in `$defs`, under its name. JSON Schema accepts a number with no fractional part, such as `1.0`, as an integer, while `check` requires `1`: from zarr_metadata.json_schema import json_schema json_schema(GzipCodecConfiguration) # {'$schema': 'https://json-schema.org/draft/2020-12/schema', # 'type': 'object', # 'properties': {'level': {'type': 'integer', 'minimum': 0, 'maximum': 9}}, # 'required': ['level'], 'additionalProperties': False} **One v3 field.** `field_json_schema_v3(kind, context=None)` writes one field read as `kind` in the scope `context`: a field that a definition in scope reads, or a name that no definition claims, with any configuration. **A whole `zarr.json`.** `node_metadata_json_schema_v3(context=None)` writes a whole v3 `zarr.json`, array or group, as the scope reads it (`CORE_AND_EXTENSIONS` when none is given). Its fields are written as the scope reads them. For each definition, the schema accepts its field (its name, its configuration as its TypedDict declares it, including bounds, a `must_understand` of `true`, and its bare name when it needs no configuration). It also accepts a name that nothing in scope claims, with any configuration. A nested field is written in the same scope, and a member that accepts only codecs of static size accepts only those. The rules are not in the schema, so a field it accepts may still have a problem. Any field that `resolve` reads without a problem is accepted by the schema, as JSON (arrays as lists, as a parser returns them). Each configuration TypedDict and each field alias is written once, in `$defs`, under its name. The fill value must match its data type's fill value schema. """ from zarr_metadata._typed_json import JSONSchema, json_schema from zarr_metadata.v3._json_schema import field_json_schema_v3, node_metadata_json_schema_v3 __all__ = [ "JSONSchema", "field_json_schema_v3", "json_schema", "node_metadata_json_schema_v3", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/model/000077500000000000000000000000001527014137600305045ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/model/__init__.py000066400000000000000000000147721527014137600326300ustar00rootroot00000000000000"""In-memory models for Zarr metadata documents. A model holds a metadata document (as written, then refined) and the scope the document was read in. Everything a model returns is read-only. Every document type has a reader: `read_array_metadata_v3`, `read_group_metadata_v3`, `read_consolidated_metadata_v3`, `read_array_metadata_v2`, `read_group_metadata_v2` and `read_consolidated_metadata_v2`. A reader checks a document's JSON structure. In a v3 document it also reads each extension point (codecs, chunk grids, data types, ...) through the definition that claims its name in a scope. The scope is `CORE_AND_EXTENSIONS` unless a `context` is passed. A v3 reader checks the fill value against the data type the document names, the chunk grid against the shape, and the codecs as a pipeline, each codec against the chunk it receives. Each reader does one read and returns a reading: what it read, every problem in `problems` (a tuple of `ValidationProblem`, each with a machine-readable `kind`), and the model in `metadata` when there are no problems. A reader is the form of the model's constructor that returns problems as values instead of raising them. A node of either type is read by `read_node_metadata_v3`, from a `zarr.json` and its `node_type`, or by `read_node_metadata_v2`, from the node's decoded `.zarray` or `.zgroup` and `.zattrs`, which can come from a store or from a `.zmetadata`. To read a store that another writer made with a known writer bug, use `read_repaired_node_metadata_v3` or `read_repaired_node_metadata_v2`. They undo each bug with `repair_node_metadata_v3` or `repair_node_metadata_v2` before the strict read, and report what they changed. `zarr_metadata.json_schema` writes a JSON Schema of what the v3 readers read, except for the rules. One field on its own is read with `zarr_metadata.definition.resolve`. The model constructors `from_json` and `from_key_value` raise `MetadataValidationError` for every ingestion failure, including missing store keys and undecodable bytes. The v3 constructors take the same `context`. A v3 model is its document plus the scope it was read in. `ZarrV3ArrayMetadata(document, context=None)` reads the document in the scope and raises `MetadataValidationError` with every problem, so an invalid model is never built. `to_json` returns the document as written. `update` reads new members in the model's own scope. `with_context` and `refined_in` read the document in another scope. `to_key_value` writes the model as it is. A v3 group reads its `consolidated_metadata` as an envelope and does not read the documents it lists, because consolidated metadata is a convention of the reference implementation, not core metadata. `ZarrV3ConsolidatedMetadata` and `read_consolidated_metadata_v3` read those documents. The model also accepts node models as entries: an entry is accepted when its claims refine into the model's scope, and refused with a problem at its path otherwise. Every reader takes `context=None` for the default scope. """ from zarr_metadata._sentinel import UNSET from zarr_metadata.model._array_v2 import ( ZarrV2ArrayMetadata, ZarrV2ArrayMetadataReading, ZarrV2ArrayMetadataUpdate, read_array_metadata_v2, ) from zarr_metadata.model._array_v3 import ( ZarrV3ArrayMetadata, ZarrV3ArrayMetadataReading, ZarrV3ArrayMetadataUpdate, read_array_metadata_v3, ) from zarr_metadata.model._group_v2 import ( ZarrV2ConsolidatedMetadata, ZarrV2ConsolidatedMetadataReading, ZarrV2GroupMetadata, ZarrV2GroupMetadataReading, ZarrV2GroupMetadataUpdate, ZarrV2NodeMetadata, ZarrV2NodeMetadataReading, ZarrV2UnknownNodeMetadataReading, node_metadata_from_key_value_v2, read_consolidated_metadata_v2, read_group_metadata_v2, read_node_metadata_v2, ) from zarr_metadata.model._group_v3 import ( ZarrV3ConsolidatedMetadata, ZarrV3ConsolidatedMetadataInput, ZarrV3ConsolidatedMetadataReading, ZarrV3GroupMetadata, ZarrV3GroupMetadataReading, ZarrV3GroupMetadataUpdate, ZarrV3NodeMetadata, ZarrV3NodeMetadataInput, ZarrV3NodeMetadataReading, ZarrV3UnknownNodeMetadataReading, node_metadata_from_json_v3, node_metadata_from_key_value_v3, read_consolidated_metadata_v3, read_group_metadata_v3, read_node_metadata_v3, ) from zarr_metadata.model._repair import ( Repair, RepairKind, ZarrV2RepairedConsolidatedMetadataReading, ZarrV2RepairedNodeMetadataReading, ZarrV3RepairedNodeMetadataReading, read_repaired_consolidated_metadata_v2, read_repaired_node_metadata_v2, read_repaired_node_metadata_v3, repair_consolidated_metadata_v2, repair_node_metadata_v2, repair_node_metadata_v3, ) from zarr_metadata.v3._hierarchy import ( is_node_name_v3, is_node_path_v3, parse_node_name_v3, parse_node_path_v3, validate_node_name_v3, validate_node_path_v3, ) __all__ = [ "UNSET", "Repair", "RepairKind", "ZarrV2ArrayMetadata", "ZarrV2ArrayMetadataReading", "ZarrV2ArrayMetadataUpdate", "ZarrV2ConsolidatedMetadata", "ZarrV2ConsolidatedMetadataReading", "ZarrV2GroupMetadata", "ZarrV2GroupMetadataReading", "ZarrV2GroupMetadataUpdate", "ZarrV2NodeMetadata", "ZarrV2NodeMetadataReading", "ZarrV2RepairedConsolidatedMetadataReading", "ZarrV2RepairedNodeMetadataReading", "ZarrV2UnknownNodeMetadataReading", "ZarrV3ArrayMetadata", "ZarrV3ArrayMetadataReading", "ZarrV3ArrayMetadataUpdate", "ZarrV3ConsolidatedMetadata", "ZarrV3ConsolidatedMetadataInput", "ZarrV3ConsolidatedMetadataReading", "ZarrV3GroupMetadata", "ZarrV3GroupMetadataReading", "ZarrV3GroupMetadataUpdate", "ZarrV3NodeMetadata", "ZarrV3NodeMetadataInput", "ZarrV3NodeMetadataReading", "ZarrV3RepairedNodeMetadataReading", "ZarrV3UnknownNodeMetadataReading", "is_node_name_v3", "is_node_path_v3", "node_metadata_from_json_v3", "node_metadata_from_key_value_v2", "node_metadata_from_key_value_v3", "parse_node_name_v3", "parse_node_path_v3", "read_array_metadata_v2", "read_array_metadata_v3", "read_consolidated_metadata_v2", "read_consolidated_metadata_v3", "read_group_metadata_v2", "read_group_metadata_v3", "read_node_metadata_v2", "read_node_metadata_v3", "read_repaired_consolidated_metadata_v2", "read_repaired_node_metadata_v2", "read_repaired_node_metadata_v3", "repair_consolidated_metadata_v2", "repair_node_metadata_v2", "repair_node_metadata_v3", "validate_node_name_v3", "validate_node_path_v3", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py000066400000000000000000001067371527014137600327600ustar00rootroot00000000000000"""The Zarr v2 array: its reader, its reading, and its model.""" from __future__ import annotations import dataclasses from collections.abc import Mapping from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final, Literal, cast from typing_extensions import TypedDict, Unpack from zarr_metadata._common import JSONValue from zarr_metadata._json import ( MetadataValidationError, ValidationProblem, copied, frozen, is_array, is_object, is_tuple, json_text, not_an_object, outside_of, refine_json, with_input, within, ) from zarr_metadata._sentinel import UNSET from zarr_metadata.definition._context import disagreements, scoped from zarr_metadata.definition._definition import ( AcceptedField, ResolvedField, UnclaimedField, field_key, fields_of, fill_value_problems, held, resolve, resolve_refined, spelled_canonically, ) from zarr_metadata.definition._scope import ScopeConflictError, claims_of from zarr_metadata.definition._scope import refines as refines_field from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, attributes_of, check_literal, dimension_lengths, dump_store_json, load_store_json, located_conflicts, member_changes, missing_keys, other_members, reading_of, unclaimed_fields_of, ) # A runtime import: `get_type_hints` resolves the public signatures. from zarr_metadata.v2._context import ZarrV2Context # noqa: TC001 from zarr_metadata.v2._field_json import ZarrV2DataTypeMetadata from zarr_metadata.v2.array import ( ZARR_V2_ARRAY_METADATA_STORE_KEY, ZarrV2ArrayDimensionSeparator, ZarrV2ArrayMetadataJSON, ZarrV2ArrayOrder, ) from zarr_metadata.v2.attributes import ZARR_V2_ATTRIBUTES_STORE_KEY from zarr_metadata.v2.codec import ZarrV2CodecMetadata from zarr_metadata.v2.definition import ( CORE_V2, ZarrV2CodecDefinition, ZarrV2DataTypeDefinition, ) from zarr_metadata.v2.group import ZARR_V2_GROUP_METADATA_STORE_KEY if TYPE_CHECKING: from collections.abc import Iterator from zarr_metadata._typed_json import Loc from zarr_metadata.v2.array import ZarrV2ArrayMetadataStoreKey from zarr_metadata.v2.attributes import ZarrV2AttributesStoreKey _ARRAY_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( ZarrV2ArrayMetadataJSON.__required_keys__ ) _ARRAY_METADATA_OPTIONAL_KEYS_V2: Final[frozenset[str]] = frozenset( ZarrV2ArrayMetadataJSON.__optional_keys__ ) _ARRAY_METADATA_STANDARD_KEYS_V2: Final[frozenset[str]] = ( _ARRAY_METADATA_REQUIRED_KEYS_V2 | _ARRAY_METADATA_OPTIONAL_KEYS_V2 ) """The members the spec defines in a v2 array document, as the model merges it, with `attributes` from `.zattrs`: these are never given in `extra_fields`.""" def _is_canonical_dtype_v2(value: object) -> bool: """Whether a validated v2 dtype uses the tuple-backed public representation.""" if isinstance(value, str): return True if not is_tuple(value): return False for record in value: if not is_tuple(record): return False fields = record if not _is_canonical_dtype_v2(fields[1]): return False if len(fields) == 3 and not isinstance(fields[2], tuple): return False return True @dataclass(frozen=True, slots=True) class ArrayMembersV2: """The members other than fields of a v2 array document that a read found no problems with, refined as the model holds them.""" shape: tuple[int, ...] chunks: tuple[int, ...] fill_value: JSONValue order: ZarrV2ArrayOrder dimension_separator: ZarrV2ArrayDimensionSeparator attributes: dict[str, JSONValue] | UNSET extra_fields: dict[str, JSONValue] document: dict[str, JSONValue] """The refined document that the model holds: each member as the read refined it, in the order the document writes them.""" @dataclass(frozen=True, slots=True) class ZarrV2ArrayMetadataReading: """A v2 array document as a scope read it, whatever it holds: its dtype, compressor and filters, every problem, and the model when there are no problems. A field the document does not hold is `UNSET`, and so is a `filters` that is not an array, which holds no filters to read. A `compressor` or `filters` written as `null` is None. """ dtype: ResolvedField[ZarrV2DataTypeDefinition[Any]] | UNSET = UNSET """The dtype, as the scope read it.""" compressor: ResolvedField[ZarrV2CodecDefinition[Any]] | UNSET | None = UNSET """The compressor, as the scope read it; None when written as `null`.""" filters: tuple[ResolvedField[ZarrV2CodecDefinition[Any]], ...] | UNSET | None = UNSET """The filters, each as the scope read it; None when written as `null`.""" problems: tuple[ValidationProblem, ...] = () """Every reason the document is not valid.""" metadata: ZarrV2ArrayMetadata | None = None """The document's model, which holds these fields, when there is no problem; None otherwise.""" def __reduce__(self) -> str | tuple[object, ...]: # Same as the v3 reading: pickle through the model, when there is one. if self.metadata is not None: return (reading_of, (self.metadata,)) return object.__reduce__(self) def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: """Each field in the document, as the scope read it, with its location: the dtype, followed by a struct's record types, then the compressor, then each filter at its index.""" if self.dtype is not UNSET: yield from fields_of(self.dtype, ("dtype",)) if self.compressor is not UNSET and self.compressor is not None: yield from fields_of(self.compressor, ("compressor",)) if self.filters is not UNSET and self.filters is not None: for index, entry in enumerate(self.filters): yield from fields_of(entry, ("filters", index)) def read_array_v2( value: object, context: ZarrV2Context, *, at: Loc = () ) -> tuple[ZarrV2ArrayMetadataReading, ArrayMembersV2 | None]: """Read `value`, a v2 array document, once in `context`. Return the reading, and the other members when there are no problems. `dtype`, `compressor` and `filters` are read in `context`. A dtype or codec that the scope refuses is a problem. One that the scope does not claim is not a problem. `fill_value` is checked against the dtype the scope read. `compressor` and `filters` are required keys that may be `None`. `at` is the location of this document within the input document, and prefixes every problem. """ if not is_object(value): return ZarrV2ArrayMetadataReading(problems=within(not_an_object(value), at)), None doc = value # Unlike the group document ("Other keys MUST NOT be present", # https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313), the v2 array document is open: other keys "SHOULD NOT be # present within the metadata object and SHOULD be ignored by # implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), so a member outside # _ARRAY_METADATA_STANDARD_KEYS_V2 is not a problem just for being there. # Ignored does not mean unchecked: as in v3, the member must be JSON, # and its key must be a string. problems: list[ValidationProblem] = list(missing_keys(_ARRAY_METADATA_REQUIRED_KEYS_V2, doc)) extra_fields, found = other_members(doc, _ARRAY_METADATA_STANDARD_KEYS_V2) problems.extend(found) problems.extend(check_literal(doc, "zarr_format", 2)) shape, shape_problems = dimension_lengths(doc, "shape") chunks, chunks_problems = dimension_lengths(doc, "chunks") problems.extend(shape_problems) problems.extend(chunks_problems) if shape is not None and chunks is not None and len(shape) != len(chunks): problems.append( ValidationProblem( ("chunks",), "expected the same number of dimensions as shape", "invalid_value", ) ) # Each member is refined once, here, and the refined value is kept. # The document the model holds is built from these values, so nothing # is walked twice. refined: dict[str, JSONValue] = { **extra_fields, "zarr_format": 2, "shape": shape, "chunks": chunks, } dtype: ResolvedField[ZarrV2DataTypeDefinition[Any]] | UNSET = UNSET if "dtype" in doc: # A typestr is read by its family's definition, and a list of field # records as a struct. refined["dtype"], found = refine_json(doc["dtype"], ("dtype",)) dtype, found = resolve_refined( doc["dtype"], refined["dtype"], found, ZarrV2DataTypeDefinition, context, ("dtype",) ) problems.extend(found) if "order" in doc: # Checked after refinement, so a value that only compares equal to # "C" is refused, like every other value outside the set. order, found = refine_json(doc["order"], ("order",)) refined["order"] = order if len(found) != 0 or order not in ("C", "F"): problems.append(outside_of(("order",), doc["order"], ("C", "F"))) compressor: ResolvedField[ZarrV2CodecDefinition[Any]] | UNSET | None = UNSET if "compressor" in doc: compressor = None refined["compressor"] = None if doc["compressor"] is not None: refined["compressor"], found = refine_json(doc["compressor"], ("compressor",)) compressor, found = resolve_refined( doc["compressor"], refined["compressor"], found, ZarrV2CodecDefinition, context, ("compressor",), ) problems.extend(found) filters: tuple[ResolvedField[ZarrV2CodecDefinition[Any]], ...] | UNSET | None = UNSET if "filters" in doc: entries = doc["filters"] if entries is None: filters = None refined["filters"] = None elif not is_array(entries): # There are no filters to read, so `filters` stays UNSET: None # means that the document writes null. problems.append( ValidationProblem( ("filters",), "expected null or an array of codec configurations", "invalid_type", ) ) else: # "A list of JSON objects providing codec configurations, or # null" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79): an empty list is a list. read: list[ResolvedField[ZarrV2CodecDefinition[Any]]] = [] items: list[JSONValue] = [] for index, item in enumerate(entries): refined_item, found = refine_json(item, ("filters", index)) items.append(refined_item) entry, found = resolve_refined( item, refined_item, found, ZarrV2CodecDefinition, context, ("filters", index), ) read.append(entry) problems.extend(found) filters = tuple(read) refined["filters"] = tuple(items) if "dimension_separator" in doc: separator, found = refine_json(doc["dimension_separator"], ("dimension_separator",)) refined["dimension_separator"] = separator if len(found) != 0 or separator not in (".", "/"): problems.append( outside_of(("dimension_separator",), doc["dimension_separator"], (".", "/")) ) fill_value: JSONValue = None if "fill_value" in doc: # The fill value must be JSON. It is checked against the dtype the # scope read, if any. If nothing in scope claims the dtype, the # fill value is not checked. fill_value, found = refine_json(doc["fill_value"], ("fill_value",)) refined["fill_value"] = fill_value problems.extend(found) if len(found) == 0 and dtype is not UNSET: problems.extend(fill_value_problems(dtype, fill_value, ("fill_value",))) attributes: dict[str, JSONValue] | UNSET | None = UNSET if "attributes" in doc: attributes, found = attributes_of(doc["attributes"]) refined["attributes"] = attributes problems.extend(found) reading = ZarrV2ArrayMetadataReading( dtype=dtype, compressor=compressor, filters=filters, problems=within(with_input(problems, doc), at), ) if len(problems) != 0 or shape is None or chunks is None or attributes is None: return reading, None members = ArrayMembersV2( shape=shape, chunks=chunks, fill_value=fill_value, order=cast("ZarrV2ArrayOrder", refined["order"]), dimension_separator=cast( "ZarrV2ArrayDimensionSeparator", refined.get("dimension_separator", ".") ), attributes=attributes, extra_fields=extra_fields, document={key: refined[key] for key in cast("Mapping[str, object]", doc)}, ) return reading, members NODE_FILES_V2: Final = ( ZARR_V2_ARRAY_METADATA_STORE_KEY, ZARR_V2_GROUP_METADATA_STORE_KEY, ZARR_V2_ATTRIBUTES_STORE_KEY, ) """The files that describe a v2 node: `.zarray` or `.zgroup`, and `.zattrs`.""" def decoded_node_files_v2( mapping: Mapping[StoreKey, bytes], keys: tuple[str, ...] = NODE_FILES_V2 ) -> tuple[dict[str, object], tuple[ValidationProblem, ...]]: """The JSON of each of `keys` that `mapping` holds, decoded from its bytes, and the problem of each one that is not bytes of JSON. A key that `mapping` does not hold is left out.""" stored = cast("Mapping[str, bytes]", mapping) files: dict[str, object] = {} problems: list[ValidationProblem] = [] for key in keys: if key not in stored: continue try: files[key] = load_store_json(mapping, key) except MetadataValidationError as error: problems.extend(error.problems) return files, tuple(problems) def node_document_v2( files: Mapping[str, object], key: str, beside_attributes: ValidationProblem ) -> tuple[object, tuple[ValidationProblem, ...]]: """The merged v2 document of the node whose file is `key` in `files` (each file's decoded JSON, keyed by its name): the file's JSON, with the attributes at `.zattrs` when `files` holds it. Returns the document and the problems found in merging. A missing `key` gives None and that problem. A file that holds `attributes` itself gives `beside_attributes`, since attributes belong in `.zattrs`, and the document without that member, so that its reader still reports every other problem. A file that is not an object is returned as it is, for the document's reader to report. """ if key not in files: return None, (ValidationProblem((key,), "missing store key", "missing_key"),) document = files[key] if not is_object(document): return document, () problems: tuple[ValidationProblem, ...] = () if "attributes" in document: problems = with_input((beside_attributes,), document) document = {name: value for name, value in document.items() if name != "attributes"} if ZARR_V2_ATTRIBUTES_STORE_KEY not in files: return document, problems return {**document, "attributes": files[ZARR_V2_ATTRIBUTES_STORE_KEY]}, problems ARRAY_FILE_ATTRIBUTES: Final = ValidationProblem( ("attributes",), f"unexpected key 'attributes' in {ZARR_V2_ARRAY_METADATA_STORE_KEY}: an array's attributes " f"belong in {ZARR_V2_ATTRIBUTES_STORE_KEY}", "invalid_value", ) """The problem of a `.zarray` that holds `attributes`, which belong in `.zattrs`. A `.zarray` may hold members the spec does not define, so the key is not unknown, but this one is taken by the attributes.""" def read_array_metadata_v2( value: object, *, context: ZarrV2Context | None = None ) -> ZarrV2ArrayMetadataReading: """Read `value`, a v2 array document, in `context` (`CORE_V2` when none is given), whatever it holds. The result holds everything the read finds: the dtype, the compressor and each filter as the scope read them (an `AcceptedField` read by the definition that claims the typestr or id, an `UnclaimedField`, or a `RefusedField`), every problem, and, when there are no problems, the document's model. A dtype or codec that the scope refuses is a problem. One that the scope does not claim is not a problem. `fill_value` is checked against the dtype the scope read. """ context = scoped(context, CORE_V2) reading, members = read_array_v2(value, context) if members is None: return reading model = ZarrV2ArrayMetadata._of(members.document, context, reading, members) # pyright: ignore[reportPrivateUsage] return model.reading class ZarrV2ArrayMetadataUpdate(TypedDict, total=False, closed=True): """The members `ZarrV2ArrayMetadata.update` replaces: each in the form a document writes it, or `UNSET` to remove a member that a document may omit. The members a document may omit are `attributes` (meaning no `.zattrs`), `dimension_separator` (read as `"."`), and any member the spec does not define. Those go in `extra_fields`, by key: each one given is set, or removed when given as `UNSET`, and the others are kept. A member is given by its own name only when the spec defines it, so a misspelled name is refused. """ shape: tuple[int, ...] dtype: ZarrV2DataTypeMetadata chunks: tuple[int, ...] fill_value: JSONValue order: ZarrV2ArrayOrder compressor: ZarrV2CodecMetadata | None filters: tuple[ZarrV2CodecMetadata, ...] | None dimension_separator: ZarrV2ArrayDimensionSeparator | UNSET attributes: Mapping[str, JSONValue] | UNSET extra_fields: Mapping[str, JSONValue | UNSET] class ZarrV2ArrayMetadata(Keyed): """A v2 array document, and the scope it was read in. Like the v3 models, this model is a pair. `to_json` returns the merged document (the `.zarray` members, plus `attributes` when a `.zattrs` holds them) as written, refined. A `.zarray` that omits `dimension_separator` means `"."` by the v2 convention (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L81-L86): the `dimension_separator` property returns `"."`, and the model compares equal to one whose document writes `"."`, but `to_json` writes the document without the key. `dtype`, `compressor` and each filter are views of the reading: an `AcceptedField` read by the definition in scope that claims the typestr or id, or an `UnclaimedField`. `attributes` is empty when no `.zattrs` exists, as in a v3 model. The document still tells the two apart: `to_json` and `to_key_value` write `attributes` and `.zattrs` only when a `.zattrs` exists, even an empty one. A member the spec does not define is kept in `extra_fields`. A model is only built by reading: the constructor reads `document` in `context` and raises `MetadataValidationError` with every problem, so no model is invalid. Two models are equal when their documents mean the same thing in their scopes, as the model key defines. `update` reads new members in the model's own scope. `with_context` and `refined_in` read the document in another scope. A model pickles as its pair. """ __slots__ = ("_context", "_document", "_members", "_reading", "_shown") zarr_format: Final[Literal[2]] = 2 # noqa: PYI064 - a bare Final does not narrow a union of models node_type: Final[Literal["array"]] = "array" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: context = scoped(context, CORE_V2) reading, members = read_array_v2(document, context) if members is None: raise MetadataValidationError(reading.problems) self._adopt(members.document, context, reading, members) @classmethod def _of( cls, document: dict[str, JSONValue], context: ZarrV2Context, reading: ZarrV2ArrayMetadataReading, members: ArrayMembersV2, ) -> ZarrV2ArrayMetadata: """Build a model of a document that a read found no problems with, holding that reading, without reading it again. The readers in this package build models through this method; pyright reports these calls as private use.""" model = object.__new__(cls) model._adopt(document, context, reading, members) return model def _adopt( self, document: dict[str, JSONValue], context: ZarrV2Context, reading: ZarrV2ArrayMetadataReading, members: ArrayMembersV2, ) -> None: self._document = document self._context = context self._reading = dataclasses.replace(reading, metadata=self) self._members = members # The members the model exposes, read-only at every level. self._shown = ( frozen(members.fill_value), frozen({} if members.attributes is UNSET else members.attributes), frozen(members.extra_fields), ) # --- the pair --------------------------------------------------------- @property def context(self) -> ZarrV2Context: """The scope the document was read in. `update` reads new members in this scope.""" return self._context @property def reading(self) -> ZarrV2ArrayMetadataReading: """The document as the scope read it: the dtype, the compressor, and each filter.""" return self._reading def to_json(self) -> ZarrV2ArrayMetadataJSON: """The merged document as written, refined. The result shares no objects with the model. `attributes` is included when set, even empty. This is not the on-disk `.zarray`, which excludes them: `to_key_value` splits the document as a store holds it (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330). """ return cast("ZarrV2ArrayMetadataJSON", copied(self._document)) def to_key_value( self, *, indent: int | str | None = None ) -> Mapping[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]: """The document as a store holds it: `.zarray` without the attributes, and `.zattrs` with the attributes when they are set, even if empty.""" zarray = {key: value for key, value in self._document.items() if key != "attributes"} out: dict[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes] = { ZARR_V2_ARRAY_METADATA_STORE_KEY: dump_store_json(zarray, indent=indent) } if "attributes" in self._document: out[ZARR_V2_ATTRIBUTES_STORE_KEY] = dump_store_json( self._document["attributes"], indent=indent ) return out def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" def _key_of(self) -> tuple[object, ...]: """The key `==` and `hash` compare for a v2 array model: what its document means. The key holds each field's `field_key`, and the fill value as JSON text, in its canonical spelling when a definition in scope read the dtype. Every other member is included as it is, with JSON members as text. No `.zattrs` counts as empty attributes, since both mean that the array has none, as in v3. """ members = self._members return ( members.shape, members.chunks, members.order, members.dimension_separator, self._fill_value_key(), field_key(self.dtype), None if self.compressor is None else field_key(self.compressor), None if self.filters is None else tuple(field_key(entry) for entry in self.filters), json_text({} if members.attributes is UNSET else members.attributes), json_text(members.extra_fields), ) def _plain_key( self, dtype: AcceptedField[ZarrV2DataTypeDefinition[Any]] | UnclaimedField ) -> tuple[object, ...]: """The members other than fields that `refines` compares. The fill value is in the spelling of `dtype`, the dtype of the more informed side.""" members = self._members return ( members.shape, members.chunks, members.order, members.dimension_separator, json_text(spelled_canonically(dtype, members.fill_value)), json_text({} if members.attributes is UNSET else members.attributes), json_text(members.extra_fields), ) def _fill_value_key(self) -> str: """The fill value as `==` compares it, as JSON text: in its canonical spelling when a definition in scope read the dtype, and as written otherwise.""" fill_value = self._members.fill_value if isinstance(self.dtype, AcceptedField): return json_text(spelled_canonically(self.dtype, fill_value)) return json_text(fill_value) def __reduce__(self) -> tuple[type[ZarrV2ArrayMetadata], tuple[object, ZarrV2Context]]: return type(self), (self._document, self._context) # --- typed views ------------------------------------------------------ @property def shape(self) -> tuple[int, ...]: """The array's shape.""" return self._members.shape @property def chunks(self) -> tuple[int, ...]: """The shape of each chunk.""" return self._members.chunks @property def fill_value(self) -> JSONValue: """The fill value as written, refined; read-only at every level.""" return self._shown[0] @property def order(self) -> ZarrV2ArrayOrder: """The in-chunk layout, `"C"` or `"F"`.""" return self._members.order @property def dimension_separator(self) -> ZarrV2ArrayDimensionSeparator: """The separator between chunk indices in a chunk key: `"."` when the document writes none.""" return self._members.dimension_separator @property def unclaimed_fields(self) -> dict[Loc, UnclaimedField]: """Every field nothing in scope claims, its dtype or a codec, located as `reading.fields()` locates it: at its place in the document, except that a struct dtype's field types are located by record and position, as their problems are. This answers whether a reader can open the array: a reader cannot decode data with a dtype or a codec it does not understand, and the scope is what the reader understands. The model only reports these fields; refusing to open is up to the reader. """ return unclaimed_fields_of(self._reading.fields(), ()) @property def attributes(self) -> Mapping[str, JSONValue]: """The user attributes a `.zattrs` holds, read-only at every level; empty when there is no `.zattrs`.""" return self._shown[1] @property def extra_fields(self) -> Mapping[str, JSONValue]: """Every member the spec does not define, as written, read-only at every level.""" return self._shown[2] @property def dtype(self) -> AcceptedField[ZarrV2DataTypeDefinition[Any]] | UnclaimedField: """The dtype as the scope read it: read by its family's definition, or unclaimed.""" return held(self._reading.dtype) @property def compressor(self) -> AcceptedField[ZarrV2CodecDefinition[Any]] | UnclaimedField | None: """The compressor as the scope read it; None when written as `null`.""" compressor = self._reading.compressor return None if compressor is None else held(compressor) @property def filters( self, ) -> tuple[AcceptedField[ZarrV2CodecDefinition[Any]] | UnclaimedField, ...] | None: """The filters, each as the scope read it; None when written as `null`.""" filters = self._reading.filters if filters is None: return None if filters is UNSET: msg = "expected filters a read found nothing wrong with, got UNSET" raise TypeError(msg) return tuple(held(entry) for entry in filters) # --- changing --------------------------------------------------------- def update(self, **members: Unpack[ZarrV2ArrayMetadataUpdate]) -> ZarrV2ArrayMetadata: """Return this model with `members` (as JSON) replacing the document's members, read in this model's own scope. A member set to `UNSET` is removed. Raises `MetadataValidationError` when the resulting document has a problem. So members that depend on each other must be passed together, for example a `dtype` with a fill value of that dtype. """ changes = member_changes( members, ZarrV2ArrayMetadataUpdate, _ARRAY_METADATA_STANDARD_KEYS_V2 ) document: dict[str, object] = {**self._document, **changes} for key, value in changes.items(): if value is UNSET: del document[key] return type(self)(document, context=self._context) def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadata: """Return this document read in `context`, whatever that changes: a gain, a loss, or a conflict. Raises `MetadataValidationError` when the document has a problem in `context`. The existing reading is reused when `context` reads every claim identically. """ context = scoped(context, CORE_V2) if disagreements(context, claims_of(self._reading.fields())).agrees: return self._of(self._document, context, self._reading, self._members) return type(self)(self._document, context=context) def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadata: """Return this document read in `context`. `context` may claim names this scope left unclaimed, but must not contradict any claim. Raises `ScopeConflictError` when `context` reads a typestr or id with a different definition, or with no definition, where this scope read it with one. The error names each one and its location in the document. Raises `MetadataValidationError` when a definition that `context` claims rejects what was written. """ context = scoped(context, CORE_V2) found = disagreements(context, claims_of(self._reading.fields())) if len(found.conflicts) != 0: raise ScopeConflictError(located_conflicts(self._reading.fields(), found.conflicts)) return self.with_context(context) def refines(self, other: ZarrV2ArrayMetadata) -> bool: """Whether this model holds everything `other` holds. Each field must refine its counterpart. A `null` compressor or `null` filters refine only a `null`. Every other member must be the same, with the fill value compared in the spelling of the more informed dtype. If that dtype rejects the other model's fill value, this is not a refinement.""" if type(other) is not type(self): return False if (self.compressor is None) != (other.compressor is None): return False if (self.filters is None) != (other.filters is None): return False mine = () if self.filters is None else self.filters theirs = () if other.filters is None else other.filters if len(mine) != len(theirs): return False pairs = [(self.dtype, other.dtype), *zip(mine, theirs, strict=True)] if self.compressor is not None and other.compressor is not None: pairs.append((self.compressor, other.compressor)) if not all(refines_field(one, another) for one, another in pairs): return False if len(fill_value_problems(self.dtype, other._members.fill_value)) != 0: return False return self._plain_key(self.dtype) == other._plain_key(self.dtype) # --- constructors ----------------------------------------------------- @classmethod def create_default( cls, *, context: ZarrV2Context | None = None, **overrides: Unpack[ZarrV2ArrayMetadataUpdate] ) -> ZarrV2ArrayMetadata: """Return a scalar `|u1` array, with any document members in `overrides` applied, read in `context`. Raises `MetadataValidationError` when the resulting document has a problem. Overriding `shape` without `chunks` sets `chunks` equal to `shape`, so one chunk covers the array. Overriding `chunks` without `shape` keeps the scalar default shape, which chunks of another rank do not fit. A dtype given without a fill value gets a fill value of `0` when its family accepts `0`, and `null` otherwise. Every family accepts `null`. """ document: dict[str, object] = { "zarr_format": 2, "shape": (), "chunks": (), "dtype": "|u1", "fill_value": 0, "order": "C", "compressor": None, "filters": None, "dimension_separator": ".", } given = member_changes( overrides, ZarrV2ArrayMetadataUpdate, _ARRAY_METADATA_STANDARD_KEYS_V2 ) if "shape" in given and "chunks" not in given: lengths, _ = dimension_lengths(given, "shape") if lengths is not None: given["chunks"] = lengths if "dtype" in given and "fill_value" not in given: dtype, _ = resolve(given["dtype"], ZarrV2DataTypeDefinition, scoped(context, CORE_V2)) if len(fill_value_problems(dtype, 0)) != 0: given["fill_value"] = None merged = {key: value for key, value in {**document, **given}.items() if value is not UNSET} return cls(merged, context=context) @classmethod def from_json( cls, data: object, *, context: ZarrV2Context | None = None ) -> ZarrV2ArrayMetadata: """Return the model of `data`, a v2 array document with its attributes under `attributes`, read in `context`. Raises `MetadataValidationError` with every problem the read finds. `read_array_metadata_v2` returns the reading this model is built from, and the problems of a document that has any. """ return cls(data, context=context) @classmethod def from_key_value( cls, mapping: Mapping[StoreKey, bytes], *, context: ZarrV2Context | None = None ) -> ZarrV2ArrayMetadata: """Return the model of the array at `.zarray` in `mapping`, with the attributes at `.zattrs` when that key exists, read in `context`. Raises `MetadataValidationError` when `.zarray` is missing, bytes are not JSON, `.zarray` holds `attributes`, or the document is not valid, with every problem the read finds. Raises `TypeError` for a scope of another format before any bytes are decoded. """ context = scoped(context, CORE_V2) files, problems = decoded_node_files_v2( mapping, (ZARR_V2_ARRAY_METADATA_STORE_KEY, ZARR_V2_ATTRIBUTES_STORE_KEY) ) if len(problems) != 0: raise MetadataValidationError(problems) document, problems = node_document_v2( files, ZARR_V2_ARRAY_METADATA_STORE_KEY, ARRAY_FILE_ATTRIBUTES ) if len(problems) != 0: if document is not None: problems = (*problems, *read_array_metadata_v2(document, context=context).problems) raise MetadataValidationError(problems) return cls(document, context=context) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py000066400000000000000000001226761527014137600327610ustar00rootroot00000000000000"""The Zarr v3 array: its reader, its reading, and its model.""" from __future__ import annotations import dataclasses from collections.abc import Mapping from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, cast, get_args, get_origin from typing_extensions import TypedDict, Unpack from zarr_metadata._common import JSONValue from zarr_metadata._json import ( MetadataValidationError, ValidationProblem, copied, frozen, is_array, is_object, json_text, nested_past_the_levels, not_an_object, refine_json, shown_key, with_input, within, ) from zarr_metadata._sentinel import UNSET from zarr_metadata._typed_json import typeddict_keys from zarr_metadata.definition._context import disagreements, scoped from zarr_metadata.definition._definition import ( AcceptedField, Definition, ResolvedField, UnclaimedField, field_key, field_kind, fields_of, fill_value_problems, held, resolve_refined, spelled_canonically, ) from zarr_metadata.definition._scope import ScopeConflictError, claims_of from zarr_metadata.definition._scope import refines as refines_field from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, attributes_of, check_literal, dimension_lengths, dump_store_json, load_store_json, located_conflicts, member_changes, missing_keys, reading_of, unclaimed_fields_of, ) from zarr_metadata.v3._common import ZarrV3ExtensionField, ZarrV3MetadataFieldJSON # A runtime import: `get_type_hints` resolves the public signatures. from zarr_metadata.v3._context import ZarrV3Context # noqa: TC001 from zarr_metadata.v3._core import CORE_AND_EXTENSIONS from zarr_metadata.v3._definition import ( ArrayExtraFieldDefinition, Chunk, ChunkGridDefinition, ChunkKeyEncodingDefinition, DataTypeDefinition, Lengths, StorageTransformerDefinition, chunk_grid_lengths, ) from zarr_metadata.v3._pipeline import Stage, read_pipeline from zarr_metadata.v3.array import ( ZARR_V3_ARRAY_METADATA_STORE_KEY, ZarrV3ArrayMetadataJSON, ) if TYPE_CHECKING: from collections.abc import Iterator from zarr_metadata._typed_json import Loc from zarr_metadata.v3._definition import CodecDefinition, ExtraFieldDefinition from zarr_metadata.v3.array import ZarrV3ArrayMetadataStoreKey # The members a v3 array document must have. The members the spec defines # are `ArrayExtraFieldDefinition.spec_members`; any other is an extra field. _ARRAY_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( ZarrV3ArrayMetadataJSON.__required_keys__ ) def members_past_the_levels( doc: Mapping[object, object], at: Loc ) -> dict[object, ValidationProblem]: """Each member of `doc` (a document located at `at`) that is a container nested deeper than a reader walks, mapped to its problem, located in the document. A reader reports these members and does not walk into them. This matches what `refine_json` would report for the whole document: a document one level before the depth cap can hold scalars, and nothing else. """ past: dict[object, ValidationProblem] = {} for key, item in doc.items(): if isinstance(key, str): problem = nested_past_the_levels(item, (*at, key)) if problem is not None: (past[key],) = within((problem,), at) return past _MEMBERS_V3: Final = typeddict_keys(ZarrV3ArrayMetadataJSON).members _EXTENSION_POINTS_V3: Final[tuple[tuple[str, type[Definition[Any]]], ...]] = tuple( (key, kind) for key, (annotation, _) in _MEMBERS_V3.items() if (kind := field_kind(annotation)) is not None ) """The single extension points of a v3 array document, and the kind each is read as: each member that the TypedDict annotates with a field alias.""" _EXTENSION_LISTS_V3: Final[tuple[tuple[str, type[Definition[Any]]], ...]] = tuple( (key, kind) for key, (annotation, _) in _MEMBERS_V3.items() if get_origin(annotation) is tuple and (kind := field_kind(get_args(annotation)[0])) is not None ) """The lists of extension points of a v3 array document, and the kind each entry is read as: each member that the TypedDict annotates as a tuple of a field alias.""" def no_extra_fields() -> Mapping[str, ResolvedField[Any]]: """The extra fields of a reading of a document that has none.""" return MappingProxyType({}) @dataclass(frozen=True, slots=True) class ZarrV3ArrayMetadataReading: """A v3 array document as a scope read it, whatever it holds: each extension point, its codecs as a pipeline, every problem, and the model when there are no problems. A field the document does not have is `UNSET`. A list of fields that the document does not have as a list is empty. """ data_type: ResolvedField[DataTypeDefinition[Any]] | UNSET = UNSET """The data type, as the scope read it.""" chunk_grid: ResolvedField[ChunkGridDefinition[Any]] | UNSET = UNSET """The chunk grid, as the scope read it.""" chunk_key_encoding: ResolvedField[ChunkKeyEncodingDefinition[Any]] | UNSET = UNSET """The chunk key encoding, as the scope read it.""" chunk: Chunk = dataclasses.field(default_factory=Chunk) """The chunks the first codec receives: the length of the grid's chunks along each axis of the shape, and the data type.""" pipeline: tuple[Stage, ...] = () """The codecs, read as a pipeline: each as the scope read it, with the chunk it receives.""" storage_transformers: tuple[ResolvedField[StorageTransformerDefinition[Any]], ...] = () """The storage transformers, each as the scope read it.""" extra_fields: Mapping[str, ResolvedField[ArrayExtraFieldDefinition[Any]]] = dataclasses.field( default_factory=no_extra_fields ) """Each member the spec does not define, as the scope read it, by key: an `UnclaimedField` when no definition in scope claims the key.""" problems: tuple[ValidationProblem, ...] = () """Every reason the document is not valid.""" metadata: ZarrV3ArrayMetadata | None = None """The document's model, which holds these fields, when there is no problem; None otherwise.""" def __reduce__(self) -> str | tuple[object, ...]: # A reading that holds a model pickles and copies the same way the # model does (as the pair, read once on load). It is restored as # that model's own reading, so there is still one model per # document. if self.metadata is not None: return (reading_of, (self.metadata,)) # A read-only view does not pickle, so the extra fields go as a dict. values = {item.name: getattr(self, item.name) for item in dataclasses.fields(self)} return (_array_reading, ({**values, "extra_fields": dict(self.extra_fields)},)) def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: """Each field in the document, as the scope read it, with its location in the document. The order is: the extension points, then each codec and storage transformer at its index, then each extra field at its key. Each is followed by the fields nested in it, as `fields_of` returns them: a shard's codecs, or a struct's field types. `with_problems` returns each field with its problems. """ own: tuple[tuple[str, ResolvedField[Any] | UNSET], ...] = ( ("data_type", self.data_type), ("chunk_grid", self.chunk_grid), ("chunk_key_encoding", self.chunk_key_encoding), ) for key, field in own: if field is not UNSET: yield from fields_of(field, (key,)) for index, stage in enumerate(self.pipeline): yield from fields_of(stage.codec, ("codecs", index)) for index, transformer in enumerate(self.storage_transformers): yield from fields_of(transformer, ("storage_transformers", index)) for key, extra in self.extra_fields.items(): yield from fields_of(extra, (key,)) def _array_reading(values: dict[str, Any]) -> ZarrV3ArrayMetadataReading: """Rebuild an array reading from the values `ZarrV3ArrayMetadataReading.__reduce__` returns, with its extra fields held read-only.""" values["extra_fields"] = MappingProxyType(values["extra_fields"]) return ZarrV3ArrayMetadataReading(**values) @dataclass(frozen=True, slots=True) class ArrayMembersV3: """The members other than fields of a v3 array document that a read found no problems with, refined as the model holds them.""" shape: tuple[int, ...] fill_value: JSONValue dimension_names: tuple[str | None, ...] | UNSET attributes: dict[str, JSONValue] extra_fields: dict[str, JSONValue] document: dict[str, JSONValue] """The refined document that the model holds: each member as the read refined it, in the order the document writes them.""" def read_array_v3( value: object, context: ZarrV3Context, *, at: Loc = () ) -> tuple[ZarrV3ArrayMetadataReading, ArrayMembersV3 | None]: """Read `value`, a v3 array document, in `context`, without building its model. Return the reading and the other members, refined; the members are None when the document has a problem. `read_array_metadata_v3` builds the model from these two results. A field object in the document (an `AcceptedField` built by hand) is not JSON, and is refused as such. `at` is the location of this document within the input document; a document in consolidated metadata is three levels below its group's document. Nesting depth is counted from the root of the input document. The problems are located relative to this document. """ if not is_object(value): return ZarrV3ArrayMetadataReading(problems=not_an_object(value)), None doc = value problems: list[ValidationProblem] = list(missing_keys(_ARRAY_METADATA_REQUIRED_KEYS_V3, doc)) past = members_past_the_levels(doc, at) problems.extend(past.values()) whole, doc = doc, {key: item for key, item in doc.items() if key not in past} extra_fields, extra, found = read_extra_fields_v3(doc, ArrayExtraFieldDefinition, context, at) problems.extend(found) problems.extend(check_literal(doc, "zarr_format", 3)) problems.extend(check_literal(doc, "node_type", "array")) shape, shape_problems = dimension_lengths(doc, "shape") problems.extend(shape_problems) # Each extension point is read by `resolve`. It first checks the # envelope. Every extension *point* must be understood, so a # `must_understand` of `false` is refused at each one: ignoring a codec # gives wrong bytes just as ignoring a data type gives wrong values. # The spec names only the first three # (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581), # which this reader treats as an oversight, not as permission. Then # `resolve` checks the configuration against the definition in # `context` that claims the name. `must_understand: false` keeps its # meaning where it has one: on an unknown top-level extension *field*, # which a reader really can skip. # Each member is refined once, here, and the refined value is kept. # The document the model holds is built from these values, so nothing # is walked twice. refined: dict[str, JSONValue] = { **extra_fields, "zarr_format": 3, "node_type": "array", "shape": shape, } read: dict[str, ResolvedField[Any]] = {} for key, kind in _EXTENSION_POINTS_V3: if key in doc: refined[key], found = refine_json(doc[key], (*at, key)) read[key], found = resolve_refined( doc[key], refined[key], found, kind, context, (*at, key) ) problems.extend(within(found, at)) # The fill value must be JSON. It is checked against the data type the # scope read, if any. If nothing in scope claims the data type, the # fill value is not checked. fill_value: JSONValue = None if "fill_value" in doc: fill_value, found = refine_json(doc["fill_value"], (*at, "fill_value")) refined["fill_value"] = fill_value problems.extend(within(found, at)) if len(found) == 0 and "data_type" in read: problems.extend(fill_value_problems(read["data_type"], fill_value, ("fill_value",))) # Once both are read, the chunk grid is checked against the shape. The # grid gives the chunk lengths the first codec receives: one entry for # each dimension of the shape, None where the length is unknown. lengths: Lengths | None = None if shape is None else (None,) * len(shape) if "chunk_grid" in read and shape is not None: lengths, found = chunk_grid_lengths(read["chunk_grid"], shape, ("chunk_grid",)) problems.extend(found) listed: dict[str, list[ResolvedField[Any]]] = {} for key, kind in _EXTENSION_LISTS_V3: if key in doc: entries = doc[key] if not is_array(entries): problems.append(ValidationProblem((key,), "expected an array", "invalid_type")) else: listed[key] = [] items: list[JSONValue] = [] for index, entry in enumerate(entries): item, found = refine_json(entry, (*at, key, index)) items.append(item) resolved, found = resolve_refined( entry, item, found, kind, context, (*at, key, index) ) listed[key].append(resolved) problems.extend(within(found, at)) refined[key] = tuple(items) # The codecs are read as a pipeline. The first codec receives the # grid's chunks, of the array's data type. The codecs are checked in # order, each against the chunk it receives. A valid pipeline holds # one array -> bytes codec, so it is not empty. chunk = Chunk(lengths, read.get("data_type")) pipeline: tuple[Stage, ...] = () if "codecs" in listed: pipeline, found = read_pipeline(listed["codecs"], chunk, ("codecs",)) problems.extend(found) attributes: dict[str, JSONValue] | None = {} if "attributes" in doc: attributes, found = attributes_of(doc["attributes"], at) refined["attributes"] = attributes problems.extend(found) if "dimension_names" in doc: # Simple typed sequences (dimension_names, shape, chunks) report a single # field-level loc, not per-bad-item locs; per-index locs are reserved for # the metadata-field lists (codecs, storage_transformers). names = doc["dimension_names"] if not is_array(names): problems.append( ValidationProblem(("dimension_names",), "expected an array", "invalid_type") ) elif not all(item is None or isinstance(item, str) for item in names): problems.append( ValidationProblem( ("dimension_names",), "expected an array of strings or null", "invalid_type" ) ) elif shape is not None and len(names) != len(shape): problems.append( ValidationProblem( ("dimension_names",), "expected one name per dimension of shape", "invalid_value", ) ) reading = ZarrV3ArrayMetadataReading( data_type=read.get("data_type", UNSET), chunk_grid=read.get("chunk_grid", UNSET), chunk_key_encoding=read.get("chunk_key_encoding", UNSET), chunk=chunk, pipeline=pipeline, storage_transformers=tuple(listed.get("storage_transformers", ())), extra_fields=MappingProxyType(extra), problems=with_input(problems, whole), ) if len(problems) != 0 or shape is None or attributes is None: return reading, None dimension_names: tuple[str | None, ...] | UNSET = UNSET if "dimension_names" in doc: # An array of strings and nulls, as checked above. Refine it, so # each string is as JSON writes it. dimension_names = cast("tuple[str | None, ...]", refine_json(doc["dimension_names"])[0]) refined["dimension_names"] = dimension_names members = ArrayMembersV3( shape=shape, fill_value=fill_value, dimension_names=dimension_names, attributes=attributes, extra_fields=extra_fields, document={key: refined[key] for key in cast("Mapping[str, object]", doc)}, ) return reading, members def read_extra_fields_v3( doc: Mapping[object, object], kind: type[ExtraFieldDefinition[Any]], context: ZarrV3Context, at: Loc, ) -> tuple[dict[str, JSONValue], dict[str, ResolvedField[Any]], tuple[ValidationProblem, ...]]: """Read each member of `doc` (a document at `at`) that is not one of the `spec_members` of `kind` as an extra field of `kind`. Returns three values: each member that is JSON, refined; each member as the scope read it, by key; and every problem, located in `doc`. A member that a definition in `context` claims is read by that definition, and must be an object whose numbers are finite, unless the definition reads user data. Any other member is not read: it is an `UnclaimedField`, and, like attributes, need only be JSON as Python's `json` module reads it. """ values: dict[str, JSONValue] = {} fields: dict[str, ResolvedField[Any]] = {} problems: list[ValidationProblem] = [] for key, value in doc.items(): if not isinstance(key, str): problems.append( ValidationProblem((), f"non-string top-level key {shown_key(key)}", "invalid_type") ) continue if key in kind.spec_members: continue member = {key: value} definition = context.claimant(kind, key) refined, found = kind._refined(member, (*at, key), definition) # pyright: ignore[reportPrivateUsage] if isinstance(refined, dict): values[key] = refined[key] if definition is None: fields[key] = UnclaimedField(json=refined, name=key, read_as=kind) continue fields[key], found = resolve_refined(member, refined, found, kind, context, (*at, key)) problems.extend(within(found, at)) return values, fields, tuple(problems) def must_understand_subset( fields: Mapping[str, AcceptedField[Any] | UnclaimedField], values: Mapping[str, ZarrV3ExtensionField], ) -> dict[str, ZarrV3ExtensionField]: """The extra fields, by key and as written, that no definition in scope reads and that a reader is obligated to understand. Per the v3 spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578), an extension field is implicitly `must_understand: True` unless it explicitly says otherwise, and an implementation MUST fail to open a group or array carrying fields it does not recognize that are not explicitly `must_understand: false`. A field that a definition in scope read is recognized. A non-mapping field value cannot carry the explicit waiver, so it always requires understanding (the runtime isinstance check defends against values looser than the declared `ZarrV3ExtensionField`). """ return { name: values[name] for name, field in fields.items() if isinstance(field, UnclaimedField) and not _waived(values[name]) } def waived_members(values: Mapping[str, ZarrV3ExtensionField]) -> list[str]: """The keys of the extra fields in `values` (by key, as written) that say `"must_understand": false`, which a reader may ignore.""" return [key for key, value in values.items() if _waived(value)] def _waived(value: ZarrV3ExtensionField) -> bool: """Whether `value`, an extra field as written, is an object that says `"must_understand": false`.""" return isinstance(value, Mapping) and value.get("must_understand") is False class ZarrV3ArrayMetadataUpdate(TypedDict, total=False, closed=True): """The members `ZarrV3ArrayMetadata.update` replaces: each in the form a document writes it, or `UNSET` to remove a member that a document may omit. The members a document may omit are `dimension_names`, `attributes`, `storage_transformers`, and any member the spec does not define. Those go in `extra_fields`, by key: each one given is set, or removed when given as `UNSET`, and the others are kept. A member is given by its own name only when the spec defines it, so a misspelled name is refused. """ shape: tuple[int, ...] data_type: ZarrV3MetadataFieldJSON chunk_grid: ZarrV3MetadataFieldJSON chunk_key_encoding: ZarrV3MetadataFieldJSON fill_value: JSONValue codecs: tuple[ZarrV3MetadataFieldJSON, ...] attributes: Mapping[str, JSONValue] | UNSET storage_transformers: tuple[ZarrV3MetadataFieldJSON, ...] | UNSET dimension_names: tuple[str | None, ...] | UNSET extra_fields: Mapping[str, ZarrV3ExtensionField | UNSET] class ZarrV3ArrayMetadata(Keyed): """A v3 array document, and the scope it was read in. The model is this pair. `to_json` returns the document as written, refined (arrays as tuples, string keys), and `context` is the scope. Every typed member is a view of the reading of the document in the scope. `data_type`, `chunk_grid`, `chunk_key_encoding`, and each codec and storage transformer are fields as the scope read them: an `AcceptedField` (read by the definition that claims its name) or an `UnclaimedField`, and so is each member in `extra_fields`, which the spec does not define. `shape`, `fill_value`, `dimension_names` and `attributes` are the values the read refined. A model is only built by reading: the constructor reads `document` in `context` and raises `MetadataValidationError` with every problem, so no model is invalid. Two models are equal when their documents mean the same thing in their scopes, as the model key defines. The scope itself is not compared. `update` reads new members in the model's own scope. `with_context` and `refined_in` read the document in another scope. A model pickles as its pair, provided the definitions in its scope can be pickled, which requires their functions to be defined at the top level of a module. """ __slots__ = ("_context", "_document", "_members", "_reading", "_shown") zarr_format: Final[Literal[3]] = 3 # noqa: PYI064 - a bare Final does not narrow a union of models node_type: Final[Literal["array"]] = "array" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV3Context | None = None) -> None: context = scoped(context, CORE_AND_EXTENSIONS) reading, members = read_array_v3(document, context) if members is None: raise MetadataValidationError(reading.problems) self._adopt(members.document, context, reading, members) @classmethod def _of( cls, document: dict[str, JSONValue], context: ZarrV3Context, reading: ZarrV3ArrayMetadataReading, members: ArrayMembersV3, ) -> ZarrV3ArrayMetadata: """Build a model of a document that a read found no problems with, holding that reading, without reading it again. The readers in this package build models through this method; pyright reports these calls as private use.""" model = object.__new__(cls) model._adopt(document, context, reading, members) return model def _adopt( self, document: dict[str, JSONValue], context: ZarrV3Context, reading: ZarrV3ArrayMetadataReading, members: ArrayMembersV3, ) -> None: self._document = document self._context = context # The reading holds this model, in the form `read_array_metadata_v3` # returns it, no matter how the model was built. self._reading = dataclasses.replace(reading, metadata=self) self._members = members # The members the model exposes, read-only at every level. self._shown = ( frozen(members.fill_value), frozen(members.attributes), frozen(members.extra_fields), MappingProxyType({key: held(extra) for key, extra in reading.extra_fields.items()}), ) # --- the pair --------------------------------------------------------- @property def context(self) -> ZarrV3Context: """The scope the document was read in. `update` reads new members in this scope.""" return self._context @property def reading(self) -> ZarrV3ArrayMetadataReading: """The document as the scope read it: each field, the pipeline, and the chunk each codec receives.""" return self._reading def to_json(self) -> ZarrV3ArrayMetadataJSON: """The document as written, refined. The result shares no objects with the model.""" return cast("ZarrV3ArrayMetadataJSON", copied(self._document)) def to_key_value( self, *, indent: int | str | None = None ) -> Mapping[ZarrV3ArrayMetadataStoreKey, bytes]: """The document as a store holds it: JSON bytes at `zarr.json`, indented by `indent`. `NaN`, `Infinity` and `-Infinity` in `attributes` are written as bare tokens, as zarr-python writes them. A strict JSON parser rejects these tokens. """ return {ZARR_V3_ARRAY_METADATA_STORE_KEY: dump_store_json(self._document, indent=indent)} def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" def _key_of(self) -> tuple[object, ...]: """The key `==` and `hash` compare for a v3 array model: what its document means, as computed by `array_key_of`.""" return array_key_of(self._reading, self._members) def _plain_key( self, data_type: AcceptedField[DataTypeDefinition[Any]] | UnclaimedField ) -> tuple[object, ...]: """The members other than fields that `refines` compares. The fill value is in the spelling of `data_type`, the data type of the more informed side.""" members = self._members return ( members.shape, json_text(spelled_canonically(data_type, members.fill_value)), members.dimension_names, json_text(members.attributes), ) def __reduce__(self) -> tuple[type[ZarrV3ArrayMetadata], tuple[object, ZarrV3Context]]: # Pickle the pair and read it again on load, so a model's reading # always matches its document. return type(self), (self._document, self._context) # --- typed views ------------------------------------------------------ @property def shape(self) -> tuple[int, ...]: """The array's shape.""" return self._members.shape @property def fill_value(self) -> JSONValue: """The fill value as written, read-only at every level.""" return self._shown[0] @property def dimension_names(self) -> tuple[str | None, ...] | UNSET: """The dimension names; `UNSET` when the document writes none.""" return self._members.dimension_names @property def attributes(self) -> Mapping[str, JSONValue]: """The attributes, read-only at every level; empty when the document writes none.""" return self._shown[1] @property def extra_fields( self, ) -> Mapping[str, AcceptedField[ArrayExtraFieldDefinition[Any]] | UnclaimedField]: """Each member the spec does not define, by key, as the scope read it: an `AcceptedField` when a definition in scope claims the key, and an `UnclaimedField` otherwise.""" return self._shown[3] @property def unclaimed_fields(self) -> dict[Loc, UnclaimedField]: """Every field nothing in scope claims, at any depth, by its location in the document, except an extra member that says `"must_understand": false` and every field inside it, claimed or not, since a reader may ignore such a member whole. This answers whether a reader can open the array: the spec requires a reader to refuse a node with a field it does not understand, and the scope is what the reader understands. A codec inside a shard is located inside the shard's configuration. The model only reports these fields; refusing to open is up to the reader. """ return unclaimed_fields_of(self._reading.fields(), waived_members(self._shown[2])) @property def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: """The extra fields, by key and as written, that no definition in scope reads and that a reader is obligated to understand. Each unclaimed member of `extra_fields` not explicitly waived with `must_understand: false` (the spec's implicit-true rule, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578). The scope is what the reader recognizes, so a compliant reader MUST fail to open the array when this is not empty. The model only reports these fields; refusing to open is up to the reader. """ return must_understand_subset(self.extra_fields, self._shown[2]) @property def data_type(self) -> AcceptedField[DataTypeDefinition[Any]] | UnclaimedField: """The data type, as the scope read it.""" return held(self._reading.data_type) @property def chunk_grid(self) -> AcceptedField[ChunkGridDefinition[Any]] | UnclaimedField: """The chunk grid, as the scope read it.""" return held(self._reading.chunk_grid) @property def chunk_key_encoding(self) -> AcceptedField[ChunkKeyEncodingDefinition[Any]] | UnclaimedField: """The chunk key encoding, as the scope read it.""" return held(self._reading.chunk_key_encoding) @property def codecs(self) -> tuple[AcceptedField[CodecDefinition[Any]] | UnclaimedField, ...]: """The codecs, each as the scope read it, in pipeline order.""" return tuple(held(stage.codec) for stage in self._reading.pipeline) @property def storage_transformers( self, ) -> tuple[AcceptedField[StorageTransformerDefinition[Any]] | UnclaimedField, ...]: """The storage transformers, each as the scope read it.""" return tuple(held(entry) for entry in self._reading.storage_transformers) # --- changing --------------------------------------------------------- def update(self, **members: Unpack[ZarrV3ArrayMetadataUpdate]) -> ZarrV3ArrayMetadata: """Return this model with `members` (as JSON) replacing the document's members, read in this model's own scope. A member set to `UNSET` is removed. A member the spec does not define is given in `extra_fields`. Raises `TypeError` for a member the update does not take. Raises `MetadataValidationError` when the resulting document has a problem. So members that depend on each other must be passed together, for example a `shape` with a grid that fits it. """ changes = member_changes( members, ZarrV3ArrayMetadataUpdate, ArrayExtraFieldDefinition.spec_members ) document: dict[str, object] = {**self._document, **changes} for key, value in changes.items(): if value is UNSET: del document[key] return type(self)(document, context=self._context) def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadata: """Return this document read in `context`, whatever that changes: a gain, a loss, or a conflict. Raises `MetadataValidationError` when the document has a problem in `context`. The existing reading is reused when `context` reads every claim identically. """ context = scoped(context, CORE_AND_EXTENSIONS) if disagreements(context, claims_of(self._reading.fields())).agrees: return self._of(self._document, context, self._reading, self._members) return type(self)(self._document, context=context) def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadata: """Return this document read in `context`. `context` may claim names this scope left unclaimed, but must not contradict any claim. Raises `ScopeConflictError` when `context` reads a name with a different definition, or with no definition, where this scope read it with one. A loss of meaning is refused in the same way as a conflict. The error names each such name and its location in the document. Raises `MetadataValidationError` when a definition that `context` newly claims rejects what was written under its name: a gain can expose a problem. `with_context` reads the document in any scope. """ context = scoped(context, CORE_AND_EXTENSIONS) found = disagreements(context, claims_of(self._reading.fields())) if len(found.conflicts) != 0: raise ScopeConflictError(located_conflicts(self._reading.fields(), found.conflicts)) return self.with_context(context) def refines(self, other: ZarrV3ArrayMetadata) -> bool: """Whether this model holds everything `other` holds. Each field must refine its counterpart, in the order `refines` defines for fields (including the fields nested in each field). Every other member must be the same, with the fill value compared in the spelling of the more informed data type. If that data type rejects the other model's fill value, this is not a refinement.""" if type(other) is not type(self): return False if len(self.codecs) != len(other.codecs) or len(self.storage_transformers) != len( other.storage_transformers ): return False if self.extra_fields.keys() != other.extra_fields.keys(): return False pairs = ( (self.data_type, other.data_type), (self.chunk_grid, other.chunk_grid), (self.chunk_key_encoding, other.chunk_key_encoding), *zip(self.codecs, other.codecs, strict=True), *zip(self.storage_transformers, other.storage_transformers, strict=True), *((self.extra_fields[key], other.extra_fields[key]) for key in self.extra_fields), ) if not all(refines_field(mine, theirs) for mine, theirs in pairs): return False if len(fill_value_problems(self.data_type, other._members.fill_value)) != 0: return False return self._plain_key(self.data_type) == other._plain_key(self.data_type) # --- constructors ----------------------------------------------------- @classmethod def create_default( cls, *, context: ZarrV3Context | None = None, **overrides: Unpack[ZarrV3ArrayMetadataUpdate], ) -> ZarrV3ArrayMetadata: """Return a scalar `uint8` array, with any document members in `overrides` applied, read in `context`. `overrides` takes what `update` takes, including `UNSET` to drop a member. Raises `MetadataValidationError` when the resulting document has a problem. So members that depend on each other must be passed together: a data type with a fill value of that type, or a grid with the shape it fits. The default codec is `bytes` with little `endian`, which accepts a data type of any fixed size. Overriding `shape` without `chunk_grid` derives a consistent default grid: one regular chunk covering the array (`chunk_shape` equal to `shape`, with a length of 1 for a dimension of length 0, since "Chunk sizes must be greater than zero", https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-grids/regular-grid/index.rst#L40). """ # The grid is derived only from a valid shape. An invalid shape # derives no grid, and the read reports it. changes = member_changes( overrides, ZarrV3ArrayMetadataUpdate, ArrayExtraFieldDefinition.spec_members ) lengths, _ = dimension_lengths(changes, "shape") document: dict[str, object] = { "zarr_format": 3, "node_type": "array", "shape": (), "fill_value": 0, "data_type": "uint8", "chunk_grid": { "name": "regular", "configuration": {"chunk_shape": tuple(max(length, 1) for length in lengths or ())}, }, "codecs": ({"name": "bytes", "configuration": {"endian": "little"}},), "chunk_key_encoding": {"name": "default"}, } merged = { key: value for key, value in {**document, **changes}.items() if value is not UNSET } return cls(merged, context=context) @classmethod def from_json( cls, data: object, *, context: ZarrV3Context | None = None ) -> ZarrV3ArrayMetadata: """Return the model of `data`, a v3 array document read in `context`. Raises `MetadataValidationError` with every problem the read finds. `read_array_metadata_v3` returns the reading this model is built from, and the problems of a document that has any. """ return cls(data, context=context) @classmethod def from_key_value( cls, mapping: Mapping[StoreKey, bytes], *, context: ZarrV3Context | None = None ) -> ZarrV3ArrayMetadata: """Return the model of the array document at `zarr.json` in `mapping`, read in `context`. Raises `MetadataValidationError` when the key is missing, its bytes are not JSON, or the document is not valid. Raises `TypeError` for a scope of another format before any bytes are decoded. """ context = scoped(context, CORE_AND_EXTENSIONS) return cls(load_store_json(mapping, ZARR_V3_ARRAY_METADATA_STORE_KEY), context=context) def array_key_of( reading: ZarrV3ArrayMetadataReading, members: ArrayMembersV3 ) -> tuple[object, ...]: """A key for what a v3 array document means, from its `reading` and refined `members`. `==` and `hash` compare this key for the model, and two listings of one node are compared by it. The key holds each field's `field_key`, and the fill value as JSON text: in its canonical spelling when a definition in scope read the data type, and as written otherwise. Every other member is included as it is, with JSON members as text. """ data_type = held(reading.data_type) fill_value = ( spelled_canonically(data_type, members.fill_value) if isinstance(data_type, AcceptedField) else members.fill_value ) return ( members.shape, json_text(fill_value), field_key(data_type), field_key(held(reading.chunk_grid)), tuple(field_key(held(stage.codec)) for stage in reading.pipeline), field_key(held(reading.chunk_key_encoding)), members.dimension_names, json_text(members.attributes), tuple(field_key(held(entry)) for entry in reading.storage_transformers), extra_fields_key(reading.extra_fields), ) def extra_fields_key(extra_fields: Mapping[str, ResolvedField[Any]]) -> tuple[object, ...]: """A key for what a document's extra fields mean: each key with its field's `field_key`, sorted by key.""" return tuple( (key, field_key(extra)) for key, extra in sorted(extra_fields.items(), key=lambda item: item[0]) ) def read_array_metadata_v3( value: object, *, context: ZarrV3Context | None = None ) -> ZarrV3ArrayMetadataReading: """Read `value`, a v3 array document, in `context`, whatever it holds. The result holds everything the read finds: each extension point as `context` read it (an `AcceptedField` read by the definition that claims its name, an `UnclaimedField`, or a `RefusedField`), the chunks the codecs receive, each codec with the chunk it receives, every problem, and, when there are no problems, the document's model, which holds the same reading. The read checks the structure, and reads each extension point through the definition that claims its name in `context`. Examples of problems: a gzip `level` out of range, or a key that a codec's configuration does not declare. The fill value is checked against the data type as `context` read it (for example, an `int8` fill value of 300). The chunk grid is checked against the shape (for example, a regular grid with chunk lengths for two dimensions, over an array with three). The codecs are read as a pipeline, in order, each checked against the chunk it receives (for example, a `transpose` whose `order` has a different number of axes, or a shard that its inner chunks do not divide). A shard's inner and index codecs are checked too. A name that nothing in `context` claims is not checked, and neither is a fill value of that data type. After a codec with such a name, the next codec receives a chunk with unknown shape and data type. Unknown top-level keys are allowed (they map to `extra_fields`). A reader must understand each one that does not declare `must_understand: false`; the model reports these as `must_understand_fields`. To apply a policy over the fields (for example, allow only the core spec's fields), iterate over the reading's `fields()`. A value that is not an object has no fields. """ context = scoped(context, CORE_AND_EXTENSIONS) reading, members = read_array_v3(value, context) if members is None: return reading model = ZarrV3ArrayMetadata._of(members.document, context, reading, members) # pyright: ignore[reportPrivateUsage] return model.reading zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py000066400000000000000000001163731527014137600327730ustar00rootroot00000000000000"""The Zarr v2 group and its consolidated metadata.""" from __future__ import annotations import dataclasses from collections.abc import Mapping from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, cast from typing_extensions import TypeAliasType, TypedDict, Unpack from zarr_metadata._common import JSONValue from zarr_metadata._hierarchy import NodeType, tree_problems from zarr_metadata._json import ( MetadataValidationError, ValidationProblem, copied, frozen, is_json_object, is_object, json_text, not_an_object, object_at, refine_json, refine_user_data, refined_object, shown, with_input, ) from zarr_metadata._sentinel import UNSET from zarr_metadata.definition._context import disagreements, scoped from zarr_metadata.definition._scope import Conflict, ScopeConflictError, claims_of from zarr_metadata.model._array_v2 import ( ARRAY_FILE_ATTRIBUTES, ZarrV2ArrayMetadata, ZarrV2ArrayMetadataReading, decoded_node_files_v2, node_document_v2, read_array_metadata_v2, read_array_v2, ) from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, check_literal, dump_store_json, load_store_json, located_conflicts, member_changes, missing_keys, reading_of, unexpected_keys, validate_attributes, ) # A runtime import: `get_type_hints` resolves the public signatures. from zarr_metadata.v2._context import ZarrV2Context # noqa: TC001 from zarr_metadata.v2.array import ZARR_V2_ARRAY_METADATA_STORE_KEY from zarr_metadata.v2.attributes import ZARR_V2_ATTRIBUTES_STORE_KEY from zarr_metadata.v2.consolidated import ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY from zarr_metadata.v2.definition import CORE_V2 from zarr_metadata.v2.group import ZARR_V2_GROUP_METADATA_STORE_KEY, ZarrV2GroupMetadataJSON if TYPE_CHECKING: from collections.abc import Iterator from zarr_metadata._typed_json import Loc from zarr_metadata.definition._definition import ResolvedField, UnclaimedField from zarr_metadata.v2.attributes import ZarrV2AttributesStoreKey from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataStoreKey from zarr_metadata.v2.group import ZarrV2GroupMetadataStoreKey _GROUP_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( ZarrV2GroupMetadataJSON.__required_keys__ ) _GROUP_METADATA_OPTIONAL_KEYS_V2: Final[frozenset[str]] = frozenset( ZarrV2GroupMetadataJSON.__optional_keys__ ) _GROUP_METADATA_STANDARD_KEYS_V2: Final[frozenset[str]] = ( _GROUP_METADATA_REQUIRED_KEYS_V2 | _GROUP_METADATA_OPTIONAL_KEYS_V2 ) def _group_problems_v2(value: object) -> tuple[ValidationProblem, ...]: """Every reason `value` is not a v2 group document in the merged form: the `.zgroup` members, plus an optional `attributes` object from `.zattrs`. A group has no fields for a scope to read, so no scope is needed.""" if not is_object(value): return not_an_object(value) doc = value problems: list[ValidationProblem] = list(missing_keys(_GROUP_METADATA_REQUIRED_KEYS_V2, doc)) problems.extend(unexpected_keys(_GROUP_METADATA_STANDARD_KEYS_V2, doc)) problems.extend(check_literal(doc, "zarr_format", 2)) if "attributes" in doc: problems.extend(validate_attributes(doc["attributes"])) return with_input(problems, doc) _GROUP_FILE_ATTRIBUTES: Final = ValidationProblem( ("attributes",), f"unexpected key 'attributes' in {ZARR_V2_GROUP_METADATA_STORE_KEY}: a group's attributes " f"belong in {ZARR_V2_ATTRIBUTES_STORE_KEY}", "unknown_key", ) """The problem of a `.zgroup` that holds `attributes`, which `.zgroup` does not declare: a group's attributes belong in `.zattrs`.""" @dataclass(frozen=True, slots=True) class ZarrV2GroupMetadataReading: """A v2 group document as a scope read it: every problem, and the model when there are no problems. A group has no fields for a scope to read.""" problems: tuple[ValidationProblem, ...] = () """Every reason the document is not valid.""" metadata: ZarrV2GroupMetadata | None = None """The document's model when there is no problem; None otherwise.""" def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: """Its fields as read: none, since a group has no fields.""" return iter(()) def __reduce__(self) -> str | tuple[object, ...]: # Same as the other readings: pickle through the model, when there is one. if self.metadata is not None: return (reading_of, (self.metadata,)) return object.__reduce__(self) def read_group_metadata_v2( value: object, *, context: ZarrV2Context | None = None ) -> ZarrV2GroupMetadataReading: """Read `value`, a v2 group document in the merged form (`.zgroup` plus `attributes` from `.zattrs`), in `context` (`CORE_V2` when none is given), whatever it holds: every problem, and the model when there are none. A group has no fields for a scope to read; `context` is taken for consistency with every other v2 reader.""" context = scoped(context, CORE_V2) problems = _group_problems_v2(value) if len(problems) != 0: return ZarrV2GroupMetadataReading(problems=problems) return ZarrV2GroupMetadata(value, context=context).reading @dataclass(frozen=True, slots=True) class ZarrV2UnknownNodeMetadataReading: """A store that holds no v2 node: neither a `.zarray` nor a `.zgroup`, or both. The problems say which.""" problems: tuple[ValidationProblem, ...] """Why the store holds no node.""" @property def metadata(self) -> None: """Its model: always None, since there is no node to model.""" return None def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: """Its fields as read: none, since no document is read.""" return iter(()) ZarrV2NodeMetadataReading = TypeAliasType( "ZarrV2NodeMetadataReading", ZarrV2ArrayMetadataReading | ZarrV2GroupMetadataReading | ZarrV2UnknownNodeMetadataReading, ) """A v2 node as `read_node_metadata_v2` reads it from a store: as the array or group its files declare, or as neither.""" def read_node_metadata_v2( files: Mapping[str, object], *, context: ZarrV2Context | None = None ) -> ZarrV2NodeMetadataReading: """Read the v2 node described by `files`, in `context` (`CORE_V2` when none is given), as the node type its files declare. `files` maps each of the node's files to its decoded JSON: `.zarray` or `.zgroup`, and `.zattrs` when there is one. They can come from a store, decoded (as `node_metadata_from_key_value_v2` does), or from the entries of a `.zmetadata`, which hold the same JSON. A v2 document does not name its node type; the file does. A `.zarray` makes the node an array, read as `read_array_metadata_v2` reads it, and a `.zgroup` makes it a group, read as `read_group_metadata_v2` reads it, with the attributes at `.zattrs` in either case. `files` with neither, or with both, give a `ZarrV2UnknownNodeMetadataReading` with the problem. A `.zarray` or `.zgroup` that writes `attributes` itself gives the reading of its node type with no model, and with that problem followed by the problems of the rest of the file. This is the v2 counterpart of `read_node_metadata_v3`, which reads one `zarr.json` and its `node_type`. """ context = scoped(context, CORE_V2) is_array = ZARR_V2_ARRAY_METADATA_STORE_KEY in files is_group = ZARR_V2_GROUP_METADATA_STORE_KEY in files if is_array and is_group: both = ValidationProblem( (ZARR_V2_GROUP_METADATA_STORE_KEY,), f"a node is an array or a group, not both: a {ZARR_V2_ARRAY_METADATA_STORE_KEY} is beside it", "invalid_value", ) return ZarrV2UnknownNodeMetadataReading((both,)) if is_array: document, problems = node_document_v2( files, ZARR_V2_ARRAY_METADATA_STORE_KEY, ARRAY_FILE_ATTRIBUTES ) reading = read_array_metadata_v2(document, context=context) if len(problems) != 0: return dataclasses.replace( reading, problems=(*problems, *reading.problems), metadata=None ) return reading if is_group: document, problems = node_document_v2( files, ZARR_V2_GROUP_METADATA_STORE_KEY, _GROUP_FILE_ATTRIBUTES ) reading = read_group_metadata_v2(document, context=context) if len(problems) != 0: return ZarrV2GroupMetadataReading(problems=(*problems, *reading.problems)) return reading neither = ValidationProblem( (), f"expected a {ZARR_V2_ARRAY_METADATA_STORE_KEY} or a {ZARR_V2_GROUP_METADATA_STORE_KEY}", "missing_key", ) return ZarrV2UnknownNodeMetadataReading((neither,)) def node_metadata_from_key_value_v2( mapping: Mapping[StoreKey, bytes], *, context: ZarrV2Context | None = None ) -> ZarrV2NodeMetadata: """Return the model of the v2 node whose files are in `mapping`, a store's keys and bytes: each file is decoded, then read as `read_node_metadata_v2` reads it. Raises `MetadataValidationError` with every problem, including a file whose bytes are not JSON, and `TypeError` for a scope of another format before any bytes are decoded.""" context = scoped(context, CORE_V2) files, problems = decoded_node_files_v2(mapping) if len(problems) != 0: raise MetadataValidationError(problems) reading = read_node_metadata_v2(files, context=context) if reading.metadata is None: raise MetadataValidationError(reading.problems) return reading.metadata class ZarrV2GroupMetadataUpdate(TypedDict, total=False): """The members `ZarrV2GroupMetadata.update` replaces: `attributes` in the form a `.zattrs` writes them, or `UNSET` for no `.zattrs`.""" attributes: Mapping[str, JSONValue] | UNSET class ZarrV2GroupMetadata(Keyed): """A v2 group document, and the scope it was read in. Like the v3 models, this model is a pair. `to_json` returns the merged document (`.zgroup`, plus `attributes` when a `.zattrs` holds them) as written, refined. `attributes` is empty when no `.zattrs` exists, as in a v3 model. The document still tells the two apart: `to_json` and `to_key_value` write `attributes` and `.zattrs` only when a `.zattrs` exists, even an empty one. A group has no fields for a scope to read, so the scope is held only for consistency with the other models: `update` reads new attributes in it, and no other scope conflicts with the reading. A model is only built by reading: the constructor raises `MetadataValidationError` with every problem. Two groups are equal when their attributes are written the same way, as `_key_of` defines; no `.zattrs` and an empty one are equal. """ __slots__ = ("_attributes", "_context", "_document", "_shown") _attributes: dict[str, JSONValue] | UNSET _context: ZarrV2Context _document: dict[str, JSONValue] _shown: Mapping[str, JSONValue] zarr_format: Final[Literal[2]] = 2 # noqa: PYI064 - a bare Final does not narrow a union of models node_type: Final[Literal["group"]] = "group" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: context = scoped(context, CORE_V2) problems = _group_problems_v2(document) if len(problems) != 0: raise MetadataValidationError(problems) # Refine once. The attributes the model holds are taken from the # refined document, which shares no objects with the input. refined = refined_object(document) attributes: dict[str, JSONValue] | UNSET = ( object_at(refined, "attributes") if "attributes" in refined else UNSET ) self._adopt(refined, context, attributes) @classmethod def _of( cls, document: dict[str, JSONValue], context: ZarrV2Context, attributes: dict[str, JSONValue] | UNSET, ) -> ZarrV2GroupMetadata: """Build a model of a document that a read found no problems with, without reading it again. The readers in this package build models through this method; pyright reports these calls as private use.""" model = object.__new__(cls) model._adopt(document, context, attributes) return model def _adopt( self, document: dict[str, JSONValue], context: ZarrV2Context, attributes: dict[str, JSONValue] | UNSET, ) -> None: self._document = document self._context = context self._attributes = attributes self._shown = frozen({} if attributes is UNSET else attributes) @property def reading(self) -> ZarrV2GroupMetadataReading: """The document as the scope read it: no problems, and this model.""" return ZarrV2GroupMetadataReading(metadata=self) @property def context(self) -> ZarrV2Context: """The scope the document was read in. `update` reads new attributes in this scope.""" return self._context @property def unclaimed_fields(self) -> dict[Loc, UnclaimedField]: """Every field nothing in scope claims. A v2 group holds no fields, so this is always empty; it is here so a reader can ask any node the same question.""" return {} @property def attributes(self) -> Mapping[str, JSONValue]: """The user attributes a `.zattrs` holds, read-only at every level; empty when there is no `.zattrs`.""" return self._shown def to_json(self) -> ZarrV2GroupMetadataJSON: """The merged document as written, refined. The result shares no objects with the model. `attributes` is included when set, even if empty. This is not the on-disk `.zgroup`, which excludes them: `to_key_value` splits the document as a store holds it (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313; https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330). """ return cast("ZarrV2GroupMetadataJSON", copied(self._document)) def to_key_value( self, *, indent: int | str | None = None ) -> Mapping[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]: """The document as a store holds it: `.zgroup` without the attributes, and `.zattrs` with the attributes when they are set, even if empty.""" zgroup = {key: value for key, value in self._document.items() if key != "attributes"} out: dict[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes] = { ZARR_V2_GROUP_METADATA_STORE_KEY: dump_store_json(zgroup, indent=indent) } if "attributes" in self._document: out[ZARR_V2_ATTRIBUTES_STORE_KEY] = dump_store_json( self._document["attributes"], indent=indent ) return out def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" def _key_of(self) -> tuple[object, ...]: """The key `==` and `hash` compare for a v2 group model: its attributes as JSON text. No `.zattrs` counts as empty attributes, since both mean that the group has none, as in v3.""" attributes = self._attributes return (json_text({} if attributes is UNSET else attributes),) def __reduce__(self) -> tuple[type[ZarrV2GroupMetadata], tuple[object, ZarrV2Context]]: return type(self), (self._document, self._context) def update(self, **members: Unpack[ZarrV2GroupMetadataUpdate]) -> ZarrV2GroupMetadata: """Return this model with `attributes` replacing the document's attributes, read in this model's own scope. `UNSET` removes them. Raises `MetadataValidationError` when the resulting document has a problem.""" changes = member_changes(members, ZarrV2GroupMetadataUpdate, frozenset()) document: dict[str, object] = {**self._document, **changes} for key, value in changes.items(): if value is UNSET: del document[key] return type(self)(document, context=self._context) def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadata: """Return this document read in `context`: the same group, holding that scope.""" context = scoped(context, CORE_V2) return self._of(self._document, context, self._attributes) def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadata: """Return this document read in `context`. A group has no fields, so no scope conflicts with its reading, and this is the same as `with_context`.""" return self.with_context(context) def refines(self, other: ZarrV2GroupMetadata) -> bool: """Whether this group holds everything `other` holds, which means its attributes are written the same way, no `.zattrs` counting as empty attributes. False when `other` is not a v2 group.""" return type(other) is type(self) and self._model_key == other._model_key @classmethod def create_default( cls, *, context: ZarrV2Context | None = None, **overrides: Unpack[ZarrV2GroupMetadataUpdate] ) -> ZarrV2GroupMetadata: """Return a group with no `.zattrs`, with `overrides` applied, read in `context`. Raises `MetadataValidationError` when the resulting document has a problem.""" changes = member_changes(overrides, ZarrV2GroupMetadataUpdate, frozenset()) given = {key: value for key, value in changes.items() if value is not UNSET} return cls({"zarr_format": 2, **given}, context=context) @classmethod def from_json( cls, data: object, *, context: ZarrV2Context | None = None ) -> ZarrV2GroupMetadata: """Return the model of `data`, a v2 group document with its attributes under `attributes`, read in `context`. Raises `MetadataValidationError` with every problem.""" return cls(data, context=context) @classmethod def from_key_value( cls, mapping: Mapping[StoreKey, bytes], *, context: ZarrV2Context | None = None ) -> ZarrV2GroupMetadata: """Return the model of the group at `.zgroup` in `mapping`, with the attributes at `.zattrs` when that key exists, read in `context`. Raises `MetadataValidationError` when `.zgroup` is missing, bytes are not JSON, `.zgroup` holds `attributes`, or the document is not valid, with every problem the read finds. Raises `TypeError` for a scope of another format before any bytes are decoded. """ context = scoped(context, CORE_V2) files, problems = decoded_node_files_v2( mapping, (ZARR_V2_GROUP_METADATA_STORE_KEY, ZARR_V2_ATTRIBUTES_STORE_KEY) ) if len(problems) != 0: raise MetadataValidationError(problems) document, problems = node_document_v2( files, ZARR_V2_GROUP_METADATA_STORE_KEY, _GROUP_FILE_ATTRIBUTES ) if len(problems) != 0: if document is not None: problems = (*problems, *_group_problems_v2(document)) raise MetadataValidationError(problems) return cls(document, context=context) ZarrV2NodeMetadata = TypeAliasType("ZarrV2NodeMetadata", ZarrV2ArrayMetadata | ZarrV2GroupMetadata) """The model of one v2 node, as `node_metadata_from_key_value_v2` returns it and a `.zmetadata` holds it: an array or a group.""" @dataclass(frozen=True, slots=True) class ZarrV2ConsolidatedMetadataReading: """A v2 `.zmetadata` document as a scope read it: every problem, and the model when there are no problems.""" problems: tuple[ValidationProblem, ...] = () """Every reason the document is not valid.""" metadata: ZarrV2ConsolidatedMetadata | None = None """The document's model when there is no problem; None otherwise.""" def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: """Each field of each array node, as read, located in the `.zmetadata`: under the node's `.zarray` entry, at the key the document writes for it. Nothing when there is no model, and nothing for a group, which has no fields.""" if self.metadata is None: return yield from self.metadata._located_fields() # pyright: ignore[reportPrivateUsage] def __reduce__(self) -> str | tuple[object, ...]: # Same as the other readings: pickle through the model, when there is one. if self.metadata is not None: return (reading_of, (self.metadata,)) return object.__reduce__(self) def read_consolidated_metadata_v2( value: object, *, context: ZarrV2Context | None = None ) -> ZarrV2ConsolidatedMetadataReading: """Read `value`, a v2 `.zmetadata` document, in `context` (`CORE_V2` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV2ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" context = scoped(context, CORE_V2) document, nodes, files, problems = _read_consolidated_v2(value, context) if len(problems) != 0: return ZarrV2ConsolidatedMetadataReading(problems=problems) model = ZarrV2ConsolidatedMetadata._of(document, context, nodes, files) # pyright: ignore[reportPrivateUsage] return model.reading class ZarrV2ConsolidatedMetadata(Keyed): """A v2 `.zmetadata` document, and the scope its nodes were read in. `metadata` maps the path of each node to its model in this scope, as `ZarrV3ConsolidatedMetadata.metadata` does. A path is written without the leading `/`, and the root's path is `""`: the node at `/a/b` is at `a/b`. Each node is a `.zarray` or `.zgroup` entry, merged with its sibling `.zattrs`. A `.zattrs` with no sibling is kept but does not form a node. Any other entry must be JSON, and is kept. `to_json` returns the entries keyed by file (`"a/.zarray"`, `"a/.zattrs"`, ...) as written, refined, so the document still tells which nodes have a `.zattrs` at all. A model is only built by reading: the constructor raises `MetadataValidationError` with every problem, each located under its entry. Two documents are equal when each node means the same thing and the other entries are written the same way, as the model key defines. `refines`, `with_context` and `refined_in` work through the nodes. """ __slots__ = ("_context", "_document", "_files", "_metadata") _context: ZarrV2Context _document: dict[str, JSONValue] _files: dict[str, dict[str, str]] _metadata: dict[str, ZarrV2NodeMetadata] zarr_consolidated_format: Final = 1 def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: context = scoped(context, CORE_V2) read, nodes, files, problems = _read_consolidated_v2(document, context) if len(problems) != 0: raise MetadataValidationError(problems) self._adopt(read, context, nodes, files) @classmethod def _of( cls, document: dict[str, JSONValue], context: ZarrV2Context, metadata: dict[str, ZarrV2NodeMetadata], files: dict[str, dict[str, str]], ) -> ZarrV2ConsolidatedMetadata: """Build a model of a document that a read found no problems with, holding the nodes that read built and the entry key of each node's files. The readers in this package build models through this method; pyright reports these calls as private use.""" model = object.__new__(cls) model._adopt(document, context, metadata, files) return model def _adopt( self, document: dict[str, JSONValue], context: ZarrV2Context, metadata: dict[str, ZarrV2NodeMetadata], files: dict[str, dict[str, str]], ) -> None: self._document = document self._context = context self._metadata = metadata # The entry key of each file of each node, by node path and then by # file name, as the document writes it. self._files = files @property def reading(self) -> ZarrV2ConsolidatedMetadataReading: """The document as the scope read it: no problems, and this model.""" return ZarrV2ConsolidatedMetadataReading(metadata=self) @property def context(self) -> ZarrV2Context: """The scope the nodes were read in.""" return self._context @property def metadata(self) -> Mapping[str, ZarrV2NodeMetadata]: """A read-only view of the model of each node, keyed by its path without the leading `/` (`""` for the root).""" return MappingProxyType(self._metadata) def to_json(self) -> dict[str, JSONValue]: """The `.zmetadata` document as written, refined. The result shares no objects with the model.""" return cast("dict[str, JSONValue]", copied(self._document)) def to_key_value( self, *, indent: int | str | None = None ) -> Mapping[ZarrV2ConsolidatedMetadataStoreKey, bytes]: """The document as a store holds it: JSON bytes at `.zmetadata`, indented by `indent`.""" return { ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY: dump_store_json(self._document, indent=indent) } def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" def _key_of(self) -> tuple[object, ...]: """The key `==` and `hash` compare for v2 consolidated metadata: each node's path and key, and every other entry as JSON text.""" return ( tuple(sorted((path, node._model_key) for path, node in self._metadata.items())), self._other_entries_text(), ) def _other_entries_text(self) -> str: """The entries that no node is read from (an orphan `.zattrs`, or any other key) as JSON text, which `==` compares.""" entries = object_at(self._document, "metadata") consumed = {key for names in self._files.values() for key in names.values()} return json_text({key: value for key, value in entries.items() if key not in consumed}) def _located_fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: """Each field of each array node, as read, located in the document: under the node's `.zarray` entry, at the key the document writes for it.""" for path, node in self._metadata.items(): if isinstance(node, ZarrV2ArrayMetadata): key = self._files[path][ZARR_V2_ARRAY_METADATA_STORE_KEY] for loc, field in node.reading.fields(): yield ("metadata", key, *loc), field def __reduce__(self) -> tuple[type[ZarrV2ConsolidatedMetadata], tuple[object, ZarrV2Context]]: return type(self), (self._document, self._context) def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ConsolidatedMetadata: """Return this document with every node read in `context`, whatever that changes. Raises `MetadataValidationError` when a node has a problem in `context`.""" context = scoped(context, CORE_V2) return type(self)(self._document, context=context) def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2ConsolidatedMetadata: """Return this document with every node read in `context`. `context` may claim names this scope left unclaimed, but must not contradict any claim. Raises `ScopeConflictError` naming each conflict, located at the node's entry. Raises `MetadataValidationError` when a gain exposes a problem. """ context = scoped(context, CORE_V2) conflicts: list[Conflict] = [] for path, node in self._metadata.items(): if not isinstance(node, ZarrV2ArrayMetadata): continue found = disagreements(context, claims_of(node.reading.fields())) key = self._files[path][ZARR_V2_ARRAY_METADATA_STORE_KEY] conflicts.extend( dataclasses.replace( conflict, loc=("metadata", key, *(() if conflict.loc is None else conflict.loc)), ) for conflict in located_conflicts(node.reading.fields(), found.conflicts) ) if len(conflicts) != 0: raise ScopeConflictError(conflicts) return self.with_context(context) def refines(self, other: ZarrV2ConsolidatedMetadata) -> bool: """Whether every node this holds refines the node `other` holds at the same path, both hold the same paths, and the other entries are written the same way. False when `other` is not v2 consolidated metadata.""" if type(other) is not type(self): return False if self._metadata.keys() != other._metadata.keys(): return False if self._other_entries_text() != other._other_entries_text(): return False return all( _v2_node_refines(self._metadata[path], other._metadata[path]) for path in self._metadata ) @classmethod def from_json( cls, data: object, *, context: ZarrV2Context | None = None ) -> ZarrV2ConsolidatedMetadata: """Return the model of `data`, a `.zmetadata` document, with its nodes read in `context`. Raises `MetadataValidationError` with every problem.""" return cls(data, context=context) @classmethod def from_key_value( cls, mapping: Mapping[StoreKey, bytes], *, context: ZarrV2Context | None = None ) -> ZarrV2ConsolidatedMetadata: """Return the model of the document at `.zmetadata` in `mapping`, read in `context`. Raises `MetadataValidationError` when the key is missing, its bytes are not JSON, or the document is not valid. Raises `TypeError` for a scope of another format before the bytes are decoded. """ context = scoped(context, CORE_V2) return cls( load_store_json(mapping, ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY), context=context ) def _v2_node_refines(node: ZarrV2NodeMetadata, other: ZarrV2NodeMetadata) -> bool: """Whether `node` refines `other`. Models of the same kind use their own `refines`; an array never refines a group, or the reverse.""" if isinstance(node, ZarrV2ArrayMetadata): return isinstance(other, ZarrV2ArrayMetadata) and node.refines(other) return isinstance(other, ZarrV2GroupMetadata) and node.refines(other) _NODE_FILES: Final = ( ZARR_V2_ARRAY_METADATA_STORE_KEY, ZARR_V2_GROUP_METADATA_STORE_KEY, ZARR_V2_ATTRIBUTES_STORE_KEY, ) def _entries_by_path( entries: Mapping[str, JSONValue], ) -> tuple[dict[str, dict[str, str]], list[ValidationProblem]]: """The store key of each `.zarray`, `.zgroup` and `.zattrs` entry, by node path and then by file name. Also a problem for each extra key that names a file of a node already seen, since that entry would otherwise not be read.""" by_path: dict[str, dict[str, str]] = {} problems: list[ValidationProblem] = [] for key in entries: path, _, name = key.rpartition("/") if name not in _NODE_FILES: continue # A node's path, as a store names it: segments joined by a single `/`. segments = [segment for segment in path.split("/") if segment != ""] if any(segment in (".", "..") for segment in segments): problems.append( ValidationProblem( ("metadata", key), f"expected the path of a node, got {key!r}, which has a '.' or '..' segment", "invalid_value", ) ) continue path = "/".join(segments) files = by_path.setdefault(path, {}) if name in files: problems.append( ValidationProblem( ("metadata", key), f"a second {name} for the node at {path!r}, which {files[name]!r} is", "invalid_value", ) ) continue files[name] = key return by_path, problems def _read_consolidated_v2( data: object, context: ZarrV2Context ) -> tuple[ dict[str, JSONValue], dict[str, ZarrV2NodeMetadata], dict[str, dict[str, str]], tuple[ValidationProblem, ...], ]: """Read `data`, a `.zmetadata` document. Return the document as read, with its keys in the order it writes them and each entry as read; the model of each node read in `context`, by path; the entry key of each node's files, by path and then by file name; and every problem, located in the document. Each entry is read as the kind of document its key names. A `.zattrs` entry is user data. Any other entry must be JSON by RFC 8259. A `.zarray` or `.zgroup` entry is read as that document, merged with its sibling `.zattrs`. Its problems are located under its own entry, and problems with the attributes under the `.zattrs` entry. The nodes and their files are empty when there is any problem. """ if not is_object(data): return {}, {}, {}, not_an_object(data) doc = data problems: list[ValidationProblem] = [ ValidationProblem((key,), "missing required key", "missing_key") for key in ("zarr_consolidated_format", "metadata") if key not in doc ] problems.extend(unexpected_keys(frozenset({"zarr_consolidated_format", "metadata"}), doc)) problems.extend(check_literal(doc, "zarr_consolidated_format", 1)) refined: dict[str, JSONValue] = {} if "metadata" in doc: entries = doc["metadata"] if not is_object(entries) or not all(isinstance(k, str) for k in entries): problems.append( ValidationProblem( ("metadata",), "expected an object with string keys", "invalid_type" ) ) else: for key, value in entries.items(): if not isinstance(key, str): continue refine = ( refine_user_data if key.rsplit("/", 1)[-1] == ZARR_V2_ATTRIBUTES_STORE_KEY else refine_json ) entry, found = refine(value, ("metadata", key)) problems.extend(found) refined[key] = entry nodes: dict[str, ZarrV2NodeMetadata] = {} by_path: dict[str, dict[str, str]] = {} if len(problems) == 0: # A second key for one file is reported along with the node # problems. Every node is still read, so a user sees all problems # at once. by_path, doubled = _entries_by_path(refined) problems.extend(doubled) for path, names in by_path.items(): node, found = _read_node_v2(path, names, refined, context) problems.extend(found) if node is not None: nodes[path] = node problems.extend(_hierarchy_problems_v2(by_path)) if len(problems) != 0: nodes = {} files = {path: by_path[path] for path in nodes} members: dict[str, JSONValue] = {"zarr_consolidated_format": 1, "metadata": refined} # The keys in the order the document writes them, as the v3 models keep them. document = {key: members[key] for key in doc if isinstance(key, str) and key in members} return document, nodes, files, with_input(problems, doc) def _hierarchy_problems_v2(by_path: Mapping[str, Mapping[str, str]]) -> list[ValidationProblem]: """Why the nodes in a `.zmetadata` do not form a valid hierarchy, as `tree_problems` checks it; the paths are already checked against the v2 rules. A node below an array is reported at its own entry. A missing group above a node is reported at the entry that group would have. An orphan `.zattrs` is not a node.""" # The root is a group unless an entry says otherwise: a `.zmetadata` # does not have to list its own `.zgroup`. nodes: dict[str, NodeType] = {"/": "group"} for path, names in by_path.items(): if ZARR_V2_ARRAY_METADATA_STORE_KEY in names: nodes[f"/{path}"] = "array" elif ZARR_V2_GROUP_METADATA_STORE_KEY in names: nodes[f"/{path}"] = "group" problems: list[ValidationProblem] = [] for found in tree_problems(nodes): at = found.loc[0] path = at[1:] if isinstance(at, str) else "" names = by_path.get(path, {}) key = names.get( ZARR_V2_ARRAY_METADATA_STORE_KEY, names.get( ZARR_V2_GROUP_METADATA_STORE_KEY, f"{path}/{ZARR_V2_GROUP_METADATA_STORE_KEY}" if path != "" else ZARR_V2_GROUP_METADATA_STORE_KEY, ), ) problems.append(ValidationProblem(("metadata", key), found.message, found.kind)) return problems def _read_node_v2( path: str, names: Mapping[str, str], entries: Mapping[str, JSONValue], context: ZarrV2Context ) -> tuple[ZarrV2NodeMetadata | None, list[ValidationProblem]]: """Read the node at `path` from its `.zarray` or `.zgroup` entry merged with its `.zattrs`. Return the node and every problem, located under the entries. Returns None and no problems when there is only a `.zattrs`.""" zarray = names.get(ZARR_V2_ARRAY_METADATA_STORE_KEY) zgroup = names.get(ZARR_V2_GROUP_METADATA_STORE_KEY) zattrs = names.get(ZARR_V2_ATTRIBUTES_STORE_KEY) if zarray is not None and zgroup is not None: return None, [ ValidationProblem( ("metadata", zgroup), f"a node is an array or a group, not both: a {ZARR_V2_ARRAY_METADATA_STORE_KEY} is at " f"{path!r} too", "invalid_value", ) ] key = zarray if zarray is not None else zgroup if key is None: return None, [] document = entries[key] if not is_json_object(document): return None, [ ValidationProblem( ("metadata", key), f"expected an object, got {shown(document)}", "invalid_type" ) ] merged = dict(document) beside: list[ValidationProblem] = [] if "attributes" in merged: # The same problem the node reader reports for the file, at the # entry. The rest of the entry is still read, for its own problems. found = ARRAY_FILE_ATTRIBUTES if zarray is not None else _GROUP_FILE_ATTRIBUTES beside.append(dataclasses.replace(found, loc=("metadata", key, *found.loc))) del merged["attributes"] if zattrs is not None: merged["attributes"] = entries[zattrs] def located(found: tuple[ValidationProblem, ...]) -> list[ValidationProblem]: placed: list[ValidationProblem] = [] for problem in found: if zattrs is not None and problem.loc[:1] == ("attributes",): placed.append( dataclasses.replace(problem, loc=("metadata", zattrs, *problem.loc[1:])) ) else: placed.append(dataclasses.replace(problem, loc=("metadata", key, *problem.loc))) return placed if zarray is not None: reading, members = read_array_v2(merged, context) if members is None or len(beside) != 0: return None, [*beside, *located(reading.problems)] return ZarrV2ArrayMetadata._of(members.document, context, reading, members), [] # pyright: ignore[reportPrivateUsage] found = _group_problems_v2(merged) if len(found) != 0 or len(beside) != 0: return None, [*beside, *located(found)] document = refined_object(merged) attributes: dict[str, JSONValue] | UNSET = ( object_at(document, "attributes") if "attributes" in document else UNSET ) return ZarrV2GroupMetadata._of(document, context, attributes), [] # pyright: ignore[reportPrivateUsage] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py000066400000000000000000001714211527014137600327670ustar00rootroot00000000000000"""The Zarr v3 group, its consolidated metadata, and the reader of a `zarr.json` of either node type. These three are one module because they are mutually recursive: consolidated metadata holds the documents of arrays and groups, each of those is read as a node of either type, and a group's `update` takes a consolidated metadata model. Split into modules, they would import each other. """ from __future__ import annotations import dataclasses from collections.abc import Mapping from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, cast from typing_extensions import TypeAliasType, TypedDict, Unpack from zarr_metadata._common import JSONValue from zarr_metadata._json import ( MetadataValidationError, ValidationProblem, copied, frozen, is_object, json_text, nested_past_the_levels, not_an_object, outside_of, prefixed, refine_user_data, shown, shown_key, with_input, within, ) from zarr_metadata._json import prefixed as _prefix from zarr_metadata._sentinel import UNSET from zarr_metadata.definition._context import disagreements, scoped from zarr_metadata.definition._definition import UnclaimedField, fields_of, held from zarr_metadata.definition._scope import ( Conflict, ScopeConflictError, claims_of, definition_said, kind_name, ) from zarr_metadata.definition._scope import refines as refines_field from zarr_metadata.model._array_v3 import ( ArrayMembersV3, ZarrV3ArrayMetadata, ZarrV3ArrayMetadataReading, array_key_of, extra_fields_key, members_past_the_levels, must_understand_subset, no_extra_fields, read_array_metadata_v3, read_array_v3, read_extra_fields_v3, waived_members, ) from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, attributes_of, check_literal, dump_store_json, load_store_json, located_conflicts, member_changes, missing_keys, reading_of, unclaimed_fields_of, unexpected_keys, ) from zarr_metadata.v3._common import ZarrV3ExtensionField # A runtime import: `get_type_hints` resolves the public signatures. from zarr_metadata.v3._context import ZarrV3Context # noqa: TC001 from zarr_metadata.v3._core import CORE_AND_EXTENSIONS from zarr_metadata.v3._definition import GroupExtraFieldDefinition from zarr_metadata.v3._hierarchy import hierarchy_problems, path_faults, said from zarr_metadata.v3.consolidated import ( ZARR_V3_CONSOLIDATED_METADATA_KEY, ZarrV3ConsolidatedMetadataJSON, ) from zarr_metadata.v3.group import ZARR_V3_GROUP_METADATA_STORE_KEY, ZarrV3GroupMetadataJSON if TYPE_CHECKING: from collections.abc import Iterator from zarr_metadata._hierarchy import NodeType from zarr_metadata._typed_json import Loc from zarr_metadata.definition._definition import AcceptedField, ResolvedField from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.group import ZarrV3GroupMetadataStoreKey # The members a v3 group document must have. The members the spec defines # are `GroupExtraFieldDefinition.spec_members`; any other is an extra field. _GROUP_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( ZarrV3GroupMetadataJSON.__required_keys__ ) # Unlike the node aliases below, this one holds its union as a string: the # JSON TypedDicts are imported only for type checking, and the models are # defined later in this module. ZarrV3NodeMetadataInput = TypeAliasType( "ZarrV3NodeMetadataInput", "ZarrV3ArrayMetadataJSON | ZarrV3GroupMetadataJSON | ZarrV3ArrayMetadata | ZarrV3GroupMetadata", ) """An entry of consolidated metadata, at a path, as `ZarrV3ConsolidatedMetadata` accepts it: a document, or a model of one. An `Input` type is what a constructor accepts. A `JSON` TypedDict is what a document writes, and an `Update` type is what `update` replaces. Of these three, only an `Input` may hold a model. """ class ZarrV3ConsolidatedMetadataInput(TypedDict, closed=True): """The `consolidated_metadata` member as `ZarrV3ConsolidatedMetadata` accepts it: in the form a document writes it, with each entry a document or a node model. A node model is accepted when the scope of the consolidated metadata reads every one of its claims identically, or claims names the model's scope left unclaimed. In the second case the model's document is read again in that scope. A node model is refused, with a problem at its path, when the two scopes read a name differently, or when the scope of the consolidated metadata leaves a claimed name unclaimed. """ kind: Literal["inline"] must_understand: Literal[False] metadata: Mapping[str, ZarrV3NodeMetadataInput] class ZarrV3GroupMetadataUpdate(TypedDict, total=False, closed=True): """The members `ZarrV3GroupMetadata.update` replaces: each in the form a document writes it, or `UNSET` to remove it. `consolidated_metadata` is typed as the reference implementation writes it, or as any JSON object, such as another group's `consolidated_metadata`, since a group checks only its envelope. It can also be a `ZarrV3ConsolidatedMetadata`: its models are read in the group's scope, as a listing of node models is read, and its document is written. A group does not read the documents its consolidated metadata lists, so build that model to check them before they are written. Any other member the spec does not define goes in `extra_fields`, by key: each one given is set, or removed when given as `UNSET`, and the others are kept. """ attributes: Mapping[str, JSONValue] | UNSET consolidated_metadata: ( ZarrV3ConsolidatedMetadataJSON | Mapping[str, JSONValue] | ZarrV3ConsolidatedMetadata | UNSET ) extra_fields: Mapping[str, ZarrV3ExtensionField | UNSET] class ZarrV3GroupMetadata(Keyed): """A v3 group document, and the scope it was read in. Like `ZarrV3ArrayMetadata`, the model is this pair. `to_json` returns the document as written, refined, and `context` is the scope. `attributes` and `consolidated_metadata` are views of the values the read refined. Each member the spec does not define is an extra field, in `extra_fields`, as the scope read it. `CORE` reads a `consolidated_metadata` member as the envelope the core spec fixes, and no definition reads the documents the member lists: they follow a convention of the reference implementation, not the core spec. To read them, pass the member to `ZarrV3ConsolidatedMetadata` or `read_consolidated_metadata_v3`, or read one listed document with `read_node_metadata_v3`. A model is only built by reading: the constructor reads `document` in `context` and raises `MetadataValidationError` with every problem. """ __slots__ = ("_context", "_document", "_members", "_reading", "_shown") zarr_format: Final[Literal[3]] = 3 # noqa: PYI064 - a bare Final does not narrow a union of models node_type: Final[Literal["group"]] = "group" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV3Context | None = None) -> None: context = scoped(context, CORE_AND_EXTENSIONS) reading, members = read_group_v3(document, context) if members is None or len(reading.problems) != 0: raise MetadataValidationError(reading.problems) self._adopt(_document_of(members), context, reading, members) @classmethod def _of( cls, document: dict[str, JSONValue], context: ZarrV3Context, reading: ZarrV3GroupMetadataReading, members: GroupMembersV3, ) -> ZarrV3GroupMetadata: """Build a model of a document that a read found no problems with, holding that reading, without reading it again. The readers in this package build models through this method; pyright reports these calls as private use.""" model = object.__new__(cls) model._adopt(document, context, reading, members) return model def _adopt( self, document: dict[str, JSONValue], context: ZarrV3Context, reading: ZarrV3GroupMetadataReading, members: GroupMembersV3, ) -> None: self._document = document self._context = context self._members = members if members.attributes is None: msg = "a group model holds members a read found nothing wrong with" raise TypeError(msg) # The members the model exposes, read-only at every level. self._shown = ( frozen(members.attributes), frozen(members.extra_fields), UNSET if members.consolidated is UNSET else frozen(members.consolidated), MappingProxyType({key: held(extra) for key, extra in reading.extra_fields.items()}), ) # The reading holds this model, no matter how the model was built. self._reading = dataclasses.replace(reading, metadata=self) # --- the pair --------------------------------------------------------- @property def context(self) -> ZarrV3Context: """The scope the document was read in. `update` reads new members in this scope.""" return self._context @property def reading(self) -> ZarrV3GroupMetadataReading: """The document as the scope read it: its extra fields, no problems, and this model.""" return self._reading def to_json(self) -> ZarrV3GroupMetadataJSON: """The document as written, refined. The result shares no objects with the model.""" return cast("ZarrV3GroupMetadataJSON", copied(self._document)) def to_key_value( self, *, indent: int | str | None = None ) -> Mapping[ZarrV3GroupMetadataStoreKey, bytes]: """The document as a store holds it: JSON bytes at `zarr.json`, indented by `indent`. `NaN`, `Infinity` and `-Infinity` in `attributes`, or in the documents the consolidated metadata lists, are written as bare tokens, as zarr-python writes them. A strict JSON parser rejects these tokens. """ return {ZARR_V3_GROUP_METADATA_STORE_KEY: dump_store_json(self._document, indent=indent)} def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" def _key_of(self) -> tuple[object, ...]: """The key `==` and `hash` compare for a v3 group model: its attributes as JSON text, and what each extra field means, as `extra_fields_key` computes it.""" return ( json_text(self._members.attributes), extra_fields_key(self._reading.extra_fields), ) def __reduce__(self) -> tuple[type[ZarrV3GroupMetadata], tuple[object, ZarrV3Context]]: # Pickle the pair, and read it again on load. return type(self), (self._document, self._context) # --- typed views ------------------------------------------------------ @property def attributes(self) -> Mapping[str, JSONValue]: """The attributes, read-only at every level; empty when the document writes none.""" return self._shown[0] @property def extra_fields( self, ) -> Mapping[str, AcceptedField[GroupExtraFieldDefinition[Any]] | UnclaimedField]: """Each member the spec does not define, by key, as the scope read it: an `AcceptedField` when a definition in scope claims the key (as `CORE` claims `consolidated_metadata`), and an `UnclaimedField` otherwise.""" return self._shown[3] @property def consolidated_metadata(self) -> Mapping[str, JSONValue] | UNSET: """The `consolidated_metadata` member as written, refined, and read-only at every level. `UNSET` when the document writes none, or writes one that is not an object, which only a scope that leaves the member unclaimed allows. The documents it lists are not read. Pass the member to `ZarrV3ConsolidatedMetadata` to read them all, or read one with `read_node_metadata_v3`. """ return self._shown[2] @property def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: """The extra fields, by key and as written, that no definition in scope reads and that a reader is obligated to understand. Each unclaimed member of `extra_fields` not explicitly waived with `must_understand: false` (the spec's implicit-true rule, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578). The scope is what the reader recognizes, so a compliant reader MUST fail to open the group when this is not empty. The model only reports these fields; refusing to open is up to the reader. """ return must_understand_subset(self.extra_fields, self._shown[1]) @property def unclaimed_fields(self) -> dict[Loc, UnclaimedField]: """Every field nothing in scope claims, at any depth, by its location in the document, except an extra member that says `"must_understand": false` and every field inside it, claimed or not, since a reader may ignore such a member whole. This answers whether a reader can open the group: the spec requires a reader to refuse a node with a field it does not understand, and the scope is what the reader understands. The fields of the documents that the group's `consolidated_metadata` lists are not included, because the group does not read those documents; read them with `consolidated()`. The model only reports these fields; refusing to open is up to the reader. """ return unclaimed_fields_of(self._reading.fields(), waived_members(self._shown[1])) def consolidated(self) -> ZarrV3ConsolidatedMetadata | None: """The documents this group's `consolidated_metadata` lists, read in this group's own scope; None when the group writes no such member. A group opens without reading them, so a document with a problem fails only here. Raises `MetadataValidationError` with every problem in them, located in this group's document, under `consolidated_metadata`. A `consolidated_metadata` that is not an object, which only a scope that leaves it unclaimed opens, is such a problem too. That includes `"consolidated_metadata": null`, which zarr-python 3.0 and 3.1 wrote; `read_repaired_node_metadata_v3` removes it. """ if ZARR_V3_CONSOLIDATED_METADATA_KEY not in self._document: return None try: return ZarrV3ConsolidatedMetadata( self._document[ZARR_V3_CONSOLIDATED_METADATA_KEY], context=self._context ) except MetadataValidationError as error: raise MetadataValidationError( prefixed(ZARR_V3_CONSOLIDATED_METADATA_KEY, error.problems) ) from error # --- changing --------------------------------------------------------- def update(self, **members: Unpack[ZarrV3GroupMetadataUpdate]) -> ZarrV3GroupMetadata: """Return this model with `members` (as JSON) replacing the document's members, read in this model's own scope. A member set to `UNSET` is removed. A member the spec does not define is given in `extra_fields`. A `ZarrV3ConsolidatedMetadata` given as `consolidated_metadata` is read in this model's scope and written as its document. Raises `TypeError` for a member the update does not take, and `MetadataValidationError` when the resulting document has a problem, such as a listed model that conflicts with this scope. """ changes = member_changes( members, ZarrV3GroupMetadataUpdate, GroupExtraFieldDefinition.spec_members ) document: dict[str, object] = {**self._document, **_written(changes, self._context)} for key, value in changes.items(): if value is UNSET: del document[key] return type(self)(document, context=self._context) def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadata: """Return this document read in `context`, whatever that changes. Raises `MetadataValidationError` when the document has a problem in `context`. The existing reading is reused when `context` reads every claim identically.""" context = scoped(context, CORE_AND_EXTENSIONS) if disagreements(context, claims_of(self._reading.fields())).agrees: return self._of(self._document, context, self._reading, self._members) return type(self)(self._document, context=context) def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadata: """Return this document read in `context`. `context` may claim names this scope left unclaimed, but must not contradict any claim. Raises `ScopeConflictError` when `context` reads an extra field with a different definition, or with no definition, naming each one and its key. Raises `MetadataValidationError` when a definition that `context` claims rejects what was written under its name. The group does not read the documents its `consolidated_metadata` lists, so this does not check them: `consolidated()` on the result reads them in `context`, and `ZarrV3ConsolidatedMetadata.refined_in` checks that they read the same there. """ context = scoped(context, CORE_AND_EXTENSIONS) found = disagreements(context, claims_of(self._reading.fields())) if len(found.conflicts) != 0: raise ScopeConflictError(located_conflicts(self._reading.fields(), found.conflicts)) return self.with_context(context) def refines(self, other: ZarrV3GroupMetadata) -> bool: """Whether this group holds everything `other` holds: the same attributes, and the same extra fields, each refining its counterpart in the order `refines` defines for fields. False when `other` is not a v3 group.""" if type(other) is not type(self): return False if json_text(self._members.attributes) != json_text(other._members.attributes): return False mine, theirs = self.extra_fields, other.extra_fields if mine.keys() != theirs.keys(): return False return all(refines_field(mine[key], theirs[key]) for key in mine) # --- constructors ----------------------------------------------------- @classmethod def create_default( cls, *, context: ZarrV3Context | None = None, **overrides: Unpack[ZarrV3GroupMetadataUpdate], ) -> ZarrV3GroupMetadata: """Return a group with no attributes, with any document members in `overrides` applied, read in `context`. `overrides` takes what `update` takes, including `UNSET` to drop a member and a `ZarrV3ConsolidatedMetadata` as `consolidated_metadata`. Raises `MetadataValidationError` when the resulting document has a problem.""" context = scoped(context, CORE_AND_EXTENSIONS) changes = member_changes( overrides, ZarrV3GroupMetadataUpdate, GroupExtraFieldDefinition.spec_members ) given = {key: value for key, value in changes.items() if value is not UNSET} document = {"zarr_format": 3, "node_type": "group", **_written(given, context)} return cls(document, context=context) @classmethod def from_json( cls, data: object, *, context: ZarrV3Context | None = None ) -> ZarrV3GroupMetadata: """Return the model of `data`, a v3 group document read in `context`. Raises `MetadataValidationError` with every problem the read finds. A `consolidated_metadata` of `null`, which zarr-python 3.0 and 3.1 wrote, is a value the document wrote that is not an object. In a scope that claims the member, as `CORE` does, this is a problem, because the spec requires an object. Use `read_repaired_node_metadata_v3` to read such a store. A member the spec does not define is kept in `extra_fields`. """ return cls(data, context=context) @classmethod def from_key_value( cls, mapping: Mapping[StoreKey, bytes], *, context: ZarrV3Context | None = None ) -> ZarrV3GroupMetadata: """Return the model of the group document at `zarr.json` in `mapping`, read in `context`. Raises `MetadataValidationError` when the key is missing, its bytes are not JSON, or the document is not valid. Raises `TypeError` for a scope of another format before any bytes are decoded. """ context = scoped(context, CORE_AND_EXTENSIONS) return cls(load_store_json(mapping, ZARR_V3_GROUP_METADATA_STORE_KEY), context=context) def _written(members: Mapping[str, object], context: ZarrV3Context) -> dict[str, object]: """`members`, the members given to `update` or `create_default` of a group read in `context`, with a `ZarrV3ConsolidatedMetadata` given as `consolidated_metadata` replaced by its document. The models the consolidated metadata lists are read in `context` first, as a listing of node models is read: a gain is read again, and a conflict or a loss is a problem at the child's path. So the group never writes a listing that its own scope reads differently. Raises `MetadataValidationError` with those problems, located in the group. """ given = members.get(ZARR_V3_CONSOLIDATED_METADATA_KEY) if not isinstance(given, ZarrV3ConsolidatedMetadata): return dict(members) if given.context != context: try: given = ZarrV3ConsolidatedMetadata( {**given.to_json(), "metadata": dict(given.metadata)}, context=context ) except MetadataValidationError as error: raise MetadataValidationError( prefixed(ZARR_V3_CONSOLIDATED_METADATA_KEY, error.problems) ) from error return {**members, ZARR_V3_CONSOLIDATED_METADATA_KEY: given.to_json()} class ZarrV3ConsolidatedMetadata(Keyed): """A group's inline `consolidated_metadata` member with every document it lists read, and the scope they were read in. Models the reference-implementation convention where consolidated metadata is embedded in a group's `zarr.json`. A group's own read checks only the member's envelope. This model reads each document the member lists, once, as the array or group its `node_type` declares, and checks that the documents and the group form a hierarchy with the group as its root. Build it to check consolidated metadata before `ZarrV3GroupMetadata.update` writes it into a group. `metadata` maps each path to the model, in this scope, of the document at that path. Each document is stored at its node's path without the leading `/`: the node at `/a/b` is at `a/b`. `kind` is `inline` and `must_understand` is `False`, as the convention declares. A model is only built by reading: the constructor raises `MetadataValidationError` with every problem, located in the member. """ __slots__ = ("_context", "_document", "_metadata") kind: Final = "inline" must_understand: Final = False def __init__(self, member: object, context: ZarrV3Context | None = None) -> None: context = scoped(context, CORE_AND_EXTENSIONS) # The member is always under a group's key, even when read on its # own, so nesting depth is counted from there, as in the group. readings, members, document, problems = _read_consolidated_v3( member, context, (ZARR_V3_CONSOLIDATED_METADATA_KEY,) ) if len(problems) != 0 or document is None: raise MetadataValidationError(problems) self._adopt(document, context, _nested_models(context, readings, members)) @classmethod def _of( cls, document: dict[str, JSONValue], context: ZarrV3Context, metadata: dict[str, ZarrV3NodeMetadata], ) -> ZarrV3ConsolidatedMetadata: """Build the model of a member that a read found no problems with, holding the models that read built. The readers in this package build models through this method; pyright reports these calls as private use.""" model = object.__new__(cls) model._adopt(document, context, metadata) return model def _adopt( self, document: dict[str, JSONValue], context: ZarrV3Context, metadata: dict[str, ZarrV3NodeMetadata], ) -> None: self._document = document self._context = context self._metadata = metadata @property def reading(self) -> ZarrV3ConsolidatedMetadataReading: """The member as the scope read it: no problems, and this model.""" return ZarrV3ConsolidatedMetadataReading(metadata=self) @property def context(self) -> ZarrV3Context: """The scope the documents were read in.""" return self._context @property def metadata(self) -> Mapping[str, ZarrV3NodeMetadata]: """A read-only view of the model of each document, keyed by its path below the group.""" return MappingProxyType(self._metadata) def to_json(self) -> ZarrV3ConsolidatedMetadataJSON: """The member as written, refined. The result shares no objects with the model.""" return cast("ZarrV3ConsolidatedMetadataJSON", copied(self._document)) def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" def _key_of(self) -> tuple[object, ...]: """The key `==` and `hash` compare for consolidated metadata: each document's path and key, sorted by path.""" return tuple( (path, node._model_key) for path, node in sorted(self.metadata.items(), key=lambda item: item[0]) ) def __reduce__(self) -> tuple[type[ZarrV3ConsolidatedMetadata], tuple[object, ZarrV3Context]]: return type(self), (self._document, self._context) def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ConsolidatedMetadata: """Return this member with every document read in `context`, whatever that changes. Raises `MetadataValidationError` when a document has a problem in `context`. The existing readings are reused when `context` reads every claim of every document identically.""" context = scoped(context, CORE_AND_EXTENSIONS) if all( disagreements(context, claims_of(node.reading.fields())).agrees for node in self._metadata.values() ): moved = {path: node.with_context(context) for path, node in self._metadata.items()} return self._of(self._document, context, moved) return type(self)(self._document, context=context) def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3ConsolidatedMetadata: """Return this member with every document read in `context`. `context` may claim names this scope left unclaimed, but must not contradict any claim. Raises `ScopeConflictError` naming each conflict, located at the document's path. Raises `MetadataValidationError` when a gain exposes a problem. """ context = scoped(context, CORE_AND_EXTENSIONS) conflicts = [ dataclasses.replace( conflict, loc=("metadata", path, *(() if conflict.loc is None else conflict.loc)) ) for path, node in self._metadata.items() for conflict in located_conflicts( node.reading.fields(), disagreements(context, claims_of(node.reading.fields())).conflicts, ) ] if len(conflicts) != 0: raise ScopeConflictError(conflicts) return self.with_context(context) def refines(self, other: ZarrV3ConsolidatedMetadata) -> bool: """Whether every document this holds refines the document `other` holds at the same path, and both hold the same paths. False when `other` is not consolidated metadata.""" if type(other) is not type(self): return False if self._metadata.keys() != other._metadata.keys(): return False return all( _node_refines(self._metadata[path], other._metadata[path]) for path in self._metadata ) @classmethod def from_json( cls, data: object, *, context: ZarrV3Context | None = None ) -> ZarrV3ConsolidatedMetadata: """Return the model of `data`, a group's `consolidated_metadata` member. Each document is read once in `context`, as the array or group its `node_type` declares. Raises `MetadataValidationError` with every problem found.""" return cls(data, context=context) def _node_refines(node: ZarrV3NodeMetadata, other: ZarrV3NodeMetadata) -> bool: """Whether `node` refines `other`. Models of the same kind use their own `refines`; an array never refines a group, or the reverse.""" if isinstance(node, ZarrV3ArrayMetadata): return isinstance(other, ZarrV3ArrayMetadata) and node.refines(other) return isinstance(other, ZarrV3GroupMetadata) and node.refines(other) @dataclass(frozen=True, slots=True) class ZarrV3GroupMetadataReading: """A v3 group document as a scope read it: each extra field, every problem, and the model when there are no problems.""" problems: tuple[ValidationProblem, ...] = () """Every reason the document is not valid.""" metadata: ZarrV3GroupMetadata | None = None """The document's model when there is no problem; None otherwise.""" extra_fields: Mapping[str, ResolvedField[GroupExtraFieldDefinition[Any]]] = dataclasses.field( default_factory=no_extra_fields ) """Each member the spec does not define, as the scope read it, by key: an `UnclaimedField` when no definition in scope claims the key.""" def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: """Each field in the document, as the scope read it, with its location: the extra fields, each at its key.""" for key, extra in self.extra_fields.items(): yield from fields_of(extra, (key,)) def __reduce__(self) -> str | tuple[object, ...]: # Same as the other readings: pickle through the model, when there # is one. A read-only view does not pickle, so the extra fields go # as a dict. if self.metadata is not None: return (reading_of, (self.metadata,)) return (_group_reading, (self.problems, dict(self.extra_fields))) def _group_reading( problems: tuple[ValidationProblem, ...], extra_fields: dict[str, ResolvedField[GroupExtraFieldDefinition[Any]]], ) -> ZarrV3GroupMetadataReading: """Rebuild a group reading without a model from the values `ZarrV3GroupMetadataReading.__reduce__` returns, with its extra fields held read-only.""" return ZarrV3GroupMetadataReading( problems=problems, extra_fields=MappingProxyType(extra_fields) ) @dataclass(frozen=True, slots=True) class ZarrV3ConsolidatedMetadataReading: """A group's `consolidated_metadata` member as a scope read it: every problem, and the model when there are no problems.""" problems: tuple[ValidationProblem, ...] = () """Every reason the member is not valid.""" metadata: ZarrV3ConsolidatedMetadata | None = None """The member's model when there is no problem; None otherwise.""" def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: """Each field of each document, as read, located in the member: under `metadata` and the document's path. Nothing when there is no model.""" if self.metadata is None: return for path, node in self.metadata.metadata.items(): for loc, field in node.reading.fields(): yield ("metadata", path, *loc), field def __reduce__(self) -> str | tuple[object, ...]: # Same as the other readings: pickle through the model, when there is one. if self.metadata is not None: return (reading_of, (self.metadata,)) return object.__reduce__(self) def read_consolidated_metadata_v3( value: object, *, context: ZarrV3Context | None = None ) -> ZarrV3ConsolidatedMetadataReading: """Read `value`, a group's `consolidated_metadata` member, in `context` (`CORE_AND_EXTENSIONS` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV3ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" context = scoped(context, CORE_AND_EXTENSIONS) readings, members, document, problems = _read_consolidated_v3( value, context, (ZARR_V3_CONSOLIDATED_METADATA_KEY,) ) if len(problems) != 0 or document is None: return ZarrV3ConsolidatedMetadataReading(problems=problems) model = ZarrV3ConsolidatedMetadata._of( # pyright: ignore[reportPrivateUsage] document, context, _nested_models(context, readings, members) ) return model.reading @dataclass(frozen=True, slots=True) class ZarrV3UnknownNodeMetadataReading: """A v3 document with no node type the spec defines (its `node_type` is missing, or is neither `"array"` nor `"group"`), or a value that is not an object. Only its `zarr_format` is read, and the problems report it. This way, a document in another format is reported as such: zarr-python 2's draft of v3 wrote a root `zarr.json` whose `zarr_format` is a URL, and a v2 document names format 2. """ problems: tuple[ValidationProblem, ...] """Why the document is not a node.""" @property def metadata(self) -> None: """Its model: always None, since without a node type there is no model to choose.""" return None def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: """Its fields as read: none, since no field is read.""" return iter(()) ZarrV3NodeMetadataReading = TypeAliasType( "ZarrV3NodeMetadataReading", ZarrV3ArrayMetadataReading | ZarrV3GroupMetadataReading | ZarrV3UnknownNodeMetadataReading, ) """A v3 `zarr.json` as `read_node_metadata_v3` reads it: as the array or group its `node_type` declares, or as neither.""" def read_node_metadata_v3( value: object, *, context: ZarrV3Context | None = None ) -> ZarrV3NodeMetadataReading: """Read `value`, a v3 `zarr.json`, in `context` as the node type its `node_type` declares. The node type is the tag of a union, like pydantic's discriminator and zod's discriminated union. An array is read by `read_array_metadata_v3`, and a group by `read_group_metadata_v3`. A document that declares neither, or a value that is not an object, gives a `ZarrV3UnknownNodeMetadataReading` with the problems, including any problem with its `zarr_format`. So no caller has to read `node_type` from unvalidated JSON, and a document in another format is reported as not v3. """ context = scoped(context, CORE_AND_EXTENSIONS) node_type, problems = _node_type(value) if node_type == "array": return read_array_metadata_v3(value, context=context) if node_type == "group": return read_group_metadata_v3(value, context=context) return ZarrV3UnknownNodeMetadataReading(problems) ZarrV3NodeMetadata = TypeAliasType("ZarrV3NodeMetadata", ZarrV3ArrayMetadata | ZarrV3GroupMetadata) """The model of a v3 `zarr.json`: an array model or a group model, as its `node_type` declares.""" def node_metadata_from_json_v3( data: object, *, context: ZarrV3Context | None = None ) -> ZarrV3NodeMetadata: """Return the model of `data`, a v3 `zarr.json` read in `context`, as the node type its `node_type` declares. The result is what `ZarrV3ArrayMetadata.from_json` or `ZarrV3GroupMetadata.from_json` returns. This works like pydantic's `TypeAdapter` validating a discriminated union. Raises `MetadataValidationError` with every problem `read_node_metadata_v3` finds, including a `node_type` that is neither array nor group. """ context = scoped(context, CORE_AND_EXTENSIONS) reading = read_node_metadata_v3(data, context=context) if reading.metadata is None: raise MetadataValidationError(reading.problems) return reading.metadata def node_metadata_from_key_value_v3( mapping: Mapping[StoreKey, bytes], *, context: ZarrV3Context | None = None ) -> ZarrV3NodeMetadata: """Return the model of the document at `zarr.json` in `mapping`, read in `context` as the node type its `node_type` declares, as `node_metadata_from_json_v3` reads it. Raises `MetadataValidationError` when the key is missing, its bytes are not JSON, or the document is not a valid array or group. """ context = scoped(context, CORE_AND_EXTENSIONS) # Array documents and group documents are both stored at `zarr.json`. document = load_store_json(mapping, ZARR_V3_GROUP_METADATA_STORE_KEY) return node_metadata_from_json_v3(document, context=context) def _read_node_v3( value: object, context: ZarrV3Context, at: tuple[str | int, ...] = () ) -> tuple[ZarrV3NodeMetadataReading, ArrayMembersV3 | GroupMembersV3 | None]: """Read `value` as `read_node_metadata_v3` does, without building models, and return its refined members. `at` is the location of `value` within the input document, so nesting depth is counted from the root of the input document. A document nested too deep is not read. Its problem is the one `_refine` reports for a container at that depth. """ past = nested_past_the_levels(value, at) if past is not None: return ZarrV3UnknownNodeMetadataReading(within((past,), at)), None node_type, problems = _node_type(value) if node_type == "array": return read_array_v3(value, context, at=at) if node_type == "group": return read_group_v3(value, context, at=at) return ZarrV3UnknownNodeMetadataReading(problems), None _NODE_TYPES: Final = ("array", "group") """The node types the spec defines.""" def _node_type(value: object) -> tuple[str | None, tuple[ValidationProblem, ...]]: """The node type `value` declares, one of `_NODE_TYPES`. None, with the problems, when it declares none of them or is not an object. When a document declares no valid node type, its `zarr_format` is checked too, so a document in another format is reported as not v3. """ if not is_object(value): return None, not_an_object(value) document = value node_type = document.get("node_type") if isinstance(node_type, str) and node_type in _NODE_TYPES: return node_type, () problems = [ *missing_keys(frozenset({"zarr_format"}), document), *check_literal(document, "zarr_format", 3), ] if "node_type" not in document: problems.append(ValidationProblem(("node_type",), "missing required key", "missing_key")) else: problems.append(outside_of(("node_type",), node_type, _NODE_TYPES)) return None, with_input(problems, document) @dataclass(frozen=True, slots=True) class GroupMembersV3: """The refined members of a v3 group document, as the model holds them.""" attributes: dict[str, JSONValue] | None """The attributes; None when they have a problem.""" extra_fields: dict[str, JSONValue] """Each member the spec does not define that is JSON.""" consolidated: dict[str, JSONValue] | UNSET """The `consolidated_metadata` member, refined, with the documents it lists not read; UNSET when the document writes none, or the member has a problem.""" document: dict[str, JSONValue] | None """The refined document that the model holds: each member as the read refined it, in the order the document writes them; None when the read found a problem.""" def read_group_metadata_v3( value: object, *, context: ZarrV3Context | None = None ) -> ZarrV3GroupMetadataReading: """Read `value`, a v3 group document, in `context`, whatever it holds: every problem, and the model when there are none. Unknown top-level keys are allowed (they map to `extra_fields`). A reader must understand each one that does not declare `must_understand: false`; the model reports these as `must_understand_fields`. A `consolidated_metadata` member, if present and claimed by the scope, as `CORE` claims it, is checked against the envelope the core spec fixes. The documents it lists are not read; `read_consolidated_metadata_v3` reads them. This is the form of `ZarrV3GroupMetadata(value, context)` that returns problems instead of raising them. """ context = scoped(context, CORE_AND_EXTENSIONS) reading, members = read_group_v3(value, context) if members is None or members.document is None: return reading return ZarrV3GroupMetadata._of(members.document, context, reading, members).reading # pyright: ignore[reportPrivateUsage] def read_group_v3( value: object, context: ZarrV3Context, *, at: tuple[str | int, ...] = () ) -> tuple[ZarrV3GroupMetadataReading, GroupMembersV3 | None]: """Read `value`, a v3 group document, in `context`, without building a model. Return the reading and the refined members; the members are None when `value` is not an object. `at` is the location of this document within the input document, as in `read_array_v3`, so nesting depth is counted from the outermost root. Each member the spec does not define is read as an extra field of the group, by the definition in `context` that claims its key. """ if not is_object(value): return ZarrV3GroupMetadataReading(problems=not_an_object(value)), None doc = value found: list[ValidationProblem] = list(missing_keys(_GROUP_METADATA_REQUIRED_KEYS_V3, doc)) past = members_past_the_levels(doc, at) found.extend(past.values()) whole, doc = doc, {key: item for key, item in doc.items() if key not in past} extra_fields, extra, others = read_extra_fields_v3(doc, GroupExtraFieldDefinition, context, at) found.extend(others) found.extend(check_literal(doc, "zarr_format", 3)) found.extend(check_literal(doc, "node_type", "group")) attributes: dict[str, JSONValue] | None = {} if "attributes" in doc: attributes, problems = attributes_of(doc["attributes"], at) found.extend(problems) member = extra_fields.get(ZARR_V3_CONSOLIDATED_METADATA_KEY) consolidated: dict[str, JSONValue] | UNSET = member if isinstance(member, dict) else UNSET reading = ZarrV3GroupMetadataReading( problems=with_input(found, whole), extra_fields=MappingProxyType(extra) ) document: dict[str, JSONValue] | None = None if len(found) == 0 and attributes is not None: # The document the model holds: each member as refined above, in # the document's order, built from what was read instead of being # refined again. refined: dict[str, JSONValue] = { **extra_fields, "zarr_format": 3, "node_type": "group", "attributes": attributes, } document = {key: refined[key] for key in cast("Mapping[str, object]", doc)} return reading, GroupMembersV3(attributes, extra_fields, consolidated, document) _CONSOLIDATED_MEMBERS: Final = ("kind", "must_understand", "metadata") """The members of an inline `consolidated_metadata`, in the order the convention declares them.""" _CONSOLIDATED_ENVELOPE: Final[dict[str, JSONValue]] = {"kind": "inline", "must_understand": False} """The fixed values the convention declares for the member.""" def _envelope_problems(member: Mapping[object, object]) -> list[ValidationProblem]: """Why `member`, a `consolidated_metadata` object, is not the envelope the core spec fixes: `kind` `"inline"`, `must_understand` false and an object at `metadata`, and nothing else. The entries of `metadata` are not checked. Each problem is located in the member.""" # Missing members are reported in the order the envelope declares them. problems: list[ValidationProblem] = [ ValidationProblem((key,), "missing required key", "missing_key") for key in _CONSOLIDATED_MEMBERS if key not in member ] problems.extend(unexpected_keys(frozenset(_CONSOLIDATED_MEMBERS), member)) problems.extend(check_literal(member, "kind", "inline")) problems.extend(check_literal(member, "must_understand", False)) if "metadata" in member and not is_object(member["metadata"]): problems.append(ValidationProblem(("metadata",), "expected an object", "invalid_type")) return problems def _read_node_model( entry: ZarrV3ArrayMetadata | ZarrV3GroupMetadata, context: ZarrV3Context, at: Loc ) -> tuple[ZarrV3NodeMetadataReading, ArrayMembersV3 | GroupMembersV3 | None]: """Read `entry`, a node model given in place of a listed document, in `context`. Returns the model's own reading when `context` reads every one of its claims identically, and a new read of its document when `context` claims more. Returns problems when `context` reads a name with a different definition, or with none. A model may join consolidated metadata when its claims refine into the scope of the consolidated metadata. On a gain, the document is read again in that scope, so a problem found by a newly claiming definition is reported at its location. """ document = entry.to_json() found = disagreements(context, claims_of(entry.reading.fields())) if len(found.conflicts) != 0: # Read the document again in this scope, so the reading belongs to # it, like every other reading the consolidated metadata holds. # The conflicts become its problems, each at the field's location. reading, _ = _read_node_v3(document, context, at) written = {loc: field.name for loc, field in entry.reading.fields()} problems = tuple( ValidationProblem( conflict.loc if conflict.loc is not None else (), _conflict_said( conflict, None if conflict.loc is None else written.get(conflict.loc) ), "invalid_value", ) for conflict in located_conflicts(entry.reading.fields(), found.conflicts) ) # List the conflicts first: they are the cause of what the re-read finds. return _with_problems(reading, (*problems, *reading.problems)), None if found.agrees: # Use the model's own reading, unless the document is nested too # deep at its listed location. In that case, a read from that # location reports the problem. _, past = refine_user_data(document, at) if len(past) != 0: return _read_node_v3(document, context, at) moved = entry.with_context(context) return moved.reading, moved._members # pyright: ignore[reportPrivateUsage] return _read_node_v3(document, context, at) def _conflict_said(conflict: Conflict, written: str | None) -> str: """The message for a conflict between a model entry's scope and the scope of the consolidated metadata. It states the kind, the name as the document writes it, the definition that read it, and how the scope of the consolidated metadata reads it: with another definition (distinguished from the model's when the two print the same), or with none.""" kind, filed = conflict.key name = filed if written is None else written claimed = definition_said(conflict.claimed) head = ( "expected a document read in the scope of the consolidated metadata, got a model that " f"reads the {kind_name(kind)} {name!r}" ) if conflict.found is None: return f"{head} by {claimed}, which that scope leaves unclaimed" found = definition_said(conflict.found) if claimed == found: return f"{head} by another definition than the one that scope reads it by, {found}" return f"{head} by {claimed}, which that scope reads by {found}" def _with_problems( reading: ZarrV3NodeMetadataReading, problems: tuple[ValidationProblem, ...] ) -> ZarrV3NodeMetadataReading: """`reading`, a read of a document in the scope of the consolidated metadata, with `problems` as its problems and no model.""" if isinstance(reading, (ZarrV3GroupMetadataReading, ZarrV3ArrayMetadataReading)): return dataclasses.replace(reading, problems=problems, metadata=None) return dataclasses.replace(reading, problems=problems) @dataclass(frozen=True, slots=True) class _Listing: """The documents a group listed in consolidated metadata lists in its own `consolidated_metadata`, as read: the reading of each, and the members of each that a model can be built from, keyed by path.""" readings: Mapping[str, ZarrV3NodeMetadataReading] members: Mapping[str, ArrayMembersV3 | GroupMembersV3] def _read_consolidated_v3( value: object, context: ZarrV3Context, at: tuple[str | int, ...] = () ) -> tuple[ dict[str, ZarrV3NodeMetadataReading], dict[str, ArrayMembersV3 | GroupMembersV3], dict[str, JSONValue] | None, tuple[ValidationProblem, ...], ]: """Read an inline `consolidated_metadata` member in `context`, with every document it lists. Returns four values: the reading of each document it holds, read once as `read_node_metadata_v3` reads it, keyed by path; the members of each document that a model can be built from; the refined member (each document as its read refined it) when there are no problems, else None; and every problem, located in the member. A listed group that has its own listing has that listing read the same way, so that it can be checked against this one. `at` is the location of the member within the input document. Each container the reader descends into (the member, its `metadata`, and each document) is checked at its location, as `_refine` checks a container, so a chain of documents is bounded by the maximum nesting depth. """ past = nested_past_the_levels(value, at) if past is not None: return {}, {}, None, within((past,), at) if not is_object(value): return {}, {}, None, (ValidationProblem((), "expected an object", "invalid_type"),) env = value problems = _envelope_problems(env) readings: dict[str, ZarrV3NodeMetadataReading] = {} members: dict[str, ArrayMembersV3 | GroupMembersV3] = {} listings: dict[str, _Listing] = {} node_types: dict[str, NodeType | None] = {} entries = env.get("metadata") claims_listings = ( context.claimant(GroupExtraFieldDefinition, ZARR_V3_CONSOLIDATED_METADATA_KEY) is not None ) past = nested_past_the_levels(entries, (*at, "metadata")) if is_object(entries) and past is not None: problems.extend(within((past,), at)) elif is_object(entries): for key, entry in entries.items(): if not isinstance(key, str): problems.append( ValidationProblem( ("metadata",), f"non-string key {shown_key(key)}", "invalid_type" ) ) continue faults = _key_problems(key) problems.extend(faults) here = (*at, "metadata", key) if isinstance(entry, (ZarrV3ArrayMetadata, ZarrV3GroupMetadata)): readings[key], child = _read_node_model(entry, context, here) else: readings[key], child = _read_node_v3(entry, context, here) if child is not None: members[key] = child problems.extend(_prefix("metadata", _prefix(key, readings[key].problems))) if ( isinstance(child, GroupMembersV3) and child.document is not None and child.consolidated is not UNSET and claims_listings ): # A listed group with no problem of its own, which lists # documents too, in a scope that reads a listing: read its # listing, located in its document. In a scope that leaves # the member unclaimed, the listed group does not read it, # so neither does this. nested, nested_members, _, inside = _read_consolidated_v3( child.consolidated, context, (*here, ZARR_V3_CONSOLIDATED_METADATA_KEY) ) listings[key] = _Listing(nested, nested_members) problems.extend( _prefix( "metadata", _prefix(key, _prefix(ZARR_V3_CONSOLIDATED_METADATA_KEY, inside)) ) ) if len(faults) == 0: node_types[key] = _node_type_of(readings[key]) problems.extend(_hierarchy_problems(node_types)) for key, listing in listings.items(): if key in node_types: problems.extend(_nested_listing_problems(key, listing, readings, members, node_types)) refined: dict[str, JSONValue] | None = None if len(problems) == 0: # There are no problems, so every entry has members and a # document. The envelope has the declared values, in the member's # key order. documents = { path: _document_of(members[path]) for path in cast("Mapping[str, object]", entries) } held: dict[str, JSONValue] = {**_CONSOLIDATED_ENVELOPE, "metadata": documents} refined = {key: held[key] for key in cast("Mapping[str, object]", env)} return readings, members, refined, tuple(problems) def _document_of(members: ArrayMembersV3 | GroupMembersV3) -> dict[str, JSONValue]: """The refined document held by `members`, the members of a node that a read found no problems with. Raises `TypeError` for a group's members with a problem, which such a read rules out.""" if members.document is None: msg = "expected the members of a document a read found nothing wrong with" raise TypeError(msg) return members.document def _key_problems(key: str) -> list[ValidationProblem]: """Why `key` is not a valid key for a document in consolidated metadata, as one problem at the key. Consolidated metadata holds the hierarchy below its group, with the group as the root. The reference implementation stores the document of each node at the node's path in that hierarchy, without the leading `/`: the node at `/a/b` is at the key `a/b`. """ faults = _below_faults(key) if len(faults) == 0: return [] message = f"expected the path of a node below the group, got {shown(key)}, which {said(faults)}" return [ValidationProblem(("metadata", key), message, "invalid_value")] def _nested_listing_problems( key: str, listing: _Listing, readings: Mapping[str, ZarrV3NodeMetadataReading], members: Mapping[str, ArrayMembersV3 | GroupMembersV3], node_types: Mapping[str, NodeType | None], ) -> list[ValidationProblem]: """Check `listing`, the own consolidated listing of the group at `key`, against the outer group's flat listing (`readings` and their `node_types`). Each problem is located at the nested entry. The reference implementation lists every node below the group in the group's own listing, as a flat mapping, and gives each group it lists an empty listing of its own. So every node that a listed group lists must also be listed by the outer group, at the joined key, with the same node type and the same document, as each is read. The reference reader would drop a node that only the nested listing has. A node listed with another type contradicts the tree. A node listed with a different document would give a reader two answers for one node. Only the entries of this listing are checked. Deeper listings are checked when the listing that holds them is read, and a document with its own problem is reported for that problem. """ problems: list[ValidationProblem] = [] for path, entry in listing.readings.items(): if len(_below_faults(path)) != 0: # The nested listing's own read reports a key that is not a node path. continue joined = f"{key}/{path}" here = ("metadata", key, ZARR_V3_CONSOLIDATED_METADATA_KEY, "metadata", path) if joined not in node_types: message = ( f"expected a node the group lists, got {shown(f'/{joined}')}, which " f"{shown(f'/{key}')} lists alone" ) problems.append(ValidationProblem(here, message, "invalid_value")) continue flat, nested = node_types[joined], _node_type_of(entry) if flat is not None and nested is not None and flat != nested: message = ( f"expected {_an(flat)}, as the group lists {shown(f'/{joined}')}, got {_an(nested)}" ) problems.append(ValidationProblem(here, message, "invalid_value")) continue own = listing.members.get(path) flat_members = members.get(joined) if ( own is not None and flat_members is not None and not _read_alike(readings[joined], flat_members, entry, own) ): message = ( f"expected the document the group lists at {shown(f'/{joined}')}, got one " "that reads otherwise: one node, one document" ) problems.append(ValidationProblem(here, message, "invalid_value")) return problems def _read_alike( reading: ZarrV3NodeMetadataReading, members: ArrayMembersV3 | GroupMembersV3, other: ZarrV3NodeMetadataReading, other_members: ArrayMembersV3 | GroupMembersV3, ) -> bool: """Whether two documents of one node read the same. Arrays are compared by their model keys. Groups are compared by their attributes and extra fields only; each group's listing is checked separately, at its own location.""" if isinstance(reading, ZarrV3ArrayMetadataReading) and isinstance( other, ZarrV3ArrayMetadataReading ): if not isinstance(members, ArrayMembersV3) or not isinstance(other_members, ArrayMembersV3): return True return array_key_of(reading, members) == array_key_of(other, other_members) if isinstance(members, GroupMembersV3) and isinstance(other_members, GroupMembersV3): return (json_text(members.attributes), json_text(_unlisted(members))) == ( json_text(other_members.attributes), json_text(_unlisted(other_members)), ) return True def _unlisted(members: GroupMembersV3) -> dict[str, JSONValue]: """The extra fields of a group's members, without its own consolidated listing.""" return { key: value for key, value in members.extra_fields.items() if key != ZARR_V3_CONSOLIDATED_METADATA_KEY } def _an(node_type: NodeType) -> str: return "an array" if node_type == "array" else "a group" def _hierarchy_problems(node_types: Mapping[str, NodeType | None]) -> list[ValidationProblem]: """Why the documents in consolidated metadata and their group do not form a valid hierarchy with the group as root, as `hierarchy_problems` checks it. Each problem is at the key it is about. `node_types` gives the node type of each document by its key, or None for a document with no node type the spec defines. It holds only keys that `_key_problems` finds no problems with. """ nodes: dict[str, NodeType | None] = {"/": "group"} nodes.update((f"/{key}", node_type) for key, node_type in node_types.items()) return [ ValidationProblem(("metadata", cast("str", found.loc[0])[1:]), found.message, found.kind) for found in hierarchy_problems(nodes) ] def _node_type_of(reading: ZarrV3NodeMetadataReading) -> NodeType | None: """The node type a document declares, as its reading shows; None when it declares none.""" if isinstance(reading, ZarrV3ArrayMetadataReading): return "array" if isinstance(reading, ZarrV3GroupMetadataReading): return "group" return None def _below_faults(path: str) -> list[str]: """Each reason `path` is not a valid path, relative to a group, of a node below that group.""" if path == "": return ["is the group's own"] if path.startswith("/"): return ['starts with "/"'] return path_faults(f"/{path}") def _nested_models( context: ZarrV3Context, readings: Mapping[str, ZarrV3NodeMetadataReading], members: Mapping[str, ArrayMembersV3 | GroupMembersV3], ) -> dict[str, ZarrV3NodeMetadata]: """The model, in `context`, of each document in `members` (the members of a `metadata` member) that a model can be built from. Reuses the model its reading already holds when that model is in `context`. Otherwise builds the model from its reading and members, which hold the refined document, without reading it again.""" models: dict[str, ZarrV3NodeMetadata] = {} for path, child in members.items(): reading = readings[path] held = reading.metadata if held is not None and held.context is context: models[path] = held continue if isinstance(reading, ZarrV3ArrayMetadataReading) and isinstance(child, ArrayMembersV3): models[path] = ZarrV3ArrayMetadata._of( # pyright: ignore[reportPrivateUsage] child.document, context, reading, child ) elif ( isinstance(reading, ZarrV3GroupMetadataReading) and isinstance(child, GroupMembersV3) and child.document is not None ): models[path] = ZarrV3GroupMetadata._of( # pyright: ignore[reportPrivateUsage] child.document, context, reading, child ) return models zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/model/_keyed.py000066400000000000000000000026331527014137600323220ustar00rootroot00000000000000"""The key every model uses for comparison and hashing. The key covers everything the model shows, and is computed once, the first time the model is compared.""" class Keyed: """A model whose `==` and `hash` compare a cached key that describes what its document means. Each model computes its own key in `_key_of`. `_model_key` computes the key the first time something asks for it (`==`, `hash`, or a model that holds this one) and caches it. A model that is only read never pays for the key. Two models are equal when they have the same class and equal keys. A model of another class, or any other object, is never equal to a model. """ __slots__ = ("_key",) _key: tuple[object, ...] def _key_of(self) -> tuple[object, ...]: """The key `==` and `hash` compare for this model. Each model class defines its own.""" raise NotImplementedError @property def _model_key(self) -> tuple[object, ...]: """The key. It is computed on first access and cached.""" try: return self._key except AttributeError: key = self._key_of() self._key = key return key def __eq__(self, other: object) -> bool: if type(other) is not type(self): return NotImplemented return self._model_key == other._model_key def __hash__(self) -> int: return hash(self._model_key) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/model/_repair.py000066400000000000000000000544341527014137600325110ustar00rootroot00000000000000"""Repairing v2 and v3 metadata that a known writer bug made invalid. The readers are strict: a document the spec does not allow has problems, no matter who wrote it. Some writers have written documents the spec does not allow, but where the intended meaning is clear. Repair is a separate step that runs before a strict read, and a caller must request it by name: `read_repaired_node_metadata_v3` instead of `read_node_metadata_v3`, or `read_repaired_node_metadata_v2` instead of `read_node_metadata_v2`. The set of repairs is closed, and each repair targets one writer bug. A TypedDict describes the shape of the JSON the buggy writer produced. A document must match that TypedDict for the repair to apply, and the repair turns it into correct JSON. Anything else (a missing `data_type`, a chunk length of 0 along a dimension that is not empty) is left as it is, and the strict read reports it. """ from __future__ import annotations import math from collections.abc import Mapping from dataclasses import dataclass from typing import TYPE_CHECKING, Annotated, Literal, TypeAlias, TypedDict, cast from annotated_types import Ge from zarr_metadata._common import ( JSONValue, ) from zarr_metadata._json import ( JSON_DEPTH, is_json_object, is_list_or_tuple, is_object, ) from zarr_metadata._typed_json import Loc, check from zarr_metadata.definition._context import ( scoped, ) from zarr_metadata.model._group_v2 import ( ZarrV2ConsolidatedMetadataReading, ZarrV2NodeMetadataReading, read_consolidated_metadata_v2, read_node_metadata_v2, ) from zarr_metadata.model._group_v3 import ZarrV3NodeMetadataReading, read_node_metadata_v3 from zarr_metadata.v2.array import ZARR_V2_ARRAY_METADATA_STORE_KEY from zarr_metadata.v2.definition import CORE_V2 from zarr_metadata.v2.group import ZARR_V2_GROUP_METADATA_STORE_KEY from zarr_metadata.v3._core import CORE_AND_EXTENSIONS from zarr_metadata.v3.consolidated import ZARR_V3_CONSOLIDATED_METADATA_KEY if TYPE_CHECKING: from zarr_metadata.v2._context import ZarrV2Context from zarr_metadata.v3._context import ZarrV3Context RepairKind: TypeAlias = Literal[ "zero_chunk_length", "null_consolidated_metadata", "consolidated_metadata_in_zgroup_entry", "non_finite_fill_value", ] """The name of each writer bug that a repair undoes.""" @dataclass(frozen=True, slots=True) class Repair: """One change a repair made to a document: its location, the bug it undid, and what it changed.""" loc: Loc """The location of the change in the document: the value that changed, or the key that was removed.""" kind: RepairKind """The writer bug the change undoes.""" message: str """What was written, which writer wrote it, and what it was changed to.""" _Length = Annotated[int, Ge(0)] class ZarrV3ZeroChunkRegularGridConfigurationJSON(TypedDict): """A `regular` grid's configuration as zarr-python 3.0 and 3.1 wrote it: chunk lengths that may be 0.""" chunk_shape: tuple[_Length, ...] class ZarrV3ZeroChunkRegularGridJSON(TypedDict): """A `regular` chunk grid whose chunk lengths may be 0.""" name: Literal["regular"] configuration: ZarrV3ZeroChunkRegularGridConfigurationJSON class ZarrV3ZeroChunkArrayMetadataJSON(TypedDict): """The members of an array document the `zero_chunk_length` repair reads. zarr-python 3.0 and 3.1 wrote a chunk length of 0 along a dimension of length 0, which the regular grid refuses: "Chunk sizes must be greater than zero". A chunk along an empty dimension holds nothing, whatever its length, so the repair writes 1 there. zarr-python has also written 1 since https://github.com/zarr-developers/zarr-python/pull/4328. """ node_type: Literal["array"] shape: tuple[_Length, ...] chunk_grid: ZarrV3ZeroChunkRegularGridJSON class ZarrV3NullConsolidatedGroupMetadataJSON(TypedDict): """The members of a group document the `null_consolidated_metadata` repair reads. zarr-python 3.0 and 3.1 wrote `"consolidated_metadata": null` on a group they had not consolidated. The convention does not allow this: the member is an object, or absent. The repair removes the member, which is what the writer meant. """ node_type: Literal["group"] consolidated_metadata: None class ZarrV3NonFiniteFillValueArrayMetadataJSON(TypedDict): """The members of an array document the v3 `non_finite_fill_value` repair reads. zarr-python 3.0.0 to 3.0.6 wrote the fill value of a float64 array, in the documents a group's consolidated metadata lists, as a bare `NaN`, `Infinity` or `-Infinity`, which is not JSON. The spec writes these as the strings `"NaN"`, `"Infinity"` and `"-Infinity"`. The repair writes each such fill value, or complex fill value component, of a float or complex data type by its name. """ node_type: Literal["array"] data_type: Literal["float16", "float32", "float64", "complex64", "complex128"] def repair_node_metadata_v3(value: object) -> tuple[object, tuple[Repair, ...]]: """Undo each known writer bug in `value`, a v3 `zarr.json`, and return the result with the list of changes. Each document in the consolidated metadata is repaired too. A group does not read those documents, but `ZarrV3ConsolidatedMetadata` reads them from the repaired group's member. Content that no repair applies to is left as it is. `value` is not modified: a repaired document is a new object, which shares its unchanged parts with `value`. A document with none of the bugs is returned as it is, with no repairs. """ return _repaired(value, ()) def _repaired(value: object, at: Loc) -> tuple[object, tuple[Repair, ...]]: if not is_json_object(value) or len(at) >= JSON_DEPTH: # Not a document, or nested deeper than a reader walks. The strict # read reports both. return value, () original = value repairs: list[Repair] = [] document = _zero_chunk_length(original, at, repairs) if len(at) != 0: # The writer bug is in the documents a listing holds only. Those # versions wrote a node's own `zarr.json` correctly. document = _non_finite_fill_value_v3(document, at, repairs) document = _null_consolidated_metadata(document, at, repairs) document = _consolidated(document, at, repairs) return (original if len(repairs) == 0 else dict(document)), tuple(repairs) def _non_finite_fill_value_v3( document: Mapping[str, object], at: Loc, repairs: list[Repair] ) -> Mapping[str, object]: """`document` with a non-finite fill value of a float data type, or complex fill value component, written by its name.""" array, problems = check( _members(document, ("node_type", "data_type")), ZarrV3NonFiniteFillValueArrayMetadataJSON ) if array is None or len(problems) != 0 or "fill_value" not in document: return document return _non_finite_named(document, array["data_type"].startswith("float"), at, repairs) def _members(document: Mapping[str, object], keys: tuple[str, ...]) -> dict[str, object]: """Only the members of `document` that a repair reads, so the rest is not walked.""" return {key: document[key] for key in keys if key in document} def _zero_chunk_length( document: Mapping[str, object], at: Loc, repairs: list[Repair] ) -> Mapping[str, object]: array, problems = check( _members(document, ("node_type", "shape", "chunk_grid")), ZarrV3ZeroChunkArrayMetadataJSON ) if array is None or len(problems) != 0: return document shape = array["shape"] chunk_shape = array["chunk_grid"]["configuration"]["chunk_shape"] zeros = [axis for axis, length in enumerate(chunk_shape) if length == 0] if len(zeros) == 0 or len(chunk_shape) != len(shape) or any(shape[a] != 0 for a in zeros): # Nothing to repair, or a 0 that is not a known writer bug. The # strict read reports the latter. return document repairs.extend( Repair( (*at, "chunk_grid", "configuration", "chunk_shape", axis), "zero_chunk_length", "a regular grid's chunk length of 0 along a dimension of length 0, as zarr-python " "3.0 and 3.1 wrote it, written as 1", ) for axis in zeros ) grid = cast("Mapping[str, object]", document["chunk_grid"]) configuration = cast("Mapping[str, object]", grid["configuration"]) repaired = tuple(max(length, 1) for length in chunk_shape) return { **document, "chunk_grid": {**grid, "configuration": {**configuration, "chunk_shape": repaired}}, } def _null_consolidated_metadata( document: Mapping[str, object], at: Loc, repairs: list[Repair] ) -> Mapping[str, object]: group, problems = check( _members(document, ("node_type", ZARR_V3_CONSOLIDATED_METADATA_KEY)), ZarrV3NullConsolidatedGroupMetadataJSON, ) if group is None or len(problems) != 0: return document repairs.append( Repair( (*at, ZARR_V3_CONSOLIDATED_METADATA_KEY), "null_consolidated_metadata", "a consolidated_metadata of null, as zarr-python 3.0 and 3.1 wrote it, removed", ) ) return {key: item for key, item in document.items() if key != ZARR_V3_CONSOLIDATED_METADATA_KEY} def _consolidated( document: Mapping[str, object], at: Loc, repairs: list[Repair] ) -> Mapping[str, object]: """`document` with each document in its consolidated metadata repaired, if it has any.""" member = document.get(ZARR_V3_CONSOLIDATED_METADATA_KEY) if not is_object(member): return document envelope = member entries = envelope.get("metadata") if not is_object(entries): return document held: dict[object, object] = {} found = len(repairs) for key, entry in entries.items(): if isinstance(key, str): entry, inside = _repaired( entry, (*at, ZARR_V3_CONSOLIDATED_METADATA_KEY, "metadata", key) ) repairs.extend(inside) held[key] = entry if len(repairs) == found: return document return {**document, ZARR_V3_CONSOLIDATED_METADATA_KEY: {**envelope, "metadata": held}} @dataclass(frozen=True, slots=True) class ZarrV3RepairedNodeMetadataReading: """A v3 `zarr.json` read after its known writer bugs were undone: the strict reading of the repaired document, and the repairs.""" reading: ZarrV3NodeMetadataReading """The repaired document, as `read_node_metadata_v3` reads it. The problems are those of the repaired document. The model is present when there are no problems.""" repairs: tuple[Repair, ...] """The changes that produced the document that was read.""" def read_repaired_node_metadata_v3( value: object, *, context: ZarrV3Context | None = None ) -> ZarrV3RepairedNodeMetadataReading: """Read `value`, a v3 `zarr.json`, in `context` as `read_node_metadata_v3` does, after `repair_node_metadata_v3` has undone each known writer bug in it. Use this to read stores that other writers made. A caller requests repairs by calling this function instead of `read_node_metadata_v3`. Content that no repair applies to is read as it is, and reported the same way `read_node_metadata_v3` reports it. """ context = scoped(context, CORE_AND_EXTENSIONS) repaired, repairs = repair_node_metadata_v3(value) return ZarrV3RepairedNodeMetadataReading( read_node_metadata_v3(repaired, context=context), repairs ) class ZarrV2ZGroupWithConsolidatedMetadataJSON(TypedDict): """A `.zgroup` entry of a `.zmetadata`, below the root, as zarr-python 3.x writes it: with a `consolidated_metadata` member, which a v2 group document does not allow.""" zarr_format: Literal[2] consolidated_metadata: Mapping[str, JSONValue] def repair_consolidated_metadata_v2(value: object) -> tuple[object, tuple[Repair, ...]]: """Undo each known writer bug in `value`, a v2 `.zmetadata`, and return the result with the list of changes. zarr-python 3.x writes a `consolidated_metadata` member into each `.zgroup` entry below the root. This function removes it. The root's entry is left alone, since no writer puts the member there. Each `.zarray` entry is repaired as `repair_node_metadata_v2` repairs a `.zarray`. Content that no repair applies to is left as it is, and `value` is not modified. A document with none of the bugs is returned as it is, with no repairs. """ if not is_json_object(value): return value, () document = value entries = document.get("metadata") if not is_object(entries): return value, () repairs: list[Repair] = [] held: dict[object, object] = {} for key, entry in entries.items(): if isinstance(key, str): path, _, name = key.rpartition("/") # Below the root only: no writer puts the member in the root's .zgroup. if name == ZARR_V2_GROUP_METADATA_STORE_KEY and path.strip("/") != "": entry = _without_consolidated_metadata(entry, ("metadata", key), repairs) if name == ZARR_V2_ARRAY_METADATA_STORE_KEY: entry = _repaired_zarray(entry, ("metadata", key), repairs) held[key] = entry if len(repairs) == 0: return value, () return {**document, "metadata": held}, tuple(repairs) def _without_consolidated_metadata(entry: object, at: Loc, repairs: list[Repair]) -> object: """`entry`, a `.zgroup` entry, without the member zarr-python 3.x writes into it, if the entry has that member.""" if not is_json_object(entry): return entry group = entry shaped, problems = check( _members(group, ("zarr_format", ZARR_V3_CONSOLIDATED_METADATA_KEY)), ZarrV2ZGroupWithConsolidatedMetadataJSON, ) if shaped is None or len(problems) != 0: return entry repairs.append( Repair( (*at, ZARR_V3_CONSOLIDATED_METADATA_KEY), "consolidated_metadata_in_zgroup_entry", "a consolidated_metadata, as zarr-python 3.x writes into a .zgroup entry of a " ".zmetadata, removed", ) ) kept: dict[str, object] = { key: item for key, item in group.items() if key != ZARR_V3_CONSOLIDATED_METADATA_KEY } return kept _ZarrV2FloatTypestr: TypeAlias = Literal[ "f2", "f4", "f8", "c8", "c16" ] class ZarrV2NonFiniteFillValueArrayMetadataJSON(TypedDict): """The members of a `.zarray` that the `non_finite_fill_value` repair reads, other than the fill value: a float or complex `dtype`. zarr-python 3.0.0 to 3.0.6 wrote a non-finite fill value of a float array as a bare token, `"fill_value": NaN`, and each non-finite component of a complex one the same way, `[NaN, Infinity]`. Neither is JSON. The v2 spec spells these values as strings, `"NaN"`, `"Infinity"` and `"-Infinity"` (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L178-L190), and so does zarr-python since 3.0.7. Python's `json` module reads the tokens as floats, which `check` refuses, so the fill value is not a member here: the repair reads a float, or a pair for a complex dtype. """ zarr_format: Literal[2] dtype: _ZarrV2FloatTypestr def repair_node_metadata_v2( files: Mapping[str, object], ) -> tuple[Mapping[str, object], tuple[Repair, ...]]: """Undo each known writer bug in `files`, a v2 node's files as `read_node_metadata_v2` takes them, and return the result with the list of changes. zarr-python 3.0.0 to 3.0.6 wrote a non-finite fill value in a `.zarray` as a bare token, `NaN`, which Python's `json` module reads as a float. This function writes it as the string the v2 spec uses, `"NaN"`, and does the same for each component of a complex fill value. Each repair is located in `files`, under the name of the file. Content that no repair applies to is left as it is, and `files` is not modified. Files with none of the bugs are returned as they are, with no repairs. """ if ZARR_V2_ARRAY_METADATA_STORE_KEY not in files: return files, () repairs: list[Repair] = [] zarray = _repaired_zarray( files[ZARR_V2_ARRAY_METADATA_STORE_KEY], (ZARR_V2_ARRAY_METADATA_STORE_KEY,), repairs ) if len(repairs) == 0: return files, () return {**files, ZARR_V2_ARRAY_METADATA_STORE_KEY: zarray}, tuple(repairs) def _repaired_zarray(entry: object, at: Loc, repairs: list[Repair]) -> object: """`entry`, a `.zarray` document, with each known writer bug in it undone.""" if not is_json_object(entry): return entry return _non_finite_fill_value(entry, at, repairs) def _non_finite_fill_value( document: Mapping[str, object], at: Loc, repairs: list[Repair] ) -> Mapping[str, object]: """`document` with a non-finite float fill value, or complex fill value component, written by its name.""" array, problems = check( _members(document, ("zarr_format", "dtype")), ZarrV2NonFiniteFillValueArrayMetadataJSON ) if array is None or len(problems) != 0 or "fill_value" not in document: return document return _non_finite_named(document, array["dtype"][1] == "f", at, repairs) def _non_finite_named( document: Mapping[str, object], is_float: bool, at: Loc, repairs: list[Repair] ) -> Mapping[str, object]: """`document`, an array of either format with a `fill_value` of a float data type when `is_float` and of a complex one otherwise, with a non-finite fill value, or complex fill value component, written by its name. Each name written is added to `repairs`, located under `at`.""" fill_value = document["fill_value"] if is_float: name = _non_finite_name(fill_value) if name is None: return document repairs.append(_non_finite_repair((*at, "fill_value"), name, "a fill value")) return {**document, "fill_value": name} if not is_list_or_tuple(fill_value) or len(fill_value) != 2: return document names = [_non_finite_name(component) for component in fill_value] if all(name is None for name in names): return document repairs.extend( _non_finite_repair((*at, "fill_value", index), name, "a fill value component") for index, name in enumerate(names) if name is not None ) repaired = tuple( component if name is None else name for component, name in zip(fill_value, names, strict=True) ) return {**document, "fill_value": repaired} def _non_finite_name(value: object) -> str | None: """The name the v2 spec gives `value`, a non-finite float: `"NaN"`, `"Infinity"` or `"-Infinity"`. None for any other value.""" if not isinstance(value, float) or math.isfinite(value): return None if math.isnan(value): return "NaN" return "Infinity" if value > 0 else "-Infinity" def _non_finite_repair(loc: Loc, name: str, what: str) -> Repair: return Repair( loc, "non_finite_fill_value", f'{what} of {name} as a bare token, as zarr-python 3.0.0 to 3.0.6 wrote it, written as "{name}"', ) @dataclass(frozen=True, slots=True) class ZarrV2RepairedNodeMetadataReading: """A v2 node's files read after their known writer bugs were undone: the strict reading of the repaired files, and the repairs.""" reading: ZarrV2NodeMetadataReading """The repaired files, as `read_node_metadata_v2` reads them. The problems are those of the repaired files. The model is present when there are no problems.""" repairs: tuple[Repair, ...] """The changes that produced the files that were read.""" def read_repaired_node_metadata_v2( files: Mapping[str, object], *, context: ZarrV2Context | None = None ) -> ZarrV2RepairedNodeMetadataReading: """Read `files`, a v2 node's files, in `context` as `read_node_metadata_v2` does, after `repair_node_metadata_v2` has undone each known writer bug in them. Use this to read stores that other writers made. A caller requests repairs by calling this function instead of `read_node_metadata_v2`. Content that no repair applies to is read as it is, and reported the same way `read_node_metadata_v2` reports it. """ context = scoped(context, CORE_V2) repaired, repairs = repair_node_metadata_v2(files) return ZarrV2RepairedNodeMetadataReading( read_node_metadata_v2(repaired, context=context), repairs ) @dataclass(frozen=True, slots=True) class ZarrV2RepairedConsolidatedMetadataReading: """A v2 `.zmetadata` read after its known writer bugs were undone: the strict reading of the repaired document, and the repairs.""" reading: ZarrV2ConsolidatedMetadataReading """The repaired document, as `read_consolidated_metadata_v2` reads it. The problems are those of the repaired document. The model is present when there are no problems.""" repairs: tuple[Repair, ...] """The changes that produced the document that was read.""" def read_repaired_consolidated_metadata_v2( value: object, *, context: ZarrV2Context | None = None ) -> ZarrV2RepairedConsolidatedMetadataReading: """Read `value`, a v2 `.zmetadata`, in `context` as `read_consolidated_metadata_v2` does, after `repair_consolidated_metadata_v2` has undone each known writer bug in it. Use this to read stores that other writers made. A caller requests repairs by calling this function instead of `read_consolidated_metadata_v2`. Content that no repair applies to is read as it is, and reported the same way `read_consolidated_metadata_v2` reports it. """ context = scoped(context, CORE_V2) repaired, repairs = repair_consolidated_metadata_v2(value) return ZarrV2RepairedConsolidatedMetadataReading( read_consolidated_metadata_v2(repaired, context=context), repairs ) __all__ = [ "Repair", "RepairKind", "ZarrV2NonFiniteFillValueArrayMetadataJSON", "ZarrV2RepairedConsolidatedMetadataReading", "ZarrV2RepairedNodeMetadataReading", "ZarrV2ZGroupWithConsolidatedMetadataJSON", "ZarrV3NonFiniteFillValueArrayMetadataJSON", "ZarrV3NullConsolidatedGroupMetadataJSON", "ZarrV3RepairedNodeMetadataReading", "ZarrV3ZeroChunkArrayMetadataJSON", "ZarrV3ZeroChunkRegularGridConfigurationJSON", "ZarrV3ZeroChunkRegularGridJSON", "read_repaired_consolidated_metadata_v2", "read_repaired_node_metadata_v2", "read_repaired_node_metadata_v3", "repair_consolidated_metadata_v2", "repair_node_metadata_v2", "repair_node_metadata_v3", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/model/_shared.py000066400000000000000000000340301527014137600324630ustar00rootroot00000000000000"""Helpers that the v2 and v3 readers share: checking the keys of a document, refining its members and attributes, and reading and writing store JSON.""" from __future__ import annotations import dataclasses import json from typing import TYPE_CHECKING, Any, Protocol, TypeGuard, TypeVar, cast from zarr_metadata._json import ( MetadataValidationError, ValidationProblem, is_array, is_object, outside_of, refine_json, refine_user_data, shown_key, with_input, within, ) from zarr_metadata._typed_json import typeddict_keys from zarr_metadata.definition._definition import RefusedField, UnclaimedField, is_field from zarr_metadata.definition._scope import Conflict, claim_key from zarr_metadata.v3._definition import ExtraFieldDefinition if TYPE_CHECKING: from collections.abc import Iterable, Mapping, Sequence from zarr_metadata._common import JSONValue from zarr_metadata._typed_json import Loc from zarr_metadata.definition._definition import ResolvedField def missing_keys( required: frozenset[str], doc: Mapping[object, object] ) -> tuple[ValidationProblem, ...]: """One `missing_key` problem per required key absent from `doc`.""" return tuple( ValidationProblem((key,), "missing required key", "missing_key") for key in sorted(required - doc.keys()) ) def unexpected_keys( allowed: frozenset[str], doc: Mapping[object, object] ) -> tuple[ValidationProblem, ...]: """One problem per member outside a closed document's declared shape. The kind is `unknown_key`, as a closed TypedDict's checker reports it, so a caller who tolerates a member another writer added can tell it apart from a wrong value.""" problems: list[ValidationProblem] = [] for key in doc: if not isinstance(key, str): problems.append( ValidationProblem((), f"non-string document key {shown_key(key)}", "invalid_type") ) elif key not in allowed: problems.append(ValidationProblem((key,), f"unexpected key {key!r}", "unknown_key")) return tuple(problems) def check_literal( doc: Mapping[object, object], key: str, expected: object ) -> tuple[ValidationProblem, ...]: """One problem if `doc[key]` is present but not equal to `expected`. The problem is about the type when the value is not of `expected`'s JSON type, and about the value otherwise. The value is compared after refinement, so a `StrEnum` member counts as its string.""" if key not in doc: return () value, found = refine_json(doc[key], (key,)) if len(found) != 0 or type(value) is not type(expected) or value != expected: return (outside_of((key,), doc[key], (expected,)),) return () def other_members( doc: Mapping[object, object], standard_keys: frozenset[str], *, additional_reserved_keys: frozenset[str] = frozenset(), at: Loc = (), ) -> tuple[dict[str, JSONValue], tuple[ValidationProblem, ...]]: """Each member outside `standard_keys`, refined to JSON, and every problem found in them. A member that is not JSON is left out of the result.""" members: dict[str, JSONValue] = {} problems: list[ValidationProblem] = [] reserved_keys = standard_keys | additional_reserved_keys for key, value in doc.items(): if not isinstance(key, str): problems.append( ValidationProblem((), f"non-string top-level key {shown_key(key)}", "invalid_type") ) continue if key in reserved_keys: continue refined, found = refine_json(value, (*at, key)) problems.extend(within(found, at)) if len(found) == 0: members[key] = refined return members, tuple(problems) def member_changes( members: Mapping[str, object], update: type, spec_members: frozenset[str] ) -> dict[str, object]: """The document members that `members`, given to a model's `update` or `create_default`, set: each member that `update` (the model's update TypedDict) declares, and each member in its `extra_fields`. A value of `UNSET` removes its member. Raises `TypeError` for a member that `update` does not declare, and for a member of `extra_fields` that the spec defines (one of `spec_members`) or that `update` takes by name. So a misspelled member is never written as a member the spec does not define, which a reader that must understand every such member would refuse. Also raises `TypeError` for a field given where a member's JSON goes. """ declared = typeddict_keys(update).members unknown = [key for key in members if key not in declared] if len(unknown) != 0: names = ", ".join(map(repr, unknown)) msg = ( f"unknown member {names}: give each member the spec defines by its name, and any " "other member in extra_fields" if "extra_fields" in declared else f"unknown member {names}" ) raise TypeError(msg) changes = {key: value for key, value in members.items() if key != "extra_fields"} for key, value in changes.items(): _refuse_fields(key, value) extra = members.get("extra_fields", {}) if not is_object(extra): msg = f"extra_fields maps members to their values, got {extra!r}" raise TypeError(msg) for key, value in extra.items(): if not isinstance(key, str): msg = ( f"extra_fields maps each member's key, a string, to its value, got the key {key!r}" ) raise TypeError(msg) # `extra_fields` itself is the mapping, so a member of that name is # given in it like any other member the spec does not define. if key in declared and key != "extra_fields": msg = f"{key!r} is a member that update takes by name; give it by name, not in extra_fields" raise TypeError(msg) if key in spec_members: msg = f"{key!r} is a member the spec defines, which extra_fields does not hold" raise TypeError(msg) _refuse_fields(key, value) changes[key] = value return changes def unclaimed_fields_of( fields: Iterable[tuple[Loc, ResolvedField[Any]]], waived: Iterable[str] ) -> dict[Loc, UnclaimedField]: """Each of `fields` (a reading's fields, at every depth) that nothing in scope claims, by its location, except one inside an extra member whose key is in `waived`: a member that says `"must_understand": false`, which a reader may ignore whole, whether or not a definition claims it.""" ignored = frozenset(waived) return { loc: field for loc, field in fields if isinstance(field, UnclaimedField) and not (len(loc) != 0 and loc[0] in ignored) } def _refuse_fields(key: str, value: object) -> None: """Raise `TypeError` when `value`, given as the member at `key`, is a field or holds fields, rather than the JSON a document writes, with the JSON to give instead.""" for item in value if is_array(value) else (value,): if not is_field(item): continue if isinstance(item, RefusedField): advice = "a refused field has no JSON to write; give the member as JSON" elif issubclass(item.read_as, ExtraFieldDefinition): # An extra field is read as the member alone, `{key: value}`. advice = f"give the member's value, field.to_json()[{item.name!r}]" else: advice = "give the field as JSON, field.to_json()" msg = f"{key}: a member is given as a document writes it, got a field; {advice}" raise TypeError(msg) def _is_int_sequence(value: object) -> TypeGuard[Sequence[int]]: """Whether `value` is a JSON array of integers. JSON booleans decode to `bool`, which is an `int` subclass in Python but is not an integer in a metadata document, so booleans are excluded. This is a `TypeGuard`, not a `TypeIs`, because it returns False for `bytes`, which is a `Sequence[int]`. """ return is_array(value) and all( isinstance(item, int) and not isinstance(item, bool) for item in value ) def dimension_lengths( doc: Mapping[object, object] | Mapping[str, object], key: str ) -> tuple[tuple[int, ...] | None, tuple[ValidationProblem, ...]]: """The dimension lengths `doc` holds at `key` (`shape`, `chunks`), and every problem with them. Dimension lengths are non-negative integers. The lengths are None when `doc` has nothing at `key`, or when the value at `key` has a problem. """ if key not in doc: return None, () value, found = refine_json(doc[key], (key,)) if len(found) != 0: # An integer too large to write as JSON text, or a value nested # too deep. return None, found if not _is_int_sequence(value): return None, (ValidationProblem((key,), "expected an array of integers", "invalid_type"),) if any(item < 0 for item in value): return None, (ValidationProblem((key,), "expected non-negative integers", "invalid_value"),) return tuple(value), () def validate_attributes(value: object) -> tuple[ValidationProblem, ...]: """Validate an `attributes` value: a mapping with string keys. Returns a problem at `("attributes",)` if it is not, else `()`. Shared by the v2 and v3 readers. The loc is relative to the document, not to the value, since this function is only called with a document's `attributes` value. """ return attributes_of(value)[1] def attributes_of( value: object, at: Loc = () ) -> tuple[dict[str, JSONValue] | None, tuple[ValidationProblem, ...]]: """An `attributes` value refined as user data, and every problem `validate_attributes` finds. The value is None when it is not an object with string keys, or holds a value that is not JSON. `at` is the location, within the input document, of the document that holds the attributes. Nesting depth is counted from the root of the input document. The problems are located relative to the document that holds the attributes. """ if not is_object(value) or not all(isinstance(k, str) for k in value): return None, ( ValidationProblem( ("attributes",), "expected an object with string keys", "invalid_type" ), ) attributes: dict[str, JSONValue] = {} problems: list[ValidationProblem] = [] for key, item in value.items(): if not isinstance(key, str): continue refined, found = refine_user_data(item, (*at, "attributes", key)) problems.extend(found) attributes[key] = refined return (attributes if len(problems) == 0 else None), within(problems, at) class _HoldsReading(Protocol): """A model: an object that holds the reading it was built from.""" @property def reading(self) -> object: ... def reading_of(model: _HoldsReading) -> object: """The reading that `model` holds. Unpickling a reading that held a model rebuilds it with this function.""" return model.reading StoreKey = TypeVar("StoreKey", bound=str) """The key type of a mapping from store keys to bytes: `str`, or the literal keys of one document.""" def load_store_json(mapping: Mapping[StoreKey, bytes], key: str) -> object: """Decode the JSON document stored at `key` in `mapping`. Returns `object`, not `Any`: what a store holds is unknown until a reader says otherwise, and `Any` would let unchecked values flow into typed positions silently. Read the result with a reader. Decoding uses Python's `json` module, so `NaN`, `Infinity` and `-Infinity` are read as the corresponding floats, which matches how zarr-python writes attributes. The document's reader decides where such values are allowed. Every ingestion failure here surfaces as `MetadataValidationError`: a missing store key is a `missing_key` problem, a value that is not `bytes` an `invalid_type` problem, and undecodable bytes an `invalid_json` problem, rather than leaking `KeyError`, `TypeError` or `json.JSONDecodeError` to callers. """ # Look up by a `str` key, whatever narrower key type the mapping # declares. A key the mapping does not have is simply absent. stored = cast("Mapping[str, bytes]", mapping) if key not in stored: raise MetadataValidationError( [ValidationProblem((key,), "missing store key", "missing_key")] ) # Check the annotation at runtime: `json.loads` also decodes a `str`, # and raises `TypeError` for most other types. raw = cast("object", stored[key]) if not isinstance(raw, bytes): refused = ValidationProblem( (key,), f"expected bytes, got {type(raw).__name__}", "invalid_type" ) raise MetadataValidationError(with_input((refused,), stored)) try: return json.loads(raw) except (UnicodeDecodeError, ValueError, RecursionError) as exc: raise MetadataValidationError( [ValidationProblem((key,), f"invalid JSON: {exc}", "invalid_json")] ) from exc def dump_store_json(value: object, *, indent: int | str | None = None) -> bytes: """Encode a document that its reader has accepted as JSON bytes. A non-finite number is written as Python's `json` module writes it (`NaN`, `Infinity`, `-Infinity`), which matches how zarr-python writes attributes. The reader keeps such numbers out of every other place in the document. """ return json.dumps(value, indent=indent, allow_nan=True).encode("utf-8") def located_conflicts( fields: Iterable[tuple[Loc, ResolvedField[Any]]], conflicts: Sequence[Conflict] ) -> tuple[Conflict, ...]: """Locate each of `conflicts` (found against a reading's claims) in the document, as a problem is located. A conflict appears once for each place in `fields` where its name occurs.""" located: list[Conflict] = [] placed = list(fields) for conflict in conflicts: places = [(loc, field) for loc, field in placed if claim_key(field) == conflict.key] if len(places) == 0: located.append(conflict) located.extend( dataclasses.replace(conflict, loc=loc, written=field.name) for loc, field in places ) return tuple(located) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/problems.py000066400000000000000000000017571527014137600316130ustar00rootroot00000000000000"""The problems every reader in this package reports, and the error that carries them. A reader returns problems as values: `typed_json.check`, `definition.resolve` and the model readers each return a tuple of `ValidationProblem`. Each problem has a `loc` (where it is), a `kind` (one of the `ProblemKind` values, for code that handles problems by kind), a `message`, and the `input` and `ctx` its message is made from. A constructor that cannot return problems, such as a model's `from_json`, raises `MetadataValidationError` with every problem in `.problems`. `shown` formats a value for a problem's message, as the package's own messages do: as JSON (`null`, `[1, 2]`, `"C"`). An extension's rules use it so their messages read like the package's. """ from zarr_metadata._json import ( MetadataValidationError, ProblemKind, Problems, ValidationProblem, shown, ) __all__ = [ "MetadataValidationError", "ProblemKind", "Problems", "ValidationProblem", "shown", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/py.typed000066400000000000000000000000001527014137600310710ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/pydantic.py000066400000000000000000000503751527014137600316030ustar00rootroot00000000000000"""Optional pydantic (v2) integration: field types over the core models. Importing this module requires pydantic; the core package deliberately does not depend on it, so this module is never imported by `zarr_metadata` itself. Each exported name is an `Annotated` field type over the corresponding core model class — the instances ARE the core classes, so values interoperate freely with non-pydantic code (equality, isinstance, nesting). Validation delegates to the library: a raw document routes through `from_json` (the single source of truth for validation and normalization, so pydantic's field-level coercion can never bypass it). A field type reads the fields of its document in the scope given by pydantic's validation context (pydantic passes this context to every validator). The scope is chosen as follows: - If the context is a `Context`, every field type of that format reads in it. A field type of the other format raises `TypeError`. - If the context is a mapping, the v3 field types read in its `"zarr_metadata_context"` item and the v2 field types read in its `"zarr_metadata_context_v2"` item. This lets a model with fields of both formats give a scope for each format. - If no scope is given for a format, its field types read in the default scope for that format, `CORE_AND_EXTENSIONS` or `CORE_V2`. A mapping key that starts like these keys but is not one of them raises `TypeError`, so a misspelled key is not read as no scope. To fix a field's scope whatever the validation context holds, annotate the model class with `InScope`: `Annotated[ZarrV3ArrayMetadata, InScope(CONTEXT)]`. For example: TypeAdapter(zmp.ZarrV3ArrayMetadataField).validate_python(document, context=CONTEXT) ArrayManifest.model_validate(data, context={"zarr_metadata_context": CONTEXT}) Mixed.model_validate(data, context={"zarr_metadata_context": V3, "zarr_metadata_context_v2": V2}) An existing model instance passes through unchanged when no scope is given. When one is, the instance is read again in it with `refined_in`, so an instance that scope reads differently is refused. Serialization emits the document as it was written, via `to_json`. A failed parse raises a pydantic `ValidationError` with one line error per problem, in the same format pydantic uses for its own errors. Each line error has the problem's `kind` as its `type`, the problem's `loc` under the field's `loc`, the `input` found there, and the problem's `ctx`. If the message contains a ctx placeholder, the message is passed in the ctx as `message`. Annotate a field with these types, or with a core class and `InScope`, not with a bare core class, because pydantic cannot build a schema for the core classes. A node's attributes may hold `NaN`, `Infinity` or `-Infinity`, which the models read and write as zarr-python does. Pydantic writes JSON by its own rules, and by default writes such a number as `null`. To keep these numbers, a `BaseModel` with one of these fields needs `ser_json_inf_nan="constants"` in its `model_config`. A `TypeAdapter` over a field type takes no `config`, so write its value with the model's `to_key_value` instead. Pydantic's own JSON code paths limit nesting to less than the 256 levels the readers support: `validate_json` rejects a document nested about 200 levels deep, and `dump_json` rejects one nested about 255 levels deep, each with its own error. For a document that deep, call `validate_python` on the parsed JSON, and write it with the model's `to_key_value`. Usage: import zarr_metadata.pydantic as zmp class ArrayManifest(BaseModel): path: str metadata: zmp.ZarrV3ArrayMetadataField Static type checkers see each field type as its core model class, so `manifest.metadata` is a `zarr_metadata.model.ZarrV3ArrayMetadata`. """ from __future__ import annotations from dataclasses import dataclass from types import MappingProxyType from typing import ( TYPE_CHECKING, Annotated, Any, Final, LiteralString, Protocol, TypeGuard, TypeVar, cast, ) from pydantic import ( BeforeValidator, GetCoreSchemaHandler, InstanceOf, PlainSerializer, ValidationInfo, ) from pydantic_core import ( CoreSchema, InitErrorDetails, PydanticCustomError, ValidationError, core_schema, ) from zarr_metadata import model as _model from zarr_metadata._json import MetadataValidationError, ValidationProblem, is_object, value_at from zarr_metadata._pydantic_schema import ( ZarrV2ArrayMetadataJSON as _ZarrV2ArrayMetadataSchema, ) from zarr_metadata._pydantic_schema import ( ZarrV2ConsolidatedMetadataJSON as _ZarrV2ConsolidatedMetadataSchema, ) from zarr_metadata._pydantic_schema import ( ZarrV2GroupMetadataJSON as _ZarrV2GroupMetadataSchema, ) from zarr_metadata._pydantic_schema import ( ZarrV3ArrayMetadataJSON as _ZarrV3ArrayMetadataSchema, ) from zarr_metadata._pydantic_schema import ( ZarrV3ConsolidatedMetadataJSON as _ZarrV3ConsolidatedMetadataSchema, ) from zarr_metadata._pydantic_schema import ( ZarrV3GroupMetadataJSON as _ZarrV3GroupMetadataSchema, ) from zarr_metadata._sentinel import UNSET from zarr_metadata.definition._context import scoped from zarr_metadata.definition._definition import Context from zarr_metadata.definition._scope import ScopeConflictError from zarr_metadata.v2.definition import CORE_V2, ZarrV2Context from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, ZarrV3Context if TYPE_CHECKING: from collections.abc import Callable, Mapping class _Written(Protocol): """A model, which writes its document.""" def to_json(self) -> object: ... _M = TypeVar("_M", bound=_Written) def _as_pydantic_raises(cls: type[_M], value: object, read: Callable[[], _M]) -> _M: """Return the result of `read` for `value`, or convert the `MetadataValidationError` it raises into a `pydantic_core.ValidationError`: one line error per problem, with the problem's kind as its type, at the problem's loc, with its input and ctx. For a missing key, the line error reports the object that lacks the key, as pydantic's own `missing` error does. For a problem with no `input` (because the value found was not JSON a reader can walk), the line error reports the value at the problem's loc. """ try: return read() except MetadataValidationError as error: raise _validation_error(cls, error, value) from error def _validation_error( cls: type[object], error: MetadataValidationError, value: object ) -> ValidationError: """The `pydantic_core.ValidationError` for `error`, raised while reading `value` as `cls`: one line error per problem, located in `value`.""" line_errors = [_line_error(problem, value) for problem in error.problems] return ValidationError.from_exception_data(cls.__name__, line_errors) def _line_error(problem: ValidationProblem, value: object) -> InitErrorDetails: """Convert `problem` to a `ValidationError` line error: the problem's kind as its type, at its loc, with its input and ctx, and its message as `msg`. pydantic treats a message as a template over its ctx. It replaces each `{key}` that the ctx holds, one key at a time in ctx order, and has no way to escape a placeholder. A problem message can contain such a placeholder, for example when it shows a document value written as `{expected}`. In that case the whole message is put in the ctx as `message`, as the last item, so no later key is substituted into it, and the template becomes `{message}`. This way `msg` is always the problem's message, whatever it contains. An existing ctx item named `message` is replaced. """ template, ctx = problem.message, dict(problem.ctx) if any(f"{{{key}}}" in template for key in ctx): ctx.pop("message", None) template, ctx = "{message}", {**ctx, "message": problem.message} return InitErrorDetails( type=PydanticCustomError(problem.kind, cast("LiteralString", template), ctx), loc=problem.loc, input=_input_of(problem, value), ) def _input_of(problem: ValidationProblem, value: object) -> object: """The input a line error reports. This is the problem's `input` if it has one. For a missing key, it is the value in `value` at the parent loc, as pydantic's own `missing` error reports the object. For a problem whose value was not JSON a reader can walk (so the problem could not hold it), it is the value at the problem's loc. If nothing is found there either, it is `value` itself.""" if problem.input is not UNSET: return problem.input if problem.kind == "missing_key": above = value_at(value, problem.loc[:-1]) if len(problem.loc) != 0 else UNSET return value if above is UNSET else above there = value_at(value, problem.loc) return value if there is UNSET else there CONTEXT_KEY: Final = "zarr_metadata_context" """The key in a mapping validation context that holds the scope for the v3 field types.""" CONTEXT_KEY_V2: Final = "zarr_metadata_context_v2" """The key in a mapping validation context that holds the scope for the v2 field types.""" _Read_co = TypeVar("_Read_co", covariant=True) _Context_contra = TypeVar("_Context_contra", bound=Context, contravariant=True) _C = TypeVar("_C", bound=Context) class _Reads(Protocol[_Read_co, _Context_contra]): def __call__(self, data: object, /, *, context: _Context_contra) -> _Read_co: ... def _read_in_context( cls: type[_M], read: _Reads[_M, _C], refine: Callable[[_M, _C], _M], default: _C, key: str ) -> Callable[[object, ValidationInfo], _M]: """A validator that reads a value in the `Context` stored in the validation context under `key`, or in `default` if there is none. An instance of `cls` is read again in a scope the validation context gives, with `refine` (its `refined_in`), and passes through unchanged when none is given.""" def coerce(value: object, info: ValidationInfo) -> _M: given = _context_of(info.context, default, key) if isinstance(value, cls): return value if given is None else _refined(cls, value, refine, given) context = default if given is None else given return _as_pydantic_raises(cls, value, lambda: read(value, context=context)) return coerce def _read_in( cls: type[_M], read: _Reads[_M, _C], refine: Callable[[_M, _C], _M], context: _C ) -> Callable[[object], _M]: """A validator that reads a value in `context`, and an instance of `cls` again in `context`, with `refine` (its `refined_in`).""" def coerce(value: object) -> _M: if isinstance(value, cls): return _refined(cls, value, refine, context) return _as_pydantic_raises(cls, value, lambda: read(value, context=context)) return coerce def _refined(cls: type[_M], value: _M, refine: Callable[[_M, _C], _M], context: _C) -> _M: """`value`, an instance of `cls`, read again in `context` with `refine` (its `refined_in`), or a `pydantic_core.ValidationError` with one line error for each problem the scope finds, as for a document, and one `scope_conflict` line error for each conflict or lost meaning, at the field it is in.""" try: return refine(value, context) except MetadataValidationError as error: # `to_json` copies the document, so it is built only to report an error. raise _validation_error(cls, error, value.to_json()) from error except ScopeConflictError as error: document = value.to_json() line_errors = [ InitErrorDetails( type=PydanticCustomError("scope_conflict", "{message}", {"message": str(conflict)}), loc=() if conflict.loc is None else conflict.loc, input=document if conflict.loc is None else _found_at(document, conflict.loc), ) for conflict in error.conflicts ] raise ValidationError.from_exception_data(cls.__name__, line_errors) from error def _found_at(document: object, loc: tuple[str | int, ...]) -> object: """The value at `loc` in `document`, or `document` itself when nothing is there.""" found = value_at(document, loc) return document if found is UNSET else found def _is_context(value: object) -> TypeGuard[ZarrV3Context | ZarrV2Context]: """Whether `value` is a scope of either format.""" return isinstance(value, (ZarrV3Context, ZarrV2Context)) def _context_of(validation_context: object, default: _C, key: str) -> _C | None: """The `Context` a pydantic validation context gives for one format, of `default`'s class. If the validation context is a `Context`, it is the context for every field type of its format. If it is a mapping with a `key` item, that item is the context for this format. Otherwise it gives none. A bare `Context` of the other format raises `TypeError`, as it does in every reader. A model with fields of both formats must give each format's context under its own key. A mapping key that starts like this module's keys but is not one of them raises `TypeError` too, so a misspelled key is not read as no scope at all. """ if _is_context(validation_context): return scoped(validation_context, default) if not is_object(validation_context): return None unknown = [ name for name in validation_context if isinstance(name, str) and name.startswith("zarr_metadata") and name not in (CONTEXT_KEY, CONTEXT_KEY_V2) ] if len(unknown) != 0: msg = ( f"the validation context gives a scope under {CONTEXT_KEY!r} or " f"{CONTEXT_KEY_V2!r}, got {', '.join(map(repr, unknown))}" ) raise TypeError(msg) if key not in validation_context: return None context = validation_context[key] if not _is_context(context): msg = ( f"{key}: the context to read in is a ZarrV3Context or a ZarrV2Context, got {context!r}" ) raise TypeError(msg) return scoped(context, default) ZarrV3ArrayMetadataField = Annotated[ InstanceOf[_model.ZarrV3ArrayMetadata], BeforeValidator( _read_in_context( _model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json, _model.ZarrV3ArrayMetadata.refined_in, CORE_AND_EXTENSIONS, CONTEXT_KEY, ), json_schema_input_type=_ZarrV3ArrayMetadataSchema, ), PlainSerializer(_model.ZarrV3ArrayMetadata.to_json, return_type=_ZarrV3ArrayMetadataSchema), ] """Field type for a v3 array metadata document (`zarr.json` content).""" ZarrV2ArrayMetadataField = Annotated[ InstanceOf[_model.ZarrV2ArrayMetadata], BeforeValidator( _read_in_context( _model.ZarrV2ArrayMetadata, _model.ZarrV2ArrayMetadata.from_json, _model.ZarrV2ArrayMetadata.refined_in, CORE_V2, CONTEXT_KEY_V2, ), json_schema_input_type=_ZarrV2ArrayMetadataSchema, ), PlainSerializer(_model.ZarrV2ArrayMetadata.to_json, return_type=_ZarrV2ArrayMetadataSchema), ] """Field type for a v2 array metadata document (merged `.zarray` + `.zattrs` form).""" ZarrV3GroupMetadataField = Annotated[ InstanceOf[_model.ZarrV3GroupMetadata], BeforeValidator( _read_in_context( _model.ZarrV3GroupMetadata, _model.ZarrV3GroupMetadata.from_json, _model.ZarrV3GroupMetadata.refined_in, CORE_AND_EXTENSIONS, CONTEXT_KEY, ), json_schema_input_type=_ZarrV3GroupMetadataSchema, ), PlainSerializer(_model.ZarrV3GroupMetadata.to_json, return_type=_ZarrV3GroupMetadataSchema), ] """Field type for a v3 group metadata document (`zarr.json` content).""" ZarrV2GroupMetadataField = Annotated[ InstanceOf[_model.ZarrV2GroupMetadata], BeforeValidator( _read_in_context( _model.ZarrV2GroupMetadata, _model.ZarrV2GroupMetadata.from_json, _model.ZarrV2GroupMetadata.refined_in, CORE_V2, CONTEXT_KEY_V2, ), json_schema_input_type=_ZarrV2GroupMetadataSchema, ), PlainSerializer(_model.ZarrV2GroupMetadata.to_json, return_type=_ZarrV2GroupMetadataSchema), ] """Field type for a v2 group metadata document (merged `.zgroup` + `.zattrs` form).""" ZarrV3ConsolidatedMetadataField = Annotated[ InstanceOf[_model.ZarrV3ConsolidatedMetadata], BeforeValidator( _read_in_context( _model.ZarrV3ConsolidatedMetadata, _model.ZarrV3ConsolidatedMetadata.from_json, _model.ZarrV3ConsolidatedMetadata.refined_in, CORE_AND_EXTENSIONS, CONTEXT_KEY, ), json_schema_input_type=_ZarrV3ConsolidatedMetadataSchema, ), PlainSerializer( _model.ZarrV3ConsolidatedMetadata.to_json, return_type=_ZarrV3ConsolidatedMetadataSchema, ), ] """Field type for v3 inline consolidated metadata.""" ZarrV2ConsolidatedMetadataField = Annotated[ InstanceOf[_model.ZarrV2ConsolidatedMetadata], BeforeValidator( _read_in_context( _model.ZarrV2ConsolidatedMetadata, _model.ZarrV2ConsolidatedMetadata.from_json, _model.ZarrV2ConsolidatedMetadata.refined_in, CORE_V2, CONTEXT_KEY_V2, ), json_schema_input_type=_ZarrV2ConsolidatedMetadataSchema, ), PlainSerializer( _model.ZarrV2ConsolidatedMetadata.to_json, return_type=_ZarrV2ConsolidatedMetadataSchema, ), ] """Field type for a v2 `.zmetadata` document.""" _FIELD_PARTS: Final[Mapping[type[Any], tuple[ZarrV3Context | ZarrV2Context, object]]] = ( MappingProxyType( { _model.ZarrV3ArrayMetadata: (CORE_AND_EXTENSIONS, _ZarrV3ArrayMetadataSchema), _model.ZarrV2ArrayMetadata: (CORE_V2, _ZarrV2ArrayMetadataSchema), _model.ZarrV3GroupMetadata: (CORE_AND_EXTENSIONS, _ZarrV3GroupMetadataSchema), _model.ZarrV2GroupMetadata: (CORE_V2, _ZarrV2GroupMetadataSchema), _model.ZarrV3ConsolidatedMetadata: ( CORE_AND_EXTENSIONS, _ZarrV3ConsolidatedMetadataSchema, ), _model.ZarrV2ConsolidatedMetadata: (CORE_V2, _ZarrV2ConsolidatedMetadataSchema), } ) ) """For each model class, the default scope of its format and the pydantic schema of its document.""" @dataclass(frozen=True, slots=True) class InScope: """A field type's scope, fixed: `Annotated[ZarrV3ArrayMetadata, InScope(CONTEXT)]` is a field of that model class read in `CONTEXT`, whatever the validation context holds. A document is read in `context`, and a model instance is read again in it with `refined_in`, so an instance that `context` reads differently is refused rather than passed through in its own scope. The field writes the document as written, and a type checker sees the model class. A scope of the other format raises `TypeError` when pydantic builds the schema. `InScope` builds the field's whole schema, as pydantic's `PlainValidator` does, so `Annotated` metadata before it is not applied: put a validator of your own after it. """ context: ZarrV3Context | ZarrV2Context def __post_init__(self) -> None: if not _is_context(self.context): msg = f"InScope takes a ZarrV3Context or a ZarrV2Context, got {self.context!r}" raise TypeError(msg) def __get_pydantic_core_schema__( self, source: Any, # noqa: ANN401 - pydantic passes the annotated class handler: GetCoreSchemaHandler, ) -> CoreSchema: parts = _FIELD_PARTS.get(source) if parts is None: msg = f"InScope annotates a model class of zarr_metadata.model, got {source!r}" raise TypeError(msg) default, schema = parts cls: type[Any] = source document = handler.generate_schema(schema) return core_schema.no_info_before_validator_function( _read_in(cls, cls.from_json, cls.refined_in, scoped(self.context, default)), core_schema.is_instance_schema(cls), json_schema_input_schema=document, serialization=core_schema.plain_serializer_function_ser_schema( cls.to_json, return_schema=document ), ) __all__ = [ "CONTEXT_KEY", "CONTEXT_KEY_V2", "InScope", "ZarrV2ArrayMetadataField", "ZarrV2ConsolidatedMetadataField", "ZarrV2GroupMetadataField", "ZarrV3ArrayMetadataField", "ZarrV3ConsolidatedMetadataField", "ZarrV3GroupMetadataField", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/typed_json.py000066400000000000000000000113641527014137600321410ustar00rootroot00000000000000"""JSON checked against a TypedDict: the public entry point. This package describes every Zarr document and configuration as a TypedDict, such as `ZarrV3ArrayMetadataJSON` or `GzipCodecConfiguration`. `check` turns any of them into a runtime check. It type-checks a JSON value against the TypedDict and returns a value of the TypedDict (or None) and every problem, each located in the value: import json from zarr_metadata import ZarrV3ArrayMetadataJSON from zarr_metadata.typed_json import check document, problems = check(json.loads(raw), ZarrV3ArrayMetadataJSON) for problem in problems: print(problem.loc, problem.kind, problem.message) `check` reads a TypedDict as the typing spec defines it, regardless of how its module writes annotations: - A key is required when its `Required` or `NotRequired` qualifier says so. Otherwise the `total` setting of the class that declared the key decides. The runtime's `__required_keys__` cannot see a qualifier written as a string (and `from __future__ import annotations` makes every annotation a string). `check` reads the evaluated annotations, so it can see the qualifier. - A key the TypedDict does not declare is handled according to `closed` or `extra_items`, or, when the class sets neither, according to its bases. In a closed TypedDict the key is reported as `unknown_key` and left out, and the value is still returned. With `extra_items=` the key is checked as that type. In an open TypedDict the key is kept. - Each annotation is evaluated in the module of the class that wrote it, as the spec requires. `get_type_hints`, by contrast, evaluates inherited annotations in the subclass's module. - `Required`, `NotRequired` and `ReadOnly` are removed wherever they are written. A `NewType` is checked as its supertype, and a type alias as the type it stands for. A TypedDict or alias that refers to itself is checked as deep as the value goes. - A number type may carry bounds, written with annotated-types as pydantic reads them: `Gt`, `Ge`, `Lt`, `Le` and `Interval`, at any depth. For example, `tuple[Annotated[int, Ge(1)], ...]` bounds each element. A value outside the bounds is an `invalid_value` problem. Its message states what the type allows, and its `ctx` holds the bounds. `Annotated` may also carry a note (a string or a `Doc`). Any other metadata raises `TypeError`, because `check` would not enforce a constraint it does not support. - In a union of TypedDicts where each requires a key as a `Literal` of values unique to that branch, the value of that key selects the branch, and the problems are that branch's. Otherwise the value is checked by the first branch that reports no problem (among TypedDicts, the one that declares the most of the value's keys). If every branch reports problems, the result is from the branch with the fewest problems. The returned value is a value of the TypedDict: arrays as tuples, and each object a new dict of the keys its type allows. Problems are values, not exceptions: `ValidationProblem(loc, message, kind)`, where `kind` is one of `missing_key`, `invalid_type`, `invalid_value`, `unknown_key` and `invalid_json`. A caller that tolerates undeclared keys can therefore distinguish them from wrong values. Each problem carries the data in its message, as pydantic's errors and zod's issues do: `input` is the JSON in the value at `loc`, and `ctx` is what was expected there (a type's bounds, or the values of a `Literal`). `zarr_metadata.json_schema.json_schema` writes the rules `check` applies as a JSON Schema. `check` supports only annotations that describe JSON shapes: `int`, `float` for any number, `bool`, `str`, `None`, `JSONValue`, `UserJSONValue` (a `JSONValue` whose numbers may also be non-finite, for user data such as attributes), a `Literal`, `tuple[T, ...]` and `tuple[T1, T2]`, a union, a TypedDict, `Mapping[str, V]`, a `NewType` and a type alias. A TypedDict that contains anything else raises `TypeError`, naming the member and the TypedDict that contains it. The same happens for a TypedDict the spec itself forbids, such as one with a key that is both `Required` and `NotRequired`, and for a generic TypedDict, because no runtime records how its type arguments bind in its bases' keys. A string annotation resolves only in its module, so a TypedDict written inside a function cannot refer to a class defined in that function. On Python 3.11, `typing.TypedDict` does not record a class's bases, so a subclass evaluates its inherited annotations in its own module. `typing_extensions.TypedDict` records the bases on every version. """ from zarr_metadata._common import JSONValue, UserJSONValue from zarr_metadata._typed_json import ( Loc, check, ) __all__ = [ "JSONValue", "Loc", "UserJSONValue", "check", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/000077500000000000000000000000001527014137600277335ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py000066400000000000000000000053101527014137600320430ustar00rootroot00000000000000"""Zarr v2 metadata types, and the front door to the v2 definitions. Each document type is in the submodule for its node or file: `array` (`.zarray`), `group` (`.zgroup`), `attributes` (`.zattrs`) and `consolidated` (`.zmetadata`). The codecs and data types are in `codec` and `data_type`, and the v2 kinds and scope are in `definition`. This module also exports every v2 definition (`GZIP_CODEC_V2`, `FLOAT_DATA_TYPE_V2`, ...), the scope `CORE_V2`, and the scope class `ZarrV2Context`, so that one import builds or updates a scope. Each is the same object as in its own submodule, where a definition's configuration types and constants are. """ from zarr_metadata.v2.codec.checksum import ( ADLER32_CODEC_V2, CRC32_CODEC_V2, CRC32C_CODEC_V2, FLETCHER32_CODEC_V2, ) from zarr_metadata.v2.codec.compression import ( BLOSC_CODEC_V2, BZ2_CODEC_V2, GZIP_CODEC_V2, LZ4_CODEC_V2, LZMA_CODEC_V2, ZLIB_CODEC_V2, ZSTD_CODEC_V2, ) from zarr_metadata.v2.codec.filters import ( ASTYPE_CODEC_V2, BITROUND_CODEC_V2, DELTA_CODEC_V2, FIXEDSCALEOFFSET_CODEC_V2, PACKBITS_CODEC_V2, QUANTIZE_CODEC_V2, SHUFFLE_CODEC_V2, ) from zarr_metadata.v2.codec.vlen import ( VLEN_ARRAY_CODEC_V2, VLEN_BYTES_CODEC_V2, VLEN_UTF8_CODEC_V2, ) from zarr_metadata.v2.data_type.fixed_width import ( BYTES_DATA_TYPE_V2, STR_DATA_TYPE_V2, VOID_DATA_TYPE_V2, ) from zarr_metadata.v2.data_type.object import OBJECT_DATA_TYPE_V2 from zarr_metadata.v2.data_type.scalar import ( BOOL_DATA_TYPE_V2, COMPLEX_DATA_TYPE_V2, FLOAT_DATA_TYPE_V2, INT_DATA_TYPE_V2, UINT_DATA_TYPE_V2, ) from zarr_metadata.v2.data_type.struct import STRUCT_DATA_TYPE_V2 from zarr_metadata.v2.data_type.time import ( DATETIME64_DATA_TYPE_V2, TIMEDELTA64_DATA_TYPE_V2, ) from zarr_metadata.v2.definition import CORE_V2, ZarrV2Context __all__ = [ "ADLER32_CODEC_V2", "ASTYPE_CODEC_V2", "BITROUND_CODEC_V2", "BLOSC_CODEC_V2", "BOOL_DATA_TYPE_V2", "BYTES_DATA_TYPE_V2", "BZ2_CODEC_V2", "COMPLEX_DATA_TYPE_V2", "CORE_V2", "CRC32C_CODEC_V2", "CRC32_CODEC_V2", "DATETIME64_DATA_TYPE_V2", "DELTA_CODEC_V2", "FIXEDSCALEOFFSET_CODEC_V2", "FLETCHER32_CODEC_V2", "FLOAT_DATA_TYPE_V2", "GZIP_CODEC_V2", "INT_DATA_TYPE_V2", "LZ4_CODEC_V2", "LZMA_CODEC_V2", "OBJECT_DATA_TYPE_V2", "PACKBITS_CODEC_V2", "QUANTIZE_CODEC_V2", "SHUFFLE_CODEC_V2", "STRUCT_DATA_TYPE_V2", "STR_DATA_TYPE_V2", "TIMEDELTA64_DATA_TYPE_V2", "UINT_DATA_TYPE_V2", "VLEN_ARRAY_CODEC_V2", "VLEN_BYTES_CODEC_V2", "VLEN_UTF8_CODEC_V2", "VOID_DATA_TYPE_V2", "ZLIB_CODEC_V2", "ZSTD_CODEC_V2", "ZarrV2Context", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/_context.py000066400000000000000000000173671527014137600321460ustar00rootroot00000000000000"""The scope a Zarr v2 document is read in: one slot for each kind of v2 field.""" from __future__ import annotations from collections.abc import Iterable, Mapping from typing import ( TYPE_CHECKING, Any, ClassVar, Final, Literal, TypeAlias, final, get_args, get_origin, ) from typing_extensions import TypedDict, TypeIs, TypeVar, Unpack from zarr_metadata.definition._context import ( Tables, claimant_in, definitions_in, filed, joined_tables, refuse_others, slot_contents, slot_kinds, tables_key, updated, ) from zarr_metadata.v2._definition import ZarrV2CodecDefinition, ZarrV2DataTypeDefinition if TYPE_CHECKING: from zarr_metadata.definition._definition import D, Definition ZarrV2Kind: TypeAlias = ZarrV2DataTypeDefinition[Any] | ZarrV2CodecDefinition[Any] """A definition of a kind of v2 field.""" _K = TypeVar("_K", bound=ZarrV2Kind) """A kind of v2 field, so a type checker refuses a kind of the other format.""" _KINDS: Final[frozenset[type[Definition[Any]]]] = frozenset( get_origin(member) for member in get_args(ZarrV2Kind) ) def is_v2_kind(kind: type[Definition[Any]]) -> TypeIs[type[ZarrV2Kind]]: """Whether `kind` is a kind of v2 field: one of the kinds in `ZarrV2Kind`.""" return kind in _KINDS class ZarrV2ContextSlots(TypedDict, total=False, closed=True): """The definitions in each slot of a v2 scope, as `ZarrV2Context` and its `update` take them. This is where a v2 scope's slots are declared: each key is a slot, and its value the definitions of the kind that slot holds, as an iterable of them or as a mapping from each one's name to it. `ZarrV2Context.kinds` is read from these annotations, so a type checker checks every slot a caller names, and every definition given in it. """ data_types: ( Iterable[ZarrV2DataTypeDefinition[Any]] | Mapping[str, ZarrV2DataTypeDefinition[Any]] ) """Data types, read from `dtype`.""" codecs: Iterable[ZarrV2CodecDefinition[Any]] | Mapping[str, ZarrV2CodecDefinition[Any]] """Codecs, read from `compressor` and `filters`.""" @final class ZarrV2Context: """A scope that reads Zarr v2 documents. Every v2 reader takes one. It implements `Context`, and has one slot for each kind of field a v2 document holds, as `ZarrV2ContextSlots` declares them: `data_types`, read from `dtype`, and `codecs`, read from `compressor` and `filters`. Each slot is a read-only mapping from name to definition. The constructor and `update` take, for each slot, an iterable of definitions or such a mapping, so one scope's slot can be given to another. A definition of another kind than its slot holds, and a key that is not its definition's name, raise `TypeError`, and two different definitions under one name in a slot raise `ScopeConflictError`. To replace a definition, or a whole slot, use `update`. `ZarrV2Context()` is the scope that claims nothing. Two scopes are equal when they file the same definitions in each slot, and equal scopes have equal hashes. """ __slots__ = ("_tables",) kinds: ClassVar[Mapping[str, type[ZarrV2Kind]]] = slot_kinds(ZarrV2ContextSlots, is_v2_kind) """The kind of definition each slot holds, by slot, as `ZarrV2ContextSlots` declares them.""" _tables: Tables def __init__(self, **slots: Unpack[ZarrV2ContextSlots]) -> None: self._tables = filed(self.kinds, slots, 2) @classmethod def _of(cls, tables: Tables) -> ZarrV2Context: """A scope of `tables`, which were filed for this class's slots.""" context = object.__new__(cls) context._tables = tables return context @classmethod def _restored(cls, contents: Mapping[str, Iterable[Definition[Any]]]) -> ZarrV2Context: """A scope rebuilt from the definitions in each slot, as `__reduce__` gives them.""" return cls._of(filed(cls.kinds, contents, 2)) @property def zarr_format(self) -> Literal[2]: """The Zarr format of the documents this scope reads: 2.""" return 2 @property def data_types(self) -> Mapping[str, ZarrV2DataTypeDefinition[Any]]: """The data type definitions in scope, by the NumPy family each reads.""" return self.definitions_of(ZarrV2DataTypeDefinition) @property def codecs(self) -> Mapping[str, ZarrV2CodecDefinition[Any]]: """The codec definitions in scope, by numcodecs id.""" return self.definitions_of(ZarrV2CodecDefinition) def definitions_of(self, kind: type[_K]) -> Mapping[str, _K]: """The definitions of `kind` in scope, by the name each is filed under: the slot that holds `kind`. Raises `TypeError` for a kind of the other format.""" return definitions_in(self._tables, kind) def claimant(self, kind: type[D], name: str) -> D | None: """The definition of `kind` in scope that reads `name` (a name as a document uses it), or None if there is none. Raises `TypeError` for a kind of the other format. This is the definition filed under the name that `filed_under` maps `name` to. That is `name` itself, except for raw bits, where `r16` is read by the definition of `r*`. """ return claimant_in(self._tables, kind, name) def update(self, **slots: Unpack[ZarrV2ContextSlots]) -> ZarrV2Context: """This scope, with each slot given here holding only the definitions given for it. The other slots are kept. A slot can be given as a mapping from each definition's name to it, so a slot of this scope passes back in changed: `context.update(codecs={**context.codecs, "gzip": MY_GZIP})` replaces the gzip definition and keeps the other codecs, and `context.update(codecs=[*context.codecs.values(), MY_CODEC])` adds one, raising `ScopeConflictError` if the scope already files another definition under its name. """ return self._of(updated(self._tables, self.kinds, slots, 2)) def union(self, *others: ZarrV2Context) -> ZarrV2Context: """The smallest scope that files everything this scope and each of `others` file. Raises `ScopeConflictError` when two of them file different definitions under one name in a slot. To replace a definition on purpose, use `update`. Raises `TypeError` for a scope of another format. """ refuse_others(others, ZarrV2Context) return self._of( joined_tables(self.kinds, [self._tables, *(other._tables for other in others)]) ) def __repr__(self) -> str: # Keep this short, because it appears in signatures as a default # argument. A full repr would list every definition, and `help` on a # validator would run to pages. count = sum(len(table) for table in self._tables.values()) return f"ZarrV2Context(<{count} definitions>)" def __reduce__(self) -> tuple[object, tuple[dict[str, tuple[Definition[Any], ...]]]]: # A scope is defined by the definitions in its slots, so pickle it as # them. This lets a scope go to another process along with the # documents it will read there. return (type(self)._restored, (slot_contents(self._tables, self.kinds),)) def __copy__(self) -> ZarrV2Context: return self def __deepcopy__(self, memo: dict[int, object]) -> ZarrV2Context: # A scope is immutable, so a copy can be the same object. return self def __eq__(self, other: object) -> bool: if not isinstance(other, ZarrV2Context): return NotImplemented return self._tables == other._tables def __hash__(self) -> int: return hash((ZarrV2Context, tables_key(self._tables))) __all__ = ["ZarrV2Context", "ZarrV2ContextSlots", "ZarrV2Kind", "is_v2_kind"] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py000066400000000000000000000262671527014137600326110ustar00rootroot00000000000000"""The kinds of field in a Zarr v2 array document: a dtype, and a codec in `compressor` or `filters`. A v2 dtype is written as a NumPy typestr, which is a byte order, a type code and a size, such as `|])([A-Za-z])([0-9]*)(?:\[([0-9]*)([A-Za-z\u03bc]+)\])?") """A typestr: a byte order, a type code, an optional size in bytes, and an optional bracketed time unit with its multiplier.""" def parse_typestr(name: str) -> tuple[str, dict[str, JSONValue]] | None: """Split `name` into its type code and the values it carries. Return None if `name` is not a typestr. The carried values are the byte order, the item size (only `|O` omits it), and, when a unit is in brackets, the unit and its multiplier. The multiplier is 1 when none is written. """ match = TYPESTR_PATTERN.fullmatch(name) if match is None: return None byteorder, code, size, factor, unit = match.groups() if size == "" and code != "O": # "An integer specifying the number of bytes": only `|O` omits it. return None carried: dict[str, JSONValue] = {"byteorder": byteorder} if size != "": carried["itemsize"] = int(size) if unit is not None: carried["unit"] = unit carried["scale_factor"] = 1 if factor == "" else int(factor) return code, carried CODE_POINT_BYTES: Final = 4 """The bytes that one code point of a `U` type takes. The size in a `U` typestr counts code points, as NumPy writes it, so ` int | None: """The number of bytes that one item of the typestr `name` takes: its size, or four bytes per code point for `U`. None when `name` is not a typestr with a size, such as `|O`.""" parsed = parse_typestr(name) if parsed is None: return None code, carried = parsed size = carried.get("itemsize") if not isinstance(size, int): return None return size * CODE_POINT_BYTES if code == "U" else size def typestr_problem(name: str, at: Loc) -> ValidationProblem | None: """Return the problem at `at` if `name` is not a typestr. Return None if it is a typestr or `struct`.""" if name == STRUCT_NAME: return None if TYPESTR_PATTERN.fullmatch(name) is not None: if parse_typestr(name) is not None: return None # "An integer specifying the number of bytes": a listed code without a size. return ValidationProblem( at, f"expected a size in bytes after the type code, '' or '|', a type code and a size in " f"bytes, ' str | None: if parse_typestr(self.name) is not None: return ( f"{self.name!r} is how a document writes one type of a family, which reads as " "the family; define the family" ) # Use the explicit form, not a bare `super()`, because a dataclass with slots is rebuilt. return super(ZarrV2DataTypeDefinition, self)._refusal() @classmethod def _name_problem(cls, name: str, at: Loc) -> ValidationProblem | None: return typestr_problem(name, at) @classmethod def _filed_under(cls, name: str) -> tuple[str | None, dict[str, JSONValue] | None]: if name == STRUCT_NAME: return name, None if name in FAMILIES.values(): return None, None parsed = parse_typestr(name) if parsed is None: return name, None code, carried = parsed family = FAMILIES.get(code) if family is None: return name, None return family, carried def _carrying_name(self, configuration: Mapping[str, JSONValue]) -> str | None: if self.name == STRUCT_NAME: return None code = next(code for code, family in FAMILIES.items() if family == self.name) name = f"{configuration['byteorder']}{code}{configuration.get('itemsize', '')}" if "unit" in configuration: factor = configuration.get("scale_factor", 1) name += f"[{'' if factor == 1 else factor}{configuration['unit']}]" return name @classmethod def _named_configuration( cls, value: object ) -> tuple[str | None, Mapping[str, object] | None, Problems]: if isinstance(value, str): return value, None, () if isinstance(value, (list, tuple)): return STRUCT_NAME, {"fields": cast("JSONValue", value)}, () return None, None, () @classmethod def _envelope_problems(cls, value: object) -> Problems: if isinstance(value, str): bad = typestr_problem(value, ()) return () if bad is None else (bad,) if isinstance(value, (list, tuple)): return () return ( ValidationProblem( (), "expected a v2 dtype -- a NumPy typestr, or an array of field records -- got " f"{shown(value)}", "invalid_type", ), ) @classmethod def _envelope_json(cls, name: str, configuration: Mapping[str, JSONValue]) -> JSONValue: if name == STRUCT_NAME and "fields" in configuration: return configuration["fields"] return name @classmethod def _configuration_loc(cls, loc: Loc) -> Loc: return loc @classmethod def _name_loc(cls, loc: Loc) -> Loc: return loc @dataclass(frozen=True, kw_only=True, slots=True, repr=False) class ZarrV2CodecDefinition(Definition[C], kind=True, format=2): """A v2 codec: a numcodecs id, and the TypedDict that describes its parameters. A document writes `{"id": name, **parameters}`. The configuration of the definition is the parameters, read at the location of the field itself. """ label: ClassVar[str] = "v2 codec" @classmethod def _name_problem(cls, name: str, at: Loc) -> ValidationProblem | None: if len(name) != 0: return None return ValidationProblem(at, "expected a codec id, got ''", "invalid_value") @classmethod def _named_configuration( cls, value: object ) -> tuple[str | None, Mapping[str, object] | None, Problems]: if not is_object(value): return None, None, () name = value.get("id") if not isinstance(name, str): return None, None, () return ( name, {key: item for key, item in value.items() if isinstance(key, str) and key != "id"}, (), ) @classmethod def _envelope_problems(cls, value: object) -> Problems: if not is_object(value): return ( ValidationProblem( (), "expected a codec configuration with a string 'id'", "invalid_type" ), ) entry = value if "id" not in entry: return (ValidationProblem(("id",), "missing required key", "missing_key"),) if not isinstance(entry["id"], str): return ( ValidationProblem( ("id",), f"expected a string codec id, got {shown(entry['id'])}", "invalid_type", ), ) bad = cls._name_problem(entry["id"], ("id",)) return () if bad is None else (bad,) @classmethod def _envelope_json(cls, name: str, configuration: Mapping[str, JSONValue]) -> JSONValue: return {"id": name, **configuration} @classmethod def _configuration_loc(cls, loc: Loc) -> Loc: return loc @classmethod def _name_loc(cls, loc: Loc) -> Loc: return (*loc, "id") __all__ = [ "CODE_POINT_BYTES", "FAMILIES", "STRUCT_NAME", "TYPESTR_PATTERN", "ZarrV2CodecDefinition", "ZarrV2DataTypeDefinition", "ZarrV2DataTypeField", "item_bytes", "parse_typestr", "typestr_problem", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/_field_json.py000066400000000000000000000030761527014137600325660ustar00rootroot00000000000000"""The JSON shapes of a v2 dtype and a v2 codec. They are in their own module so that both the array document and the definitions can import them without a cycle; `zarr_metadata.v2.data_type` and `zarr_metadata.v2.codec` export them.""" from typing_extensions import TypeAliasType, TypedDict from zarr_metadata._common import JSONValue ZarrV2DataTypeMetadata = TypeAliasType( "ZarrV2DataTypeMetadata", str | tuple[ tuple[str, "ZarrV2DataTypeMetadata"] | tuple[str, "ZarrV2DataTypeMetadata", tuple[int, ...]], ..., ], ) """The v2 dtype representation. Either a numpy-style dtype string (e.g. `"/.zarray` for a v2 array. User attributes live in a sibling `.zattrs` file and are NOT part of this type; see `ZarrV2ZAttrsJSON`. See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ zarr_format: Literal[2] shape: tuple[Annotated[int, Ge(0)], ...] chunks: tuple[Annotated[int, Ge(0)], ...] dtype: ZarrV2DataTypeMetadata compressor: ZarrV2CodecMetadata | None fill_value: JSONValue order: ZarrV2ArrayOrder filters: tuple[ZarrV2CodecMetadata, ...] | None dimension_separator: NotRequired[ZarrV2ArrayDimensionSeparator] class ZarrV2ArrayMetadataJSON(TypedDict, extra_items=JSONValue): """ Zarr v2 array metadata document, in-memory merged form. Models the union of `.zarray` (the spec-defined fields, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L51-L92) and `.zattrs` (user attributes, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330). On disk, attributes live in a sibling `.zattrs` file and are not part of `.zarray`; this type folds them in as the `attributes` field so a single TypedDict represents the complete in-memory state of a v2 array node. Consumers that read or write a real `.zarray` file should split / merge `attributes` accordingly, or use `ZarrV2ZArrayJSON` (strict on-disk) plus `ZarrV2ZAttrsJSON` directly. The type is open: other keys "SHOULD NOT be present ... and SHOULD be ignored by implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), and each other member is a JSON value. See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ zarr_format: Literal[2] shape: tuple[Annotated[int, Ge(0)], ...] chunks: tuple[Annotated[int, Ge(0)], ...] dtype: ZarrV2DataTypeMetadata compressor: ZarrV2CodecMetadata | None fill_value: JSONValue order: ZarrV2ArrayOrder filters: tuple[ZarrV2CodecMetadata, ...] | None dimension_separator: NotRequired[ZarrV2ArrayDimensionSeparator] attributes: NotRequired[Mapping[str, JSONValue]] """User attributes from the sibling `.zattrs` file (not part of `.zarray`). See the class docstring for the rationale behind the merged representation. """ ZarrV2ArrayMetadataStoreKey = Literal[".zarray"] """Literal type of the store key holding a v2 array's metadata document.""" ZARR_V2_ARRAY_METADATA_STORE_KEY: Final[ZarrV2ArrayMetadataStoreKey] = ".zarray" """The store key a v2 array's metadata document is persisted under.""" __all__ = [ "ZARR_V2_ARRAY_DIMENSION_SEPARATOR", "ZARR_V2_ARRAY_METADATA_STORE_KEY", "ZARR_V2_ARRAY_ORDER", "ZarrV2ArrayDimensionSeparator", "ZarrV2ArrayMetadataJSON", "ZarrV2ArrayMetadataStoreKey", "ZarrV2ArrayOrder", "ZarrV2ZArrayJSON", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py000066400000000000000000000022351527014137600324750ustar00rootroot00000000000000"""Zarr v2 user-attributes file content. See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ from collections.abc import Mapping from typing import Final, Literal from zarr_metadata._common import JSONValue ZarrV2ZAttrsJSON = Mapping[str, JSONValue] """On-disk `.zattrs` file content. A JSON object holding user-defined attributes for a v2 array or group. Spec-defined keys for arrays / groups live in sibling `.zarray` / `.zgroup` files (modeled by `ZarrV2ZArrayJSON` / `ZarrV2ZGroupJSON`). This type does not constrain the keys or values of the attributes mapping. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330 """ ZarrV2AttributesStoreKey = Literal[".zattrs"] """Literal type of the store key holding a v2 node's user attributes.""" ZARR_V2_ATTRIBUTES_STORE_KEY: Final[ZarrV2AttributesStoreKey] = ".zattrs" """The store key a v2 node's user attributes are persisted under. Shared by arrays and groups: both node types keep their attributes in a sibling `.zattrs` file. """ __all__ = [ "ZARR_V2_ATTRIBUTES_STORE_KEY", "ZarrV2AttributesStoreKey", "ZarrV2ZAttrsJSON", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/codec/000077500000000000000000000000001527014137600310105ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/codec/__init__.py000066400000000000000000000035561527014137600331320ustar00rootroot00000000000000"""Zarr v2 codecs: the configuration shape, and one definition per numcodecs id that this package models. In v2, compressors and filters are numcodecs configuration dicts: a required `id` field naming the codec, plus codec-specific parameters. Each codec's definition and parameter TypedDict are in its submodule (`checksum`, `compression`, `filters`, `vlen`). This module holds the configuration shape and `CODECS_V2`, the tuple of every definition. """ from typing import Any, Final from zarr_metadata.v2._definition import ZarrV2CodecDefinition from zarr_metadata.v2._field_json import ZarrV2CodecMetadata from zarr_metadata.v2.codec.checksum import ( ADLER32_CODEC_V2, CRC32_CODEC_V2, CRC32C_CODEC_V2, FLETCHER32_CODEC_V2, ) from zarr_metadata.v2.codec.compression import ( BLOSC_CODEC_V2, BZ2_CODEC_V2, GZIP_CODEC_V2, LZ4_CODEC_V2, LZMA_CODEC_V2, ZLIB_CODEC_V2, ZSTD_CODEC_V2, ) from zarr_metadata.v2.codec.filters import ( ASTYPE_CODEC_V2, BITROUND_CODEC_V2, DELTA_CODEC_V2, FIXEDSCALEOFFSET_CODEC_V2, PACKBITS_CODEC_V2, QUANTIZE_CODEC_V2, SHUFFLE_CODEC_V2, ) from zarr_metadata.v2.codec.vlen import VLEN_ARRAY_CODEC_V2, VLEN_BYTES_CODEC_V2, VLEN_UTF8_CODEC_V2 CODECS_V2: Final[tuple[ZarrV2CodecDefinition[Any], ...]] = ( ZLIB_CODEC_V2, GZIP_CODEC_V2, BZ2_CODEC_V2, LZMA_CODEC_V2, BLOSC_CODEC_V2, ZSTD_CODEC_V2, LZ4_CODEC_V2, SHUFFLE_CODEC_V2, DELTA_CODEC_V2, FIXEDSCALEOFFSET_CODEC_V2, QUANTIZE_CODEC_V2, BITROUND_CODEC_V2, ASTYPE_CODEC_V2, PACKBITS_CODEC_V2, VLEN_UTF8_CODEC_V2, VLEN_BYTES_CODEC_V2, VLEN_ARRAY_CODEC_V2, CRC32_CODEC_V2, CRC32C_CODEC_V2, ADLER32_CODEC_V2, FLETCHER32_CODEC_V2, ) """Every codec that numcodecs 0.16 configures and that this package models.""" __all__ = [ "CODECS_V2", "ZarrV2CodecMetadata", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/codec/_dtype.py000066400000000000000000000075501527014137600326550ustar00rootroot00000000000000"""A rule for the dtype parameters of a v2 codec.""" from __future__ import annotations from dataclasses import dataclass, replace from typing import TYPE_CHECKING, Final from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import canonicalize, resolve from zarr_metadata.v2._context import ZarrV2Context from zarr_metadata.v2._definition import ZarrV2DataTypeDefinition, parse_typestr from zarr_metadata.v2.data_type import DATA_TYPES_V2 from zarr_metadata.v2.data_type._rules import time_rules from zarr_metadata.v2.data_type.time import DATETIME64_DATA_TYPE_V2, TIMEDELTA64_DATA_TYPE_V2 if TYPE_CHECKING: from collections.abc import Iterator, Mapping from zarr_metadata.definition._definition import Nested _DTYPE_CONTEXT: Final = ZarrV2Context( data_types=[ *( definition for definition in DATA_TYPES_V2 if definition.name not in {DATETIME64_DATA_TYPE_V2.name, TIMEDELTA64_DATA_TYPE_V2.name} ), replace(DATETIME64_DATA_TYPE_V2, rules=time_rules("M", unit_required=False)), replace(TIMEDELTA64_DATA_TYPE_V2, rules=time_rules("m", unit_required=False)), ] ) """The scope a dtype parameter is read in: the data types that zarr-python 2.x writes, `DATA_TYPES_V2`, whatever scope the array is read in, because numcodecs reads the parameter with NumPy. A time type may leave out its unit here, as in ` Iterator[ValidationProblem]: for key in self.keys: written = configuration.get(key) if not isinstance(written, str): # Omitted, or null for an `astype` that numcodecs takes from `dtype`. continue _, problems = resolve(written, ZarrV2DataTypeDefinition, _DTYPE_CONTEXT, (key,)) if len(problems) != 0: yield from problems continue parsed = parse_typestr(written) if self.float_only and (parsed is None or parsed[0] != "f"): yield ValidationProblem( (key,), f"expected a float typestr, ' _DtypeParameter: """Return a rule that checks that each of `keys`, when present and not null, is a typestr that an array's dtype could be, by the rules of its family in `DATA_TYPES_V2`. So ` str: """Return `written`, a dtype parameter that `dtype_parameter` accepts, in the canonical spelling of its data type, which is how an array's dtype compares. This is how NumPy writes it: ` ZarrV2Checksum32Parameters: """Fill in a `location` that is omitted or null with `default`.""" location = configuration.get("location") return {"location": default if location is None else location} def _checksum_canonical(configuration: ZarrV2Checksum32Parameters) -> ZarrV2Checksum32Parameters: """Fill in a `location` that is omitted or null with `"start"`, the default of `numcodecs.CRC32` and `numcodecs.Adler32`.""" return _fill_location(configuration, "start") def _crc32c_canonical(configuration: ZarrV2Checksum32Parameters) -> ZarrV2Checksum32Parameters: """Fill in a `location` that is omitted or null with `"end"`, the default of `numcodecs.CRC32C`.""" return _fill_location(configuration, "end") CRC32_CODEC_V2: Final = ZarrV2CodecDefinition( name="crc32", configuration=ZarrV2Checksum32Parameters, canonical=_checksum_canonical ) """`numcodecs.CRC32`.""" CRC32C_CODEC_V2: Final = ZarrV2CodecDefinition( name="crc32c", configuration=ZarrV2Checksum32Parameters, canonical=_crc32c_canonical ) """`numcodecs.CRC32C`. Its `location` defaults to the end.""" ADLER32_CODEC_V2: Final = ZarrV2CodecDefinition( name="adler32", configuration=ZarrV2Checksum32Parameters, canonical=_checksum_canonical ) """`numcodecs.Adler32`.""" FLETCHER32_CODEC_V2: Final = ZarrV2CodecDefinition( name="fletcher32", configuration=EmptyConfiguration ) """`numcodecs.Fletcher32`. It has no configuration.""" __all__ = [ "ADLER32_CODEC_V2", "CRC32C_CODEC_V2", "CRC32_CODEC_V2", "FLETCHER32_CODEC_V2", "ZarrV2Checksum32Parameters", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/codec/compression.py000066400000000000000000000165241527014137600337330ustar00rootroot00000000000000"""The v2 compressors that numcodecs 0.16 configures: `zlib`, `gzip`, `bz2`, `lzma`, `blosc`, `zstd` and `lz4`. Each parameter that numcodecs gives a default is optional. numcodecs writes the full object with `get_config()`. The canonical form of each codec fills in those defaults, so a configuration that leaves out a parameter and one that writes its default are one codec. """ from __future__ import annotations from typing import Annotated, Final, Literal, NotRequired from annotated_types import Ge, Interval from typing_extensions import ReadOnly, TypedDict from zarr_metadata._common import ( JSONValue, ) from zarr_metadata.v2._definition import ZarrV2CodecDefinition ZarrV2CompressionLevel = Annotated[int, Interval(ge=-1, le=9)] """A zlib-style compression level: 0 to 9, or -1 for zlib's default. numcodecs writes the level as given.""" _CInt = Annotated[int, Interval(ge=-(2**31), le=2**31 - 1)] """An integer that numcodecs passes to C as an `int`: one that fits in 32 bits, with a sign. numcodecs raises `OverflowError` for any other.""" class ZarrV2ZlibParameters(TypedDict, closed=True): """`numcodecs.Zlib(level=1)`.""" level: NotRequired[ReadOnly[ZarrV2CompressionLevel]] class ZarrV2GzipParameters(TypedDict, closed=True): """`numcodecs.GZip(level=1)`.""" level: NotRequired[ReadOnly[ZarrV2CompressionLevel]] class ZarrV2Bz2Parameters(TypedDict, closed=True): """`numcodecs.BZ2(level=1)`: 1 to 9.""" level: NotRequired[ReadOnly[Annotated[int, Interval(ge=1, le=9)]]] class ZarrV2LzmaParameters(TypedDict, closed=True): """`numcodecs.LZMA(format=1, check=-1, preset=None, filters=None)`, with the parameters as the `lzma` module accepts them.""" format: NotRequired[ReadOnly[Annotated[int, Interval(ge=0, le=3)]]] check: NotRequired[ReadOnly[int]] preset: NotRequired[ReadOnly[Annotated[int, Ge(0)] | None]] filters: NotRequired[ReadOnly[tuple[JSONValue, ...] | None]] ZarrV2BloscCName = Literal["blosclz", "lz4", "lz4hc", "snappy", "zlib", "zstd"] """The compressors inside Blosc.""" class ZarrV2BloscParameters(TypedDict, closed=True): """`numcodecs.Blosc(cname='lz4', clevel=5, shuffle=1, blocksize=0, typesize=None)`. For `shuffle`, -1 is automatic, 0 is none, 1 is byte shuffle, and 2 is bit shuffle. numcodecs writes a `shuffle` given as `True` or `False` as it was given, and reads it as 1 or 0. A `blocksize` of 0 lets Blosc choose the block size. numcodecs reads any `blocksize` that fits in a C `int`, negative ones included. Blosc reads any `blocksize` other than 0 as a size in bytes, and raises one below its minimum of 128 bytes to that minimum, so a negative `blocksize` does not mean automatic. """ cname: NotRequired[ReadOnly[ZarrV2BloscCName]] clevel: NotRequired[ReadOnly[Annotated[int, Interval(ge=0, le=9)]]] shuffle: NotRequired[ReadOnly[Literal[-1, 0, 1, 2] | bool]] blocksize: NotRequired[ReadOnly[_CInt]] typesize: NotRequired[ReadOnly[Annotated[int, Interval(ge=1, le=2**31 - 1)] | None]] class ZarrV2ZstdParameters(TypedDict, closed=True): """`numcodecs.Zstd(level=0, checksum=False)`. Level 0 is zstd's default level, and negative levels give up compression ratio for speed. zstd clamps a level outside its own range, -131072 to 22, to that range, so numcodecs reads any level that fits in a C `int`.""" level: NotRequired[ReadOnly[_CInt]] checksum: NotRequired[ReadOnly[bool]] class ZarrV2Lz4Parameters(TypedDict, closed=True): """`numcodecs.LZ4(acceleration=1)`. numcodecs passes `acceleration` to C as an `int`, and lz4 reads one below 1 as 1.""" acceleration: NotRequired[ReadOnly[_CInt]] def _zlib_canonical(configuration: ZarrV2ZlibParameters) -> ZarrV2ZlibParameters: """Fill in the default `level`, 1, as numcodecs writes it.""" return {"level": 1, **configuration} def _gzip_canonical(configuration: ZarrV2GzipParameters) -> ZarrV2GzipParameters: """Fill in the default `level`, 1, as numcodecs writes it.""" return {"level": 1, **configuration} def _bz2_canonical(configuration: ZarrV2Bz2Parameters) -> ZarrV2Bz2Parameters: """Fill in the default `level`, 1, as numcodecs writes it.""" return {"level": 1, **configuration} def _lzma_canonical(configuration: ZarrV2LzmaParameters) -> ZarrV2LzmaParameters: """Fill in the defaults, as numcodecs writes them: `format` 1 (`lzma.FORMAT_XZ`), `check` -1 (the format's own check), and null `preset` and `filters`.""" return {"format": 1, "check": -1, "preset": None, "filters": None, **configuration} def _blosc_canonical(configuration: ZarrV2BloscParameters) -> ZarrV2BloscParameters: """Fill in the defaults, as numcodecs writes them, and write a `shuffle` of `true` or `false` as 1 or 0, which numcodecs reads it as. numcodecs 0.16 reads `typesize` but never writes it. A null `typesize` is the default, and is dropped. Any other `typesize` is kept, whatever the `shuffle`: Blosc writes it into each chunk's header and splits blocks by it, so it changes the encoded chunk even when nothing is shuffled. """ shuffle = configuration.get("shuffle", 1) filled: ZarrV2BloscParameters = { "cname": configuration.get("cname", "lz4"), "clevel": configuration.get("clevel", 5), "shuffle": (1 if shuffle else 0) if isinstance(shuffle, bool) else shuffle, "blocksize": configuration.get("blocksize", 0), } typesize = configuration.get("typesize") if typesize is None: return filled return {**filled, "typesize": typesize} def _zstd_canonical(configuration: ZarrV2ZstdParameters) -> ZarrV2ZstdParameters: """Fill in the defaults, `level` 0 and `checksum` false, as numcodecs writes them.""" return {"level": 0, "checksum": False, **configuration} def _lz4_canonical(configuration: ZarrV2Lz4Parameters) -> ZarrV2Lz4Parameters: """Fill in the default `acceleration`, 1, as numcodecs writes it.""" return {"acceleration": 1, **configuration} ZLIB_CODEC_V2: Final = ZarrV2CodecDefinition( name="zlib", configuration=ZarrV2ZlibParameters, canonical=_zlib_canonical ) """`numcodecs.Zlib`.""" GZIP_CODEC_V2: Final = ZarrV2CodecDefinition( name="gzip", configuration=ZarrV2GzipParameters, canonical=_gzip_canonical ) """`numcodecs.GZip`.""" BZ2_CODEC_V2: Final = ZarrV2CodecDefinition( name="bz2", configuration=ZarrV2Bz2Parameters, canonical=_bz2_canonical ) """`numcodecs.BZ2`.""" LZMA_CODEC_V2: Final = ZarrV2CodecDefinition( name="lzma", configuration=ZarrV2LzmaParameters, canonical=_lzma_canonical ) """`numcodecs.LZMA`.""" BLOSC_CODEC_V2: Final = ZarrV2CodecDefinition( name="blosc", configuration=ZarrV2BloscParameters, canonical=_blosc_canonical ) """`numcodecs.Blosc`.""" ZSTD_CODEC_V2: Final = ZarrV2CodecDefinition( name="zstd", configuration=ZarrV2ZstdParameters, canonical=_zstd_canonical ) """`numcodecs.Zstd`.""" LZ4_CODEC_V2: Final = ZarrV2CodecDefinition( name="lz4", configuration=ZarrV2Lz4Parameters, canonical=_lz4_canonical ) """`numcodecs.LZ4`.""" __all__ = [ "BLOSC_CODEC_V2", "BZ2_CODEC_V2", "GZIP_CODEC_V2", "LZ4_CODEC_V2", "LZMA_CODEC_V2", "ZLIB_CODEC_V2", "ZSTD_CODEC_V2", "ZarrV2BloscCName", "ZarrV2BloscParameters", "ZarrV2Bz2Parameters", "ZarrV2CompressionLevel", "ZarrV2GzipParameters", "ZarrV2Lz4Parameters", "ZarrV2LzmaParameters", "ZarrV2ZlibParameters", "ZarrV2ZstdParameters", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/codec/filters.py000066400000000000000000000151631527014137600330400ustar00rootroot00000000000000"""The v2 filters that numcodecs 0.16 configures: `shuffle`, `delta`, `fixedscaleoffset`, `quantize`, `bitround`, `astype` and `packbits`. A dtype parameter is a NumPy typestr, as numcodecs writes it. A parameter that numcodecs gives a default is optional, and the canonical form of each codec fills it in, as `get_config()` writes it. The canonical form writes each dtype parameter as NumPy spells it, as an array's dtype compares, so ` ZarrV2ShuffleParameters: """Fill in the default `elementsize`, 4, as numcodecs writes it.""" return {"elementsize": 4, **configuration} def _typestrs(dtype: str, astype: str | None) -> tuple[str, str]: """`dtype` and `astype` as NumPy spells them, with an `astype` that is omitted or null filled in with the `dtype`, as numcodecs writes them.""" canonical = canonical_typestr(dtype) return canonical, canonical if astype is None else canonical_typestr(astype) def _delta_canonical(configuration: ZarrV2DeltaParameters) -> ZarrV2DeltaParameters: """Fill in an `astype` that is omitted or null with the `dtype`, and write each dtype as NumPy spells it, as numcodecs writes them.""" dtype, astype = _typestrs(configuration["dtype"], configuration.get("astype")) return {**configuration, "dtype": dtype, "astype": astype} def _fixedscaleoffset_canonical( configuration: ZarrV2FixedScaleOffsetParameters, ) -> ZarrV2FixedScaleOffsetParameters: """Fill in an `astype` that is omitted or null with the `dtype`, and write each dtype as NumPy spells it, as numcodecs writes them. `offset` and `scale` are kept as written. NumPy computes with an integer and a float differently for integer and half-precision dtypes, so numcodecs encodes a `scale` of 10 and of 10.0 differently. """ dtype, astype = _typestrs(configuration["dtype"], configuration.get("astype")) return {**configuration, "dtype": dtype, "astype": astype} def _quantize_canonical(configuration: ZarrV2QuantizeParameters) -> ZarrV2QuantizeParameters: """Fill in an `astype` that is omitted or null with the `dtype`, and write each dtype as NumPy spells it, as numcodecs writes them.""" dtype, astype = _typestrs(configuration["dtype"], configuration.get("astype")) return {**configuration, "dtype": dtype, "astype": astype} def _astype_canonical(configuration: ZarrV2AsTypeParameters) -> ZarrV2AsTypeParameters: """Write each dtype as NumPy spells it, as numcodecs writes them.""" return { **configuration, "encode_dtype": canonical_typestr(configuration["encode_dtype"]), "decode_dtype": canonical_typestr(configuration["decode_dtype"]), } SHUFFLE_CODEC_V2: Final = ZarrV2CodecDefinition( name="shuffle", configuration=ZarrV2ShuffleParameters, canonical=_shuffle_canonical ) """`numcodecs.Shuffle`.""" DELTA_CODEC_V2: Final = ZarrV2CodecDefinition( name="delta", configuration=ZarrV2DeltaParameters, rules=dtype_parameter("dtype", "astype"), canonical=_delta_canonical, ) """`numcodecs.Delta`.""" FIXEDSCALEOFFSET_CODEC_V2: Final = ZarrV2CodecDefinition( name="fixedscaleoffset", configuration=ZarrV2FixedScaleOffsetParameters, rules=dtype_parameter("dtype", "astype"), canonical=_fixedscaleoffset_canonical, ) """`numcodecs.FixedScaleOffset`.""" QUANTIZE_CODEC_V2: Final = ZarrV2CodecDefinition( name="quantize", configuration=ZarrV2QuantizeParameters, rules=dtype_parameter("dtype", "astype", float_only=True), canonical=_quantize_canonical, ) """`numcodecs.Quantize`.""" BITROUND_CODEC_V2: Final = ZarrV2CodecDefinition( name="bitround", configuration=ZarrV2BitRoundParameters ) """`numcodecs.BitRound`.""" ASTYPE_CODEC_V2: Final = ZarrV2CodecDefinition( name="astype", configuration=ZarrV2AsTypeParameters, rules=dtype_parameter("encode_dtype", "decode_dtype"), canonical=_astype_canonical, ) """`numcodecs.AsType`.""" PACKBITS_CODEC_V2: Final = ZarrV2CodecDefinition(name="packbits", configuration=EmptyConfiguration) """`numcodecs.PackBits`. It has no configuration.""" __all__ = [ "ASTYPE_CODEC_V2", "BITROUND_CODEC_V2", "DELTA_CODEC_V2", "FIXEDSCALEOFFSET_CODEC_V2", "PACKBITS_CODEC_V2", "QUANTIZE_CODEC_V2", "SHUFFLE_CODEC_V2", "ZarrV2AsTypeParameters", "ZarrV2BitRoundParameters", "ZarrV2DeltaParameters", "ZarrV2FixedScaleOffsetParameters", "ZarrV2QuantizeParameters", "ZarrV2ShuffleParameters", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/codec/vlen.py000066400000000000000000000027641527014137600323370ustar00rootroot00000000000000"""The v2 variable-length filters that numcodecs 0.16 configures: `vlen-utf8`, `vlen-bytes` and `vlen-array`.""" from __future__ import annotations from typing import Final from typing_extensions import ReadOnly, TypedDict from zarr_metadata.definition._definition import EmptyConfiguration from zarr_metadata.v2._definition import ZarrV2CodecDefinition from zarr_metadata.v2.codec._dtype import canonical_typestr, dtype_parameter class ZarrV2VLenArrayParameters(TypedDict, closed=True): """`numcodecs.VLenArray(dtype)`: the element type of each array.""" dtype: ReadOnly[str] def _vlen_array_canonical(configuration: ZarrV2VLenArrayParameters) -> ZarrV2VLenArrayParameters: """Write the `dtype` as NumPy spells it, as numcodecs writes it.""" return {**configuration, "dtype": canonical_typestr(configuration["dtype"])} VLEN_UTF8_CODEC_V2: Final = ZarrV2CodecDefinition( name="vlen-utf8", configuration=EmptyConfiguration ) """`numcodecs.VLenUTF8`. It has no configuration.""" VLEN_BYTES_CODEC_V2: Final = ZarrV2CodecDefinition( name="vlen-bytes", configuration=EmptyConfiguration ) """`numcodecs.VLenBytes`. It has no configuration.""" VLEN_ARRAY_CODEC_V2: Final = ZarrV2CodecDefinition( name="vlen-array", configuration=ZarrV2VLenArrayParameters, rules=dtype_parameter("dtype"), canonical=_vlen_array_canonical, ) """`numcodecs.VLenArray`.""" __all__ = [ "VLEN_ARRAY_CODEC_V2", "VLEN_BYTES_CODEC_V2", "VLEN_UTF8_CODEC_V2", "ZarrV2VLenArrayParameters", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py000066400000000000000000000037171527014137600327650ustar00rootroot00000000000000"""Zarr v2 consolidated metadata (`.zmetadata` file). This module models the de-facto `.zmetadata` file used by the reference Python implementation of Zarr v2. **This is NOT a spec artifact.** There is no Zarr v2 specification that defines `.zmetadata`; it is a canonical-implementation convention. """ from collections.abc import Mapping from typing import Final, Literal from typing_extensions import TypedDict from zarr_metadata.v2.array import ZarrV2ZArrayJSON from zarr_metadata.v2.attributes import ZarrV2ZAttrsJSON from zarr_metadata.v2.group import ZarrV2ZGroupJSON class ZarrV2ConsolidatedMetadataJSON(TypedDict): """ `.zmetadata` file contents. The `metadata` map uses flat path keys (`"foo/bar/.zarray"`, `"foo/.zattrs"`, etc.) pointing to the JSON contents of the file at that path. The keys include the filename suffix, not just the node path; the value's shape is determined by which file the key points at: - `/.zarray` -> `ZarrV2ZArrayJSON` - `/.zgroup` -> `ZarrV2ZGroupJSON` - `/.zattrs` -> `ZarrV2ZAttrsJSON` The TypedDict cannot discriminate the value shape on the key suffix at the type level; consumers should narrow at runtime by inspecting `key.endswith(".zarray")` etc. """ zarr_consolidated_format: Literal[1] metadata: Mapping[str, ZarrV2ZArrayJSON | ZarrV2ZGroupJSON | ZarrV2ZAttrsJSON] ZarrV2ConsolidatedMetadataStoreKey = Literal[".zmetadata"] """Literal type of the store key holding a v2 hierarchy's consolidated metadata.""" ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY: Final[ZarrV2ConsolidatedMetadataStoreKey] = ".zmetadata" """The store key a v2 hierarchy's consolidated metadata is persisted under. Like the document it names, this is a reference-implementation convention rather than a spec artifact; see the module docstring. """ __all__ = [ "ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY", "ZarrV2ConsolidatedMetadataJSON", "ZarrV2ConsolidatedMetadataStoreKey", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/data_type/000077500000000000000000000000001527014137600317055ustar00rootroot00000000000000__init__.py000066400000000000000000000026641527014137600337470ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/data_type"""The data types that zarr-python 2.x writes, with one definition per NumPy family. Each family's definition is in its submodule (`fixed_width`, `object`, `scalar`, `struct`, `time`). This module holds the dtype's JSON shape, `ZarrV2DataTypeMetadata`, and `DATA_TYPES_V2`, the tuple of every definition. """ from typing import Any, Final from zarr_metadata.v2._definition import ZarrV2DataTypeDefinition from zarr_metadata.v2._field_json import ZarrV2DataTypeMetadata from zarr_metadata.v2.data_type.fixed_width import ( BYTES_DATA_TYPE_V2, STR_DATA_TYPE_V2, VOID_DATA_TYPE_V2, ) from zarr_metadata.v2.data_type.object import OBJECT_DATA_TYPE_V2 from zarr_metadata.v2.data_type.scalar import ( BOOL_DATA_TYPE_V2, COMPLEX_DATA_TYPE_V2, FLOAT_DATA_TYPE_V2, INT_DATA_TYPE_V2, UINT_DATA_TYPE_V2, ) from zarr_metadata.v2.data_type.struct import STRUCT_DATA_TYPE_V2 from zarr_metadata.v2.data_type.time import DATETIME64_DATA_TYPE_V2, TIMEDELTA64_DATA_TYPE_V2 DATA_TYPES_V2: Final[tuple[ZarrV2DataTypeDefinition[Any], ...]] = ( BOOL_DATA_TYPE_V2, INT_DATA_TYPE_V2, UINT_DATA_TYPE_V2, FLOAT_DATA_TYPE_V2, COMPLEX_DATA_TYPE_V2, BYTES_DATA_TYPE_V2, STR_DATA_TYPE_V2, VOID_DATA_TYPE_V2, DATETIME64_DATA_TYPE_V2, TIMEDELTA64_DATA_TYPE_V2, OBJECT_DATA_TYPE_V2, STRUCT_DATA_TYPE_V2, ) """Every v2 data type, by family.""" __all__ = [ "DATA_TYPES_V2", "ZarrV2DataTypeMetadata", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/data_type/_rules.py000066400000000000000000000145341527014137600335570ustar00rootroot00000000000000"""Rules that several v2 dtype families share: the sizes and byte orders a family allows, the unit of a time type, and a base64 fill value of the item's size.""" from __future__ import annotations import base64 from dataclasses import dataclass from typing import TYPE_CHECKING, Literal from zarr_metadata._json import ValidationProblem, shown from zarr_metadata._numpy import NUMPY_TIME_UNITS, base64_bytes if TYPE_CHECKING: from collections.abc import Iterator from zarr_metadata.definition._definition import Nested from zarr_metadata.v2.data_type.fixed_width import ZarrV2Base64FillValue from zarr_metadata.v2.data_type.scalar import ZarrV2ScalarConfiguration from zarr_metadata.v2.data_type.time import ZarrV2TimeConfiguration @dataclass(frozen=True, slots=True) class _Sized: """The rules of a family whose types are `sizes` bytes wide. This is a value rather than a closure, so a scope that holds it can be pickled.""" sizes: tuple[int, ...] | None orderless: frozenset[int] | None unit: int def __call__( self, configuration: ZarrV2ScalarConfiguration, nested: Nested ) -> Iterator[ValidationProblem]: size = configuration["itemsize"] if self.sizes is not None and size not in self.sizes: yield ValidationProblem( ("itemsize",), f"expected a size of {', '.join(map(str, self.sizes))} bytes, got {size}", "invalid_value", ) return if ( configuration["byteorder"] == "|" and self.orderless is not None and size not in self.orderless ): yield ValidationProblem( ("byteorder",), f"expected a byte order '<' or '>' for a type of {size * self.unit} bytes, got '|'", "invalid_value", ) def sized( sizes: tuple[int, ...] | None, orderless: frozenset[int] | None, *, unit: int = 1 ) -> _Sized: """Return the rules of a family whose types are `sizes` bytes wide. If `sizes` is None, any width is allowed. `orderless` is the set of sizes at which the type has no byte order, which NumPy writes as `|`. This is one byte for an integer, every size for bytes and void, and no size for a float. None means every size. At any other size, the typestr must state which end comes first, `<` or `>`. `unit` is the number of bytes that one unit of the size takes: one, or four for `U`, whose size counts code points. """ return _Sized(sizes, orderless, unit) @dataclass(frozen=True, slots=True) class _OrderlessAt: """The canonical spelling for a family whose types have no byte order at `sizes` bytes.""" sizes: frozenset[int] | None def __call__(self, configuration: ZarrV2ScalarConfiguration) -> ZarrV2ScalarConfiguration: orderless = self.sizes is None or configuration["itemsize"] in self.sizes if orderless and configuration["byteorder"] != "|": return {**configuration, "byteorder": "|"} return configuration def orderless_at(sizes: frozenset[int] | None) -> _OrderlessAt: """Return the canonical spelling for a family whose types have no byte order at `sizes` bytes. At those sizes the byte order is `|`, as NumPy writes it. None means every size.""" return _OrderlessAt(sizes) @dataclass(frozen=True, slots=True) class _TimeRules: """The rules that `time_rules` returns. This is a value rather than a closure, so a scope that holds it can be pickled.""" code: Literal["M", "m"] unit_required: bool def __call__( self, configuration: ZarrV2TimeConfiguration, nested: Nested ) -> Iterator[ValidationProblem]: size = configuration["itemsize"] if size != 8: yield ValidationProblem( ("itemsize",), f"expected a size of 8 bytes, got {size}", "invalid_value" ) return if configuration["byteorder"] == "|": yield ValidationProblem( ("byteorder",), "expected a byte order '<' or '>' for a type of 8 bytes, got '|'", "invalid_value", ) if "unit" not in configuration: if self.unit_required: yield ValidationProblem( ("unit",), f"expected a time unit in brackets, '<{self.code}8[ns]': the v2 spec " "requires one", "missing_key", ) return if configuration["unit"] not in NUMPY_TIME_UNITS: yield ValidationProblem( ("unit",), f"expected a NumPy time unit -- {', '.join(NUMPY_TIME_UNITS)} -- got {configuration['unit']!r}", "invalid_value", ) def time_rules(code: Literal["M", "m"], *, unit_required: bool = True) -> _TimeRules: """Return the rules of the time family whose type code is `code`: eight bytes with a byte order, and a NumPy time unit in brackets. The v2 spec requires the unit of an array's dtype (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L147-L150). With `unit_required` False, the unit may be left out, as in ` Iterator[ValidationProblem]: """Check that a fill value that is not null is standard-alphabet base64 of exactly `size` bytes, or of at most `size` bytes when `exact` is False. Any size passes when `size` is None, which happens only when the item size is unknown.""" if value is None: return try: base64_bytes(value) except ValueError: yield ValidationProblem( (), f"expected standard-alphabet base64, got {shown(value)}", "invalid_value" ) return if size is None: return held = base64.b64decode(value) if (len(held) != size) if exact else (len(held) > size): bound = "" if exact else "at most " yield ValidationProblem( (), f"expected base64 of {bound}{size} bytes, the item's size, got {len(held)} bytes", "invalid_value", ) __all__ = ["orderless_at", "sized", "sized_base64_rules", "time_rules"] fixed_width.py000066400000000000000000000052431527014137600345020ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/data_type"""The v2 families with a fixed width per item, of any size: `bytes` (`S`), `str` (`U`) and `void` (`V`).""" from __future__ import annotations from typing import TYPE_CHECKING, Final from zarr_metadata.v2._definition import CODE_POINT_BYTES, ZarrV2DataTypeDefinition from zarr_metadata.v2.data_type._rules import orderless_at, sized, sized_base64_rules from zarr_metadata.v2.data_type.scalar import ZarrV2ScalarConfiguration if TYPE_CHECKING: from collections.abc import Iterator from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import Nested ZarrV2Base64FillValue = str | None """A fill value that the v2 spec encodes as base64, for a fixed-length byte string, void, or a structured type (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L191-L193), or null.""" def _void_fill_value_rules( configuration: ZarrV2ScalarConfiguration, nested: Nested, value: ZarrV2Base64FillValue ) -> Iterator[ValidationProblem]: """Check that the fill value is base64 of exactly the item's size in bytes.""" yield from sized_base64_rules(value, configuration["itemsize"], exact=True) def _bytes_fill_value_rules( configuration: ZarrV2ScalarConfiguration, nested: Nested, value: ZarrV2Base64FillValue ) -> Iterator[ValidationProblem]: """Check that the fill value is base64 of at most the item's size in bytes. NumPy pads a shorter byte string with zeros.""" yield from sized_base64_rules(value, configuration["itemsize"], exact=False) BYTES_DATA_TYPE_V2: Final = ZarrV2DataTypeDefinition( name="bytes", configuration=ZarrV2ScalarConfiguration, rules=sized(None, None), canonical=orderless_at(None), fill_value=ZarrV2Base64FillValue, fill_value_rules=_bytes_fill_value_rules, ) """`|S`: byte strings of `n` bytes. The fill value is base64 of at most `n` bytes.""" STR_DATA_TYPE_V2: Final = ZarrV2DataTypeDefinition( name="str", configuration=ZarrV2ScalarConfiguration, rules=sized(None, frozenset(), unit=CODE_POINT_BYTES), fill_value=str | None, ) """``: strings of `n` code points. Each code point takes four bytes, in the written byte order. The fill value is a string.""" VOID_DATA_TYPE_V2: Final = ZarrV2DataTypeDefinition( name="void", configuration=ZarrV2ScalarConfiguration, rules=sized(None, None), canonical=orderless_at(None), fill_value=ZarrV2Base64FillValue, fill_value_rules=_void_fill_value_rules, ) """`|V`: `n` untyped bytes. The fill value is base64 of exactly `n` bytes.""" __all__ = [ "BYTES_DATA_TYPE_V2", "STR_DATA_TYPE_V2", "VOID_DATA_TYPE_V2", "ZarrV2Base64FillValue", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/data_type/object.py000066400000000000000000000020101527014137600335160ustar00rootroot00000000000000"""The v2 `object` family, `|O`: Python objects. The fill value may be any JSON value.""" from __future__ import annotations from typing import Final from typing_extensions import ReadOnly, TypedDict from zarr_metadata.v2._definition import ZarrV2DataTypeDefinition from zarr_metadata.v2.data_type.scalar import ( ZarrV2ByteOrder, ) class ZarrV2ObjectConfiguration(TypedDict, closed=True): """The configuration that `|O` carries: a byte order, which the type ignores.""" byteorder: ReadOnly[ZarrV2ByteOrder] def _canonical(configuration: ZarrV2ObjectConfiguration) -> ZarrV2ObjectConfiguration: """Set the byte order to `|` (no byte order), as NumPy writes it.""" return {**configuration, "byteorder": "|"} OBJECT_DATA_TYPE_V2: Final = ZarrV2DataTypeDefinition( name="object", configuration=ZarrV2ObjectConfiguration, canonical=_canonical ) """`|O`: Python objects, each encoded by a filter. The fill value may be any JSON value.""" __all__ = ["OBJECT_DATA_TYPE_V2", "ZarrV2ObjectConfiguration"] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/data_type/scalar.py000066400000000000000000000146071527014137600335340ustar00rootroot00000000000000"""The v2 scalar families: `bool`, `int`, `uint`, `float` and `complex`. Each definition is filed under its family and is read for every typestr of the family.""" from __future__ import annotations import math import struct from dataclasses import dataclass from typing import TYPE_CHECKING, Annotated, Final, Literal from annotated_types import Ge from typing_extensions import ReadOnly, TypedDict from zarr_metadata._json import ValidationProblem from zarr_metadata.v2._definition import ZarrV2DataTypeDefinition from zarr_metadata.v2.data_type._rules import orderless_at, sized if TYPE_CHECKING: from collections.abc import Iterator, Mapping from zarr_metadata.definition._definition import Nested ZarrV2ByteOrder = Literal["<", ">", "|"] """The byte orders that a typestr writes: little-endian, big-endian, and not relevant.""" class ZarrV2ScalarConfiguration(TypedDict, closed=True): """The configuration that a scalar typestr carries: its byte order and its size in bytes, such as `{"byteorder": "<", "itemsize": 4}` for ` Iterator[ValidationProblem]: if value is None: return bits = 8 * configuration["itemsize"] if self.signed: low, high = -(2 ** (bits - 1)), 2 ** (bits - 1) - 1 else: low, high = 0, 2**bits - 1 if not low <= value <= high: yield ValidationProblem( (), f"expected an integer in [{low}, {high}], got {value}", "invalid_value" ) ZarrV2FloatSpecial = Literal["NaN", "Infinity", "-Infinity"] """The non-finite values that the v2 spec spells by name (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L178-L190).""" ZarrV2FloatFillValue = float | int | ZarrV2FloatSpecial | None """A v2 float fill value: a number, a named non-finite value, or null.""" ZarrV2ComplexComponent = float | int | ZarrV2FloatSpecial """One component of a complex fill value: a float fill value that is not null.""" ZarrV2ComplexFillValue = tuple[ZarrV2ComplexComponent, ZarrV2ComplexComponent] | None """A v2 complex fill value: `[real, imag]`, each a float fill value, or null.""" def _float_canonical( configuration: ZarrV2ScalarConfiguration, nested: Nested, value: ZarrV2FloatFillValue ) -> ZarrV2FloatFillValue: """Return the canonical spelling of a float fill value. An integer written for a float becomes the float, so `0` and `0.0` are one value. A number past the largest float of the dtype's width becomes the infinity of its sign. This matches how the v3 float types read a number and how NumPy stores it.""" return None if value is None else _held_in(value, configuration["itemsize"]) def _held_in(value: ZarrV2ComplexComponent, itemsize: int) -> ZarrV2ComplexComponent: """Return `value` as a float of `itemsize` bytes holds it: the value as a float, or the infinity of its sign if it is past the largest such float. If no float has that width, the value is kept.""" if isinstance(value, bool) or not isinstance(value, (int, float)): return value try: held = float(value) narrowed = held if itemsize in _FLOAT_CODES: # `struct` raises for a float16 past the largest value, and packs # a float32 past the largest value as an infinity. code = _FLOAT_CODES[itemsize] (narrowed,) = struct.unpack(code, struct.pack(code, held)) except OverflowError: return "-Infinity" if value < 0 else "Infinity" if math.isinf(narrowed): return "-Infinity" if value < 0 else "Infinity" return held _FLOAT_CODES: Final[Mapping[int, str]] = {2: "e", 4: "f", 8: "d"} """The `struct` code for the float of each size in bytes. Packing with it detects a value past the largest float.""" def _complex_canonical( configuration: ZarrV2ScalarConfiguration, nested: Nested, value: ZarrV2ComplexFillValue ) -> ZarrV2ComplexFillValue: """Return each component in the canonical spelling of its float type.""" if value is None: return None real, imag = value width = configuration["itemsize"] // 2 return ( _held_in(real, width), _held_in(imag, width), ) BOOL_DATA_TYPE_V2: Final = ZarrV2DataTypeDefinition( name="bool", configuration=ZarrV2ScalarConfiguration, rules=sized((1,), None), canonical=orderless_at(None), fill_value=bool | None, ) """`|b1`: one byte, true or false.""" INT_DATA_TYPE_V2: Final = ZarrV2DataTypeDefinition( name="int", configuration=ZarrV2ScalarConfiguration, rules=sized((1, 2, 4, 8), _ONE_BYTE), canonical=orderless_at(_ONE_BYTE), fill_value=int | None, fill_value_rules=_InRange(True), ) """`i1` to `i8`: signed integers. The fill value must be in the type's range.""" UINT_DATA_TYPE_V2: Final = ZarrV2DataTypeDefinition( name="uint", configuration=ZarrV2ScalarConfiguration, rules=sized((1, 2, 4, 8), _ONE_BYTE), canonical=orderless_at(_ONE_BYTE), fill_value=int | None, fill_value_rules=_InRange(False), ) """`u1` to `u8`: unsigned integers. The fill value must be in the type's range.""" FLOAT_DATA_TYPE_V2: Final = ZarrV2DataTypeDefinition( name="float", configuration=ZarrV2ScalarConfiguration, rules=sized((2, 4, 8), frozenset()), fill_value=ZarrV2FloatFillValue, fill_value_canonical=_float_canonical, ) """`f2`, `f4`, `f8`: IEEE 754 floats. The fill value is a number or a named non-finite value.""" COMPLEX_DATA_TYPE_V2: Final = ZarrV2DataTypeDefinition( name="complex", configuration=ZarrV2ScalarConfiguration, rules=sized((8, 16), frozenset()), fill_value=ZarrV2ComplexFillValue, fill_value_canonical=_complex_canonical, ) """`c8`, `c16`: complex floats. The fill value is `[real, imag]`.""" __all__ = [ "BOOL_DATA_TYPE_V2", "COMPLEX_DATA_TYPE_V2", "FLOAT_DATA_TYPE_V2", "INT_DATA_TYPE_V2", "UINT_DATA_TYPE_V2", "ZarrV2ByteOrder", "ZarrV2ComplexComponent", "ZarrV2ComplexFillValue", "ZarrV2FloatFillValue", "ZarrV2FloatSpecial", "ZarrV2ScalarConfiguration", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/data_type/struct.py000066400000000000000000000106271527014137600336110ustar00rootroot00000000000000"""The v2 `struct` type: an array of field records. The type of each record is a field of its own.""" from __future__ import annotations import math from typing import TYPE_CHECKING, Annotated, Any, Final, cast from annotated_types import Ge from typing_extensions import ReadOnly, TypedDict from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import AcceptedField from zarr_metadata.v2._definition import ( STRUCT_NAME, ZarrV2DataTypeDefinition, ZarrV2DataTypeField, item_bytes, ) from zarr_metadata.v2.data_type._rules import sized_base64_rules from zarr_metadata.v2.data_type.fixed_width import ZarrV2Base64FillValue if TYPE_CHECKING: from collections.abc import Iterator from zarr_metadata.definition._definition import Nested # List the longer record first. A record that matches neither form is # reported against the first branch, and a wrong shape is a deeper # problem than a wrong length. ZarrV2StructRecord = ( tuple[str, ZarrV2DataTypeField, tuple[Annotated[int, Ge(0)], ...]] | tuple[str, ZarrV2DataTypeField] ) """A field record: `[name, dtype]` or `[name, dtype, shape]`. The dtype is a nested field (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L152-L174).""" class ZarrV2StructConfiguration(TypedDict, closed=True): """The records of a structured dtype. The document writes the records as the dtype itself.""" fields: ReadOnly[tuple[ZarrV2StructRecord, ...]] def _rules(configuration: ZarrV2StructConfiguration, nested: Nested) -> Iterator[ValidationProblem]: """Check that the names are distinct. The name `""` may repeat: NumPy uses it for the padding of an aligned struct, and zarr-python 2.x writes it. A struct with no records is allowed: NumPy's `np.dtype([])` takes no bytes, and zarr-python 2.x writes it as `[]` and reads it back.""" fields = configuration["fields"] seen: dict[str, int] = {} for index, record in enumerate(fields): name = record[0] if name == "": continue first = seen.setdefault(name, index) if first != index: yield ValidationProblem( ("fields", index, 0), f"duplicate field name {name!r}, already used by record {first}", "invalid_value", ) def _fill_value_rules( configuration: ZarrV2StructConfiguration, nested: Nested, value: ZarrV2Base64FillValue ) -> Iterator[ValidationProblem]: """Check that the fill value is base64 of exactly one record, when the size of every field is known.""" yield from sized_base64_rules(value, _record_size(configuration, nested), exact=True) def _record_size(configuration: ZarrV2StructConfiguration, nested: Nested) -> int | None: """Return the number of bytes that one record of `configuration` takes. Each field takes its item size in bytes times the product of its subarray shape, and a nested struct takes the size of its own record. Return None when the size of a field is not known (as for `|O`) or its dtype was not read.""" total = 0 for index, record in enumerate(configuration["fields"]): field = nested.get(("fields", index, 1)) if not isinstance(field, AcceptedField): return None size = _item_size(field) if size is None: return None shape = record[2] if len(record) == 3 else () total += size * math.prod(shape) return total def _item_size(field: AcceptedField[Any]) -> int | None: """Return the number of bytes that one item of the dtype read as `field` takes, or None when that is not known. A typestr gives the size of its type, which for `U` counts code points of four bytes each.""" if field.name == STRUCT_NAME or "fields" in field.configuration: return _record_size(cast("ZarrV2StructConfiguration", field.configuration), field.nested) return item_bytes(field.name) STRUCT_DATA_TYPE_V2: Final = ZarrV2DataTypeDefinition( name=STRUCT_NAME, configuration=ZarrV2StructConfiguration, rules=_rules, fill_value=ZarrV2Base64FillValue, fill_value_rules=_fill_value_rules, ) """A structured type: an array of field records. The fill value is base64 of one record (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L191-L193).""" __all__ = ["STRUCT_DATA_TYPE_V2", "ZarrV2StructConfiguration", "ZarrV2StructRecord"] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v2/data_type/time.py000066400000000000000000000051211527014137600332140ustar00rootroot00000000000000"""The v2 time families, `datetime64` (`M`) and `timedelta64` (`m`): eight bytes, in a unit that the typestr gives in brackets.""" from __future__ import annotations from typing import TYPE_CHECKING, Annotated, Final, Literal, NotRequired from annotated_types import Ge from typing_extensions import ReadOnly, TypedDict from zarr_metadata._numpy import ( NumpyTimeScaleFactor, NumpyTimeTicks, numpy_time_fill_value_canonical, ) from zarr_metadata.v2._definition import ZarrV2DataTypeDefinition from zarr_metadata.v2.data_type._rules import time_rules from zarr_metadata.v2.data_type.scalar import ( ZarrV2ByteOrder, ) if TYPE_CHECKING: from zarr_metadata.definition._definition import Nested class ZarrV2TimeConfiguration(TypedDict, closed=True): """The configuration that a time typestr carries. For example, ` ZarrV2TimeConfiguration: """Write `μs` as `us`, as NumPy writes it.""" if configuration.get("unit") == "μs": return {**configuration, "unit": "us"} return configuration ZarrV2TimeFillValue = NumpyTimeTicks | Literal["NaT"] | None """A v2 time fill value: a count of ticks, `"NaT"`, or null.""" def _fill_value_canonical( configuration: ZarrV2TimeConfiguration, nested: Nested, value: ZarrV2TimeFillValue ) -> ZarrV2TimeFillValue: """Return `NaT` for not-a-time however it is written, and any other value as written.""" if value is None: return None return numpy_time_fill_value_canonical(configuration, nested, value) DATETIME64_DATA_TYPE_V2: Final = ZarrV2DataTypeDefinition( name="datetime64", configuration=ZarrV2TimeConfiguration, rules=time_rules("M"), canonical=_canonical, fill_value=ZarrV2TimeFillValue, fill_value_canonical=_fill_value_canonical, ) """`/.zgroup` for a v2 group. The spec defines exactly one field and forbids others. User attributes live in a sibling `.zattrs` file and are NOT part of this type; see `ZarrV2ZAttrsJSON`. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L306-L313 See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ zarr_format: Literal[2] class ZarrV2GroupMetadataJSON(TypedDict, closed=True): """ Zarr v2 group metadata document, in-memory merged form. Models the union of `.zgroup` (the spec-defined `zarr_format` field, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L306-L313) and `.zattrs` (user attributes, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330). On disk these are persisted as two separate files; this type folds them so a single TypedDict represents the complete in-memory state of a v2 group node. Consumers that read or write the real on-disk files should use `ZarrV2ZGroupJSON` (strict `.zgroup`) plus `ZarrV2ZAttrsJSON` directly. The type is closed: `.zgroup` holds no other keys ("Other keys MUST NOT be present", https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313), and `attributes` is the only member folded in. See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ zarr_format: Literal[2] attributes: NotRequired[Mapping[str, JSONValue]] ZarrV2GroupMetadataStoreKey = Literal[".zgroup"] """Literal type of the store key holding a v2 group's metadata document.""" ZARR_V2_GROUP_METADATA_STORE_KEY: Final[ZarrV2GroupMetadataStoreKey] = ".zgroup" """The store key a v2 group's metadata document is persisted under.""" __all__ = [ "ZARR_V2_GROUP_METADATA_STORE_KEY", "ZarrV2GroupMetadataJSON", "ZarrV2GroupMetadataStoreKey", "ZarrV2ZGroupJSON", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/000077500000000000000000000000001527014137600277345ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/__init__.py000066400000000000000000000103641527014137600320510ustar00rootroot00000000000000"""Zarr v3 metadata types, and the front door to the v3 definitions. This module holds the JSON of any metadata field (a bare name or a `ZarrV3NamedConfigJSON` envelope) and the node name and path types. Each document type is in the submodule for its node: `array`, `group` and `consolidated`. The extensions are in `codec`, `data_type`, `chunk_grid` and `chunk_key_encoding`, and the v3 kinds and scopes are in `definition`. This module also exports every v3 definition (`BYTES_CODEC`, `FLOAT64_DATA_TYPE`, `REGULAR_CHUNK_GRID`, ...), the scopes `CORE` and `CORE_AND_EXTENSIONS`, and the scope class `ZarrV3Context`, so that one import builds or updates a scope. Each is the same object as in its own submodule, where a definition's configuration types and constants are. """ from zarr_metadata._common import ZarrV3NamedConfigJSON from zarr_metadata.v3._common import ZarrV3ExtensionField, ZarrV3MetadataFieldJSON from zarr_metadata.v3._hierarchy import NodeName, NodePath from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID from zarr_metadata.v3.chunk_key_encoding.default import DEFAULT_CHUNK_KEY_ENCODING from zarr_metadata.v3.chunk_key_encoding.v2 import V2_CHUNK_KEY_ENCODING from zarr_metadata.v3.codec.blosc import BLOSC_CODEC from zarr_metadata.v3.codec.bytes import BYTES_CODEC from zarr_metadata.v3.codec.cast_value import CAST_VALUE_CODEC from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC from zarr_metadata.v3.codec.gzip import GZIP_CODEC from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC from zarr_metadata.v3.codec.sharding_indexed import SHARDING_INDEXED_CODEC from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC from zarr_metadata.v3.codec.zstd import ZSTD_CODEC from zarr_metadata.v3.consolidated import CONSOLIDATED_METADATA_EXTRA_FIELD from zarr_metadata.v3.data_type.bool import BOOL_DATA_TYPE from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE from zarr_metadata.v3.data_type.int8 import INT8_DATA_TYPE from zarr_metadata.v3.data_type.int16 import INT16_DATA_TYPE from zarr_metadata.v3.data_type.int32 import INT32_DATA_TYPE from zarr_metadata.v3.data_type.int64 import INT64_DATA_TYPE from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE from zarr_metadata.v3.data_type.string import STRING_DATA_TYPE from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE from zarr_metadata.v3.data_type.uint8 import UINT8_DATA_TYPE from zarr_metadata.v3.data_type.uint16 import UINT16_DATA_TYPE from zarr_metadata.v3.data_type.uint32 import UINT32_DATA_TYPE from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE from zarr_metadata.v3.definition import ( CORE, CORE_AND_EXTENSIONS, ZarrV3Context, ) __all__ = [ "BLOSC_CODEC", "BOOL_DATA_TYPE", "BYTES_CODEC", "BYTES_DATA_TYPE", "CAST_VALUE_CODEC", "COMPLEX64_DATA_TYPE", "COMPLEX128_DATA_TYPE", "CONSOLIDATED_METADATA_EXTRA_FIELD", "CORE", "CORE_AND_EXTENSIONS", "CRC32C_CODEC", "DEFAULT_CHUNK_KEY_ENCODING", "FLOAT16_DATA_TYPE", "FLOAT32_DATA_TYPE", "FLOAT64_DATA_TYPE", "GZIP_CODEC", "INT8_DATA_TYPE", "INT16_DATA_TYPE", "INT32_DATA_TYPE", "INT64_DATA_TYPE", "NUMPY_DATETIME64_DATA_TYPE", "NUMPY_TIMEDELTA64_DATA_TYPE", "RAW_BYTES_DATA_TYPE", "RECTILINEAR_CHUNK_GRID", "REGULAR_CHUNK_GRID", "SCALE_OFFSET_CODEC", "SHARDING_INDEXED_CODEC", "STRING_DATA_TYPE", "STRUCT_DATA_TYPE", "TRANSPOSE_CODEC", "UINT8_DATA_TYPE", "UINT16_DATA_TYPE", "UINT32_DATA_TYPE", "UINT64_DATA_TYPE", "V2_CHUNK_KEY_ENCODING", "ZSTD_CODEC", "NodeName", "NodePath", "ZarrV3Context", "ZarrV3ExtensionField", "ZarrV3MetadataFieldJSON", "ZarrV3NamedConfigJSON", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/_common.py000066400000000000000000000061521527014137600317410ustar00rootroot00000000000000"""The v3 metadata field: its JSON type, and the type aliases for members that hold a field. This module is private. It sits below both code paths that read a field: the model, which checks the fields of a document, and the definitions, which read a field's configuration. Public consumers import `ZarrV3MetadataFieldJSON` from `zarr_metadata.v3`, and the aliases from `zarr_metadata.v3.definition`. One field is read with `zarr_metadata.definition.resolve`. """ from typing import TypeAlias from typing_extensions import TypeAliasType from zarr_metadata._common import JSONValue, ZarrV3NamedConfigJSON ZarrV3ExtensionField: TypeAlias = JSONValue """The JSON value of an unknown top-level v3 metadata field. An object carrying the literal member `must_understand: false` may be ignored. Every other JSON shape implicitly requires understanding; recognition itself belongs to the reader rather than this structural type. """ ZarrV3MetadataFieldJSON: TypeAlias = str | ZarrV3NamedConfigJSON """The JSON shape of any v3 metadata extension-point entry: either a bare short-hand name string or a `{name, configuration, must_understand}` envelope. Used for `data_type`, `chunk_grid`, `chunk_key_encoding`, individual codec entries, and `storage_transformers` in v3 array metadata, and for the inner `codecs` / `index_codecs` lists of the `sharding_indexed` codec. """ # A member that holds a metadata field is annotated with the alias for its # kind. A scope uses that alias to decide which kind to read the field as. # Each alias is defined as the field's JSON type, so a type checker and # `check` both see it as `ZarrV3MetadataFieldJSON`. ZarrV3DataTypeField = TypeAliasType("ZarrV3DataTypeField", ZarrV3MetadataFieldJSON) """A member that holds a data type: a document's `data_type`, or a struct field's data type. It is read in the same scope as the object that contains it.""" ZarrV3ChunkGridField = TypeAliasType("ZarrV3ChunkGridField", ZarrV3MetadataFieldJSON) """A member that holds a chunk grid: a document's `chunk_grid`.""" ZarrV3ChunkKeyEncodingField = TypeAliasType("ZarrV3ChunkKeyEncodingField", ZarrV3MetadataFieldJSON) """A member that holds a chunk key encoding: a document's `chunk_key_encoding`.""" ZarrV3CodecField = TypeAliasType("ZarrV3CodecField", ZarrV3MetadataFieldJSON) """A member that holds a codec. A document's `codecs` is `tuple[ZarrV3CodecField, ...]`, and so is a shard's.""" ZarrV3StaticCodecField = TypeAliasType("ZarrV3StaticCodecField", ZarrV3MetadataFieldJSON) """A member that holds a codec with a static encoded size. A shard's `index_codecs` uses this, because a reader locates the index from a size it knows before reading it. """ ZarrV3StorageTransformerField = TypeAliasType( "ZarrV3StorageTransformerField", ZarrV3MetadataFieldJSON ) """A member that holds a storage transformer. A document's `storage_transformers` is `tuple[ZarrV3StorageTransformerField, ...]`.""" __all__ = [ "ZarrV3ChunkGridField", "ZarrV3ChunkKeyEncodingField", "ZarrV3CodecField", "ZarrV3DataTypeField", "ZarrV3ExtensionField", "ZarrV3MetadataFieldJSON", "ZarrV3StaticCodecField", "ZarrV3StorageTransformerField", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/_context.py000066400000000000000000000246511527014137600321410ustar00rootroot00000000000000"""The scope a Zarr v3 document is read in: one slot for each kind of v3 field.""" from __future__ import annotations from collections.abc import Iterable, Mapping from typing import ( TYPE_CHECKING, Any, ClassVar, Final, Literal, TypeAlias, final, get_args, get_origin, ) from typing_extensions import TypedDict, TypeIs, TypeVar, Unpack from zarr_metadata.definition._context import ( Tables, claimant_in, definitions_in, filed, joined_tables, refuse_others, slot_contents, slot_kinds, tables_key, updated, ) from zarr_metadata.v3._definition import ( ArrayExtraFieldDefinition, ChunkGridDefinition, ChunkKeyEncodingDefinition, CodecDefinition, DataTypeDefinition, GroupExtraFieldDefinition, StorageTransformerDefinition, ) if TYPE_CHECKING: from zarr_metadata.definition._definition import D, Definition ZarrV3Kind: TypeAlias = ( DataTypeDefinition[Any] | ChunkGridDefinition[Any] | ChunkKeyEncodingDefinition[Any] | CodecDefinition[Any] | StorageTransformerDefinition[Any] | ArrayExtraFieldDefinition[Any] | GroupExtraFieldDefinition[Any] ) """A definition of a kind of v3 field.""" _K = TypeVar("_K", bound=ZarrV3Kind) """A kind of v3 field, so a type checker refuses a kind of the other format.""" _KINDS: Final[frozenset[type[Definition[Any]]]] = frozenset( get_origin(member) for member in get_args(ZarrV3Kind) ) def is_v3_kind(kind: type[Definition[Any]]) -> TypeIs[type[ZarrV3Kind]]: """Whether `kind` is a kind of v3 field: one of the kinds in `ZarrV3Kind`.""" return kind in _KINDS class ZarrV3ContextSlots(TypedDict, total=False, closed=True): """The definitions in each slot of a v3 scope, as `ZarrV3Context` and its `update` take them. This is where a v3 scope's slots are declared: each key is a slot, and its value the definitions of the kind that slot holds, as an iterable of them or as a mapping from each one's name to it. `ZarrV3Context.kinds` is read from these annotations, so a type checker checks every slot a caller names, and every definition given in it. """ data_types: Iterable[DataTypeDefinition[Any]] | Mapping[str, DataTypeDefinition[Any]] """Data types, read from `data_type`.""" chunk_grids: Iterable[ChunkGridDefinition[Any]] | Mapping[str, ChunkGridDefinition[Any]] """Chunk grids, read from `chunk_grid`.""" chunk_key_encodings: ( Iterable[ChunkKeyEncodingDefinition[Any]] | Mapping[str, ChunkKeyEncodingDefinition[Any]] ) """Chunk key encodings, read from `chunk_key_encoding`.""" codecs: Iterable[CodecDefinition[Any]] | Mapping[str, CodecDefinition[Any]] """Codecs, read from `codecs` and from the codecs a codec holds.""" storage_transformers: ( Iterable[StorageTransformerDefinition[Any]] | Mapping[str, StorageTransformerDefinition[Any]] ) """Storage transformers, read from `storage_transformers`.""" array_extra_fields: ( Iterable[ArrayExtraFieldDefinition[Any]] | Mapping[str, ArrayExtraFieldDefinition[Any]] ) """Members of an array document that the spec does not define, by key.""" group_extra_fields: ( Iterable[GroupExtraFieldDefinition[Any]] | Mapping[str, GroupExtraFieldDefinition[Any]] ) """Members of a group document that the spec does not define, by key.""" @final class ZarrV3Context: """A scope that reads Zarr v3 documents. Every v3 reader takes one. It implements `Context`, and has one slot for each kind of field a v3 document holds, as `ZarrV3ContextSlots` declares them. The slots for the spec's extension points are named like the document member that holds such fields: `data_types`, `chunk_grids`, `chunk_key_encodings`, `codecs` and `storage_transformers`. `array_extra_fields` and `group_extra_fields` hold the definitions of members the spec does not define, by key. Each slot is a read-only mapping from the name each definition is filed under (`r*` for raw bits) to the definition. The constructor and `update` take, for each slot, an iterable of definitions or such a mapping, so one scope's slot can be given to another. A definition of another kind than its slot holds, and a key that is not its definition's name, raise `TypeError`, and two different definitions under one name in a slot raise `ScopeConflictError`. To replace a definition, or a whole slot, use `update`. `ZarrV3Context()` is the scope that claims nothing. Two scopes are equal when they file the same definitions in each slot, and equal scopes have equal hashes. """ __slots__ = ("_tables",) kinds: ClassVar[Mapping[str, type[ZarrV3Kind]]] = slot_kinds(ZarrV3ContextSlots, is_v3_kind) """The kind of definition each slot holds, by slot, as `ZarrV3ContextSlots` declares them.""" _tables: Tables def __init__(self, **slots: Unpack[ZarrV3ContextSlots]) -> None: self._tables = filed(self.kinds, slots, 3) @classmethod def _of(cls, tables: Tables) -> ZarrV3Context: """A scope of `tables`, which were filed for this class's slots.""" context = object.__new__(cls) context._tables = tables return context @classmethod def _restored(cls, contents: Mapping[str, Iterable[Definition[Any]]]) -> ZarrV3Context: """A scope rebuilt from the definitions in each slot, as `__reduce__` gives them.""" return cls._of(filed(cls.kinds, contents, 3)) @property def zarr_format(self) -> Literal[3]: """The Zarr format of the documents this scope reads: 3.""" return 3 @property def data_types(self) -> Mapping[str, DataTypeDefinition[Any]]: """The data type definitions in scope, by the name each is filed under (`r*` for raw bits).""" return self.definitions_of(DataTypeDefinition) @property def chunk_grids(self) -> Mapping[str, ChunkGridDefinition[Any]]: """The chunk grid definitions in scope, by name.""" return self.definitions_of(ChunkGridDefinition) @property def chunk_key_encodings(self) -> Mapping[str, ChunkKeyEncodingDefinition[Any]]: """The chunk key encoding definitions in scope, by name.""" return self.definitions_of(ChunkKeyEncodingDefinition) @property def codecs(self) -> Mapping[str, CodecDefinition[Any]]: """The codec definitions in scope, by name.""" return self.definitions_of(CodecDefinition) @property def storage_transformers(self) -> Mapping[str, StorageTransformerDefinition[Any]]: """The storage transformer definitions in scope, by name.""" return self.definitions_of(StorageTransformerDefinition) @property def array_extra_fields(self) -> Mapping[str, ArrayExtraFieldDefinition[Any]]: """The definitions of members of an array document that the spec does not define, by key.""" return self.definitions_of(ArrayExtraFieldDefinition) @property def group_extra_fields(self) -> Mapping[str, GroupExtraFieldDefinition[Any]]: """The definitions of members of a group document that the spec does not define, by key.""" return self.definitions_of(GroupExtraFieldDefinition) def definitions_of(self, kind: type[_K]) -> Mapping[str, _K]: """The definitions of `kind` in scope, by the name each is filed under: the slot that holds `kind`. Raises `TypeError` for a kind of the other format.""" return definitions_in(self._tables, kind) def claimant(self, kind: type[D], name: str) -> D | None: """The definition of `kind` in scope that reads `name` (a name as a document uses it), or None if there is none. Raises `TypeError` for a kind of the other format. This is the definition filed under the name that `filed_under` maps `name` to. That is `name` itself, except for raw bits, where `r16` is read by the definition of `r*`. """ return claimant_in(self._tables, kind, name) def update(self, **slots: Unpack[ZarrV3ContextSlots]) -> ZarrV3Context: """This scope, with each slot given here holding only the definitions given for it. The other slots are kept. A slot can be given as a mapping from each definition's name to it, so a slot of this scope passes back in changed: `context.update(codecs={**context.codecs, "gzip": MY_GZIP})` replaces the gzip definition and keeps the other codecs, and `context.update(codecs=[*context.codecs.values(), MY_CODEC])` adds one, raising `ScopeConflictError` if the scope already files another definition under its name. """ return self._of(updated(self._tables, self.kinds, slots, 3)) def union(self, *others: ZarrV3Context) -> ZarrV3Context: """The smallest scope that files everything this scope and each of `others` file. Raises `ScopeConflictError` when two of them file different definitions under one name in a slot. To replace a definition on purpose, use `update`. Raises `TypeError` for a scope of another format. """ refuse_others(others, ZarrV3Context) return self._of( joined_tables(self.kinds, [self._tables, *(other._tables for other in others)]) ) def __repr__(self) -> str: # Keep this short, because it appears in signatures as a default # argument. A full repr would list every definition, and `help` on a # validator would run to pages. count = sum(len(table) for table in self._tables.values()) return f"ZarrV3Context(<{count} definitions>)" def __reduce__(self) -> tuple[object, tuple[dict[str, tuple[Definition[Any], ...]]]]: # A scope is defined by the definitions in its slots, so pickle it as # them. This lets a scope go to another process along with the # documents it will read there. return (type(self)._restored, (slot_contents(self._tables, self.kinds),)) def __copy__(self) -> ZarrV3Context: return self def __deepcopy__(self, memo: dict[int, object]) -> ZarrV3Context: # A scope is immutable, so a copy can be the same object. return self def __eq__(self, other: object) -> bool: if not isinstance(other, ZarrV3Context): return NotImplemented return self._tables == other._tables def __hash__(self) -> int: return hash((ZarrV3Context, tables_key(self._tables))) __all__ = ["ZarrV3Context", "ZarrV3ContextSlots", "ZarrV3Kind", "is_v3_kind"] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/_core.py000066400000000000000000000103761527014137600314040ustar00rootroot00000000000000"""The scopes this package provides for Zarr v3, because "is this valid?" has two useful answers. `CORE` contains what the Zarr v3 specification itself defines, so a document read in it uses nothing that an implementation could reject as optional. `CORE_AND_EXTENSIONS` adds the extensions that `zarr-extensions` registers and this package defines. A name in neither scope is not rejected, because that is what keeps the format open. Instead it is read as an `UnclaimedField` and not checked. """ from __future__ import annotations from typing import Final from zarr_metadata.v3._context import ZarrV3Context from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID from zarr_metadata.v3.chunk_key_encoding.default import DEFAULT_CHUNK_KEY_ENCODING from zarr_metadata.v3.chunk_key_encoding.v2 import V2_CHUNK_KEY_ENCODING from zarr_metadata.v3.codec.blosc import BLOSC_CODEC from zarr_metadata.v3.codec.bytes import BYTES_CODEC from zarr_metadata.v3.codec.cast_value import CAST_VALUE_CODEC from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC from zarr_metadata.v3.codec.gzip import GZIP_CODEC from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC from zarr_metadata.v3.codec.sharding_indexed import SHARDING_INDEXED_CODEC from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC from zarr_metadata.v3.codec.zstd import ZSTD_CODEC from zarr_metadata.v3.consolidated import CONSOLIDATED_METADATA_EXTRA_FIELD from zarr_metadata.v3.data_type.bool import BOOL_DATA_TYPE from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE from zarr_metadata.v3.data_type.int8 import INT8_DATA_TYPE from zarr_metadata.v3.data_type.int16 import INT16_DATA_TYPE from zarr_metadata.v3.data_type.int32 import INT32_DATA_TYPE from zarr_metadata.v3.data_type.int64 import INT64_DATA_TYPE from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE from zarr_metadata.v3.data_type.string import STRING_DATA_TYPE from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE from zarr_metadata.v3.data_type.uint8 import UINT8_DATA_TYPE from zarr_metadata.v3.data_type.uint16 import UINT16_DATA_TYPE from zarr_metadata.v3.data_type.uint32 import UINT32_DATA_TYPE from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE CORE: Final = ZarrV3Context( data_types=( BOOL_DATA_TYPE, INT8_DATA_TYPE, INT16_DATA_TYPE, INT32_DATA_TYPE, INT64_DATA_TYPE, UINT8_DATA_TYPE, UINT16_DATA_TYPE, UINT32_DATA_TYPE, UINT64_DATA_TYPE, FLOAT16_DATA_TYPE, FLOAT32_DATA_TYPE, FLOAT64_DATA_TYPE, COMPLEX64_DATA_TYPE, COMPLEX128_DATA_TYPE, RAW_BYTES_DATA_TYPE, ), chunk_grids=(REGULAR_CHUNK_GRID,), chunk_key_encodings=(DEFAULT_CHUNK_KEY_ENCODING, V2_CHUNK_KEY_ENCODING), codecs=( BLOSC_CODEC, BYTES_CODEC, CRC32C_CODEC, GZIP_CODEC, SHARDING_INDEXED_CODEC, TRANSPOSE_CODEC, ), group_extra_fields=(CONSOLIDATED_METADATA_EXTRA_FIELD,), ) """A scope with only what the Zarr v3 specification defines, including the envelope of a group's `consolidated_metadata`, which the spec fixes.""" CORE_AND_EXTENSIONS: Final = CORE.update( data_types=[ *CORE.data_types.values(), BYTES_DATA_TYPE, STRING_DATA_TYPE, NUMPY_DATETIME64_DATA_TYPE, NUMPY_TIMEDELTA64_DATA_TYPE, STRUCT_DATA_TYPE, ], chunk_grids=[*CORE.chunk_grids.values(), RECTILINEAR_CHUNK_GRID], codecs=[*CORE.codecs.values(), CAST_VALUE_CODEC, SCALE_OFFSET_CODEC, ZSTD_CODEC], ) """A scope with what the specification defines, plus the extensions that `zarr-extensions` registers and this package defines.""" __all__ = [ "CORE", "CORE_AND_EXTENSIONS", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py000066400000000000000000000755001527014137600326040ustar00rootroot00000000000000"""The kinds of metadata in a Zarr v3 document, and what reading one needs that is specific to v3. The kinds at the spec's five extension points are `DataTypeDefinition`, `ChunkGridDefinition`, `ChunkKeyEncodingDefinition`, `CodecDefinition` and `StorageTransformerDefinition`. `ArrayExtraFieldDefinition` and `GroupExtraFieldDefinition` read members the spec does not define. This module also holds the chunk that a codec receives, how a data type's values are stored, the raw bits name `r*`, and the JSON Schema of a v3 field. The reading itself is in `zarr_metadata.definition`, which both formats share. """ from __future__ import annotations import dataclasses import re from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass from typing import ( TYPE_CHECKING, Any, ClassVar, Final, Literal, NewType, TypeAlias, TypeGuard, cast, get_args, get_origin, ) from typing_extensions import TypeAliasType, is_typeddict from zarr_metadata._json import ( ValidationProblem, is_object, is_tuple, refine_json, refine_user_data, with_input, ) from zarr_metadata._typed_json import ( Loc, alias_value, is_alias, is_union, problem, strip_annotation, typeddict_keys, ) from zarr_metadata.definition._definition import ( AcceptedField, C, Definition, Nested, Problems, ResolvedField, WithFillValue, asked, is_field, no_rules, read_only, ruled, ) from zarr_metadata.v3._common import ( ZarrV3ChunkGridField, ZarrV3ChunkKeyEncodingField, ZarrV3CodecField, ZarrV3DataTypeField, ZarrV3StaticCodecField, ZarrV3StorageTransformerField, ) from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON if TYPE_CHECKING: from zarr_metadata._common import JSONValue def unknown_lengths(configuration: object, nested: object, shape: tuple[int, ...]) -> Lengths: """The chunk lengths of a grid that does not declare them: unknown along each axis of the array.""" return (None,) * len(shape) def unknown_chunk(*_: object) -> Chunk: """The output of an array -> array codec that does not declare its output: a chunk with nothing known about it.""" return Chunk() def no_pipelines(*_: object) -> Mapping[str, Chunk]: """The pipelines of a codec that contains none: an empty mapping.""" return {} StorageClass = Literal["single_byte", "multi_byte", "variable_length"] """How a data type's values are stored: in single bytes, in several bytes at a time, or each value in as many bytes as it needs. A multi-byte number is stored in a byte order, which the `bytes` codec's `endian` sets. A value made of single bytes (a `uint8`, or a struct of `int8` fields) has no byte order. A value whose size varies needs its own codec. """ STORAGE_CLASSES: Final[tuple[StorageClass, ...]] = get_args(StorageClass) """The storage classes. `storage_of` checks a definition's result against these.""" def single_byte(*_: object) -> StorageClass: """The storage of a data type made of single bytes, where byte order does not apply, such as `uint8`.""" return "single_byte" def multi_byte(*_: object) -> StorageClass: """The storage of a data type with multi-byte numbers, where byte order applies, such as `int16`.""" return "multi_byte" def variable_length(*_: object) -> StorageClass: """The storage of a data type whose values vary in size, such as `string`.""" return "variable_length" def unknown_storage(*_: object) -> None: """The storage of a data type that does not declare it: unknown.""" RAW_BYTES_NAME: Final = "r*" """The name raw bits are filed under: `r*`, as written in the specification's table of data types. A document writes `r` followed by the size in bits (`r8`, `r16`), so raw bits are the only data type whose name encodes its configuration. `filed_under` reads such a name as `r*` with the size as its configuration. `r*` itself is the table's notation, and no document uses it as a name (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L46-L47). """ RAW_BYTES_NAME_PATTERN: Final = re.compile(r"r([0-9]{1,100})") """A raw bits name: `r` followed by a size in bits of up to a hundred digits, matched against the whole name, whether or not the size is allowed. Only ASCII digits match. `\\d` would also match every other Unicode decimal digit, so `r\uff11\uff16` would be read as sixteen bits, and a third-party name spelled that way would be mistaken for raw bits. A size the spec does not allow (`r0`, `r12`) is still raw bits, so it is reported as a bad size instead of being treated as an unknown extension. A hundred digits is more than any real size. `r` followed by more digits than that is an ordinary name, which the spec allows, and which nothing in scope claims. """ @dataclass(frozen=True, kw_only=True, slots=True, repr=False) class DataTypeDefinition(WithFillValue[C], kind=True, format=3): """A data type, and the fill value of an array with that data type. `fill_value` is the JSON shape of a fill value, such as `Int8FillValue`. It is an annotation that the checker reads the same way it reads a configuration's members, including any range. `fill_value_rules` yields the spec violations in a fill value of that shape that the type cannot express, such as a hex string of the wrong width. The rules receive the configuration, the nested fields as the scope read them (a struct's field types), and the typed fill value. A data type that does not declare its fill value accepts any JSON. `fill_value_canonical` writes a fill value without problems (well typed and allowed by the rules) in the single canonical spelling of its value. So two fill values are the same value of the type exactly when their canonical spellings are written the same way: `"NaN"` and `"0x7fc00000"` are the same `float32`, but `0.0` and `-0.0` are two different values. It receives the same arguments as the rules. A data type that does not declare it has one spelling per value: the value as written. `storage` declares how the values are stored: in single bytes, in several bytes at a time, or each value in as many bytes as it needs. The `bytes` codec uses this to decide whether the data type it receives needs an `endian` (it does for multi-byte numbers). A struct's storage depends on its fields, so `storage` receives the nested fields as the scope read them. A data type that does not declare it leaves its storage unknown. A data type named the way a document writes raw bits of one size (such as `r16`) is rejected. That name is read as `r*`, so this definition would never be used. """ label: ClassVar[str] = "data type" field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3DataTypeField,) @classmethod def _filed_under(cls, name: str) -> tuple[str | None, dict[str, JSONValue] | None]: if name == RAW_BYTES_NAME: return None, None written = RAW_BYTES_NAME_PATTERN.fullmatch(name) if written is not None: return RAW_BYTES_NAME, {"bits": int(written.group(1))} return name, None def _carrying_name(self, configuration: Mapping[str, JSONValue]) -> str | None: if self.name != RAW_BYTES_NAME: return None return f"r{configuration['bits']}" @classmethod def _envelope_json(cls, name: str, configuration: Mapping[str, JSONValue]) -> JSONValue: # A data type with nothing to configure is written as its bare name, # which is how core data types have been written since Zarr v3.0. if len(configuration) != 0: return {"name": name, "configuration": configuration} return name storage: Callable[[C, Nested], StorageClass | None] = unknown_storage """How the values are stored, given the configuration and the nested fields; None when unknown.""" def _refusal(self) -> str | None: if RAW_BYTES_NAME_PATTERN.fullmatch(self.name) is not None: return ( f"{self.name!r} is how a document writes raw bits of one size, which read as " f"{RAW_BYTES_NAME!r}; to read raw bits your own way, define {RAW_BYTES_NAME!r}" ) # Use the explicit form, not bare `super()`, because a dataclass # with slots is rebuilt. return super(DataTypeDefinition, self)._refusal() @dataclass(frozen=True, kw_only=True, slots=True, repr=False) class ChunkGridDefinition(Definition[C], kind=True, format=3): """A chunk grid, and the array shapes it fits. `shape_rules` yields the spec violations in a grid with this configuration over an array of a given shape, such as a dimension with no chunk length, or chunks that do not cover the dimension. It receives the configuration, the nested fields as the scope read them, and the shape, and it locates its problems in the configuration. A grid that does not declare shape rules fits every shape. `chunk_lengths` determines what the first codec of the array's pipeline receives: the lengths of the grid's chunks along each axis of an array whose shape the grid fits. A regular grid has one length per axis, and a rectilinear grid has every length it lists. It is called only for a grid that its shape rules accept. A grid that does not declare it leaves the lengths along every axis unknown. """ label: ClassVar[str] = "chunk grid" field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3ChunkGridField,) shape_rules: Callable[[C, Nested, tuple[int, ...]], Iterable[ValidationProblem]] = no_rules """The spec violations in this grid over an array of a given shape, located in the configuration.""" chunk_lengths: Callable[[C, Nested, tuple[int, ...]], Lengths] = unknown_lengths """The lengths of the chunks along each axis of an array whose shape the grid fits; None where unknown.""" @dataclass(frozen=True, kw_only=True, slots=True, repr=False) class ChunkKeyEncodingDefinition(Definition[C], kind=True, format=3): """A chunk key encoding.""" label: ClassVar[str] = "chunk key encoding" field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3ChunkKeyEncodingField,) CodecKind = Literal["array_array", "array_bytes", "bytes_bytes"] """What a codec converts its input to: the three positions a pipeline puts in order.""" CodecSize = Literal["static", "dynamic"] """Whether the size of a codec's output is determined by the size of its input. `static`: it is. For example, `bytes` writes each element at its width, and `crc32c` adds four bytes. `dynamic`: the size depends on the values, as for every compressor. """ _UNASKED: Final[Mapping[CodecKind, tuple[str, ...]]] = { "array_array": (), "array_bytes": ("transition",), "bytes_bytes": ("chunk_rules", "transition", "pipelines"), } """The functions that are never called for a codec of each kind. A bytes -> bytes codec receives bytes, and only an array -> array codec passes on a chunk.""" @dataclass(frozen=True, kw_only=True, slots=True, repr=False) class CodecDefinition(Definition[C], kind=True, format=3): """A codec: what it converts its input to, and whether the size of its output is static. A codec that receives an array (array -> array, array -> bytes) declares the spec violations for a given input `Chunk` with `chunk_rules`. It receives the configuration, the nested fields as the scope read them, and the chunk, and it locates its problems in the configuration. For example, a `bytes` codec without `endian` that receives a multi-byte data type is a problem. An array -> array codec also declares its output with `transition`: given its input chunk, it returns the chunk the next codec receives. For example, `transpose` permutes the axes, and `cast_value` changes the data type. These two functions correspond to the spec: a codec computes its output from the shape and data type of its input, and "If the decoded_representation_type is not supported, this algorithm must fail with an error" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L987-L994). The transition is called for every input chunk, whatever the chunk rules found, so it must return only what is true in either case. For example, a `transpose` whose `order` has the wrong number of axes passes on unknown lengths. A codec that does not declare its output passes the next codec a chunk with nothing known about it. A codec that contains its own pipelines declares what each one receives with `pipelines`. Given the codec's input chunk, it returns the chunk that the first codec of each pipeline receives, keyed by the configuration member that holds the pipeline. For example, a shard's inner codecs receive its inner chunks, and its index codecs receive the shard index. Like the transition, it is called whatever the chunk rules found, and must return only what is true in either case. A function that is never called for a codec of its kind is rejected, such as the chunk rules of a bytes -> bytes codec, which receives bytes. """ label: ClassVar[str] = "codec" field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3CodecField, ZarrV3StaticCodecField) kind: CodecKind size: CodecSize chunk_rules: Callable[[C, Nested, Chunk], Iterable[ValidationProblem]] = no_rules """The spec violations for this codec with a given input chunk, located in the configuration.""" transition: Callable[[C, Nested, Chunk], Chunk] = unknown_chunk """The chunk the next codec receives, given the chunk this array -> array codec receives.""" pipelines: Callable[[C, Nested, Chunk], Mapping[str, Chunk]] = no_pipelines """The chunk each contained pipeline receives, keyed by the configuration member that holds the pipeline.""" def _refusal(self) -> str | None: kind: object = self.kind if kind not in get_args(CodecKind): return f"{self.name!r}: kind is one of {get_args(CodecKind)!r}, got {kind!r}" size: object = self.size if size not in get_args(CodecSize): return f"{self.name!r}: size is one of {get_args(CodecSize)!r}, got {size!r}" defaults = {member.name: member.default for member in dataclasses.fields(CodecDefinition)} for member in _UNASKED[self.kind]: if getattr(self, member) is not defaults[member]: return f"{self.name!r}: {member}, which no codec of kind {kind!r} is asked" return None @classmethod def _alias_problems( cls, alias: TypeAliasType, definition: Definition[Any], loc: Loc ) -> Problems: """A codec of dynamic size in a member annotated with `ZarrV3StaticCodecField`, which admits only codecs of static size, is a problem at the field.""" if alias not in STATIC_SIZE_ALIASES: return () if not isinstance(definition, CodecDefinition) or definition.size == "static": return () return problem( loc, f"{definition.name!r} is a codec of dynamic size, and only codecs of static size " "may be used here", "invalid_value", ) @dataclass(frozen=True, kw_only=True, slots=True, repr=False) class StorageTransformerDefinition(Definition[C], kind=True, format=3): """A storage transformer.""" label: ClassVar[str] = "storage transformer" field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3StorageTransformerField,) @dataclass(frozen=True, kw_only=True, slots=True, repr=False) class ExtraFieldDefinition(Definition[C]): """A member of a v3 document that the spec does not define, an extra field: its name is the member's key, and its configuration is the member's value. This is not a kind. `ArrayExtraFieldDefinition` and `GroupExtraFieldDefinition` are, because one key may mean different things in an array document and in a group document. An extra field is not one of the spec's extension points, and no registry names it, so any key may name one. As a field it is written as the member alone, `{key: value}`, and it is located at the member's key. A definition reads a member whose value is an object. A member that no definition in scope claims is not read: a reader keeps it as it is written, and lists it in `must_understand_fields` unless it says `"must_understand": false`. `finite` says whether every number in the member must be finite, as RFC 8259 JSON requires. A member that holds user data sets it to False: consolidated metadata holds node documents, whose attributes may hold `NaN`, `Infinity` and `-Infinity`. A member that nothing claims is not read, so like attributes it may hold any number. The spec lets every extension member carry `"must_understand": false`, which tells a reader that does not understand the member that it may ignore it. So the configuration declares `must_understand`, and a definition whose configuration does not is refused. A member the spec defines, such as an array's `shape`, is never an extra field, and a definition named after one is refused too. """ finite: bool = True """Whether every number in the member must be finite.""" spec_members: ClassVar[frozenset[str]] = frozenset() """The members the spec defines in a document this kind reads extra fields of, which are never extra fields.""" def _finite_numbers(self) -> bool: return self.finite def _refusal(self) -> str | None: finite = cast("object", self.finite) if not isinstance(finite, bool): return f"{self.name!r}: finite is a boolean, got {finite!r}" if not is_typeddict(self.configuration): return None try: takes = _takes_the_waiver(self.configuration) except TypeError as error: # Name the member, as the checker names a member it cannot read. keys = typeddict_keys(self.configuration) member = "must_understand" if "must_understand" in keys.members else "extra_items" return f"{self.name!r}: {self.configuration.__name__}.{member}: {error}" if not takes: return ( f"{self.name!r}: expected a configuration that takes must_understand as a " "boolean that may be false, which the spec lets every extension member carry; " f"{self.configuration.__name__} does not" ) return None @classmethod def _name_problem(cls, name: str, at: Loc) -> ValidationProblem | None: # No registry names extra fields, so any key the spec does not # define may name one. if name in cls.spec_members: return ValidationProblem( at, f"{name!r} is a member the spec defines, not an extra field", "invalid_value" ) return None @classmethod def _named_configuration( cls, value: object ) -> tuple[str | None, Mapping[str, object] | None, Problems]: # A member whose value is not an object has no configuration. Only # a definition that claims the member reads it, and `_refined` # refuses such a value then. if not is_object(value) or len(value) != 1: return None, None, () ((name, member),) = value.items() if not isinstance(name, str): return None, None, () if not is_object(member): return name, None, () return name, {key: item for key, item in member.items() if isinstance(key, str)}, () @classmethod def _envelope_problems(cls, value: object) -> Problems: name = cls._named_configuration(value)[0] if name is None: return ( ValidationProblem( (), "expected a member, as an object with one string key", "invalid_type" ), ) bad = cls._name_problem(name, ()) return () if bad is None else (bad,) @classmethod def _refined( cls, data: object, loc: Loc, definition: Definition[Any] | None ) -> tuple[JSONValue | None, Problems]: # The member's value is at the member's location, `loc`. A member # nothing claims is not read, so like attributes it is user data. # A claimed member is user data when its definition says so, and # its value must be an object. name = cls._named_configuration(data)[0] if name is None or not is_object(data): return refine_json(data, loc) if definition is not None and not is_object(data[name]): return None, (ValidationProblem(loc, "expected an object", "invalid_type"),) refine = ( refine_json if isinstance(definition, ExtraFieldDefinition) and definition.finite else refine_user_data ) value, problems = refine(data[name], loc) return (None if len(problems) != 0 else {name: value}), problems @classmethod def _envelope_json(cls, name: str, configuration: Mapping[str, JSONValue]) -> JSONValue: return {name: configuration} @classmethod def _unclaimed_json( cls, json: JSONValue, name: str, configuration: Mapping[str, JSONValue] ) -> JSONValue: # A member no definition claims is written as it is, whatever its # value; only a definition reads it as an object. return json @classmethod def _configuration_loc(cls, loc: Loc) -> Loc: return loc @classmethod def _name_loc(cls, loc: Loc) -> Loc: return loc def _takes_the_waiver(configuration: type) -> bool: """Whether `configuration`, a TypedDict, takes the spec's `"must_understand": false`: the type it gives `must_understand`, declared or as another key, admits `false` and only booleans.""" keys = typeddict_keys(configuration) declared = keys.members.get("must_understand") booleans = _booleans(keys.extra_items if declared is None else declared[0], frozenset()) return booleans is not None and False in booleans def _booleans(annotation: object, seen: frozenset[object]) -> frozenset[bool] | None: """The booleans `annotation` admits when it admits only booleans, read through its aliases, `NewType`s, unions and `Literal`s, as the checker reads them; None when it admits anything else.""" inner = strip_annotation(annotation)[0] if isinstance(inner, NewType) or is_alias(inner): if inner in seen: # An alias that refers to itself adds nothing it has not already. return frozenset() if isinstance(inner, NewType): return _booleans(inner.__supertype__, seen | {inner}) # An alias that names what is not defined raises `TypeError` here, # which says so, as the definition's constructor does for it. return _booleans(alias_value(inner), seen | {inner}) if inner is bool: return frozenset({True, False}) if get_origin(inner) is Literal: values = get_args(inner) return frozenset(values) if all(isinstance(value, bool) for value in values) else None if not is_union(inner): return None admitted: set[bool] = set() for branch in get_args(inner): found = _booleans(branch, seen) if found is None: return None admitted |= found return frozenset(admitted) @dataclass(frozen=True, kw_only=True, slots=True, repr=False) class ArrayExtraFieldDefinition(ExtraFieldDefinition[C], kind=True, format=3): """An extra field of a v3 array document: a member the spec does not define, read by its key.""" label: ClassVar[str] = "array extra field" spec_members: ClassVar[frozenset[str]] = frozenset( typeddict_keys(ZarrV3ArrayMetadataJSON).members ) @dataclass(frozen=True, kw_only=True, slots=True, repr=False) class GroupExtraFieldDefinition(ExtraFieldDefinition[C], kind=True, format=3): """An extra field of a v3 group document: a member the spec does not define, read by its key. `consolidated_metadata` is one.""" label: ClassVar[str] = "group extra field" spec_members: ClassVar[frozenset[str]] = frozenset( typeddict_keys(ZarrV3GroupMetadataJSON).members ) STATIC_SIZE_ALIASES: Final[frozenset[TypeAliasType]] = frozenset({ZarrV3StaticCodecField}) """The field aliases whose codec must be of static size.""" Lengths: TypeAlias = tuple[frozenset[int] | None, ...] """For each axis, every chunk length along that axis (a set, because a rectilinear grid's lengths differ), or None where unknown.""" @dataclass(frozen=True, slots=True) class Chunk: """The input a codec receives: chunks with given lengths along each axis, and a data type. Unknown parts are None. This includes the lengths along an axis that the grid does not declare, and every part of the chunk output by a codec that does not declare its output. A data type field that the scope did not read is kept as written, and it also says nothing about the values. A codec's chunk rules check only what is known, so a chunk with nothing known about it, `Chunk()`, never causes a problem. """ lengths: Lengths | None = None """For each axis, the chunk lengths along it; None when even the number of axes is unknown.""" data_type: ResolvedField[DataTypeDefinition[Any]] | None = None """The data type field of the values, as a scope read it. None when no field declares the data type: a document with no data type (its reading holds `UNSET`) passes the pipeline a chunk of unknown type.""" def __post_init__(self) -> None: lengths = cast("object", self.lengths) if lengths is not None and not _is_lengths(lengths): msg = f"a chunk's lengths are a frozenset of integers or None per axis, got {lengths!r}" raise TypeError(msg) data_type = cast("object", self.data_type) if data_type is not None and not _is_data_type_field(data_type): msg = f"a chunk's data type is a data type field a scope read, got {data_type!r}" raise TypeError(msg) @property def rank(self) -> int | None: """The number of axes; None when unknown.""" return None if self.lengths is None else len(self.lengths) def _is_data_type_field(value: object) -> bool: """Whether `value` is a field that a scope read as a data type.""" return is_field(value) and value.read_as is DataTypeDefinition def _is_lengths(value: object) -> TypeGuard[Lengths]: """Whether `value` is valid chunk lengths: for each axis, a frozenset of integers or None.""" if not is_tuple(value): return False for axis in value: if axis is None: continue if not isinstance(axis, frozenset) or not all( isinstance(length, int) and not isinstance(length, bool) for length in cast("frozenset[object]", axis) ): return False return True def storage_of(data_type: ResolvedField[DataTypeDefinition[Any]]) -> StorageClass | None: """How the values of `data_type` (a data type field a scope read) are stored; None when unknown. The storage is unknown when the scope did not read the data type, or its definition does not declare storage. The definition's `storage` is the extension author's code. Its result is checked to be a storage class. If it raises an error, a note is added that says which data type's storage raised it. """ if not isinstance(data_type, AcceptedField): return None definition, configuration = data_type.definition, data_type.configuration found = asked( definition, "storage", lambda: cast("object", definition.storage(read_only(configuration), data_type.nested)), ) if found is not None and found not in STORAGE_CLASSES: msg = ( f"{definition.name!r}: its storage gives one of {STORAGE_CLASSES!r} or None, " f"got {found!r}" ) raise TypeError(msg) return found def chunk_grid_lengths( chunk_grid: ResolvedField[ChunkGridDefinition[Any]], shape: tuple[int, ...], loc: Loc = () ) -> tuple[Lengths, Problems]: """The chunk lengths of `chunk_grid` (a chunk grid field a scope read) along each axis of an array of `shape`, and the problems with the grid for that shape. A chunk has an extent "for each dimension of the array" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L277-L281), so the lengths have an entry for each dimension of `shape`, with None where they are unknown. The grid's shape rules check it first, and locate their problems in the configuration, under `loc` (the location of the field). For example, a regular grid with chunk lengths for two dimensions over an array with three dimensions is a problem. Only a grid that fits the shape returns its lengths. A grid that the scope did not read (it is out of scope or invalid) is not checked and returns no lengths. The result of a grid's `chunk_lengths` is checked: lengths of the wrong type raise `TypeError`, and lengths with a different number of axes than `shape` raise `ValueError`. Both indicate a bug in the definition, not in the field. """ unknown: Lengths = (None,) * len(shape) if not isinstance(chunk_grid, AcceptedField): return unknown, () definition, configuration = chunk_grid.definition, chunk_grid.configuration at = (*loc, "configuration") problems = ruled( definition, lambda: definition.shape_rules(read_only(configuration), chunk_grid.nested, shape), at, ) if len(problems) != 0: return unknown, with_input(problems, chunk_grid.json, loc) lengths = asked( definition, "chunk lengths", lambda: cast( "object", definition.chunk_lengths(read_only(configuration), chunk_grid.nested, shape) ), at, ) if not _is_lengths(lengths): msg = ( f"{definition.name!r}: its chunk_lengths give a frozenset of integers or None " f"per axis, got {lengths!r}" ) raise TypeError(msg) if len(lengths) != len(shape): msg = ( f"{definition.name!r}: its chunk_lengths gave {len(lengths)} axes, " f"for a shape of {len(shape)}" ) raise ValueError(msg) return lengths, () __all__ = [ "RAW_BYTES_NAME", "RAW_BYTES_NAME_PATTERN", "STATIC_SIZE_ALIASES", "ArrayExtraFieldDefinition", "Chunk", "ChunkGridDefinition", "ChunkKeyEncodingDefinition", "CodecDefinition", "CodecKind", "CodecSize", "DataTypeDefinition", "ExtraFieldDefinition", "GroupExtraFieldDefinition", "Lengths", "StorageClass", "StorageTransformerDefinition", "chunk_grid_lengths", "multi_byte", "no_pipelines", "single_byte", "storage_of", "unknown_chunk", "unknown_lengths", "unknown_storage", "variable_length", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/_hierarchy.py000066400000000000000000000203671527014137600324330ustar00rootroot00000000000000"""A Zarr v3 hierarchy: the names and paths of its nodes, and the tree they form. `NodeName` and `NodePath` are modeled on the zarrs types with the same names. They enforce the spec's rules, including the reservation of `zarr.json`. `hierarchy_problems` takes the node type of each node, keyed by path, checks each path against the v3 rules, and then checks that the nodes form a valid tree with the shared `tree_problems`. Each function here reports at most one problem per value, with all the reasons in that one problem. This keeps the size of the report proportional to the size of the input: a path with a thousand bad names is one problem, not a thousand problems that each repeat the path. This module is private. Consumers import `NodeName` and `NodePath` from `zarr_metadata.v3`, and the validators from `zarr_metadata.model`. """ from __future__ import annotations from typing import TYPE_CHECKING, NewType, TypeGuard from zarr_metadata._hierarchy import tree_problems from zarr_metadata._json import MetadataValidationError, ValidationProblem, shown, with_input if TYPE_CHECKING: from collections.abc import Mapping, Sequence from zarr_metadata._hierarchy import NodeType NodeName = NewType("NodeName", str) """The name of a node in a Zarr v3 hierarchy. The root's name is `""`. Any other name is not empty, contains no `/`, is not made only of periods (such as `.` or `..`), does not start with the reserved `__`, and is not `zarr.json`. Names are case-sensitive: `foo` and `FOO` are two different names (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L818-L837). """ NodePath = NewType("NodePath", str) """The path of a node in a Zarr v3 hierarchy. The root's path is `/`. Any other node's path is its parent's path, followed by `/` (unless the parent is the root), followed by its name. So a path starts with `/`, does not end with `/`, and has a node name between each pair of `/` (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L211-L229). """ def said(faults: Sequence[str]) -> str: """Join `faults`, each a reason, into one clause: `a`, `a and b`, `a, b and c`.""" if len(faults) <= 1: return "".join(faults) return f"{', '.join(faults[:-1])} and {faults[-1]}" def name_faults(name: str) -> list[str]: """The reasons `name` is not a node name, each as a message. Empty for a valid node name, including the root's `""`.""" faults: list[str] = [] if "/" in name: faults.append('holds "/"') if name != "" and set(name) == {"."}: faults.append("is periods alone") if name.startswith("__"): faults.append('starts with the reserved "__"') if name == "zarr.json": faults.append('is the reserved "zarr.json"') return faults def names_faults(names: list[str]) -> list[str]: """The reasons `names` (the names between a path's `/` characters) are not all valid node names below the root. Reports the first invalid name and how many more invalid names there are.""" faulty = [name for name in names if name == "" or len(name_faults(name)) != 0] if len(faulty) == 0: return [] first = faulty[0] faults = ( ['holds an empty name between two "/"'] if first == "" else [f"holds {shown(first)}, a name that {said(name_faults(first))}"] ) if len(faulty) > 1: more = len(faulty) - 1 faults.append( f"holds {more} more name{'s' if more != 1 else ''} that {'are' if more != 1 else 'is'} not a node name" ) return faults def path_faults(path: str) -> list[str]: """The reasons `path` is not a node path, each as a message. Empty for a valid node path, including the root's `/`.""" if path == "/": return [] if not path.startswith("/"): return ['does not start with "/"'] faults: list[str] = [] names = path[1:].split("/") if path.endswith("/"): faults.append('ends with "/"') names = names[:-1] return [*faults, *names_faults(names)] def hierarchy_problems(nodes: Mapping[str, NodeType | None]) -> tuple[ValidationProblem, ...]: """The reasons `nodes` (the node type of each node, keyed by path) is not a Zarr v3 hierarchy. Reports one problem per affected node, at that node's path. "A Zarr hierarchy is a tree structure, where each node in the tree is either a group or an array. Group nodes may have children but array nodes may not" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L177-L181). So each path must be a node path, and each node except the root must have its parent in `nodes`, and that parent must be a group. A node below an array is an `invalid_value` at the node's path. If groups are missing above a node, the nearest missing group is reported once as a `missing_key` at that group's path. The message says how many more groups are missing above it, including the root. A node whose type is unknown (None) is treated as a group. The cause of the unknown type is a separate problem, reported at that node's location. A mapping contains each path only once, so no two siblings can share a name. """ problems: list[ValidationProblem] = [] well_formed: list[str] = [] for path in nodes: faults = path_faults(path) if len(faults) == 0: well_formed.append(path) else: message = f"expected a node path, got {shown(path)}, which {said(faults)}" problems.append(ValidationProblem((path,), message, "invalid_value")) problems.extend(tree_problems({path: nodes[path] for path in well_formed})) return tuple(problems) def _string_problems(value: object, what: str) -> tuple[ValidationProblem, ...]: return (ValidationProblem((), f"expected {what}, got {shown(value)}", "invalid_type"),) def validate_node_name_v3(value: object) -> tuple[ValidationProblem, ...]: """Every reason `value` is not a v3 node name, reported in one `invalid_value` problem, or an `invalid_type` problem if `value` is not a string.""" if not isinstance(value, str): return with_input(_string_problems(value, "a node name"), value) faults = name_faults(value) if len(faults) == 0: return () message = f"expected a node name, got {shown(value)}, which {said(faults)}" return with_input((ValidationProblem((), message, "invalid_value"),), value) def is_node_name_v3(value: object) -> TypeGuard[NodeName]: """Whether `value` is a v3 node name, meaning `validate_node_name_v3` finds no problems.""" return len(validate_node_name_v3(value)) == 0 def parse_node_name_v3(value: object) -> NodeName: """Return `value` as a `NodeName`, or raise `MetadataValidationError` with every reason it is not one.""" if not is_node_name_v3(value): raise MetadataValidationError(validate_node_name_v3(value)) return value def validate_node_path_v3(value: object) -> tuple[ValidationProblem, ...]: """Every reason `value` is not a v3 node path, reported in one `invalid_value` problem, or an `invalid_type` problem if `value` is not a string.""" if not isinstance(value, str): return with_input(_string_problems(value, "a node path"), value) faults = path_faults(value) if len(faults) == 0: return () message = f"expected a node path, got {shown(value)}, which {said(faults)}" return with_input((ValidationProblem((), message, "invalid_value"),), value) def is_node_path_v3(value: object) -> TypeGuard[NodePath]: """Whether `value` is a v3 node path, meaning `validate_node_path_v3` finds no problems.""" return len(validate_node_path_v3(value)) == 0 def parse_node_path_v3(value: object) -> NodePath: """Return `value` as a `NodePath`, or raise `MetadataValidationError` with every reason it is not one.""" if not is_node_path_v3(value): raise MetadataValidationError(validate_node_path_v3(value)) return value __all__ = [ "NodeName", "NodePath", "hierarchy_problems", "is_node_name_v3", "is_node_path_v3", "name_faults", "names_faults", "parse_node_name_v3", "parse_node_path_v3", "path_faults", "said", "validate_node_name_v3", "validate_node_path_v3", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py000066400000000000000000000346611527014137600327500ustar00rootroot00000000000000"""The JSON Schema of a Zarr v3 field and of a whole v3 `zarr.json`, as a scope reads them.""" from __future__ import annotations from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final from zarr_metadata._typed_json import Schemas, is_alias from zarr_metadata.definition._context import scoped from zarr_metadata.definition._definition import ( Definition, EmptyConfiguration, as_kind, field_kind, ) from zarr_metadata.definition._envelope import EXTENSION_NAME_SCHEMA_PATTERN from zarr_metadata.v3._context import ZarrV3Context, ZarrV3Kind, is_v3_kind from zarr_metadata.v3._core import CORE_AND_EXTENSIONS from zarr_metadata.v3._definition import ( RAW_BYTES_NAME, RAW_BYTES_NAME_PATTERN, STATIC_SIZE_ALIASES, ArrayExtraFieldDefinition, CodecDefinition, DataTypeDefinition, ExtraFieldDefinition, GroupExtraFieldDefinition, ) from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON if TYPE_CHECKING: from zarr_metadata._common import JSONValue from zarr_metadata._typed_json import JSONSchema, SchemaLeaf def kind_field(kind: type[Definition[Any]]) -> object: """The field alias for a member that holds any field of `kind`: the first alias the kind declares that does not restrict the field. For example, `ZarrV3CodecField` instead of `ZarrV3StaticCodecField`.""" return next(alias for alias in kind.field_aliases if alias not in STATIC_SIZE_ALIASES) _RAW_BYTES_SCHEMA_PATTERN: Final = f"^{RAW_BYTES_NAME_PATTERN.pattern}(?![\\s\\S])" """`RAW_BYTES_NAME_PATTERN`, matched against the whole name, as a JSON Schema pattern. A lookahead for "no character at all" anchors the match to the end of the name. A `$` would also match before a final newline in a validator that matches patterns the way Python does, so `"r16\\n"`, which is not a valid name, would be read as raw bits. """ def field_json_schema_v3( kind: type[ZarrV3Kind], *, context: ZarrV3Context | None = None ) -> JSONSchema: """The JSON Schema of one metadata field read as `kind` in `context` (`CORE_AND_EXTENSIONS` when none is given): everything `resolve` checks, except the rules. A scope of another format raises `TypeError`, as it does for every v3 reader. An extra field is written as the member alone, `{key: value}`. The schema accepts a field that one of the definitions in scope reads (its name, its configuration as the TypedDict declares it, an optional `must_understand` of `true`, and its bare name when it needs no configuration). It also accepts a name that no definition claims, with any configuration, which is what keeps the format open. A field inside a configuration is written the same way, in the same scope, and a member that accepts only codecs of static size accepts only those. The output is JSON Schema draft 2020-12, as `json_schema` writes it. The nested fields and the configuration of each definition are in `$defs`, under the name of the field alias or TypedDict. Constraints that only a rule enforces (a blosc `typesize` checked against its `shuffle`) are not in the schema, so a field it accepts may still have a problem. """ asked = as_kind(kind) if not is_v3_kind(asked): msg = ( f"{asked.__name__} is a kind of another format; the JSON Schema writer writes " "Zarr v3 fields only" ) raise TypeError(msg) context = scoped(context, CORE_AND_EXTENSIONS) schemas = Schemas(field_schemas(context)) if issubclass(asked, (ArrayExtraFieldDefinition, GroupExtraFieldDefinition)): return schemas.document(_extra_field_schema(asked, context, schemas)) return schemas.document(schemas.of(kind_field(asked))) def field_schemas(context: ZarrV3Context) -> SchemaLeaf: """The schema leaf that `field_json_schema_v3` uses. It writes each field alias as a field of its kind, as `context` reads it.""" def leaf(annotation: object, schemas: Schemas) -> JSONSchema | None: kind = field_kind(annotation) if kind is None or not is_alias(annotation): return None alias = annotation static = annotation in STATIC_SIZE_ALIASES return schemas.defined( alias, alias.__name__, lambda: _field_schema(kind, static, context, schemas) ) return leaf def _field_schema( kind: type[Definition[Any]], static: bool, context: ZarrV3Context, schemas: Schemas ) -> JSONSchema: """The schema of a field of `kind` as `context` reads it: a field that a definition in scope reads, or a field that none of them claims. `kind` is always a v3 kind. The v3 document TypedDicts hold v3 fields only, a v3 definition whose configuration holds a field of the other format is refused when it is built, and `field_json_schema_v3` refuses a kind of the other format before it writes anything. The check below only narrows `kind` for the type checker. """ if not is_v3_kind(kind): msg = f"{kind.__name__} is a kind of another format, which a v3 schema does not hold" raise TypeError(msg) table = context.definitions_of(kind) branches: list[JSONValue] = [] for definition in table.values(): if static and isinstance(definition, CodecDefinition) and definition.size != "static": continue branches.extend(_read_by(definition, schemas)) branches.extend(_unclaimed(table)) return {"anyOf": branches} def _extra_field_schema( kind: type[ArrayExtraFieldDefinition[Any] | GroupExtraFieldDefinition[Any]], context: ZarrV3Context, schemas: Schemas, ) -> JSONSchema: """The schema of an extra field of `kind` as `context` reads it, written as the member alone, `{key: value}`: a member whose key a definition in scope claims, with the value its configuration declares, or a member with any other key that the spec does not define, and any JSON value.""" claimed = list(context.definitions_of(kind).values()) branches: list[JSONValue] = [ { "type": "object", "properties": {definition.name: schemas.of(definition.configuration)}, "required": [definition.name], "additionalProperties": False, } for definition in claimed ] unclaimed: JSONSchema = {"type": "object", "minProperties": 1, "maxProperties": 1} # Neither a claimed key nor a member the spec defines is an unclaimed member. others = sorted({definition.name for definition in claimed} | kind.spec_members) unclaimed["propertyNames"] = {"not": {"enum": [*others]}} branches.append(unclaimed) return {"anyOf": branches} def written_name(definition: Definition[Any]) -> JSONSchema: """The JSON Schema of the names a document may use for `definition`: its name, or `r` followed by a size for raw bits.""" if isinstance(definition, DataTypeDefinition) and definition.name == RAW_BYTES_NAME: return {"type": "string", "pattern": _RAW_BYTES_SCHEMA_PATTERN} return {"const": definition.name} def _read_by(definition: Definition[Any], schemas: Schemas) -> list[JSONValue]: """The schemas of the fields `definition` reads: an object with its name and configuration, and its bare name when it needs no configuration. The raw bits name encodes the configuration, so the configuration written next to it must be empty. """ carried = isinstance(definition, DataTypeDefinition) and definition.name == RAW_BYTES_NAME name = written_name(definition) bare = carried or not definition.requires_configuration envelope: JSONSchema = { "type": "object", "properties": { "name": name, "configuration": schemas.of( EmptyConfiguration if carried else definition.configuration ), "must_understand": {"const": True}, }, "required": ["name"] if bare else ["name", "configuration"], "additionalProperties": False, } return [name, envelope] if bare else [envelope] def _unclaimed(table: Mapping[str, Definition[Any]]) -> list[JSONValue]: """The schemas of the fields that no definition in `table` claims: a name that none of them uses, either bare or with any configuration.""" claimed: list[JSONValue] = [written_name(definition) for definition in table.values()] name: JSONSchema = {"type": "string", "pattern": EXTENSION_NAME_SCHEMA_PATTERN} if len(claimed) != 0: name["not"] = {"anyOf": claimed} envelope: JSONSchema = { "type": "object", "properties": { "name": name, "configuration": {"type": "object"}, "must_understand": {"const": True}, }, "required": ["name"], "additionalProperties": False, } return [name, envelope] def node_metadata_json_schema_v3(*, context: ZarrV3Context | None = None) -> JSONSchema: """The JSON Schema of a v3 `zarr.json` read in `context`: an array document or a group document, as `read_node_metadata_v3` reads it, except for the rules. Use it in an editor that validates a `zarr.json` while it is written, or in a validator written in another language. The schema is JSON Schema draft 2020-12, in the form `json_schema` writes. Each extension point is a field as the scope `context` reads it: either a name that a definition in scope claims, with the configuration that definition declares, or a name that no definition claims. When the document names a data type in scope, the fill value must have the JSON shape that the data type's definition declares (for `int8`, an integer in [-128, 127]). In a scope that claims it, as `CORE` does, a group's `consolidated_metadata` is the envelope the core spec fixes, with any JSON as the documents it lists, because the reader does not read them, and a `null` value, which zarr-python 3.0 and 3.1 wrote, is refused, because the reader refuses it too. Each document is in `$defs` under the name of its TypedDict. `ZarrV3ArrayMetadataJSON` describes only an array. A JSON Schema describes each member on its own. It does not contain the rules that relate members to each other: one dimension name per dimension of the shape, a chunk grid that fits the shape, codecs in pipeline order with each codec checked against the chunk it receives, and the checks in a definition's `rules`. So a document the schema accepts may still have a problem. Every JSON document that `read_node_metadata_v3` reads without a problem is accepted by the schema. A validator reads JSON as a parser returns it, with arrays as lists. A model's `to_json` returns tuples, and a Python validator does not accept tuples as arrays. """ context = scoped(context, CORE_AND_EXTENSIONS) schemas = Schemas(_documents(context)) array = schemas.of(ZarrV3ArrayMetadataJSON) group = schemas.of(ZarrV3GroupMetadataJSON) return schemas.document({"anyOf": [array, group]}) def _documents(context: ZarrV3Context) -> SchemaLeaf: """The schema leaf for documents read in `context`: an array's fill value constrained by its data type, each field as `context` reads it, and each extra field that a definition in `context` claims.""" fields = field_schemas(context) def leaf(annotation: object, schemas: Schemas) -> JSONSchema | None: if annotation is ZarrV3ArrayMetadataJSON: return schemas.defined( annotation, annotation.__name__, lambda: _array(context, schemas) ) if annotation is ZarrV3GroupMetadataJSON: return schemas.defined( annotation, annotation.__name__, lambda: _with_extra_fields( schemas.object_of(ZarrV3GroupMetadataJSON), context.group_extra_fields, schemas ), ) return fields(annotation, schemas) return leaf def _with_extra_fields( schema: JSONSchema, definitions: Mapping[str, ExtraFieldDefinition[Any]], schemas: Schemas ) -> JSONSchema: """`schema`, a node document's schema, with a property for each extra field in `definitions`: the member's value, as the configuration TypedDict of the definition that claims its key declares it.""" if len(definitions) == 0: return schema properties = _member_object(schema, "properties") extra: dict[str, JSONValue] = { key: schemas.of(definition.configuration) for key, definition in definitions.items() } return {**schema, "properties": {**properties, **extra}} def _member_object(schema: Mapping[str, JSONValue], key: str) -> Mapping[str, JSONValue]: """The object at member `key` of `schema`, or an empty one when the member is absent. Raises `TypeError` for a member that is not an object, which is a bug in the schema this module built.""" found = schema.get(key, {}) if not isinstance(found, Mapping): msg = f"the schema member {key!r} is an object, got {found!r}" raise TypeError(msg) return found def _array(context: ZarrV3Context, schemas: Schemas) -> JSONSchema: """An array document: its TypedDict, at least one codec, and a constraint on its fill value for each data type in scope, which applies when the document names that data type. The TypedDict cannot declare a minimum length, so the schema adds it: a pipeline needs its array -> bytes codec, and the reader rejects an empty `codecs`. """ schema = schemas.object_of(ZarrV3ArrayMetadataJSON) properties = _member_object(schema, "properties") codecs = _member_object(properties, "codecs") schema = {**schema, "properties": {**properties, "codecs": {**codecs, "minItems": 1}}} schema = _with_extra_fields(schema, context.array_extra_fields, schemas) held: list[JSONValue] = [] for definition in context.data_types.values(): fill_value = schemas.of(definition.fill_value) if len(fill_value) == 0: continue # a data type that declares no fill value shape accepts any JSON name = written_name(definition) named: JSONSchema = { "anyOf": [name, {"type": "object", "properties": {"name": name}, "required": ["name"]}] } condition: JSONSchema = { "if": {"properties": {"data_type": named}, "required": ["data_type"]}, "then": {"properties": {"fill_value": fill_value}}, } held.append(condition) return schema if len(held) == 0 else {**schema, "allOf": held} __all__ = [ "field_json_schema_v3", "field_schemas", "node_metadata_json_schema_v3", "written_name", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py000066400000000000000000000300571527014137600322570ustar00rootroot00000000000000"""Reading a codec pipeline: each codec paired with the chunk it receives. A pipeline is array -> array codecs, then one array -> bytes codec, then bytes -> bytes codecs. The array passes a `Chunk` to its first codec. The chunk has "the same data type as the Zarr array, and shape equal to the chunk shape" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1048-L1050), where the shape is the lengths of the grid's chunks along each axis. Each array -> array codec passes the next codec the chunk that its `transition` returns. Each codec that receives an array is checked by its chunk rules against that chunk. For example, a `bytes` codec that receives a multi-byte data type without an `endian` is a problem, and so is a `transpose` whose `order` has the wrong number of axes. If a codec contains its own pipelines, each one is read the same way, starting from the chunk that the codec's `pipelines` returns. For a shard, the inner codecs receive the inner chunks, and the index codecs receive the shard index. The reader does not guess. If a codec was not read (no definition in the scope claims its name, or its configuration has problems), its output is unknown, so the next codec receives a chunk with unknown shape and data type. The same applies after a codec that does not declare its output. A later codec can still declare its output itself, as `cast_value` declares its data type. An unclaimed codec could be any of the three kinds, so the order check skips it. A chunk rule checks only the properties of the chunk that are known. """ from __future__ import annotations import dataclasses from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, TypeGuard, cast from zarr_metadata._json import ValidationProblem, is_object, is_tuple, with_input from zarr_metadata.definition._definition import ( AcceptedField, ResolvedField, asked, read_only, ruled, ) from zarr_metadata.v3._definition import ( Chunk, CodecDefinition, CodecKind, ) if TYPE_CHECKING: from collections.abc import Iterator, Mapping, Sequence from zarr_metadata._typed_json import Loc from zarr_metadata.definition._definition import ( Nested, Problems, ) def _no_stages() -> Mapping[str, tuple[Stage, ...]]: """The pipelines of a codec that contains none, or that was not read: an empty mapping.""" return {} @dataclass(frozen=True, slots=True) class Stage: """One codec of a pipeline, and the chunk it receives.""" codec: ResolvedField[CodecDefinition[Any]] """The codec, as the scope read it.""" incoming: Chunk | None """The chunk the codec receives. None for a codec that receives bytes (a bytes -> bytes codec, or any codec after the array -> bytes codec). Also None for an unclaimed codec when its input is not known to be an array. """ inner: Mapping[str, tuple[Stage, ...]] = dataclasses.field(default_factory=_no_stages) """The pipelines the codec contains, keyed by the configuration member that holds each one. Each pipeline pairs each codec with the chunk it receives.""" def __post_init__(self) -> None: # Make this read-only, like everything else a reading returns. object.__setattr__(self, "inner", MappingProxyType(dict(self.inner))) def __reduce__(self) -> tuple[type[Stage], tuple[object, ...]]: # A read-only view cannot be pickled, so pickle the stage from its constructor arguments. return Stage, (self.codec, self.incoming, dict(self.inner)) _POSITIONS: Final[Mapping[CodecKind, int]] = { "array_array": 0, "array_bytes": 1, "bytes_bytes": 2, } """The position of each kind of codec in a pipeline.""" _SPOKEN: Final[Mapping[CodecKind, str]] = { "array_array": "array -> array", "array_bytes": "array -> bytes", "bytes_bytes": "bytes -> bytes", } def read_pipeline( codecs: Sequence[ResolvedField[CodecDefinition[Any]]], chunk: Chunk, loc: Loc = () ) -> tuple[tuple[Stage, ...], Problems]: """Read `codecs` (codec fields read by a context) as a pipeline that receives `chunk`. Return each codec with the chunk it receives, and the problems found. First the order is checked: array -> array codecs, then one array -> bytes codec, then bytes -> bytes codecs. Then each codec is processed in turn, with the chunk the previous codec passed on. Its chunk rules check it and locate their problems in its configuration. Each pipeline it contains is read the same way, at its location in the configuration. An array -> array codec is also asked what chunk it passes on. Problems are located under `loc`, the location of the codecs, with each codec at its index. """ problems = list(_order_problems(codecs, loc)) stages: list[Stage] = [] # The chunk the next codec receives. None once the input is not known # to be an array: after the array -> bytes codec, or after a codec of # unknown kind. handed: Chunk | None = chunk for index, codec in enumerate(codecs): definition = codec.definition if definition is None: stages.append(Stage(codec, handed)) handed = None continue if definition.kind == "bytes_bytes": stages.append(Stage(codec, None)) handed = None continue incoming = Chunk() if handed is None else handed at = (*loc, index, "configuration") inner = _no_stages() if isinstance(codec, AcceptedField): configuration, nested = codec.configuration, codec.nested problems.extend(_chunk_problems(definition, configuration, nested, incoming, at)) inner, found = _inner_pipelines(definition, configuration, nested, incoming, at) problems.extend(found) stages.append(Stage(codec, incoming, inner)) if definition.kind == "array_bytes": handed = None elif not isinstance(codec, AcceptedField): handed = Chunk() else: handed = _handed_on(definition, codec.configuration, codec.nested, incoming, at) if len(problems) == 0: return tuple(stages), () return tuple(stages), with_input(problems, tuple(codec.json for codec in codecs), loc) def _order_problems( codecs: Sequence[ResolvedField[CodecDefinition[Any]]], loc: Loc ) -> Iterator[ValidationProblem]: """Array -> array codecs, then one array -> bytes codec, then bytes -> bytes codecs. "the list of codecs must be of the following form: zero or more array -> array codecs; followed by exactly one array -> bytes codec; followed by zero or more bytes -> bytes codecs" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L974-L983). An unclaimed codec is skipped. It could be any of the three kinds, so a pipeline that contains one is not rejected for lacking an array -> bytes codec. A codec in the wrong position is reported at its location, and so is any array -> bytes codec after the first. """ furthest: CodecDefinition[Any] | None = None encoder: CodecDefinition[Any] | None = None unclaimed = False for index, codec in enumerate(codecs): definition = codec.definition if definition is None: unclaimed = True continue if furthest is not None and _POSITIONS[definition.kind] < _POSITIONS[furthest.kind]: yield ValidationProblem( (*loc, index), f"expected {_SPOKEN[definition.kind]} codecs before " f"{_SPOKEN[furthest.kind]} codecs, got {definition.name!r} after " f"{furthest.name!r}", "invalid_value", ) elif definition.kind == "array_bytes" and encoder is not None: yield ValidationProblem( (*loc, index), f"expected one array -> bytes codec, got {definition.name!r} after " f"{encoder.name!r}", "invalid_value", ) if furthest is None or _POSITIONS[definition.kind] > _POSITIONS[furthest.kind]: furthest = definition if definition.kind == "array_bytes" and encoder is None: encoder = definition if encoder is None and not unclaimed: yield ValidationProblem(loc, "expected an array -> bytes codec, got none", "invalid_value") def _chunk_problems( definition: CodecDefinition[Any], configuration: Mapping[str, object], nested: Nested, chunk: Chunk, at: Loc, ) -> Problems: """The problems `definition`'s chunk rules find in a codec that receives `chunk`, located under `at`.""" return ruled( definition, lambda: definition.chunk_rules(read_only(configuration), nested, chunk), at ) def _inner_pipelines( definition: CodecDefinition[Any], configuration: Mapping[str, object], nested: Nested, chunk: Chunk, at: Loc, ) -> tuple[Mapping[str, tuple[Stage, ...]], Problems]: """Read each pipeline contained in a codec that receives `chunk`, starting from the chunk that the codec's `pipelines` returns. A pipeline's codecs are the fields in the configuration member that holds it, as the scope read them. Its problems are located at that member, under `at`. `pipelines` is the extension author's code, so its result is checked: it must map configuration members to chunks, and each member must be a list of codecs. """ given = asked( definition, "pipelines", lambda: cast("object", definition.pipelines(read_only(configuration), nested, chunk)), at, ) if not _is_pipelines(given): msg = ( f"{definition.name!r}: its pipelines give a mapping of members to chunks, got {given!r}" ) raise TypeError(msg) read: dict[str, tuple[Stage, ...]] = {} problems: list[ValidationProblem] = [] for member, handed in given.items(): codecs = _held(definition, configuration, nested, member) read[member], found = read_pipeline(codecs, handed, (*at, member)) problems.extend(found) return read, tuple(problems) def _is_pipelines(value: object) -> TypeGuard[Mapping[str, Chunk]]: """Whether `value` maps members of a configuration to chunks.""" return is_object(value) and all( isinstance(member, str) and isinstance(chunk, Chunk) for member, chunk in value.items() ) def _held( definition: CodecDefinition[Any], configuration: Mapping[str, object], nested: Nested, member: str, ) -> tuple[ResolvedField[CodecDefinition[Any]], ...]: """The codecs in configuration member `member`, as the scope read them. If the member holds anything other than a list of fields read as codecs, this raises `TypeError`, because the definition that names the member has a bug. """ entries: object = configuration.get(member) places = [(member, index) for index in range(len(entries))] if is_tuple(entries) else None if places is None or not all( place in nested and nested[place].read_as is CodecDefinition for place in places ): msg = f"{definition.name!r}: its pipelines name {member!r}, which holds no list of codecs" raise TypeError(msg) return tuple(nested[place] for place in places) def _handed_on( definition: CodecDefinition[Any], configuration: Mapping[str, object], nested: Nested, chunk: Chunk, at: Loc, ) -> Chunk: """The chunk an array -> array codec passes on when it receives `chunk`. The codec's `transition` is the extension author's code. Its result is checked to be a chunk. If it raises an error, the error says which codec's transition raised it and where the reader was reading. """ given = asked( definition, "transition", lambda: cast("object", definition.transition(read_only(configuration), nested, chunk)), at, ) if not isinstance(given, Chunk): msg = f"{definition.name!r}: its transition gives a Chunk, got {given!r}" raise TypeError(msg) return given __all__ = ["Stage", "read_pipeline"] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/array.py000066400000000000000000000040161527014137600314250ustar00rootroot00000000000000"""Zarr v3 array metadata types.""" from collections.abc import Mapping from typing import Annotated, Final, Literal, NotRequired from annotated_types import Ge from typing_extensions import TypedDict from zarr_metadata._common import JSONValue from zarr_metadata.v3._common import ( ZarrV3ChunkGridField, ZarrV3ChunkKeyEncodingField, ZarrV3CodecField, ZarrV3DataTypeField, ZarrV3ExtensionField, ZarrV3StorageTransformerField, ) class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=ZarrV3ExtensionField): """ Zarr v3 array metadata document (the `zarr.json` content for an array). Extra keys may contain arbitrary JSON values. Each extension point is annotated with the field alias of its kind. For example, `data_type` is a `ZarrV3DataTypeField`, and each entry of `codecs` is a `ZarrV3CodecField`. The alias describes the JSON of the metadata field, and it tells a scope which kind to read the field as. See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#array-metadata """ zarr_format: Literal[3] node_type: Literal["array"] data_type: ZarrV3DataTypeField shape: tuple[Annotated[int, Ge(0)], ...] chunk_grid: ZarrV3ChunkGridField chunk_key_encoding: ZarrV3ChunkKeyEncodingField fill_value: JSONValue codecs: tuple[ZarrV3CodecField, ...] attributes: NotRequired[Mapping[str, JSONValue]] storage_transformers: NotRequired[tuple[ZarrV3StorageTransformerField, ...]] dimension_names: NotRequired[tuple[str | None, ...]] ZarrV3ArrayMetadataStoreKey = Literal["zarr.json"] """Literal type of the store key holding a v3 array's metadata document.""" ZARR_V3_ARRAY_METADATA_STORE_KEY: Final[ZarrV3ArrayMetadataStoreKey] = "zarr.json" """The store key a v3 array's metadata document is persisted under. v3 uses one key for both node types; the document's `node_type` field distinguishes an array from a group. """ __all__ = [ "ZARR_V3_ARRAY_METADATA_STORE_KEY", "ZarrV3ArrayMetadataJSON", "ZarrV3ArrayMetadataStoreKey", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/000077500000000000000000000000001527014137600320515ustar00rootroot00000000000000__init__.py000066400000000000000000000007341527014137600341070ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid"""Zarr v3 chunk grid metadata types. Each chunk grid lives in its own submodule: - `regular` -- core v3 spec - `rectilinear` -- zarr-extensions Import a grid's types and its definition from its submodule: `ChunkGridMetadata`, the canonical type for the grid's permitted JSON shapes, `ChunkGridObject`, `ChunkGridConfiguration`, and `_CHUNK_GRID`. See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-grids """ __all__: list[str] = [] rectilinear.py000066400000000000000000000162001527014137600346440ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid""" Rectilinear chunk grid (zarr-extensions). See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md """ from collections.abc import Iterator from typing import Annotated, Final, Literal, NotRequired from annotated_types import Ge from typing_extensions import TypedDict from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import Nested from zarr_metadata.v3._definition import ( ChunkGridDefinition, Lengths, ) RECTILINEAR_CHUNK_GRID_NAME: Final = "rectilinear" """The `name` field value of the rectilinear chunk grid.""" RectilinearChunkGridName = Literal["rectilinear"] """Literal type of the `name` field of the rectilinear chunk grid.""" _Positive = Annotated[int, Ge(1)] RectilinearDimSpec = _Positive | tuple[_Positive | tuple[_Positive, _Positive], ...] """JSON shape for one dimension's rectilinear spec. Either a bare integer (uniform shorthand for a regular dimension within a rectilinear grid), or a tuple of integers and/or `[value, count]` RLE pairs. Every extent, and every run's length and count, is at least 1. """ class RectilinearChunkGridConfiguration(TypedDict, closed=True): """Configuration for the rectilinear chunk grid.""" kind: Literal["inline"] chunk_shapes: tuple[RectilinearDimSpec, ...] class RectilinearChunkGridObject(TypedDict, closed=True): """Rectilinear chunk grid metadata in object form.""" name: RectilinearChunkGridName configuration: RectilinearChunkGridConfiguration must_understand: NotRequired[bool] RectilinearChunkGridMetadata = RectilinearChunkGridObject """Permitted JSON shape for rectilinear chunk grid metadata. `kind` and `chunk_shapes` are required, so only the object form is valid; the short-hand-name form is not permitted by the spec for this grid. https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md#L59-L62 https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ def canonical_dim_spec(spec: RectilinearDimSpec) -> RectilinearDimSpec: """Return one dimension's chunk sizes in their simplest equivalent form. Runs of equal sizes collapse to `[size, count]` pairs, because that is the spelling that does not grow with the number of chunks: a million equal chunks is two numbers, not a million. A run of one stays a bare size, and `[size, 1]` collapses to a bare size, because a pair adds no information there. Adjacent spellings of the same size merge, which makes this idempotent: `[[32, 2], 32]` and `[32, [32, 2]]` both become `[[32, 3]]`. A dimension-level bare integer is left alone. It is a *step* that repeats until it covers the extent, so it is not equivalent to any fixed list: expanding it would pin a grid that currently adapts, and the two would diverge the moment the array were resized. For the same reason a one-element list is never collapsed to a bare integer: `[32]` declares exactly one chunk and `32` declares as many as it takes. Assumes a spec the rules have accepted. """ if not isinstance(spec, tuple): return spec runs: list[tuple[int, int]] = [] for entry in spec: size, count = entry if isinstance(entry, tuple) else (entry, 1) if len(runs) != 0 and runs[-1][0] == size: runs[-1] = (size, runs[-1][1] + count) else: runs.append((size, count)) return tuple(size if count == 1 else (size, count) for size, count in runs) def canonical_chunk_shapes( chunk_shapes: tuple[RectilinearDimSpec, ...], ) -> tuple[RectilinearDimSpec, ...]: """Return every dimension's chunk sizes in their simplest equivalent form.""" return tuple(canonical_dim_spec(spec) for spec in chunk_shapes) def _canonical( configuration: RectilinearChunkGridConfiguration, ) -> RectilinearChunkGridConfiguration: """Run-length encode the chunk sizes, because that spelling does not grow as the array grows.""" return RectilinearChunkGridConfiguration( kind=configuration["kind"], chunk_shapes=canonical_chunk_shapes(configuration["chunk_shapes"]), ) def _shape_rules( configuration: RectilinearChunkGridConfiguration, nested: Nested, shape: tuple[int, ...] ) -> Iterator[ValidationProblem]: """Check that there are chunk lengths for each of the array's dimensions, and that they cover the dimension. "The length of `chunk_shapes` MUST match the number of dimensions of the array", and "The sum of the edge lengths MUST equal or exceed `L`" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md?plain=1#L62-L91). A bare integer repeats until it covers the dimension, so it always passes this check. """ chunk_shapes = configuration["chunk_shapes"] if len(chunk_shapes) != len(shape): yield ValidationProblem( ("chunk_shapes",), f"expected one chunk_shapes entry per dimension of shape, got {len(chunk_shapes)}", "invalid_value", ) return for axis, (spec, extent) in enumerate(zip(chunk_shapes, shape, strict=True)): if isinstance(spec, int): continue covered = sum(entry if isinstance(entry, int) else entry[0] * entry[1] for entry in spec) if covered < extent: yield ValidationProblem( ("chunk_shapes", axis), f"expected chunk lengths that cover the dimension's length {extent}, " f"got lengths summing to {covered}", "invalid_value", ) def _chunk_lengths( configuration: RectilinearChunkGridConfiguration, nested: Nested, shape: tuple[int, ...] ) -> Lengths: """Return, for each axis, every chunk length that its entry lists. A bare integer is the length of every chunk along its axis. A list gives the length of each chunk, and a `[length, count]` pair gives `count` chunks of that length (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md?plain=1#L62-L91). Every listed length counts, including the length of a chunk past the array's edge, which the array grows into when it is resized. """ return tuple( frozenset({spec}) if isinstance(spec, int) else frozenset(entry if isinstance(entry, int) else entry[0] for entry in spec) for spec in configuration["chunk_shapes"] ) RECTILINEAR_CHUNK_GRID: Final = ChunkGridDefinition( name=RECTILINEAR_CHUNK_GRID_NAME, configuration=RectilinearChunkGridConfiguration, canonical=_canonical, shape_rules=_shape_rules, chunk_lengths=_chunk_lengths, ) """The `rectilinear` chunk grid.""" __all__ = [ "RECTILINEAR_CHUNK_GRID", "RECTILINEAR_CHUNK_GRID_NAME", "RectilinearChunkGridConfiguration", "RectilinearChunkGridMetadata", "RectilinearChunkGridName", "RectilinearChunkGridObject", "RectilinearDimSpec", "canonical_chunk_shapes", "canonical_dim_spec", ] regular.py000066400000000000000000000102401527014137600340020ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid""" Regular chunk grid (Zarr v3 core spec). See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#regular-grids """ from collections.abc import Iterator from typing import Annotated, Final, Literal, NotRequired from annotated_types import Ge from typing_extensions import TypedDict from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import Nested from zarr_metadata.v3._definition import ( ChunkGridDefinition, Lengths, ) REGULAR_CHUNK_GRID_NAME: Final = "regular" """The `name` field value of the regular chunk grid.""" RegularChunkGridName = Literal["regular"] """Literal type of the `name` field of the regular chunk grid.""" class RegularChunkGridConfiguration(TypedDict, closed=True): """Configuration for the regular chunk grid. Every chunk length is at least 1, even along a dimension of length 0: "Chunk sizes must be greater than zero" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-grids/regular-grid/index.rst#L40). The core spec's "The chunk shape elements are non-zero when the corresponding dimensions of the arrays have non-zero length" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L284-L285) says less about an empty dimension, and it does not allow anything that the grid spec forbids. """ chunk_shape: tuple[Annotated[int, Ge(1)], ...] class RegularChunkGridObject(TypedDict, closed=True): """Regular chunk grid metadata in object form.""" name: RegularChunkGridName configuration: RegularChunkGridConfiguration must_understand: NotRequired[bool] RegularChunkGridMetadata = RegularChunkGridObject """Permitted JSON shape for regular chunk grid metadata. `chunk_shape` is required and has no default, so only the object form is valid; the short-hand-name form is not permitted by the spec for this grid. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L528-L537 ("must be an object with the names name and configuration") https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ def _shape_rules( configuration: RegularChunkGridConfiguration, nested: Nested, shape: tuple[int, ...] ) -> Iterator[ValidationProblem]: """Check that there is one chunk length for each of the array's dimensions. "The dimensionality of the grid is the same as the dimensionality of the array" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-grids/regular-grid/index.rst#L29-L31). """ chunk_shape = configuration["chunk_shape"] if len(chunk_shape) != len(shape): yield ValidationProblem( ("chunk_shape",), f"expected one chunk length per dimension of shape, got {len(chunk_shape)}", "invalid_value", ) def _chunk_lengths( configuration: RegularChunkGridConfiguration, nested: Nested, shape: tuple[int, ...] ) -> Lengths: """Return one length along each axis, because every chunk has the grid's `chunk_shape`. "each chunk is a hyperrectangle of the same shape" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-grids/regular-grid/index.rst#L28-L29), including a chunk at the array's edge, where "the grid will overhang the edge of the array space" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-grids/regular-grid/index.rst#L43-L46). """ return tuple(frozenset({length}) for length in configuration["chunk_shape"]) REGULAR_CHUNK_GRID: Final = ChunkGridDefinition( name=REGULAR_CHUNK_GRID_NAME, configuration=RegularChunkGridConfiguration, shape_rules=_shape_rules, chunk_lengths=_chunk_lengths, ) """The `regular` chunk grid.""" __all__ = [ "REGULAR_CHUNK_GRID", "REGULAR_CHUNK_GRID_NAME", "RegularChunkGridConfiguration", "RegularChunkGridMetadata", "RegularChunkGridName", "RegularChunkGridObject", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/000077500000000000000000000000001527014137600335625ustar00rootroot00000000000000__init__.py000066400000000000000000000016071527014137600356200ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding"""Zarr v3 chunk key encoding metadata types. Each chunk key encoding lives in its own submodule: - `default` -- v3 default encoding (`/`-separated) - `v2` -- v2-compatibility encoding (`.`-separated by default) Both are defined by the v3 core spec: https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/default/index.rst https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/v2/index.rst Import an encoding's types and its definition from its submodule: `ChunkKeyEncodingMetadata`, the canonical type for the encoding's permitted JSON shapes, `ChunkKeyEncodingObject`, `ChunkKeyEncodingConfiguration`, and `_CHUNK_KEY_ENCODING`. See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-key-encoding """ __all__: list[str] = [] default.py000066400000000000000000000062231527014137600355040ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding""" Default chunk key encoding (Zarr v3 core spec). The chunk key for a chunk with grid index `(k, j, i, ...)` is formed by appending `ckji...` (where `` is `separator`). See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-key-encoding """ from typing import Final, Literal, NotRequired from typing_extensions import TypedDict from zarr_metadata.v3._definition import ChunkKeyEncodingDefinition DEFAULT_CHUNK_KEY_ENCODING_NAME: Final = "default" """The `name` field value of the default chunk key encoding.""" DefaultChunkKeyEncodingName = Literal["default"] """Literal type of the `name` field of the default chunk key encoding.""" DefaultChunkKeyEncodingSeparator = Literal["/", "."] """Literal type of permitted `separator` values for the default chunk key encoding. Defaults to `"/"` if absent. """ DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR: Final = ("/", ".") """Tuple of permitted values for the `separator` field of the default chunk key encoding.""" class DefaultChunkKeyEncodingConfiguration(TypedDict, closed=True): """Configuration for the default chunk key encoding. `separator` is optional and defaults to `"/"` per spec. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/default/index.rst#L27-L29 """ separator: NotRequired[DefaultChunkKeyEncodingSeparator] class DefaultChunkKeyEncodingObject(TypedDict, closed=True): """Default chunk key encoding metadata in object form.""" name: DefaultChunkKeyEncodingName configuration: NotRequired[DefaultChunkKeyEncodingConfiguration] must_understand: NotRequired[bool] DefaultChunkKeyEncodingMetadata = DefaultChunkKeyEncodingObject | DefaultChunkKeyEncodingName """Permitted JSON shapes for the default chunk-key encoding metadata. The configuration has no required keys (`separator` defaults to `"/"`), so the short-hand-name form is permitted in addition to the object form. """ def _canonical( configuration: DefaultChunkKeyEncodingConfiguration, ) -> DefaultChunkKeyEncodingConfiguration: """Remove a `separator` of `/`, because an absent `separator` means the same thing. "If not specified, `separator` defaults to `/`" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/default/index.rst#L27-L29), so the two spellings describe the same encoding. """ if configuration.get("separator") != "/": return configuration return {} DEFAULT_CHUNK_KEY_ENCODING: Final = ChunkKeyEncodingDefinition( name=DEFAULT_CHUNK_KEY_ENCODING_NAME, configuration=DefaultChunkKeyEncodingConfiguration, canonical=_canonical, ) """The `default` chunk key encoding. The type of `separator` already restricts its values, so the encoding has no rules of its own.""" __all__ = [ "DEFAULT_CHUNK_KEY_ENCODING", "DEFAULT_CHUNK_KEY_ENCODING_NAME", "DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR", "DefaultChunkKeyEncodingConfiguration", "DefaultChunkKeyEncodingMetadata", "DefaultChunkKeyEncodingName", "DefaultChunkKeyEncodingObject", "DefaultChunkKeyEncodingSeparator", ] v2.py000066400000000000000000000064271527014137600344150ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding""" v2-compatibility chunk key encoding (Zarr v3 core spec). Intended only to allow existing v2 arrays to be converted to v3 without having to rename chunks. Not recommended for new arrays. Naming note: these are Zarr **v3** types. The leading `V2` in `V2ChunkKeyEncodingMetadata` (and friends) is the encoding's registered *entity name* (`"v2"`), not the format-version marker that `ZarrV2...` names carry — this package's version-prefixed names always spell it `ZarrV2` / `ZarrV3`. See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-key-encoding """ from typing import Final, Literal, NotRequired from typing_extensions import TypedDict from zarr_metadata.v3._definition import ChunkKeyEncodingDefinition V2_CHUNK_KEY_ENCODING_NAME: Final = "v2" """The `name` field value of the v2 chunk key encoding.""" V2ChunkKeyEncodingName = Literal["v2"] """Literal type of the `name` field of the v2 chunk key encoding.""" V2ChunkKeyEncodingSeparator = Literal["/", "."] """Literal type of permitted `separator` values for the v2 chunk key encoding. Defaults to `"."` if absent. """ V2_CHUNK_KEY_ENCODING_SEPARATOR: Final = ("/", ".") """Tuple of permitted values for the `separator` field of the v2 chunk key encoding.""" class V2ChunkKeyEncodingConfiguration(TypedDict, closed=True): """Configuration for the v2 chunk key encoding. `separator` is optional and defaults to `"."` per spec. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/v2/index.rst#L27-L29 """ separator: NotRequired[V2ChunkKeyEncodingSeparator] class V2ChunkKeyEncodingObject(TypedDict, closed=True): """v2-compatibility chunk key encoding metadata in object form.""" name: V2ChunkKeyEncodingName configuration: NotRequired[V2ChunkKeyEncodingConfiguration] must_understand: NotRequired[bool] V2ChunkKeyEncodingMetadata = V2ChunkKeyEncodingObject | V2ChunkKeyEncodingName """Permitted JSON shapes for the v2-compatibility chunk-key encoding metadata. The configuration has no required keys (`separator` defaults to `"."`), so the short-hand-name form is permitted in addition to the object form. """ def _canonical(configuration: V2ChunkKeyEncodingConfiguration) -> V2ChunkKeyEncodingConfiguration: """Remove a `separator` of `.`, because an absent `separator` means the same thing. "If not specified, `separator` defaults to `.`" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/v2/index.rst#L27-L29), so the two spellings describe the same encoding. """ if configuration.get("separator") != ".": return configuration return {} V2_CHUNK_KEY_ENCODING: Final = ChunkKeyEncodingDefinition( name=V2_CHUNK_KEY_ENCODING_NAME, configuration=V2ChunkKeyEncodingConfiguration, canonical=_canonical, ) """The `v2` chunk key encoding. The type of `separator` already restricts its values, so the encoding has no rules of its own.""" __all__ = [ "V2_CHUNK_KEY_ENCODING", "V2_CHUNK_KEY_ENCODING_NAME", "V2_CHUNK_KEY_ENCODING_SEPARATOR", "V2ChunkKeyEncodingConfiguration", "V2ChunkKeyEncodingMetadata", "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingObject", "V2ChunkKeyEncodingSeparator", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/codec/000077500000000000000000000000001527014137600310115ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py000066400000000000000000000021271527014137600331240ustar00rootroot00000000000000"""Zarr v3 codec spec types. Each codec defined by the spec or by zarr-extensions has its own submodule (`blosc`, `bytes`, `cast_value`, `crc32c`, `gzip`, `scale_offset`, `sharding_indexed`, `transpose`, `zstd`). Import a codec's types and its definition from its submodule: `CodecMetadata`, the canonical type for the codec's permitted JSON shapes (object form plus, where the spec allows, a bare-string short-hand form), `CodecObject`, `_CODEC`, and, for a codec that has a configuration, `CodecConfiguration`. For the JSON of any codec entry (one of the entries in array metadata's `codecs`), import `ZarrV3MetadataFieldJSON` from `zarr_metadata.v3`. A configuration member that holds a codec, such as sharding's inner pipelines, is annotated `ZarrV3CodecField` from `zarr_metadata.v3.definition` instead, so that the reader reads it as a codec. Each codec's pipeline position (`array -> array`, `array -> bytes`, `bytes -> bytes`) is its definition's `kind`, in `zarr_metadata.v3.definition`. See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/index.html """ __all__: list[str] = [] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py000066400000000000000000000056441527014137600336640ustar00rootroot00000000000000"""The data types that the arithmetic codecs, `cast_value` and `scale_offset`, accept. `cast_value` "is only defined for data types that model real numbers: floating-point and integral data types" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L5-L9), and `scale_offset` "for data types where multiplication, division, addition, and subtraction are well-defined" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L5-L9); Each spec then lists "the following data types defined in this repository", and the list is the same for both. A data type defined elsewhere may also qualify, so a data type that the list does not name is rejected only when the spec text itself excludes it. Neither codec rejects the numpy time types. Each of them "is compatible with any codec that supports arrays of signed 64-bit integers" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L120), and both codecs support those, even though neither list names the time types. """ from typing import Final from zarr_metadata.v3._definition import RAW_BYTES_NAME from zarr_metadata.v3.data_type.bool import BOOL_DATA_TYPE_NAME from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE_NAME from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE_NAME from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE_NAME from zarr_metadata.v3.data_type.string import STRING_DATA_TYPE_NAME from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE_NAME FLOATING_POINT: Final = frozenset( { "float4_e2m1fn", "float6_e2m3fn", "float6_e3m2fn", "float8_e3m4", "float8_e4m3", "float8_e4m3b11fnuz", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz", "float8_e8m0fnu", "bfloat16", "float16", "float32", "float64", } ) """The floating-point data types that the lists name. None of them is integral, so `cast_value` may not wrap to them.""" NOT_NUMBERS: Final = frozenset( { BOOL_DATA_TYPE_NAME, RAW_BYTES_NAME, STRING_DATA_TYPE_NAME, BYTES_DATA_TYPE_NAME, STRUCT_DATA_TYPE_NAME, } ) """Data types whose values are not numbers (truth values, raw bits, text, byte strings, records). Neither codec accepts them.""" COMPLEX: Final = frozenset({COMPLEX64_DATA_TYPE_NAME, COMPLEX128_DATA_TYPE_NAME}) """Complex data types. They do not model real numbers, so `cast_value` does not accept them. Addition, subtraction, multiplication and division are well-defined on them, but the `scale_offset` spec does not name them in its list. So the spec leaves open whether `scale_offset` accepts them. """ __all__ = ["COMPLEX", "FLOATING_POINT", "NOT_NUMBERS"] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py000066400000000000000000000107731527014137600324750ustar00rootroot00000000000000""" Blosc codec types. See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/blosc/index.html """ from collections.abc import Iterator from typing import Annotated, Final, Literal, NotRequired from annotated_types import Ge, Interval from typing_extensions import TypedDict from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import Nested from zarr_metadata.v3._definition import CodecDefinition BLOSC_CODEC_NAME: Final = "blosc" """The `name` field value of the `blosc` codec.""" BloscCodecName = Literal["blosc"] """Literal type of the `name` field of the `blosc` codec.""" BloscShuffle = Literal["noshuffle", "shuffle", "bitshuffle"] """Literal type of blosc shuffle mode names.""" BLOSC_SHUFFLE: Final = ("noshuffle", "shuffle", "bitshuffle") """Tuple of permitted values for the `shuffle` field of the `blosc` codec.""" BloscCName = Literal["lz4", "lz4hc", "blosclz", "snappy", "zlib", "zstd"] """Literal type of blosc compressor identifiers.""" BLOSC_CNAME: Final = ("lz4", "lz4hc", "blosclz", "snappy", "zlib", "zstd") """Tuple of permitted values for the `cname` field of the `blosc` codec.""" class BloscCodecConfiguration(TypedDict, closed=True): """Configuration for the Zarr v3 `blosc` codec.""" cname: BloscCName clevel: Annotated[int, Interval(ge=0, le=9)] shuffle: BloscShuffle blocksize: Annotated[int, Ge(0)] typesize: NotRequired[int] class BloscCodecObject(TypedDict, closed=True): """`blosc` codec metadata in object form.""" name: BloscCodecName configuration: BloscCodecConfiguration must_understand: NotRequired[bool] BloscCodecMetadata = BloscCodecObject """Permitted JSON shape for `blosc` codec metadata. The configuration has multiple required keys (`cname`, `clevel`, `shuffle`, `blocksize`), so only the object form is valid; the short-hand-name form is not permitted by the spec for this codec. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/blosc/index.rst#L57-L98 (configuration parameters) https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ BLOSC_NO_SHUFFLE: Final = "noshuffle" """The `shuffle` value under which `typesize` carries no information. The spec requires `typesize` "unless `shuffle` is `"noshuffle"`, in which case the value is ignored", so this is the one value that changes whether another member is required. """ def _rules(configuration: BloscCodecConfiguration, nested: Nested) -> Iterator[ValidationProblem]: """Check `typesize` against `shuffle`. For `noshuffle`, the spec says of `typesize` that "the value is ignored", and the canonical form drops it. For either other shuffle, `typesize` is required and must be positive. The type cannot express whether `typesize` is required, so it does not express the bound either. These rules check both. """ shuffle = configuration["shuffle"] if shuffle != BLOSC_NO_SHUFFLE: typesize = configuration.get("typesize") if typesize is None: yield ValidationProblem( ("typesize",), f"typesize is required when shuffle is {shuffle!r}", "missing_key" ) elif typesize < 1: yield ValidationProblem( ("typesize",), f"expected a positive integer, got {typesize}", "invalid_value", ctx={"ge": 1}, ) def _canonical(configuration: BloscCodecConfiguration) -> BloscCodecConfiguration: """Remove `typesize` when `shuffle` is `noshuffle`, because it has no meaning there. The spec says of that case that "the value is ignored", so two configurations that differ only in `typesize` describe the same codec. """ if configuration["shuffle"] != BLOSC_NO_SHUFFLE or "typesize" not in configuration: return configuration simpler = configuration.copy() del simpler["typesize"] return simpler BLOSC_CODEC: Final = CodecDefinition( name=BLOSC_CODEC_NAME, configuration=BloscCodecConfiguration, kind="bytes_bytes", size="dynamic", rules=_rules, canonical=_canonical, ) """The `blosc` codec.""" __all__ = [ "BLOSC_CNAME", "BLOSC_CODEC", "BLOSC_CODEC_NAME", "BLOSC_NO_SHUFFLE", "BLOSC_SHUFFLE", "BloscCName", "BloscCodecConfiguration", "BloscCodecMetadata", "BloscCodecName", "BloscCodecObject", "BloscShuffle", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py000066400000000000000000000112021527014137600325050ustar00rootroot00000000000000""" Bytes codec types. See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/bytes/index.html """ from collections.abc import Iterator from typing import Final, Literal, NotRequired from typing_extensions import TypedDict from zarr_metadata._json import ValidationProblem, shown from zarr_metadata.definition._definition import Nested from zarr_metadata.v3._definition import ( Chunk, CodecDefinition, storage_of, ) BYTES_CODEC_NAME: Final = "bytes" """The `name` field value of the `bytes` codec.""" BytesCodecName = Literal["bytes"] """Literal type of the `name` field of the `bytes` codec.""" Endianness = Literal["little", "big"] """Literal type of byte order of multi-byte numeric data.""" ENDIANNESS: Final = ("little", "big") """Tuple of permitted values for the `endian` field of the `bytes` codec.""" class BytesCodecConfiguration(TypedDict, closed=True): """ Configuration for the Zarr v3 `bytes` codec. The `endian` field is required for multi-byte data types. """ endian: NotRequired[Endianness] class BytesCodecObject(TypedDict, closed=True): """`bytes` codec metadata in object form. `configuration` is itself optional — when no configuration fields are set, the entire `configuration` key may be omitted. This matches the bare-string short-hand form (`BytesCodecName`) at the canonical data level; both encodings describe a `bytes` codec with default settings. """ name: BytesCodecName configuration: NotRequired[BytesCodecConfiguration] must_understand: NotRequired[bool] BytesCodecMetadata = BytesCodecObject | BytesCodecName """Permitted JSON shapes for `bytes` codec metadata. The configuration has no required keys (`endian` is conditionally required at runtime based on data type), so the spec's short-hand-name form is permitted in addition to the object form, and the object form may itself omit `configuration` entirely. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/bytes/index.rst#L64-L69 ("endian: Required for data types for which endianness is applicable") https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ def _chunk_rules( configuration: BytesCodecConfiguration, nested: Nested, chunk: Chunk ) -> Iterator[ValidationProblem]: """Check that `endian` is present for a data type whose values take several bytes, and that the data type has a fixed size. `endian` is "Required for data types for which endianness is applicable" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/bytes/index.rst#L64-L69): this is every core numeric type except `bool`, `int8` and `uint8`. The values of those types take one byte and do "not depend on endian" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/bytes/index.rst#L84-L101). Raw bits `r8` also take one byte. The spec does not yet say anything about wider raw bits. The codec "encodes arrays of fixed-size numeric data types" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/bytes/index.rst#L28-L30), and a type whose values vary in size needs its own codec. For example, `string` "is only compatible with the vlen-utf8 codec" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/string/README.md?plain=1#L25). These checks use the storage that the data type declares. If the storage is unknown, nothing is checked. """ if chunk.data_type is None: return storage = storage_of(chunk.data_type) written = chunk.data_type.name if storage == "multi_byte" and "endian" not in configuration: yield ValidationProblem( ("endian",), f"expected an endian, since each {written!r} value takes several bytes", "missing_key", ) elif storage == "variable_length": yield ValidationProblem( (), f"expected a data type of fixed size, got {shown(written)}, whose values vary in size", "invalid_value", ) BYTES_CODEC: Final = CodecDefinition( name=BYTES_CODEC_NAME, configuration=BytesCodecConfiguration, kind="array_bytes", size="static", chunk_rules=_chunk_rules, ) """The `bytes` codec.""" __all__ = [ "BYTES_CODEC", "BYTES_CODEC_NAME", "ENDIANNESS", "BytesCodecConfiguration", "BytesCodecMetadata", "BytesCodecName", "BytesCodecObject", "Endianness", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py000066400000000000000000000216251527014137600335170ustar00rootroot00000000000000""" Cast-value codec types. See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md """ from collections.abc import Iterator from typing import Final, Literal, NotRequired from typing_extensions import TypedDict from zarr_metadata._common import JSONValue from zarr_metadata._json import ValidationProblem, shown from zarr_metadata.definition._definition import ( AcceptedField, Nested, fill_value_problems, ) from zarr_metadata.v3._common import ZarrV3DataTypeField from zarr_metadata.v3._definition import ( Chunk, CodecDefinition, ) from zarr_metadata.v3.codec._arithmetic import COMPLEX, FLOATING_POINT, NOT_NUMBERS CAST_VALUE_CODEC_NAME: Final = "cast_value" """The `name` field value of the `cast_value` codec.""" CastValueCodecName = Literal["cast_value"] """Literal type of the `name` field of the `cast_value` codec.""" CastRoundingMode = Literal[ "nearest-even", "towards-zero", "towards-positive", "towards-negative", "nearest-away", ] """Literal type of permitted values for the `rounding` configuration field. Defaults to `"nearest-even"` if absent. """ CAST_ROUNDING_MODE: Final = ( "nearest-even", "towards-zero", "towards-positive", "towards-negative", "nearest-away", ) """Tuple of permitted values for the `rounding` field of the `cast_value` codec.""" CastOutOfRangeMode = Literal["clamp", "wrap"] """Literal type of permitted values for the `out_of_range` configuration field. If absent, out-of-range values are an encoding/decoding error. """ CAST_OUT_OF_RANGE_MODE: Final = ("clamp", "wrap") """Tuple of permitted values for the `out_of_range` field of the `cast_value` codec.""" ScalarMapEntry = tuple[JSONValue, JSONValue] """A single `[input, output]` mapping in a `scalar_map` direction. Each scalar is encoded as a fill value of its data type. For a float, the infinities are `"Infinity"` and `"-Infinity"`, as the core encoding writes them. The codec's own example writes `"+Infinity"` instead. """ class ScalarMap(TypedDict, closed=True): """Optional encode/decode scalar overrides for the cast_value codec.""" encode: NotRequired[tuple[ScalarMapEntry, ...]] decode: NotRequired[tuple[ScalarMapEntry, ...]] class CastValueCodecConfiguration(TypedDict, closed=True): """ Configuration for the Zarr v3 `cast_value` codec. `data_type` is the target data type that input values are cast to. It is the same shape as the top-level array `data_type` field: either a bare-string primitive name or a `{name, configuration}` envelope. """ data_type: ZarrV3DataTypeField rounding: NotRequired[CastRoundingMode] out_of_range: NotRequired[CastOutOfRangeMode] scalar_map: NotRequired[ScalarMap] class CastValueCodecObject(TypedDict, closed=True): """`cast_value` codec metadata in object form.""" name: CastValueCodecName configuration: CastValueCodecConfiguration must_understand: NotRequired[bool] CastValueCodecMetadata = CastValueCodecObject """Permitted JSON shape for `cast_value` codec metadata. `configuration.data_type` is required, so only the object form is valid; the short-hand-name form is not permitted by the spec for this codec. https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md#L33-L36 and #L46-L48 (required fields) https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ _NO_REAL_NUMBERS: Final = NOT_NUMBERS | COMPLEX """The data types the codec accepts neither as input nor as output, because none of them models real numbers.""" def _scalars( configuration: CastValueCodecConfiguration, side: Literal["source", "target"] ) -> Iterator[tuple[tuple[str | int, ...], JSONValue]]: """Yield each scalar of `scalar_map` that is written in the data type of one side of the cast, with its location. "For encode, the input data type is the array data type before casting and the output data type is the target data_type. For decode, the input data type is the target data_type and the output data type is the array data type before casting" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L107-L109). """ scalar_map = configuration.get("scalar_map") if scalar_map is None: return for direction, entries in ( ("encode", scalar_map.get("encode", ())), ("decode", scalar_map.get("decode", ())), ): at = int((direction == "encode") == (side == "target")) for index, entry in enumerate(entries): yield ("scalar_map", direction, index, at), entry[at] def _rules( configuration: CastValueCodecConfiguration, nested: Nested ) -> Iterator[ValidationProblem]: """Check that the target data type models real numbers, that `wrap` has an integral target, and that mapped values are valid in the target. The codec "is only defined for data types that model real numbers" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L5-L9); `wrap` is "Only permitted when data_type is an integral type" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L84); and a scalar of the target type is encoded with its fill value encoding. These rules check the target data type, which the scope read. The chunk rules check the input data type. If the target does not model real numbers, that is the only problem reported for it. """ target = nested.get(("data_type",)) if not isinstance(target, AcceptedField): return name, written = target.definition.name, target.name if name in _NO_REAL_NUMBERS: yield ValidationProblem( ("data_type",), f"expected a data type that models real numbers, got {shown(written)}", "invalid_value", ) return if configuration.get("out_of_range") == "wrap" and name in FLOATING_POINT: yield ValidationProblem( ("out_of_range",), f"expected an integral data_type to wrap to, got {shown(written)}", "invalid_value", ) for at, scalar in _scalars(configuration, "target"): yield from fill_value_problems(target, scalar, at) def _chunk_rules( configuration: CastValueCodecConfiguration, nested: Nested, chunk: Chunk ) -> Iterator[ValidationProblem]: """Check that the input data type models real numbers, and that mapped values are valid in it. "The same ordered procedure applies during decoding, with input and output data types swapped" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L13), so the input data type must meet the same requirements as the target data type. """ source = chunk.data_type if not isinstance(source, AcceptedField): return if source.definition.name in _NO_REAL_NUMBERS: yield ValidationProblem( (), f"expected a chunk of a data type that models real numbers, got {shown(source.name)}", "invalid_value", ) return for at, scalar in _scalars(configuration, "source"): yield from fill_value_problems(source, scalar, at) def _transition(configuration: CastValueCodecConfiguration, nested: Nested, chunk: Chunk) -> Chunk: """Return the chunk with its values cast to the target data type. Nothing else changes. It "converts (casts) the values of the input array to a new data type ... and it leaves all other array properties intact" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L3). The output data type is the data type field as the scope read it. If the scope did not read that field, the output data type is unknown. """ return Chunk(chunk.lengths, nested.get(("data_type",))) CAST_VALUE_CODEC: Final = CodecDefinition( name=CAST_VALUE_CODEC_NAME, configuration=CastValueCodecConfiguration, kind="array_array", size="static", rules=_rules, chunk_rules=_chunk_rules, transition=_transition, ) """The `cast_value` codec. The target data type is a nested field. It is read in the same scope as the codec, and it becomes the data type of the output chunk. """ __all__ = [ "CAST_OUT_OF_RANGE_MODE", "CAST_ROUNDING_MODE", "CAST_VALUE_CODEC", "CAST_VALUE_CODEC_NAME", "CastOutOfRangeMode", "CastRoundingMode", "CastValueCodecConfiguration", "CastValueCodecMetadata", "CastValueCodecName", "CastValueCodecObject", "ScalarMap", "ScalarMapEntry", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py000066400000000000000000000035031527014137600324430ustar00rootroot00000000000000""" CRC32C codec types. See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/crc32c/index.html The CRC32C codec has no configuration fields, so the `configuration` key is absent from the metadata. """ from typing import Final, Literal, NotRequired from typing_extensions import TypedDict from zarr_metadata.definition._definition import EmptyConfiguration from zarr_metadata.v3._definition import CodecDefinition CRC32C_CODEC_NAME: Final = "crc32c" """The `name` field value of the `crc32c` codec.""" Crc32cCodecName = Literal["crc32c"] """Literal type of the `name` field of the `crc32c` codec.""" class Crc32cCodecObject(TypedDict, closed=True): """`crc32c` codec metadata in object form. Per spec the codec has no configuration fields. `configuration` is optional and, if present, should be an empty mapping. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/crc32c/index.rst#L63-L66 """ name: Crc32cCodecName configuration: NotRequired[EmptyConfiguration] must_understand: NotRequired[bool] Crc32cCodecMetadata = Crc32cCodecObject | Crc32cCodecName """Permitted JSON shapes for `crc32c` codec metadata. The spec's Extension definition allows extensions with no required configuration to be encoded as a bare short-hand name. CRC32C has no configuration, so both forms are valid. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ CRC32C_CODEC: Final = CodecDefinition( name=CRC32C_CODEC_NAME, configuration=EmptyConfiguration, kind="bytes_bytes", size="static" ) """The `crc32c` codec. A checksum has no configuration.""" __all__ = [ "CRC32C_CODEC", "CRC32C_CODEC_NAME", "Crc32cCodecMetadata", "Crc32cCodecName", "Crc32cCodecObject", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py000066400000000000000000000040161527014137600323350ustar00rootroot00000000000000""" Gzip codec types. See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/gzip/index.html """ from typing import Annotated, Final, Literal, NotRequired from annotated_types import Interval from typing_extensions import TypedDict from zarr_metadata.v3._definition import CodecDefinition GZIP_CODEC_NAME: Final = "gzip" """The `name` field value of the `gzip` codec.""" GzipCodecName = Literal["gzip"] """Literal type of the `name` field of the `gzip` codec.""" class GzipCodecConfiguration(TypedDict, closed=True): """ Configuration for the Zarr v3 `gzip` codec. `level` is an integer in the range 0-9; 0 disables compression and 9 is slowest with the best compression ratio. The codec's compressed output depends on `level`, so metadata that omits it cannot reproducibly identify the chunk bytes produced by a writer — `level` is required for the metadata to fulfill its reproducibility role, even though the spec text does not mark it required with RFC 2119 keywords. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/gzip/index.rst#L57-L66 """ level: Annotated[int, Interval(ge=0, le=9)] class GzipCodecObject(TypedDict, closed=True): """`gzip` codec metadata in object form.""" name: GzipCodecName configuration: GzipCodecConfiguration must_understand: NotRequired[bool] GzipCodecMetadata = GzipCodecObject """Permitted JSON shape for `gzip` codec metadata. `configuration.level` is required (it determines the codec's output bytes and is therefore part of the metadata's reproducibility contract), so only the object form is valid; the short-hand-name form is not permitted. """ GZIP_CODEC: Final = CodecDefinition( name=GZIP_CODEC_NAME, configuration=GzipCodecConfiguration, kind="bytes_bytes", size="dynamic", ) """The `gzip` codec.""" __all__ = [ "GZIP_CODEC", "GZIP_CODEC_NAME", "GzipCodecConfiguration", "GzipCodecMetadata", "GzipCodecName", "GzipCodecObject", ] scale_offset.py000066400000000000000000000130341527014137600337420ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/codec""" Scale-offset codec types. See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md """ from collections.abc import Iterator from typing import Final, Literal, NotRequired from typing_extensions import TypedDict from zarr_metadata._common import JSONValue from zarr_metadata._json import ValidationProblem, shown from zarr_metadata.definition._definition import ( AcceptedField, Nested, fill_value_problems, ) from zarr_metadata.v3._definition import ( Chunk, CodecDefinition, ) from zarr_metadata.v3.codec._arithmetic import NOT_NUMBERS SCALE_OFFSET_CODEC_NAME: Final = "scale_offset" """The `name` field value of the `scale_offset` codec.""" ScaleOffsetCodecName = Literal["scale_offset"] """Literal type of the `name` field of the `scale_offset` codec.""" class ScaleOffsetCodecConfiguration(TypedDict, closed=True): """ Configuration for the Zarr v3 `scale_offset` codec. Both fields are optional. A missing `offset` is the additive identity (e.g. 0 for numeric types); a missing `scale` is the multiplicative identity (e.g. 1). Each scalar is encoded as a fill value of the input data type of the codec: an integer for an integer type, and for a float type a number, `"NaN"`, `"Infinity"`, `"-Infinity"` or a hex string. """ offset: NotRequired[JSONValue] scale: NotRequired[JSONValue] class ScaleOffsetCodecObject(TypedDict, closed=True): """`scale_offset` codec metadata in object form. `configuration` is itself optional per spec — when both `offset` and `scale` are at their identity defaults, the codec is a no-op and the entire `configuration` field may be omitted. https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md#L18 and #L35 """ name: ScaleOffsetCodecName configuration: NotRequired[ScaleOffsetCodecConfiguration] must_understand: NotRequired[bool] ScaleOffsetCodecMetadata = ScaleOffsetCodecObject | ScaleOffsetCodecName """Permitted JSON shapes for `scale_offset` codec metadata. The configuration has no required keys (both `offset` and `scale` are optional, and the configuration itself is optional), so the short-hand-name form is permitted in addition to the object form. """ def _rules( configuration: ScaleOffsetCodecConfiguration, nested: Nested ) -> Iterator[ValidationProblem]: """Check that neither scalar is null. The registry says each scalar is "JSON-encoded per the input array's fill-value rules", and no data type accepts `null` as a fill value. Checking which scalars are valid requires the input data type of the codec, so a chunk rule does that. """ if "offset" in configuration and configuration["offset"] is None: yield ValidationProblem(("offset",), "expected a scalar, got null", "invalid_value") if "scale" in configuration and configuration["scale"] is None: yield ValidationProblem(("scale",), "expected a scalar, got null", "invalid_value") def _chunk_rules( configuration: ScaleOffsetCodecConfiguration, nested: Nested, chunk: Chunk ) -> Iterator[ValidationProblem]: """Check that the input data type supports arithmetic, and that `offset` and `scale` are values of that type. The codec "is only defined for data types where multiplication, division, addition, and subtraction are well-defined" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L5-L9), and each of `offset` and `scale` "MUST be encoded to JSON using the Zarr V3 fill value encoding for the input array's data type" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L39-L43). The rules, not the chunk rules, reject a null. """ source = chunk.data_type if not isinstance(source, AcceptedField): return if source.definition.name in NOT_NUMBERS: yield ValidationProblem( (), f"expected a chunk of a data type with arithmetic, got {shown(source.name)}", "invalid_value", ) return for member in ("offset", "scale"): value = configuration.get(member) if value is not None: yield from fill_value_problems(source, value, (member,)) def _transition( configuration: ScaleOffsetCodecConfiguration, nested: Nested, chunk: Chunk ) -> Chunk: """Return the input chunk unchanged: the values change, but their shape and data type do not. Its arithmetic is "performed using the arithmetic semantics of the input array's data type" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L47), and a later codec must produce any narrower data type (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L153). """ return chunk SCALE_OFFSET_CODEC: Final = CodecDefinition( name=SCALE_OFFSET_CODEC_NAME, configuration=ScaleOffsetCodecConfiguration, kind="array_array", size="static", rules=_rules, chunk_rules=_chunk_rules, transition=_transition, ) """The `scale_offset` codec.""" __all__ = [ "SCALE_OFFSET_CODEC", "SCALE_OFFSET_CODEC_NAME", "ScaleOffsetCodecConfiguration", "ScaleOffsetCodecMetadata", "ScaleOffsetCodecName", "ScaleOffsetCodecObject", ] sharding_indexed.py000066400000000000000000000224431527014137600346100ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/codec""" Sharding-indexed codec types. See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/sharding-indexed/index.html """ from collections.abc import Iterator, Mapping from typing import Annotated, Final, Literal, NotRequired from annotated_types import Ge from typing_extensions import TypedDict from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import ( AcceptedField, Nested, ) from zarr_metadata.v3._common import ( ZarrV3CodecField, ZarrV3StaticCodecField, ) from zarr_metadata.v3._definition import ( Chunk, CodecDefinition, Lengths, ) from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE, UINT64_DATA_TYPE_NAME SHARDING_INDEXED_CODEC_NAME: Final = "sharding_indexed" """The `name` field value of the `sharding_indexed` codec.""" ShardingIndexedCodecName = Literal["sharding_indexed"] """Literal type of the `name` field of the `sharding_indexed` codec.""" ShardingIndexLocation = Literal["start", "end"] """Literal type of the position of the shard index within the encoded shard.""" SHARDING_INDEX_LOCATION: Final = ("start", "end") """Tuple of permitted values for the `index_location` field of the `sharding_indexed` codec.""" class ShardingIndexedCodecConfiguration(TypedDict, closed=True): """ Configuration for the Zarr v3 `sharding_indexed` codec. `chunk_shape` is the shape of inner chunks along each dimension; it must evenly divide the shard shape. `codecs` is the codec pipeline applied to each inner chunk; exactly one array-to-bytes codec is required. `index_codecs` is the codec pipeline applied to the shard index. It may contain only codecs of static size, because a reader locates the index by a size that it knows before reading it. A compressor in this pipeline is therefore rejected. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L147-L155 `index_location` defaults to `"end"` per the spec. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L157-L161 """ chunk_shape: tuple[Annotated[int, Ge(1)], ...] codecs: tuple[ZarrV3CodecField, ...] index_codecs: tuple[ZarrV3StaticCodecField, ...] index_location: NotRequired[ShardingIndexLocation] class ShardingIndexedCodecObject(TypedDict, closed=True): """`sharding_indexed` codec metadata in object form.""" name: ShardingIndexedCodecName configuration: ShardingIndexedCodecConfiguration must_understand: NotRequired[bool] ShardingIndexedCodecMetadata = ShardingIndexedCodecObject """Permitted JSON shape for `sharding_indexed` codec metadata. The configuration has multiple required keys (`chunk_shape`, `codecs`, `index_codecs`), so only the object form is valid; the short-hand-name form is not permitted by the spec for this codec. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L129-L138 (`chunk_shape`) https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L141-L155 (`codecs` and `index_codecs`, the members the spec marks required) https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ def _chunk_rules( configuration: ShardingIndexedCodecConfiguration, nested: Nested, chunk: Chunk ) -> Iterator[ValidationProblem]: """Check that `chunk_shape` has one inner chunk length per axis of the input shard, and that each length divides every shard length along its axis. "The length of the chunk_shape array must match the number of dimensions of the shard shape to which this sharding codec is applied, and the inner chunk shape along each dimension must evenly divide the size of the shard shape" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L131-L135). The shard is the chunk that the codec receives, after any earlier codecs have been applied. For example, a transposed shard is divided along its transposed axes. """ chunk_shape = configuration["chunk_shape"] lengths = chunk.lengths if lengths is None: return if len(chunk_shape) != len(lengths): yield ValidationProblem( ("chunk_shape",), f"expected {len(lengths)} inner chunk lengths, one per axis of the chunk the codec " f"is handed, got {len(chunk_shape)}", "invalid_value", ) return for axis, (inner, shard) in enumerate(zip(chunk_shape, lengths, strict=True)): undivided = [] if shard is None else [length for length in shard if length % inner != 0] if len(undivided) != 0: yield ValidationProblem( ("chunk_shape", axis), f"expected an inner chunk length that divides each length the shards take " f"along this axis, got {inner}, which does not divide {min(undivided)}", "invalid_value", ) _UINT64: Final = AcceptedField( json=UINT64_DATA_TYPE_NAME, name=UINT64_DATA_TYPE_NAME, definition=UINT64_DATA_TYPE, configuration={}, ) """The data type of a shard index. The spec fixes it, regardless of what the scope holds.""" def _pipelines( configuration: ShardingIndexedCodecConfiguration, nested: Nested, chunk: Chunk ) -> Mapping[str, Chunk]: """Return the input chunk of each nested pipeline: the inner codecs receive the inner chunks, and the index codecs receive the shard index. An inner chunk is "a chunk within the shard". Its shape is `chunk_shape`, and its data type is the data type of the shard (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L168-L170). The index is "an array with 64-bit unsigned integers with a shape that matches the chunks per shard tuple with an appended dimension of size 2" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L186-L187), where chunks per shard is "the element-wise division of the shard shape by the inner chunk shape" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L173-L174). The number of chunks per shard is unknown along an axis if the shard lengths there are unknown, or if the inner chunk length does not divide them. If the input shard has a different number of axes than `chunk_shape`, both pipelines receive chunks with unknown lengths, because it is not known which of the two is wrong. """ chunk_shape = configuration["chunk_shape"] lengths = chunk.lengths if lengths is not None and len(lengths) != len(chunk_shape): return {"codecs": Chunk(None, chunk.data_type), "index_codecs": Chunk(None, _UINT64)} inner = Chunk(tuple(frozenset({length}) for length in chunk_shape), chunk.data_type) index = Chunk((*_per_shard(chunk_shape, lengths), frozenset({2})), _UINT64) return {"codecs": inner, "index_codecs": index} def _per_shard(chunk_shape: tuple[int, ...], lengths: Lengths | None) -> Lengths: """Return the number of inner chunks a shard holds along each axis. An entry is None where that number is unknown.""" if lengths is None: return (None,) * len(chunk_shape) return tuple( None if shard is None or any(length % inner != 0 for length in shard) else frozenset(length // inner for length in shard) for inner, shard in zip(chunk_shape, lengths, strict=True) ) def _canonical( configuration: ShardingIndexedCodecConfiguration, ) -> ShardingIndexedCodecConfiguration: """Remove an `index_location` of `end`, because an absent `index_location` means the same thing. "If the parameter is not present, the value defaults to `end`" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L157-L161), so the two spellings describe the same codec. """ if configuration.get("index_location") != "end": return configuration simpler = configuration.copy() del simpler["index_location"] return simpler SHARDING_INDEXED_CODEC: Final = CodecDefinition( name=SHARDING_INDEXED_CODEC_NAME, configuration=ShardingIndexedCodecConfiguration, canonical=_canonical, kind="array_bytes", size="dynamic", chunk_rules=_chunk_rules, pipelines=_pipelines, ) """The `sharding_indexed` codec. Its two pipelines are nested fields. Each codec in them is read in the same scope as the shard, and each pipeline is read as a pipeline: the inner codecs receive the inner chunks, and the index codecs receive the shard index. """ __all__ = [ "SHARDING_INDEXED_CODEC", "SHARDING_INDEXED_CODEC_NAME", "SHARDING_INDEX_LOCATION", "ShardingIndexLocation", "ShardingIndexedCodecConfiguration", "ShardingIndexedCodecMetadata", "ShardingIndexedCodecName", "ShardingIndexedCodecObject", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py000066400000000000000000000103661527014137600334070ustar00rootroot00000000000000""" Transpose codec types. See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/transpose/index.html """ from collections.abc import Iterator from typing import Final, Literal, NotRequired from typing_extensions import TypedDict from zarr_metadata._json import ValidationProblem, shown from zarr_metadata.definition._definition import Nested from zarr_metadata.v3._definition import ( Chunk, CodecDefinition, ) TRANSPOSE_CODEC_NAME: Final = "transpose" """The `name` field value of the `transpose` codec.""" TransposeCodecName = Literal["transpose"] """Literal type of the `name` field of the `transpose` codec.""" class TransposeCodecConfiguration(TypedDict, closed=True): """ Configuration for the Zarr v3 `transpose` codec. `order` is a permutation of the dimension indices 0..n-1 that specifies the dimension reordering applied during encoding. """ order: tuple[int, ...] class TransposeCodecObject(TypedDict, closed=True): """`transpose` codec metadata in object form.""" name: TransposeCodecName configuration: TransposeCodecConfiguration must_understand: NotRequired[bool] TransposeCodecMetadata = TransposeCodecObject """Permitted JSON shape for `transpose` codec metadata. `order` is required, so only the object form is valid; the short-hand-name form is not permitted by the spec for this codec. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/transpose/index.rst#L60-L66 ("order: Required") https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ def _rules( configuration: TransposeCodecConfiguration, nested: Nested ) -> Iterator[ValidationProblem]: """Check that `order` is a permutation of its own indices. A chunk rule checks whether `order` matches the axes of the input chunk. """ order = configuration["order"] if sorted(order) != list(range(len(order))): yield ValidationProblem( ("order",), f"expected a permutation of 0..{len(order) - 1}, got {shown(order)}", "invalid_value", ) def _chunk_rules( configuration: TransposeCodecConfiguration, nested: Nested, chunk: Chunk ) -> Iterator[ValidationProblem]: """Check that `order` has one entry for each axis of the input chunk. "a permutation of 0, 1, ..., n-1, where n is the number of dimensions in the decoded chunk representation provided as input to this codec" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/transpose/index.rst#L63-L66). """ order = configuration["order"] if chunk.rank is not None and len(order) != chunk.rank: yield ValidationProblem( ("order",), f"expected {chunk.rank} entries, one per axis of the chunk the codec is " f"handed, got {len(order)}", "invalid_value", ) def _transition(configuration: TransposeCodecConfiguration, nested: Nested, chunk: Chunk) -> Chunk: """Return the chunk with its axes reordered by `order`. The data type does not change. "B_shape[i] = A_shape[order[i]]", where A is the input chunk and B is the output chunk, with "the same data type as A" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/transpose/index.rst#L73-L79). If `order` has a different number of axes than the chunk, the output lengths are unknown. """ lengths = chunk.lengths order = configuration["order"] if lengths is None or len(order) != len(lengths): return Chunk(None, chunk.data_type) return Chunk(tuple(lengths[axis] for axis in order), chunk.data_type) TRANSPOSE_CODEC: Final = CodecDefinition( name=TRANSPOSE_CODEC_NAME, configuration=TransposeCodecConfiguration, kind="array_array", size="static", rules=_rules, chunk_rules=_chunk_rules, transition=_transition, ) """The `transpose` codec.""" __all__ = [ "TRANSPOSE_CODEC", "TRANSPOSE_CODEC_NAME", "TransposeCodecConfiguration", "TransposeCodecMetadata", "TransposeCodecName", "TransposeCodecObject", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py000066400000000000000000000057141527014137600323560ustar00rootroot00000000000000""" Zstandard codec types. See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/zstd/README.md (the zarr-extensions registry entry; zarr-specs PR #256, which first proposed the codec, was never merged). """ from typing import Annotated, Final, Literal, NotRequired from annotated_types import Interval from typing_extensions import TypedDict from zarr_metadata.v3._definition import CodecDefinition ZSTD_CODEC_NAME: Final = "zstd" """The `name` field value of the `zstd` codec.""" ZstdCodecName = Literal["zstd"] """Literal type of the `name` field of the `zstd` codec.""" ZSTD_MIN_LEVEL: Final = -131072 """The lowest `level` zstd accepts: ZSTD_minCLevel(), -(1 << 17).""" ZSTD_MAX_LEVEL: Final = 22 """The highest `level` zstd accepts: ZSTD_maxCLevel().""" class ZstdCodecConfiguration(TypedDict, closed=True): """ Configuration for the Zarr v3 `zstd` codec. `level` is required; `checksum` is optional ("Should be omitted if false"). https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/zstd/README.md#L9-L19 """ level: Annotated[int, Interval(ge=ZSTD_MIN_LEVEL, le=ZSTD_MAX_LEVEL)] checksum: NotRequired[bool] class ZstdCodecObject(TypedDict, closed=True): """`zstd` codec metadata in object form.""" name: ZstdCodecName configuration: ZstdCodecConfiguration must_understand: NotRequired[bool] ZstdCodecMetadata = ZstdCodecObject """Permitted JSON shape for `zstd` codec metadata. `level` is required, so only the object form is valid; the short-hand-name form is not permitted by the spec for this codec. https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/zstd/README.md#L9-L19 https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ def _canonical(configuration: ZstdCodecConfiguration) -> ZstdCodecConfiguration: """Remove a `checksum` of `false`, because an absent `checksum` means the same thing. The spec says of `checksum` that it "should be omitted if false" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/zstd/README.md?plain=1#L17-L19), so the two spellings describe the same codec. """ if configuration.get("checksum") is not False: return configuration simpler = configuration.copy() del simpler["checksum"] return simpler ZSTD_CODEC: Final = CodecDefinition( name=ZSTD_CODEC_NAME, configuration=ZstdCodecConfiguration, canonical=_canonical, kind="bytes_bytes", size="dynamic", ) """The `zstd` codec.""" __all__ = [ "ZSTD_CODEC", "ZSTD_CODEC_NAME", "ZSTD_MAX_LEVEL", "ZSTD_MIN_LEVEL", "ZstdCodecConfiguration", "ZstdCodecMetadata", "ZstdCodecName", "ZstdCodecObject", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py000066400000000000000000000100301527014137600327500ustar00rootroot00000000000000"""Zarr v3 consolidated metadata types. There is no Zarr v3 specification for consolidated metadata. This module models the inline-on-group convention used by the reference Python implementation (and zarrs), where consolidated metadata is embedded as an extension field on a group's `zarr.json`. This is a known non-core interoperability extension. The core spec fixes only its `{kind, must_understand, metadata}` envelope, which `ZarrV3ConsolidatedMetadataEnvelopeJSON` types with any JSON as the entries. `CONSOLIDATED_METADATA_EXTRA_FIELD` reads a group's member as that envelope, and `CORE` holds it. `ZarrV3ConsolidatedMetadataJSON` types the entries as the reference implementation writes them. """ from collections.abc import Mapping from typing import Any, Final, Literal from typing_extensions import TypedDict from zarr_metadata._common import UserJSONValue from zarr_metadata.v3._definition import GroupExtraFieldDefinition from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON class ZarrV3ConsolidatedMetadataEnvelopeJSON(TypedDict, closed=True): """ The `consolidated_metadata` member of a v3 group document, as the core spec fixes it. The spec allows this one additional field on a group, for historical reasons, and fixes its envelope: `kind` is `"inline"`, `must_understand` is false, and `metadata` is an object. The spec leaves the entries of `metadata` to the reference implementation, so here they are any JSON, as user data: the attributes of a listed document may hold `NaN`, `Infinity` and `-Infinity`. `ZarrV3ConsolidatedMetadataJSON` types the entries as the reference implementation writes them: an array or group document at each path. See https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816 """ kind: Literal["inline"] must_understand: Literal[False] metadata: Mapping[str, UserJSONValue] class ZarrV3ConsolidatedMetadataJSON(TypedDict, closed=True): """ Inline consolidated metadata embedded in a v3 group. The `metadata` map contains only v3 array and group entries. V2 entries are excluded from this interoperability convention by design. The v3 core specification acknowledges `consolidated_metadata` as a historical additional field and fixes this envelope, but leaves the entries to the reference implementation: https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816 """ kind: Literal["inline"] must_understand: Literal[False] metadata: Mapping[str, ZarrV3ArrayMetadataJSON | ZarrV3GroupMetadataJSON] ZARR_V3_CONSOLIDATED_METADATA_KEY: Final = "consolidated_metadata" """The key under which consolidated metadata is embedded in a v3 group document. Unlike the v2 `.zmetadata` file, this is not a store key: consolidated metadata is carried as an additional field inside the group's own `zarr.json`. The core spec names the field and its envelope ("For historical reasons, group metadata documents may contain an additional field named ``consolidated_metadata``"); the entry format, like the v2 counterpart, is a reference-implementation convention. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816 """ CONSOLIDATED_METADATA_EXTRA_FIELD: Final[GroupExtraFieldDefinition[Any]] = ( GroupExtraFieldDefinition( name=ZARR_V3_CONSOLIDATED_METADATA_KEY, configuration=ZarrV3ConsolidatedMetadataEnvelopeJSON, finite=False, ) ) """A group's `consolidated_metadata` member, read as the envelope the core spec fixes. The documents it lists are not read: they may hold the non-finite numbers that node attributes may, and `ZarrV3ConsolidatedMetadata` reads them.""" __all__ = [ "CONSOLIDATED_METADATA_EXTRA_FIELD", "ZARR_V3_CONSOLIDATED_METADATA_KEY", "ZarrV3ConsolidatedMetadataEnvelopeJSON", "ZarrV3ConsolidatedMetadataJSON", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/000077500000000000000000000000001527014137600317065ustar00rootroot00000000000000__init__.py000066400000000000000000000016371527014137600337470ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type"""Zarr v3 data type spec types. Each v3 data type has its own submodule: - Core primitives: `bool`, `int8`/`16`/`32`/`64`, `uint8`/`16`/`32`/`64`, `float16`/`32`/`64`, `complex64`/`128`, `raw` (for `r`) - zarr-extensions: `bytes`, `string`, `numpy_datetime64`, `numpy_timedelta64`, `struct` Import a data type's types and its definition from its submodule: - `DataTypeName` -- the literal type of the dtype's `data_type` string (or, for named-config dtypes, the literal value of their `name` field) - `FillValue` -- the permitted JSON shape of the `fill_value` field - the envelope and configuration TypedDicts of named-config dtypes (`numpy_datetime64`, `numpy_timedelta64`, `struct`), the branded `HexFloat` / `Base64Bytes` types, and their validator functions - `_DATA_TYPE`, the definition See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ __all__: list[str] = [] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_byte.py000066400000000000000000000011571527014137600333660ustar00rootroot00000000000000"""A byte value, as used in the fill values of raw bits and of `bytes`. Each is an integer in `[0, 255]` (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L97-L99; the reference for the `bytes` type is https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/bytes/README.md?plain=1#L8). """ from typing import Annotated from annotated_types import Interval ByteValue = Annotated[int, Interval(ge=0, le=255)] """One byte of a fill value: a JSON integer in `[0, 255]`.""" __all__ = ["ByteValue"] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_float.py000066400000000000000000000255231527014137600335330ustar00rootroot00000000000000"""The fill value rules shared by floating-point types and by the complex types built from them. A float fill value is one of the following: a JSON number, which a reader rounds to the nearest value the type represents; one of `"NaN"`, `"Infinity"` and `"-Infinity"`; or `"0x"` followed by the hex digits of the value's bytes read as an unsigned integer, with as many bytes as the type has (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79). A complex fill value is a pair of such components: the real part, then the imaginary part (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L88-L91). """ import dataclasses import struct from collections.abc import Callable, Iterable, Iterator from dataclasses import dataclass from decimal import ROUND_HALF_EVEN, Decimal, localcontext from typing import Final, Literal, get_args from zarr_metadata._common import JSONValue from zarr_metadata._json import ValidationProblem, choices, shown from zarr_metadata.definition._definition import ( EmptyConfiguration, Nested, ) FloatSpecialFillValue = Literal["NaN", "Infinity", "-Infinity"] """The named non-finite fill values every IEEE 754 floating-point type takes.""" FLOAT_SPECIAL_FILL_VALUES: Final[tuple[FloatSpecialFillValue, ...]] = get_args( FloatSpecialFillValue ) """The strings that the fill value rules accept for a float fill value in place of a number.""" def float_fill_value_rules( name: str, hex_form: Callable[[str], str] ) -> Callable[[EmptyConfiguration, Nested, float | str], Iterator[ValidationProblem]]: """Return the fill value rules of the floating-point type `name`. `hex_form` accepts its hex strings. A number may have any value, because a reader rounds it. A string must be one of the named values, or a hex string of the type's own width. The type-checked shape accepts any string here, so `hex_form` raises `ValueError` for a string that is not a valid hex string. """ return _FloatFillValue(name, hex_form) @dataclass(frozen=True, slots=True) class _FloatFillValue: """The rules for a float fill value. This is a value rather than a closure, so a definition that holds it is equal to itself after a pickle or a deep copy.""" name: str hex_form: Callable[[str], str] def __call__( self, configuration: EmptyConfiguration, nested: Nested, value: float | str ) -> Iterator[ValidationProblem]: if not isinstance(value, str) or value in FLOAT_SPECIAL_FILL_VALUES: return try: self.hex_form(value) except ValueError: yield ValidationProblem( (), f"expected a number, {choices(FLOAT_SPECIAL_FILL_VALUES)}, or a " f"{self.name} hex string, got {shown(value)}", "invalid_value", ) FloatWidth = Literal[16, 32, 64] """The widths, in bits, of the IEEE 754 binary formats the float types store.""" _FORMATS: Final[dict[FloatWidth, tuple[str, int]]] = {16: ("e", 10), 32: ("f", 23), 64: ("d", 52)} """The `struct` format for each width, and the number of bits that hold the fraction.""" def float_fill_value_canonical( width: FloatWidth, ) -> Callable[[EmptyConfiguration, Nested, float | str], float | str]: """Return the canonical spelling of a fill value of the floating-point type that is `width` bits wide. A fill value spells a value of the type, as `float_bits` reads it: a number, a named value, or the bits of the value. The canonical spelling is the named value for an infinity, or for the NaN that the spec names `"NaN"`. For any other NaN, it is the hex string of its bits, in lower case. For any other value, it is the shortest number that rounds to the value; if several numbers are equally short, it is the nearest of them. This matches how numpy and `repr` spell a value, for example `0.1` for the `float32` nearest `0.1`. `-0.0` is treated as a separate value with its own spelling. """ return _FloatCanonical(width) @dataclass(frozen=True, slots=True) class _FloatCanonical: """The canonical spelling of a float fill value. This is a value rather than a closure, for the same reason as `_FloatFillValue`.""" width: FloatWidth def __call__( self, configuration: EmptyConfiguration, nested: Nested, value: float | str ) -> float | str: return _spelled(float_bits(value, self.width), self.width) def float_bits(value: float | str, width: FloatWidth) -> int: """Return the bits of the value that `value` spells in the type that is `width` bits wide. `value` is a float fill value that the rules allow. A number is read as a float64, as a JSON parser reads it. An integer with more digits than a float64 holds is rounded to a float64. The float64 is then rounded to the nearest value the type represents, with ties to even, and to an infinity if it is past the largest value. This matches how zarrs reads a number, `as_f64` then `as f32` (https://github.com/zarrs/zarrs/blob/8d68f8522b382d050b768f84bce64c2935de4523/zarrs_metadata/src/v3/array/fill_value.rs#L160), how tensorstore reads one, `static_cast` of `get()` (https://github.com/google/tensorstore/blob/692d2798c51a76d2eed0b4aee85cad5fd4be950a/tensorstore/driver/zarr3/metadata.cc#L165), and how numpy casts a float64. So an integer and a number with a fraction spell the same value if they read as the same float64. An integer past 2**53 whose float64 lies halfway between two values of the type (2**60 + 2**36 + 1, for float32) reads as the even value, 2**60, as those readers store it. It does not read as the value nearest the integer itself. """ code, fraction = _FORMATS[width] exponent = width - 1 - fraction infinity = ((1 << exponent) - 1) << fraction sign = 1 << (width - 1) if isinstance(value, str): named = { "NaN": infinity | (1 << (fraction - 1)), "Infinity": infinity, "-Infinity": sign | infinity, } return named[value] if value in named else int(value, 16) try: held = float(value) except OverflowError: # An integer past the largest float64 reads as an infinity. return (sign if value < 0 else 0) | infinity try: return int.from_bytes(struct.pack(f">{code}", held), "big") except OverflowError: # `struct` raises for a float64 that rounds past the type's largest value. return (sign if held < 0 else 0) | infinity def _spelled(bits: int, width: FloatWidth) -> float | str: """Return the canonical spelling of the value whose bits are `bits`, in the type that is `width` bits wide.""" code, fraction = _FORMATS[width] exponent = width - 1 - fraction infinity = ((1 << exponent) - 1) << fraction sign = 1 << (width - 1) if bits & infinity == infinity: if bits & ((1 << fraction) - 1) == 0: return "-Infinity" if bits & sign else "Infinity" if bits == infinity | (1 << (fraction - 1)): return "NaN" return f"0x{bits:0{width // 4}x}" value: float = struct.unpack(f">{code}", bits.to_bytes(width // 8, "big"))[0] if width == 64 or value == 0: # `repr` already writes the shortest spelling of a float64. A zero # of either sign is also already its shortest spelling. return value return _shortest(value, bits, width) def _shortest(value: float, bits: int, width: FloatWidth) -> float: """Return the shortest number that rounds to `value`. Among numbers with the fewest significant digits, return the nearest one. `bits` are the bits of `value` in the type that is `width` bits wide. For each number of digits, this function tries the nearest number first, and then the next number on the other side of `value`. At a power of two, the numbers that round to `value` reach twice as far above it as below it. So the nearest number can miss when the next one does not. """ exact = Decimal(value) for digits in range(1, 18): with localcontext() as context: context.prec = digits context.rounding = ROUND_HALF_EVEN nearest = +exact step = Decimal((0, (1,), nearest.adjusted() - digits + 1)) beyond = nearest + step if nearest < exact else nearest - step for candidate in (nearest, beyond): if float_bits(float(candidate), width) == bits: return float(candidate) # Seventeen significant digits are enough to distinguish every float64. return value def complex_fill_value_rules( component: Callable[[EmptyConfiguration, Nested, float | str], Iterable[ValidationProblem]], ) -> Callable[ [EmptyConfiguration, Nested, tuple[float | str, float | str]], Iterator[ValidationProblem] ]: """Return the fill value rules of a complex type. `component` checks each component at its index. `component` is the fill value rules of the component's own float type. """ return _ComplexFillValue(component) @dataclass(frozen=True, slots=True) class _ComplexFillValue: """The rules for a complex fill value, which check each component at its index. This is a value rather than a closure, for the same reason as `_FloatFillValue`.""" component: Callable[[EmptyConfiguration, Nested, float | str], Iterable[ValidationProblem]] def __call__( self, configuration: EmptyConfiguration, nested: Nested, value: tuple[float | str, float | str], ) -> Iterator[ValidationProblem]: for index, part in enumerate(value): for found in self.component(configuration, nested, part): yield dataclasses.replace(found, loc=(index, *found.loc)) def complex_fill_value_canonical( component: Callable[[EmptyConfiguration, Nested, float | str], JSONValue], ) -> Callable[[EmptyConfiguration, Nested, tuple[float | str, float | str]], JSONValue]: """Return the canonical spelling of a complex fill value. Each component is spelled by `component`, the canonical spelling of its float type.""" return _ComplexCanonical(component) @dataclass(frozen=True, slots=True) class _ComplexCanonical: """The canonical spelling of a complex fill value. This is a value rather than a closure, for the same reason as `_FloatFillValue`.""" component: Callable[[EmptyConfiguration, Nested, float | str], JSONValue] def __call__( self, configuration: EmptyConfiguration, nested: Nested, value: tuple[float | str, float | str], ) -> JSONValue: return tuple(self.component(configuration, nested, part) for part in value) __all__ = [ "FloatSpecialFillValue", "FloatWidth", "complex_fill_value_canonical", "complex_fill_value_rules", "float_bits", "float_fill_value_canonical", "float_fill_value_rules", ] _numpy_time.py000066400000000000000000000027551527014137600345370ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type"""The time units of the two numpy time types and how they store their values, defined once so that neither type module imports them from the other. The encodings both types share with the v2 time dtypes are in `zarr_metadata._numpy`. """ from typing import Final, Literal from zarr_metadata._numpy import NUMPY_TIME_UNITS from zarr_metadata.v3._definition import multi_byte NumpyTimeUnit = Literal[ "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", "generic" ] """The time unit codes of the numpy time types, `numpy.datetime64` and `numpy.timedelta64`.""" NUMPY_TIME_UNIT: Final = (*NUMPY_TIME_UNITS, "generic") """Runtime tuple of the permitted `numpy.timedelta64`/`numpy.datetime64` unit strings.""" numpy_time_storage: Final = multi_byte """The storage of the numpy time types: signed 64-bit integers, in the byte order that the codecs specify. Each type "is compatible with any codec that supports arrays of signed 64-bit integers" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L120), and "the endianness of numpy.datetime64 arrays is determined by the configuration of the codecs defined in metadata" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L83-L85). The `numpy.timedelta64` spec says the same. """ __all__ = ["NUMPY_TIME_UNIT", "NumpyTimeUnit", "numpy_time_storage"] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py000066400000000000000000000016571527014137600332240ustar00rootroot00000000000000""" Zarr v3 `bool` data type. See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ from typing import Final, Literal from zarr_metadata.definition._definition import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, single_byte, ) BOOL_DATA_TYPE_NAME: Final = "bool" """The `data_type` value for the `bool` type.""" BoolDataTypeName = Literal["bool"] """Literal type of the `data_type` field for `bool`.""" BoolFillValue = bool """Permitted JSON shape of the `fill_value` field for `bool`: a JSON boolean.""" BOOL_DATA_TYPE: Final = DataTypeDefinition( name=BOOL_DATA_TYPE_NAME, configuration=EmptyConfiguration, fill_value=BoolFillValue, storage=single_byte, ) """The `bool` data type. It is written as a bare name and has no configuration.""" __all__ = [ "BOOL_DATA_TYPE", "BOOL_DATA_TYPE_NAME", "BoolDataTypeName", "BoolFillValue", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py000066400000000000000000000056211527014137600334120ustar00rootroot00000000000000""" Zarr `bytes` data type (variable-length raw bytes, zarr-extensions). See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/bytes/README.md """ import base64 from collections.abc import Iterator from typing import Final, Literal from zarr_metadata._json import ValidationProblem, shown from zarr_metadata._numpy import Base64Bytes, base64_bytes from zarr_metadata.definition._definition import ( EmptyConfiguration, Nested, ) from zarr_metadata.v3._definition import ( DataTypeDefinition, variable_length, ) from zarr_metadata.v3.data_type._byte import ByteValue BYTES_DATA_TYPE_NAME: Final = "bytes" """The `data_type` value for the variable-length `bytes` type.""" BytesDataTypeName = Literal["bytes"] """Literal type of the `data_type` field for `bytes`.""" BytesFillValue = tuple[ByteValue, ...] | Base64Bytes """Permitted JSON shape of the `fill_value` field for `bytes`. Either a JSON array of integers in `[0, 255]` (one per byte), or a `Base64Bytes` string encoding the byte sequence. """ def _fill_value_rules( configuration: EmptyConfiguration, nested: Nested, value: BytesFillValue ) -> Iterator[ValidationProblem]: """Check that a string fill value is standard-alphabet base64. A fill value written as byte values is not checked here.""" if not isinstance(value, str): return try: base64_bytes(value) except ValueError: yield ValidationProblem( (), f"expected standard-alphabet base64, got {shown(value)}", "invalid_value" ) def _fill_value_canonical( configuration: EmptyConfiguration, nested: Nested, value: BytesFillValue ) -> str: """Return the bytes as the base64 string that encodes them, however they were written. The string "is more compact" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/bytes/README.md?plain=1#L11), and encoding the bytes again gives a single spelling: `"QR=="` decodes to the same byte as `"QQ=="`. """ data = base64.b64decode(value) if isinstance(value, str) else bytes(value) return base64.b64encode(data).decode("ascii") BYTES_DATA_TYPE: Final = DataTypeDefinition( name=BYTES_DATA_TYPE_NAME, configuration=EmptyConfiguration, fill_value=BytesFillValue, fill_value_rules=_fill_value_rules, fill_value_canonical=_fill_value_canonical, storage=variable_length, ) """The `bytes` data type. It is written as a bare name and has no configuration. Its values are "variable-length byte strings" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/bytes/README.md?plain=1#L3), and each value is stored in as many bytes as it needs. """ __all__ = [ "BYTES_DATA_TYPE", "BYTES_DATA_TYPE_NAME", "Base64Bytes", "BytesDataTypeName", "BytesFillValue", "base64_bytes", ] complex128.py000066400000000000000000000034341527014137600341070ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type""" Zarr v3 `complex128` data type. See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ from typing import Final, Literal from zarr_metadata.definition._definition import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) from zarr_metadata.v3.data_type._float import ( complex_fill_value_canonical, complex_fill_value_rules, ) from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE, Float64FillValue COMPLEX128_DATA_TYPE_NAME: Final = "complex128" """The `data_type` value for the `complex128` type.""" Complex128DataTypeName = Literal["complex128"] """Literal type of the `data_type` field for `complex128`.""" Complex128Component = Float64FillValue """One real or imaginary component of a `complex128` fill value. Same shape as a `float64` fill value: a JSON number, a named sentinel, or a `HexFloat64` string. """ Complex128FillValue = tuple[Complex128Component, Complex128Component] """Permitted JSON shape of the `fill_value` field for `complex128`. A two-element JSON array `[real, imag]` where each component is a `Complex128Component`. """ COMPLEX128_DATA_TYPE: Final = DataTypeDefinition( name=COMPLEX128_DATA_TYPE_NAME, configuration=EmptyConfiguration, fill_value=Complex128FillValue, fill_value_rules=complex_fill_value_rules(FLOAT64_DATA_TYPE.fill_value_rules), fill_value_canonical=complex_fill_value_canonical(FLOAT64_DATA_TYPE.fill_value_canonical), storage=multi_byte, ) """The `complex128` data type. It is written as a bare name and has no configuration. Its fill value is a pair of `float64` components.""" __all__ = [ "COMPLEX128_DATA_TYPE", "COMPLEX128_DATA_TYPE_NAME", "Complex128Component", "Complex128DataTypeName", "Complex128FillValue", ] complex64.py000066400000000000000000000034051527014137600340240ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type""" Zarr v3 `complex64` data type. See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ from typing import Final, Literal from zarr_metadata.definition._definition import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) from zarr_metadata.v3.data_type._float import ( complex_fill_value_canonical, complex_fill_value_rules, ) from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE, Float32FillValue COMPLEX64_DATA_TYPE_NAME: Final = "complex64" """The `data_type` value for the `complex64` type.""" Complex64DataTypeName = Literal["complex64"] """Literal type of the `data_type` field for `complex64`.""" Complex64Component = Float32FillValue """One real or imaginary component of a `complex64` fill value. Same shape as a `float32` fill value: a JSON number, a named sentinel, or a `HexFloat32` string. """ Complex64FillValue = tuple[Complex64Component, Complex64Component] """Permitted JSON shape of the `fill_value` field for `complex64`. A two-element JSON array `[real, imag]` where each component is a `Complex64Component`. """ COMPLEX64_DATA_TYPE: Final = DataTypeDefinition( name=COMPLEX64_DATA_TYPE_NAME, configuration=EmptyConfiguration, fill_value=Complex64FillValue, fill_value_rules=complex_fill_value_rules(FLOAT32_DATA_TYPE.fill_value_rules), fill_value_canonical=complex_fill_value_canonical(FLOAT32_DATA_TYPE.fill_value_canonical), storage=multi_byte, ) """The `complex64` data type. It is written as a bare name and has no configuration. Its fill value is a pair of `float32` components.""" __all__ = [ "COMPLEX64_DATA_TYPE", "COMPLEX64_DATA_TYPE_NAME", "Complex64Component", "Complex64DataTypeName", "Complex64FillValue", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py000066400000000000000000000063641527014137600335450ustar00rootroot00000000000000""" Zarr v3 `float16` data type. See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ import re from typing import Final, Literal, NewType from zarr_metadata.definition._definition import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) from zarr_metadata.v3.data_type._float import ( FloatSpecialFillValue, float_fill_value_canonical, float_fill_value_rules, ) FLOAT16_DATA_TYPE_NAME: Final = "float16" """The `data_type` value for the `float16` type.""" Float16DataTypeName = Literal["float16"] """Literal type of the `data_type` field for `float16`.""" Float16SpecialFillValue = FloatSpecialFillValue """Named non-finite fill values permitted by the spec for IEEE 754 floats. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 """ HexFloat16 = NewType("HexFloat16", str) """A 6-character hex string (`0x` + 4 hex digits) encoding the unsigned-integer representation of a float16.""" _HEX_FLOAT16_RE: Final = re.compile(r"^0x[0-9a-fA-F]{4}$") def hex_float16(value: str) -> HexFloat16: """Validate `value` as a HexFloat16 and brand it. Raises ValueError if `value` is not exactly `0x` followed by 4 hex digits. """ if not _HEX_FLOAT16_RE.fullmatch(value): raise ValueError(f"Expected '0x' followed by 4 hex digits, got {value!r}") return HexFloat16(value) Float16FillValue = float | int | Float16SpecialFillValue | HexFloat16 """Permitted JSON shape of the `fill_value` field for `float16`. Either a JSON number, one of the named non-finite sentinels (`"NaN"`, `"Infinity"`, `"-Infinity"`), or a `HexFloat16` (`0xYYYY` string encoding the unsigned-integer representation of the IEEE 754 value). """ CANONICAL_NAN_HEX_FLOAT16: Final = "0x7e00" """Canonical hex form of the float16 NaN sentinel `"NaN"`. Per spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L72-L74) the named `"NaN"` sentinel denotes the float with sign=0, the most significant mantissa bit set, and all other mantissa bits zero (the IEEE 754 default quiet NaN). Other NaN bit patterns must be encoded with the explicit hex-string form. """ CANONICAL_POSITIVE_INFINITY_HEX_FLOAT16: Final = "0x7c00" """Canonical hex form of the float16 `"Infinity"` sentinel.""" CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT16: Final = "0xfc00" """Canonical hex form of the float16 `"-Infinity"` sentinel.""" FLOAT16_DATA_TYPE: Final = DataTypeDefinition( name=FLOAT16_DATA_TYPE_NAME, configuration=EmptyConfiguration, fill_value=Float16FillValue, fill_value_rules=float_fill_value_rules("float16", hex_float16), fill_value_canonical=float_fill_value_canonical(16), storage=multi_byte, ) """The `float16` data type. It is written as a bare name and has no configuration. Its fill value is a number, a named value or a hex string.""" __all__ = [ "CANONICAL_NAN_HEX_FLOAT16", "CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT16", "CANONICAL_POSITIVE_INFINITY_HEX_FLOAT16", "FLOAT16_DATA_TYPE", "FLOAT16_DATA_TYPE_NAME", "Float16DataTypeName", "Float16FillValue", "Float16SpecialFillValue", "HexFloat16", "hex_float16", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py000066400000000000000000000064051527014137600335370ustar00rootroot00000000000000""" Zarr v3 `float32` data type. See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ import re from typing import Final, Literal, NewType from zarr_metadata.definition._definition import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) from zarr_metadata.v3.data_type._float import ( FloatSpecialFillValue, float_fill_value_canonical, float_fill_value_rules, ) FLOAT32_DATA_TYPE_NAME: Final = "float32" """The `data_type` value for the `float32` type.""" Float32DataTypeName = Literal["float32"] """Literal type of the `data_type` field for `float32`.""" Float32SpecialFillValue = FloatSpecialFillValue """Named non-finite fill values permitted by the spec for IEEE 754 floats. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 """ HexFloat32 = NewType("HexFloat32", str) """A 10-character hex string (`0x` + 8 hex digits) encoding the unsigned-integer representation of a float32.""" _HEX_FLOAT32_RE: Final = re.compile(r"^0x[0-9a-fA-F]{8}$") def hex_float32(value: str) -> HexFloat32: """Validate `value` as a HexFloat32 and brand it. Raises ValueError if `value` is not exactly `0x` followed by 8 hex digits. """ if not _HEX_FLOAT32_RE.fullmatch(value): raise ValueError(f"Expected '0x' followed by 8 hex digits, got {value!r}") return HexFloat32(value) Float32FillValue = float | int | Float32SpecialFillValue | HexFloat32 """Permitted JSON shape of the `fill_value` field for `float32`. Either a JSON number, one of the named non-finite sentinels (`"NaN"`, `"Infinity"`, `"-Infinity"`), or a `HexFloat32` (`0xYYYYYYYY` string encoding the unsigned-integer representation of the IEEE 754 value). """ CANONICAL_NAN_HEX_FLOAT32: Final = "0x7fc00000" """Canonical hex form of the float32 NaN sentinel `"NaN"`. Per spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L72-L74) the named `"NaN"` sentinel denotes the float with sign=0, the most significant mantissa bit set, and all other mantissa bits zero (the IEEE 754 default quiet NaN). Other NaN bit patterns must be encoded with the explicit hex-string form. """ CANONICAL_POSITIVE_INFINITY_HEX_FLOAT32: Final = "0x7f800000" """Canonical hex form of the float32 `"Infinity"` sentinel.""" CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT32: Final = "0xff800000" """Canonical hex form of the float32 `"-Infinity"` sentinel.""" FLOAT32_DATA_TYPE: Final = DataTypeDefinition( name=FLOAT32_DATA_TYPE_NAME, configuration=EmptyConfiguration, fill_value=Float32FillValue, fill_value_rules=float_fill_value_rules("float32", hex_float32), fill_value_canonical=float_fill_value_canonical(32), storage=multi_byte, ) """The `float32` data type. It is written as a bare name and has no configuration. Its fill value is a number, a named value or a hex string.""" __all__ = [ "CANONICAL_NAN_HEX_FLOAT32", "CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT32", "CANONICAL_POSITIVE_INFINITY_HEX_FLOAT32", "FLOAT32_DATA_TYPE", "FLOAT32_DATA_TYPE_NAME", "Float32DataTypeName", "Float32FillValue", "Float32SpecialFillValue", "HexFloat32", "hex_float32", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py000066400000000000000000000064521527014137600335460ustar00rootroot00000000000000""" Zarr v3 `float64` data type. See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ import re from typing import Final, Literal, NewType from zarr_metadata.definition._definition import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) from zarr_metadata.v3.data_type._float import ( FloatSpecialFillValue, float_fill_value_canonical, float_fill_value_rules, ) FLOAT64_DATA_TYPE_NAME: Final = "float64" """The `data_type` value for the `float64` type.""" Float64DataTypeName = Literal["float64"] """Literal type of the `data_type` field for `float64`.""" Float64SpecialFillValue = FloatSpecialFillValue """Named non-finite fill values permitted by the spec for IEEE 754 floats. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 """ HexFloat64 = NewType("HexFloat64", str) """An 18-character hex string (`0x` + 16 hex digits) encoding the unsigned-integer representation of a float64.""" _HEX_FLOAT64_RE: Final = re.compile(r"^0x[0-9a-fA-F]{16}$") def hex_float64(value: str) -> HexFloat64: """Validate `value` as a HexFloat64 and brand it. Raises ValueError if `value` is not exactly `0x` followed by 16 hex digits. """ if not _HEX_FLOAT64_RE.fullmatch(value): raise ValueError(f"Expected '0x' followed by 16 hex digits, got {value!r}") return HexFloat64(value) Float64FillValue = float | int | Float64SpecialFillValue | HexFloat64 """Permitted JSON shape of the `fill_value` field for `float64`. Either a JSON number, one of the named non-finite sentinels (`"NaN"`, `"Infinity"`, `"-Infinity"`), or a `HexFloat64` (`0xYYYYYYYYYYYYYYYY` string encoding the unsigned-integer representation of the IEEE 754 value). """ CANONICAL_NAN_HEX_FLOAT64: Final = "0x7ff8000000000000" """Canonical hex form of the float64 NaN sentinel `"NaN"`. Per spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L72-L74) the named `"NaN"` sentinel denotes the float with sign=0, the most significant mantissa bit set, and all other mantissa bits zero (the IEEE 754 default quiet NaN). Other NaN bit patterns must be encoded with the explicit hex-string form. """ CANONICAL_POSITIVE_INFINITY_HEX_FLOAT64: Final = "0x7ff0000000000000" """Canonical hex form of the float64 `"Infinity"` sentinel.""" CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT64: Final = "0xfff0000000000000" """Canonical hex form of the float64 `"-Infinity"` sentinel.""" FLOAT64_DATA_TYPE: Final = DataTypeDefinition( name=FLOAT64_DATA_TYPE_NAME, configuration=EmptyConfiguration, fill_value=Float64FillValue, fill_value_rules=float_fill_value_rules("float64", hex_float64), fill_value_canonical=float_fill_value_canonical(64), storage=multi_byte, ) """The `float64` data type. It is written as a bare name and has no configuration. Its fill value is a number, a named value or a hex string.""" __all__ = [ "CANONICAL_NAN_HEX_FLOAT64", "CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT64", "CANONICAL_POSITIVE_INFINITY_HEX_FLOAT64", "FLOAT64_DATA_TYPE", "FLOAT64_DATA_TYPE_NAME", "Float64DataTypeName", "Float64FillValue", "Float64SpecialFillValue", "HexFloat64", "hex_float64", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py000066400000000000000000000021341527014137600332210ustar00rootroot00000000000000""" Zarr v3 `int16` data type. See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ from typing import Annotated, Final, Literal from annotated_types import Interval from zarr_metadata.definition._definition import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) INT16_DATA_TYPE_NAME: Final = "int16" """The `data_type` value for the `int16` type.""" Int16DataTypeName = Literal["int16"] """Literal type of the `data_type` field for `int16`.""" Int16FillValue = Annotated[int, Interval(ge=-(2**15), le=2**15 - 1)] """Permitted JSON shape of the `fill_value` field for `int16`: a JSON integer in [-32768, 32767].""" INT16_DATA_TYPE: Final = DataTypeDefinition( name=INT16_DATA_TYPE_NAME, configuration=EmptyConfiguration, fill_value=Int16FillValue, storage=multi_byte, ) """The `int16` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" __all__ = [ "INT16_DATA_TYPE", "INT16_DATA_TYPE_NAME", "Int16DataTypeName", "Int16FillValue", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py000066400000000000000000000021401527014137600332140ustar00rootroot00000000000000""" Zarr v3 `int32` data type. See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ from typing import Annotated, Final, Literal from annotated_types import Interval from zarr_metadata.definition._definition import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) INT32_DATA_TYPE_NAME: Final = "int32" """The `data_type` value for the `int32` type.""" Int32DataTypeName = Literal["int32"] """Literal type of the `data_type` field for `int32`.""" Int32FillValue = Annotated[int, Interval(ge=-(2**31), le=2**31 - 1)] """Permitted JSON shape of the `fill_value` field for `int32`: a JSON integer in [-2**31, 2**31 - 1].""" INT32_DATA_TYPE: Final = DataTypeDefinition( name=INT32_DATA_TYPE_NAME, configuration=EmptyConfiguration, fill_value=Int32FillValue, storage=multi_byte, ) """The `int32` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" __all__ = [ "INT32_DATA_TYPE", "INT32_DATA_TYPE_NAME", "Int32DataTypeName", "Int32FillValue", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py000066400000000000000000000021401527014137600332210ustar00rootroot00000000000000""" Zarr v3 `int64` data type. See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ from typing import Annotated, Final, Literal from annotated_types import Interval from zarr_metadata.definition._definition import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) INT64_DATA_TYPE_NAME: Final = "int64" """The `data_type` value for the `int64` type.""" Int64DataTypeName = Literal["int64"] """Literal type of the `data_type` field for `int64`.""" Int64FillValue = Annotated[int, Interval(ge=-(2**63), le=2**63 - 1)] """Permitted JSON shape of the `fill_value` field for `int64`: a JSON integer in [-2**63, 2**63 - 1].""" INT64_DATA_TYPE: Final = DataTypeDefinition( name=INT64_DATA_TYPE_NAME, configuration=EmptyConfiguration, fill_value=Int64FillValue, storage=multi_byte, ) """The `int64` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" __all__ = [ "INT64_DATA_TYPE", "INT64_DATA_TYPE_NAME", "Int64DataTypeName", "Int64FillValue", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py000066400000000000000000000021071527014137600331420ustar00rootroot00000000000000""" Zarr v3 `int8` data type. See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ from typing import Annotated, Final, Literal from annotated_types import Interval from zarr_metadata.definition._definition import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, single_byte, ) INT8_DATA_TYPE_NAME: Final = "int8" """The `data_type` value for the `int8` type.""" Int8DataTypeName = Literal["int8"] """Literal type of the `data_type` field for `int8`.""" Int8FillValue = Annotated[int, Interval(ge=-(2**7), le=2**7 - 1)] """Permitted JSON shape of the `fill_value` field for `int8`: a JSON integer in [-128, 127].""" INT8_DATA_TYPE: Final = DataTypeDefinition( name=INT8_DATA_TYPE_NAME, configuration=EmptyConfiguration, fill_value=Int8FillValue, storage=single_byte, ) """The `int8` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" __all__ = [ "INT8_DATA_TYPE", "INT8_DATA_TYPE_NAME", "Int8DataTypeName", "Int8FillValue", ] numpy_datetime64.py000066400000000000000000000043731527014137600354060ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type""" Zarr `numpy.datetime64` data type (zarr-extensions). See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md """ from typing import Final, Literal, NotRequired from typing_extensions import ReadOnly, TypedDict from zarr_metadata._numpy import ( NumpyTimeScaleFactor, NumpyTimeTicks, numpy_time_fill_value_canonical, ) from zarr_metadata.v3._definition import DataTypeDefinition from zarr_metadata.v3.data_type._numpy_time import ( NUMPY_TIME_UNIT, NumpyTimeUnit, numpy_time_storage, ) NUMPY_DATETIME64_DATA_TYPE_NAME: Final = "numpy.datetime64" """The `name` field value of the `numpy.datetime64` data type.""" NumpyDatetime64DataTypeName = Literal["numpy.datetime64"] """Literal type of the `name` field of the `numpy.datetime64` data type.""" class NumpyDatetime64Configuration(TypedDict, closed=True): """ Configuration for the `numpy.datetime64` data type. Attributes ---------- unit A string encoding a unit of time. scale_factor The multiplier relative to the unit. """ unit: ReadOnly[NumpyTimeUnit] scale_factor: ReadOnly[NumpyTimeScaleFactor] class NumpyDatetime64(TypedDict, closed=True): """`numpy.datetime64` data type metadata.""" name: NumpyDatetime64DataTypeName configuration: NumpyDatetime64Configuration must_understand: NotRequired[bool] NumpyDatetime64FillValue = NumpyTimeTicks | Literal["NaT"] """Permitted JSON shape of the `fill_value` field for `numpy.datetime64`. Either a JSON integer (count of `unit * scale_factor` since the epoch), or the string `"NaT"` (equivalent to the integer `-2**63`). """ NUMPY_DATETIME64_DATA_TYPE: Final = DataTypeDefinition( name=NUMPY_DATETIME64_DATA_TYPE_NAME, configuration=NumpyDatetime64Configuration, fill_value=NumpyDatetime64FillValue, fill_value_canonical=numpy_time_fill_value_canonical, storage=numpy_time_storage, ) """The `numpy.datetime64` data type.""" __all__ = [ "NUMPY_DATETIME64_DATA_TYPE", "NUMPY_DATETIME64_DATA_TYPE_NAME", "NUMPY_TIME_UNIT", "NumpyDatetime64", "NumpyDatetime64Configuration", "NumpyDatetime64DataTypeName", "NumpyDatetime64FillValue", "NumpyTimeUnit", ] numpy_timedelta64.py000066400000000000000000000042721527014137600355600ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type""" Zarr `numpy.timedelta64` data type (zarr-extensions). See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.timedelta64/README.md """ from typing import Final, Literal, NotRequired from typing_extensions import ReadOnly, TypedDict from zarr_metadata._numpy import ( NumpyTimeScaleFactor, NumpyTimeTicks, numpy_time_fill_value_canonical, ) from zarr_metadata.v3._definition import DataTypeDefinition from zarr_metadata.v3.data_type._numpy_time import NumpyTimeUnit, numpy_time_storage NUMPY_TIMEDELTA64_DATA_TYPE_NAME: Final = "numpy.timedelta64" """The `name` field value of the `numpy.timedelta64` data type.""" NumpyTimedelta64DataTypeName = Literal["numpy.timedelta64"] """Literal type of the `name` field of the `numpy.timedelta64` data type.""" class NumpyTimedelta64Configuration(TypedDict, closed=True): """ Configuration for the `numpy.timedelta64` data type. Attributes ---------- unit A string encoding a unit of time. scale_factor The multiplier relative to the unit. """ unit: ReadOnly[NumpyTimeUnit] scale_factor: ReadOnly[NumpyTimeScaleFactor] class NumpyTimedelta64(TypedDict, closed=True): """`numpy.timedelta64` data type metadata.""" name: NumpyTimedelta64DataTypeName configuration: NumpyTimedelta64Configuration must_understand: NotRequired[bool] NumpyTimedelta64FillValue = NumpyTimeTicks | Literal["NaT"] """Permitted JSON shape of the `fill_value` field for `numpy.timedelta64`. Either a JSON integer (a count of `unit * scale_factor`), or the string `"NaT"` (equivalent to the integer `-2**63`). """ NUMPY_TIMEDELTA64_DATA_TYPE: Final = DataTypeDefinition( name=NUMPY_TIMEDELTA64_DATA_TYPE_NAME, configuration=NumpyTimedelta64Configuration, fill_value=NumpyTimedelta64FillValue, fill_value_canonical=numpy_time_fill_value_canonical, storage=numpy_time_storage, ) """The `numpy.timedelta64` data type.""" __all__ = [ "NUMPY_TIMEDELTA64_DATA_TYPE", "NUMPY_TIMEDELTA64_DATA_TYPE_NAME", "NumpyTimedelta64", "NumpyTimedelta64Configuration", "NumpyTimedelta64DataTypeName", "NumpyTimedelta64FillValue", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py000066400000000000000000000114631527014137600330560ustar00rootroot00000000000000""" Zarr v3 `r` raw-bytes data type (parameterised by bit count). The `data_type` value is a string of the form `r` where `N` is a positive multiple of 8 (e.g. `r8`, `r16`, `r24`). See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L46-L47; fill value: https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L97-L99) """ from collections.abc import Iterator from typing import Final, NewType from typing_extensions import TypedDict from zarr_metadata._json import ValidationProblem from zarr_metadata.definition._definition import Nested from zarr_metadata.v3._definition import ( RAW_BYTES_NAME, RAW_BYTES_NAME_PATTERN, DataTypeDefinition, StorageClass, ) from zarr_metadata.v3.data_type._byte import ByteValue RawBytesDataTypeName = NewType("RawBytesDataTypeName", str) """A spec-conformant `r` raw-bytes name (e.g. `"r8"`, `"r16"`). "raw bits, variable size given by *, limited to be a multiple of 8": https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L46-L47 """ def raw_bytes_dtype_name(value: str) -> RawBytesDataTypeName: """Validate `value` as a `r` raw-bytes name and brand it. Raises ValueError if `value` is not `r` followed by a positive multiple of 8. """ match = RAW_BYTES_NAME_PATTERN.fullmatch(value) if match is None: raise ValueError(f"Expected 'r' followed by a positive integer, got {value!r}") bits = int(match.group(1)) if bits == 0 or bits % 8 != 0: raise ValueError(f"Expected 'r' where N is a positive multiple of 8, got {value!r}") return RawBytesDataTypeName(value) RawBytesFillValue = tuple[ByteValue, ...] """Permitted JSON shape of the `fill_value` field for `r`. A JSON array of N/8 integers in `[0, 255]` (one per byte). """ class RawBytesConfiguration(TypedDict, closed=True): """The configuration that a raw-bytes name carries: its size in bits, such as `{"bits": 16}` for `r16`. A document writes the name, never this object. The definition of `r*` reads a name into this object, and the simplest spelling writes it back as the name. """ bits: int def _rules(configuration: RawBytesConfiguration, nested: Nested) -> Iterator[ValidationProblem]: """Check the size: "raw bits, variable size given by *, limited to be a multiple of 8". Zero bits is not a valid type.""" bits = configuration["bits"] if bits == 0 or bits % 8 != 0: yield ValidationProblem( ("bits",), f"expected a size in bits that is a positive multiple of 8, got {bits}", "invalid_value", ) def _fill_value_rules( configuration: RawBytesConfiguration, nested: Nested, value: RawBytesFillValue ) -> Iterator[ValidationProblem]: """Check that the fill value has one byte value for each 8 bits of the size. The spec text says `N` values for `r`, but `N` counts bits (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L897-L900), and one value is one byte. """ expected = configuration["bits"] // 8 if len(value) != expected: yield ValidationProblem( (), f"expected {expected} byte values, got {len(value)}", "invalid_value" ) def _storage(configuration: RawBytesConfiguration, nested: Nested) -> StorageClass | None: """Return single-byte storage for `r8`. Return None (unknown) for wider raw bits, because the spec does not yet say anything about their byte order. "r8, r16, and r24 should be understood as fall-back types of respectively 1, 2, and 3 byte length". About byte order, the spec says only that it looks for "more feedback and prototypes of code using the r*, raw bits, for various endianness" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L897-L908). """ return "single_byte" if configuration["bits"] == 8 else None RAW_BYTES_DATA_TYPE: Final = DataTypeDefinition( name=RAW_BYTES_NAME, configuration=RawBytesConfiguration, rules=_rules, fill_value=RawBytesFillValue, fill_value_rules=_fill_value_rules, storage=_storage, ) """Raw bits, `r*`: a single data type whose name carries its size. `r16` reads as `r*` with `{"bits": 16}`. A size that is not a positive multiple of 8 is a problem of the field that names it. The simplest spelling writes the size back into the name in decimal, so `r008` becomes `r8`. """ __all__ = [ "RAW_BYTES_DATA_TYPE", "RawBytesConfiguration", "RawBytesDataTypeName", "RawBytesFillValue", "raw_bytes_dtype_name", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py000066400000000000000000000024571527014137600335760ustar00rootroot00000000000000""" Zarr `string` data type (variable-length utf-8, zarr-extensions). See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/string/README.md """ from typing import Final, Literal from zarr_metadata.definition._definition import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, variable_length, ) STRING_DATA_TYPE_NAME: Final = "string" """The `data_type` value for the `string` type.""" StringDataTypeName = Literal["string"] """Literal type of the `data_type` field for `string`.""" StringFillValue = str """Permitted JSON shape of the `fill_value` field for `string`: a JSON unicode string.""" STRING_DATA_TYPE: Final = DataTypeDefinition( name=STRING_DATA_TYPE_NAME, configuration=EmptyConfiguration, fill_value=StringFillValue, storage=variable_length, ) """The `string` data type. It is written as a bare name and has no configuration. Its values are "variable-length UTF8 strings" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/string/README.md?plain=1#L3), and each value is stored in as many bytes as it needs. """ __all__ = [ "STRING_DATA_TYPE", "STRING_DATA_TYPE_NAME", "StringDataTypeName", "StringFillValue", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py000066400000000000000000000173611527014137600336140ustar00rootroot00000000000000""" Zarr `struct` data type (heterogeneous record, zarr-extensions). See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/struct/README.md """ from collections.abc import Iterator, Mapping from typing import Final, Literal, NotRequired from typing_extensions import ReadOnly, TypedDict from zarr_metadata._common import JSONValue from zarr_metadata._json import ValidationProblem, shown from zarr_metadata.definition._definition import ( Nested, fill_value_problems, spelled_canonically, ) from zarr_metadata.v3._common import ZarrV3DataTypeField from zarr_metadata.v3._definition import ( DataTypeDefinition, StorageClass, storage_of, ) STRUCT_DATA_TYPE_NAME: Final = "struct" """The `name` field value of the `struct` data type.""" StructDataTypeName = Literal["struct"] """Literal type of the `name` field of the `struct` data type.""" class StructField(TypedDict, closed=True): """ A single field entry inside a structured dtype. Attributes ---------- name The field name (must be unique within a struct and non-empty). data_type The field's data type. Recursive: may be a bare-string primitive or a named-config envelope including another `struct`. """ name: ReadOnly[str] data_type: ReadOnly[ZarrV3DataTypeField] class StructConfiguration(TypedDict, closed=True): """Configuration for the `struct` data type.""" fields: ReadOnly[tuple[StructField, ...]] class Struct(TypedDict, closed=True): """`struct` data type metadata.""" name: StructDataTypeName configuration: StructConfiguration must_understand: NotRequired[bool] StructFillValue = Mapping[str, JSONValue] """Permitted JSON shape of the `fill_value` field for `struct`. A JSON object mapping each field name to that field's fill value. Field fill values are themselves shaped per the field's `data_type`, recursively. """ def _rules(configuration: StructConfiguration, nested: Nested) -> Iterator[ValidationProblem]: """Check that there is at least one field, that field names are non-empty and distinct, and that field types have a fixed size. Names must be distinct because a fill value addresses fields by name. "Variable-length data types (e.g. "string") MUST NOT be used as field types, as they do not have a fixed encoded size" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/struct/README.md?plain=1#L42-L49): this is checked for each field type that the scope read. A field type whose storage is unknown is not checked. """ fields = configuration["fields"] if len(fields) == 0: yield ValidationProblem(("fields",), "expected at least one struct field", "invalid_value") seen: dict[str, int] = {} for index, member in enumerate(fields): name = member["name"] if name == "": yield ValidationProblem( ("fields", index, "name"), "expected a non-empty field name", "invalid_value" ) first = seen.setdefault(name, index) if first != index: yield ValidationProblem( ("fields", index, "name"), f"duplicate field name {name!r}, already used by field {first}", "invalid_value", ) field_type = nested.get(("fields", index, "data_type")) if field_type is not None and storage_of(field_type) == "variable_length": yield ValidationProblem( ("fields", index, "data_type"), f"expected a data type of fixed size, got {shown(field_type.name)}, whose values " "vary in size", "invalid_value", ) def _fill_value_rules( configuration: StructConfiguration, nested: Nested, value: StructFillValue ) -> Iterator[ValidationProblem]: """Check that the fill value has an entry for every field and for nothing else, and that each entry is valid for its field's type. Every field needs a fill value that is valid for its type (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/struct/README.md?plain=1#L221-L224). If the scope did not read a field type, or the reading of the struct does not hold a reading of that field type, the fill value of that field is not checked. """ names = [member["name"] for member in configuration["fields"]] declared = set(names) for index, name in enumerate(names): if name not in value: yield ValidationProblem( (name,), f"expected a fill value for struct field {name!r}", "missing_key" ) continue field_type = nested.get(("fields", index, "data_type")) if field_type is not None: yield from fill_value_problems(field_type, value[name], (name,)) for key in value: if key not in declared: yield ValidationProblem((key,), f"no struct field is named {key!r}", "unknown_key") def _fill_value_canonical( configuration: StructConfiguration, nested: Nested, value: StructFillValue ) -> dict[str, JSONValue]: """Return each field's fill value in the canonical spelling of that field's type, in the order the fields are declared. If the scope did not read a field type, or the reading of the struct does not hold a reading of that field type, the fill value of that field is kept as written. """ spelled: dict[str, JSONValue] = {} for index, member in enumerate(configuration["fields"]): name = member["name"] field_type = nested.get(("fields", index, "data_type")) held = value[name] spelled[name] = held if field_type is None else spelled_canonically(field_type, held) return spelled def _storage(configuration: StructConfiguration, nested: Nested) -> StorageClass | None: """Return the storage of the packed field values: multi-byte if any field holds multi-byte numbers, and single-byte if every field is single-byte. A nested struct "is encoded as the packed concatenation of its own sub-fields, recursively" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/struct/README.md?plain=1#L142-L144), and the `bytes` codec "MUST be configured with an explicit endian" for a struct that "contains multi-byte numeric fields", while a struct "composed entirely of single-byte fields" may omit it (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/struct/README.md?plain=1#L211-L217). If a field's storage is unknown, the struct's storage is unknown too, unless the storage of another field decides it. The rules reject a struct with a field whose values vary in size, so this function is never called for such a struct. """ found = { None if field_type is None else storage_of(field_type) for field_type in ( nested.get(("fields", index, "data_type")) for index in range(len(configuration["fields"])) ) } if "multi_byte" in found: return "multi_byte" return "single_byte" if found == {"single_byte"} else None STRUCT_DATA_TYPE: Final = DataTypeDefinition( name=STRUCT_DATA_TYPE_NAME, configuration=StructConfiguration, rules=_rules, fill_value=StructFillValue, fill_value_rules=_fill_value_rules, fill_value_canonical=_fill_value_canonical, storage=_storage, ) """The `struct` data type: a record of named fields. The type of each field is a nested field.""" __all__ = [ "STRUCT_DATA_TYPE", "STRUCT_DATA_TYPE_NAME", "Struct", "StructConfiguration", "StructDataTypeName", "StructField", "StructFillValue", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py000066400000000000000000000021411527014137600334040ustar00rootroot00000000000000""" Zarr v3 `uint16` data type. See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ from typing import Annotated, Final, Literal from annotated_types import Interval from zarr_metadata.definition._definition import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) UINT16_DATA_TYPE_NAME: Final = "uint16" """The `data_type` value for the `uint16` type.""" Uint16DataTypeName = Literal["uint16"] """Literal type of the `data_type` field for `uint16`.""" Uint16FillValue = Annotated[int, Interval(ge=0, le=2**16 - 1)] """Permitted JSON shape of the `fill_value` field for `uint16`: a JSON integer in [0, 65535].""" UINT16_DATA_TYPE: Final = DataTypeDefinition( name=UINT16_DATA_TYPE_NAME, configuration=EmptyConfiguration, fill_value=Uint16FillValue, storage=multi_byte, ) """The `uint16` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" __all__ = [ "UINT16_DATA_TYPE", "UINT16_DATA_TYPE_NAME", "Uint16DataTypeName", "Uint16FillValue", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py000066400000000000000000000021451527014137600334060ustar00rootroot00000000000000""" Zarr v3 `uint32` data type. See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ from typing import Annotated, Final, Literal from annotated_types import Interval from zarr_metadata.definition._definition import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) UINT32_DATA_TYPE_NAME: Final = "uint32" """The `data_type` value for the `uint32` type.""" Uint32DataTypeName = Literal["uint32"] """Literal type of the `data_type` field for `uint32`.""" Uint32FillValue = Annotated[int, Interval(ge=0, le=2**32 - 1)] """Permitted JSON shape of the `fill_value` field for `uint32`: a JSON integer in [0, 2**32 - 1].""" UINT32_DATA_TYPE: Final = DataTypeDefinition( name=UINT32_DATA_TYPE_NAME, configuration=EmptyConfiguration, fill_value=Uint32FillValue, storage=multi_byte, ) """The `uint32` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" __all__ = [ "UINT32_DATA_TYPE", "UINT32_DATA_TYPE_NAME", "Uint32DataTypeName", "Uint32FillValue", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py000066400000000000000000000021451527014137600334130ustar00rootroot00000000000000""" Zarr v3 `uint64` data type. See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ from typing import Annotated, Final, Literal from annotated_types import Interval from zarr_metadata.definition._definition import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, multi_byte, ) UINT64_DATA_TYPE_NAME: Final = "uint64" """The `data_type` value for the `uint64` type.""" Uint64DataTypeName = Literal["uint64"] """Literal type of the `data_type` field for `uint64`.""" Uint64FillValue = Annotated[int, Interval(ge=0, le=2**64 - 1)] """Permitted JSON shape of the `fill_value` field for `uint64`: a JSON integer in [0, 2**64 - 1].""" UINT64_DATA_TYPE: Final = DataTypeDefinition( name=UINT64_DATA_TYPE_NAME, configuration=EmptyConfiguration, fill_value=Uint64FillValue, storage=multi_byte, ) """The `uint64` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" __all__ = [ "UINT64_DATA_TYPE", "UINT64_DATA_TYPE_NAME", "Uint64DataTypeName", "Uint64FillValue", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py000066400000000000000000000021171527014137600333300ustar00rootroot00000000000000""" Zarr v3 `uint8` data type. See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ from typing import Annotated, Final, Literal from annotated_types import Interval from zarr_metadata.definition._definition import EmptyConfiguration from zarr_metadata.v3._definition import ( DataTypeDefinition, single_byte, ) UINT8_DATA_TYPE_NAME: Final = "uint8" """The `data_type` value for the `uint8` type.""" Uint8DataTypeName = Literal["uint8"] """Literal type of the `data_type` field for `uint8`.""" Uint8FillValue = Annotated[int, Interval(ge=0, le=2**8 - 1)] """Permitted JSON shape of the `fill_value` field for `uint8`: a JSON integer in [0, 255].""" UINT8_DATA_TYPE: Final = DataTypeDefinition( name=UINT8_DATA_TYPE_NAME, configuration=EmptyConfiguration, fill_value=Uint8FillValue, storage=single_byte, ) """The `uint8` data type. It is written as a bare name and has no configuration. Its fill value is an integer in its range.""" __all__ = [ "UINT8_DATA_TYPE", "UINT8_DATA_TYPE_NAME", "Uint8DataTypeName", "Uint8FillValue", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/definition.py000066400000000000000000000156221527014137600324440ustar00rootroot00000000000000"""The kinds of metadata in a Zarr v3 document, and the scopes this package provides for it. The kind of metadata a definition defines is its type: `CodecDefinition` (with the codec's `kind`, and its `size`: `"static"` if the output size is determined by the input size, or `"dynamic"` if it depends on the values), `DataTypeDefinition`, `ChunkGridDefinition`, `ChunkKeyEncodingDefinition`, `StorageTransformerDefinition`, and, for the members of a document that the spec does not define, `ArrayExtraFieldDefinition` and `GroupExtraFieldDefinition`. What a definition is and how a field is read are in `zarr_metadata.definition`, which both formats share. `ZarrV3Context` is the v3 scope, with one slot for each kind: `data_types`, `chunk_grids`, `chunk_key_encodings`, `codecs`, `storage_transformers`, `array_extra_fields` and `group_extra_fields`. `CORE` and `CORE_AND_EXTENSIONS` are the v3 scopes this package provides. A scope reads whole documents as well as fields. `read_array_metadata_v3(document, context=CONTEXT)`, from `zarr_metadata.model`, reads each extension point of a v3 array document through the definitions in `CONTEXT`, and so do the model's `from_json` and `from_key_value`. The fill value is checked against the data type it names, by that data type's definition. The chunk grid is checked against the shape, by the grid's definition. The codecs are checked as a pipeline, each by its definition against the chunk it receives. A member that holds another metadata field is annotated with the field alias of its kind (a shard's `codecs: tuple[ZarrV3CodecField, ...]`) and is read in the same scope as its parent field. A problem with the nested field belongs to that field and is reported at its location. The parent field is still read, as a document containing the nested field would be. A member that accepts only codecs of static size is annotated `ZarrV3StaticCodecField`. A shard's `index_codecs` uses this, because a reader locates the index from a size it knows before reading it. A codec of dynamic size in such a member is a problem at its location, when the field is read in a scope. A name that nothing claims is not checked, and its size is unknown, like everything else about it. `ZarrV3MetadataFieldJSON` describes the same JSON, but it is checked only as JSON, so a definition rejects a member typed with it. An extension with nothing to configure uses `EmptyConfiguration`, and is written with its name only. A data type also declares how its values are stored. `storage` is a function of the configuration and the nested fields that returns a `StorageClass`: single bytes, several bytes at a time, or each value in as many bytes as it needs. A struct's storage depends on its fields. `storage_of(data_type)` calls it for a data type field the scope read. The `bytes` codec requires an `endian` for multi-byte numbers, and a struct rejects a field whose values vary in size. A data type that does not declare storage leaves it unknown. A chunk grid declares which array shapes it fits with `shape_rules`, a function that yields the spec violations in a grid with its configuration over an array of a given shape (a dimension with no chunk length, or chunks that do not cover the dimension), located in the configuration. A grid that does not declare shape rules fits every shape. It also declares the lengths of its chunks along each axis of an array it fits, with `chunk_lengths`: a set per axis, because a rectilinear grid's chunks differ in length. A reading stores both results for the grid it read: an entry for each dimension of the shape, with None where the lengths are unknown. A codec is checked against its input. The array passes its first codec a `Chunk`: the lengths of the grid's chunks along each of the array's dimensions, and the array's data type field, with None for unknown parts. A codec that receives an array declares the spec violations for a given chunk with `chunk_rules`, located in its configuration (for example, a `transpose` whose `order` has the wrong number of axes). An array -> array codec declares what it passes to the next codec with `transition`, whatever its chunk rules found. For example, `transpose` permutes the axes. A reading reads the codec fields as a pipeline. It checks their order (array -> array codecs, one array -> bytes codec, bytes -> bytes codecs), then checks each codec against the chunk it receives, and records a `Stage` for each codec with that chunk. A codec that contains its own pipelines declares what each one receives with `pipelines`, keyed by the configuration member that holds each pipeline. For example, a shard's inner codecs receive its inner chunks, and its index codecs receive the shard index. Each pipeline is read the same way, and its stages are stored as the codec's `Stage.inner`. The reader does not guess. After a codec the scope did not read, or after a codec that does not declare its output, the next codec receives a chunk with nothing known about it, `Chunk()`, which never causes a problem. A later codec can still declare its own output. Raw bits are the only data type whose name encodes its configuration. A document writes `r` followed by the size in bits, and `r16` is read as `r*` (the notation in the specification's table of data types) with `{"bits": 16}`. A reader that wants to read raw bits its own way defines `r*`. A data type named `r16` is rejected, because that name is read as `r*`. `r*` itself is notation, and a document that uses it as a name refers to nothing in any scope. **JSON Schema.** `zarr_metadata.json_schema` writes a v3 field, or a whole `zarr.json`, as a JSON Schema. """ from zarr_metadata.v3._common import ( ZarrV3ChunkGridField, ZarrV3ChunkKeyEncodingField, ZarrV3CodecField, ZarrV3DataTypeField, ZarrV3StaticCodecField, ZarrV3StorageTransformerField, ) from zarr_metadata.v3._context import ZarrV3Context, ZarrV3ContextSlots, ZarrV3Kind from zarr_metadata.v3._core import CORE, CORE_AND_EXTENSIONS from zarr_metadata.v3._definition import ( ArrayExtraFieldDefinition, Chunk, ChunkGridDefinition, ChunkKeyEncodingDefinition, CodecDefinition, CodecKind, CodecSize, DataTypeDefinition, ExtraFieldDefinition, GroupExtraFieldDefinition, Lengths, StorageClass, StorageTransformerDefinition, storage_of, ) from zarr_metadata.v3._pipeline import Stage __all__ = [ "CORE", "CORE_AND_EXTENSIONS", "ArrayExtraFieldDefinition", "Chunk", "ChunkGridDefinition", "ChunkKeyEncodingDefinition", "CodecDefinition", "CodecKind", "CodecSize", "DataTypeDefinition", "ExtraFieldDefinition", "GroupExtraFieldDefinition", "Lengths", "Stage", "StorageClass", "StorageTransformerDefinition", "ZarrV3ChunkGridField", "ZarrV3ChunkKeyEncodingField", "ZarrV3CodecField", "ZarrV3Context", "ZarrV3ContextSlots", "ZarrV3DataTypeField", "ZarrV3Kind", "ZarrV3StaticCodecField", "ZarrV3StorageTransformerField", "storage_of", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/src/zarr_metadata/v3/group.py000066400000000000000000000024041527014137600314420ustar00rootroot00000000000000"""Zarr v3 group metadata types. See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#group-metadata """ from collections.abc import Mapping from typing import Final, Literal, NotRequired from typing_extensions import TypedDict from zarr_metadata._common import JSONValue from zarr_metadata.v3._common import ZarrV3ExtensionField class ZarrV3GroupMetadataJSON(TypedDict, extra_items=ZarrV3ExtensionField): """ Zarr v3 group metadata document (the `zarr.json` content for a group). Extra keys may contain arbitrary JSON values. See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#group-metadata """ zarr_format: Literal[3] node_type: Literal["group"] attributes: NotRequired[Mapping[str, JSONValue]] ZarrV3GroupMetadataStoreKey = Literal["zarr.json"] """Literal type of the store key holding a v3 group's metadata document.""" ZARR_V3_GROUP_METADATA_STORE_KEY: Final[ZarrV3GroupMetadataStoreKey] = "zarr.json" """The store key a v3 group's metadata document is persisted under. v3 uses one key for both node types; the document's `node_type` field distinguishes a group from an array. """ __all__ = [ "ZARR_V3_GROUP_METADATA_STORE_KEY", "ZarrV3GroupMetadataJSON", "ZarrV3GroupMetadataStoreKey", ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/000077500000000000000000000000001527014137600251415ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/__init__.py000066400000000000000000000000371527014137600272520ustar00rootroot00000000000000"""Tests for zarr-metadata.""" zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/benchmarks/000077500000000000000000000000001527014137600272565ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/benchmarks/__init__.py000066400000000000000000000001211527014137600313610ustar00rootroot00000000000000"""Timings of reading documents, and the smoke test that keeps them runnable.""" zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/benchmarks/read_documents.py000066400000000000000000000105341527014137600326270ustar00rootroot00000000000000"""Time how long it takes to read a metadata document written by zarr-python. There are three documents. Each was written by `zarr==3.4.1` and round-tripped through `json.dumps`/`json.loads`, so each holds what a reader would receive from a store: - a v3 array, `zarr.create_array(store, shape=(100, 100), chunks=(10, 10), dtype="f8", zarr_format=3)`; - the same array as v2; - a v3 group holding 200 `int32` arrays of shape `(10,)`, consolidated with `zarr.consolidate_metadata`. Run from the package directory: PYTHONHASHSEED=0 uv run --group test python -m tests.benchmarks.read_documents Each case is timed with `timeit`. The script reports the best of ten repeats, as the mean time per read in microseconds. `--profile` instead prints a cProfile of the v3 array read, sorted by cumulative time. `test_read_documents.py` runs each case once, so the script keeps working as the package changes. """ from __future__ import annotations import argparse import cProfile import json import pstats import timeit from typing import TYPE_CHECKING, Any from zarr_metadata import ZarrV2ArrayMetadata, ZarrV3ArrayMetadata, ZarrV3GroupMetadata if TYPE_CHECKING: from collections.abc import Callable V3_ARRAY: dict[str, Any] = { "shape": [100, 100], "data_type": "float64", "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [10, 10]}}, "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, "fill_value": 0.0, "codecs": [ {"name": "bytes", "configuration": {"endian": "little"}}, {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, ], "attributes": {}, "zarr_format": 3, "node_type": "array", "storage_transformers": [], } V2_ARRAY: dict[str, Any] = { "shape": [100, 100], "chunks": [10, 10], "dtype": " tuple[str, dict[str, Any]]: return f"a{index}", { "shape": [10], "data_type": "int32", "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [10]}}, "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, "fill_value": 0, "codecs": [ {"name": "bytes", "configuration": {"endian": "little"}}, {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, ], "attributes": {}, "zarr_format": 3, "node_type": "array", "storage_transformers": [], } V3_GROUP: dict[str, Any] = { "attributes": {}, "zarr_format": 3, "consolidated_metadata": { "kind": "inline", "must_understand": False, "metadata": dict(_child(index) for index in range(200)), }, "node_type": "group", } def _round_tripped(document: dict[str, Any]) -> dict[str, Any]: return json.loads(json.dumps(document)) CASES: tuple[tuple[str, Callable[[], object], int], ...] = ( ("ZarrV3ArrayMetadata.from_json", lambda: ZarrV3ArrayMetadata.from_json(_V3_ARRAY), 2000), ("ZarrV2ArrayMetadata.from_json", lambda: ZarrV2ArrayMetadata.from_json(_V2_ARRAY), 2000), ( "ZarrV3GroupMetadata.from_json, 200 consolidated arrays", lambda: ZarrV3GroupMetadata.from_json(_V3_GROUP), 10, ), ) _V3_ARRAY = _round_tripped(V3_ARRAY) _V2_ARRAY = _round_tripped(V2_ARRAY) _V3_GROUP = _round_tripped(V3_GROUP) def main() -> None: parser = argparse.ArgumentParser(description="How long reading a metadata document takes.") parser.add_argument("--profile", action="store_true", help="profile the v3 array read") parser.add_argument("--repeat", type=int, default=10, help="timeit repeats; the best is shown") arguments = parser.parse_args() if arguments.profile: profiler = cProfile.Profile() profiler.runcall(lambda: [ZarrV3ArrayMetadata.from_json(_V3_ARRAY) for _ in range(1000)]) pstats.Stats(profiler).sort_stats("cumulative").print_stats(45) return for name, read, number in CASES: read() # compile the checkers and warm every cache, as after reading many documents best = min(timeit.repeat(read, number=number, repeat=arguments.repeat)) print(f"{name}: {best / number * 1e6:.1f} us") if __name__ == "__main__": main() zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/benchmarks/test_read_documents.py000066400000000000000000000006721527014137600336700ustar00rootroot00000000000000"""Check that each benchmark case runs, so the script keeps working as the package changes.""" from __future__ import annotations import pytest from tests.benchmarks.read_documents import CASES @pytest.mark.parametrize( ("name", "read"), [(name, read) for name, read, _ in CASES], ids=[name for name, _, _ in CASES] ) def test_each_case_reads(name: str, read: object) -> None: assert callable(read) assert read() is not None zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/000077500000000000000000000000001527014137600262415ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/__init__.py000066400000000000000000000000001527014137600303400ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/_cases.py000066400000000000000000000037331527014137600300560ustar00rootroot00000000000000from __future__ import annotations import re from dataclasses import dataclass from typing import TYPE_CHECKING, Generic, TypeVar import pytest if TYPE_CHECKING: from contextlib import AbstractContextManager TIn = TypeVar("TIn") TOut = TypeVar("TOut") @dataclass(frozen=True) class Expect(Generic[TIn, TOut]): """A test case with explicit input, expected output, and a human-readable id.""" input: TIn output: TOut id: str @dataclass(frozen=True) class ExpectFail(Generic[TIn]): """A test case that should raise an exception. `msg` is a regex matched against the exception text (pytest's native `match=` semantics). Leave it `None` to assert only the exception type. Set `escape=True` when `msg` is a literal that contains regex metacharacters such as `(`, `[`, or `.`; `escape` has no effect when `msg` is `None`. """ input: TIn exception: type[Exception] id: str msg: str | None = None escape: bool = False def raises(self) -> AbstractContextManager[pytest.ExceptionInfo[Exception]]: if self.msg is None: return pytest.raises(self.exception) pattern = re.escape(self.msg) if self.escape else self.msg return pytest.raises(self.exception, match=pattern) def mutate_nested_containers(value: object) -> None: """Recursively mutate every mutable container reachable inside `value`. Adds a marker key to every dict and appends a marker to every list, descending through tuples. Used to prove a `to_json` document shares no mutable state with the model that produced it. """ if isinstance(value, dict): for item in value.values(): mutate_nested_containers(item) value["__mutated__"] = "__mutated__" elif isinstance(value, list): for item in value: mutate_nested_containers(item) value.append("__mutated__") elif isinstance(value, tuple): for item in value: mutate_nested_containers(item) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/conftest.py000066400000000000000000000011541527014137600304410ustar00rootroot00000000000000"""Fixtures for the model tests.""" from __future__ import annotations import sys from typing import TYPE_CHECKING import pytest if TYPE_CHECKING: from collections.abc import Iterator @pytest.fixture def interpreter_writes_4300_digits() -> Iterator[None]: """Pin the interpreter's default limit on converting an integer to text. An environment may lift the limit (`PYTHONINTMAXSTRDIGITS=0`), and tests of integers past the limit need it in place.""" limit = sys.get_int_max_str_digits() sys.set_int_max_str_digits(4300) try: yield finally: sys.set_int_max_str_digits(limit) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_array.py000066400000000000000000003175401527014137600310020ustar00rootroot00000000000000"""Tests for the metadata models in `zarr_metadata.model`.""" import copy import dataclasses import json import math import pickle from collections import UserDict from collections.abc import Callable from typing import TYPE_CHECKING, Any, TypeGuard, cast, get_args, get_origin, get_type_hints import pytest from typing_extensions import Unpack from tests.model._cases import Expect, ExpectFail, mutate_nested_containers from zarr_metadata._json import ( JSON_DEPTH, arrays_to_tuples, is_json, json_text, parse_json, prefixed, validate_json, ) from zarr_metadata.definition import ( AcceptedField, UnclaimedField, canonical_of, configuration_of, resolve, with_problems, ) from zarr_metadata.definition._definition import fields_of from zarr_metadata.model import ( UNSET, ZarrV2ArrayMetadata, ZarrV2ArrayMetadataReading, ZarrV2ArrayMetadataUpdate, ZarrV2GroupMetadata, ZarrV3ArrayMetadata, ZarrV3ArrayMetadataReading, ZarrV3ArrayMetadataUpdate, ZarrV3ConsolidatedMetadata, ZarrV3GroupMetadata, is_node_name_v3, is_node_path_v3, read_array_metadata_v2, read_array_metadata_v3, read_group_metadata_v2, read_group_metadata_v3, ) from zarr_metadata.problems import MetadataValidationError, ValidationProblem from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.codec.gzip import GZIP_CODEC from zarr_metadata.v3.definition import ( CORE, CORE_AND_EXTENSIONS, ArrayExtraFieldDefinition, CodecDefinition, ) if TYPE_CHECKING: from zarr_metadata.typed_json import JSONValue from zarr_metadata.v2.codec import ZarrV2CodecMetadata from zarr_metadata.v3 import ZarrV3NamedConfigJSON # --- public exports -------------------------------------------------------- # `ZARR_V3_CONSOLIDATED_METADATA_KEY` is deliberately absent: it names a key # *inside* a v3 group document, not a store key, so it has no paired `Literal` # and no `to_key_value` signature to appear in. See `test_v3_consolidated_key_ # is_not_a_store_key`, which pins that distinction. STORE_KEY_PAIRS = [ ("ZARR_V2_ARRAY_METADATA_STORE_KEY", "ZarrV2ArrayMetadataStoreKey", "zarr_metadata.v2.array"), ("ZARR_V3_ARRAY_METADATA_STORE_KEY", "ZarrV3ArrayMetadataStoreKey", "zarr_metadata.v3.array"), ("ZARR_V2_ATTRIBUTES_STORE_KEY", "ZarrV2AttributesStoreKey", "zarr_metadata.v2.attributes"), ("ZARR_V2_GROUP_METADATA_STORE_KEY", "ZarrV2GroupMetadataStoreKey", "zarr_metadata.v2.group"), ("ZARR_V3_GROUP_METADATA_STORE_KEY", "ZarrV3GroupMetadataStoreKey", "zarr_metadata.v3.group"), ( "ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY", "ZarrV2ConsolidatedMetadataStoreKey", "zarr_metadata.v2.consolidated", ), ] def test_store_key_pairs_agree() -> None: """Each store-key constant's Literal type alias holds exactly that constant, so the pair cannot drift apart.""" import importlib for const_name, alias_name, module_name in STORE_KEY_PAIRS: module = importlib.import_module(module_name) assert (getattr(module, const_name),) == get_args(getattr(module, alias_name)) def test_store_keys_are_defined_in_their_spec_modules() -> None: """Store keys are facts about the on-disk specs, so each is exported only by the `v2`/`v3` module describing that document, not by the model layer.""" import importlib import zarr_metadata.model for const_name, alias_name, module_name in STORE_KEY_PAIRS: module = importlib.import_module(module_name) for name in (const_name, alias_name): assert name in module.__all__, f"{name} should be exported by {module_name}" assert name not in zarr_metadata.model.__all__, ( f"{name} should not be exported by the model" ) def test_v3_consolidated_key_is_not_a_store_key() -> None: """v3 consolidated metadata is embedded as a field inside the group's own `zarr.json`, not persisted under its own store key. It therefore has no paired `Literal` alias, unlike every true store key — which is why it is excluded from `STORE_KEY_PAIRS` rather than merely forgotten.""" import importlib from zarr_metadata.v3 import consolidated assert "ZARR_V3_CONSOLIDATED_METADATA_KEY" in consolidated.__all__ assert not hasattr(consolidated, "ZarrV3ConsolidatedMetadataKey") assert consolidated.ZARR_V3_CONSOLIDATED_METADATA_KEY not in { getattr(importlib.import_module(module_name), const_name) for const_name, _, module_name in STORE_KEY_PAIRS } def test_v3_node_store_keys_agree() -> None: """v3 keys both node types' metadata under one store key, distinguished by the document's `node_type`. The array and group constants are separately typed but must name the same file; adjacency used to make that obvious, and they now live in different modules.""" from zarr_metadata.v3.array import ZARR_V3_ARRAY_METADATA_STORE_KEY from zarr_metadata.v3.group import ZARR_V3_GROUP_METADATA_STORE_KEY assert ZARR_V3_ARRAY_METADATA_STORE_KEY == ZARR_V3_GROUP_METADATA_STORE_KEY def test_validation_diagnostics_exported_from_package() -> None: """The readers are exported from the model, and the problem types from `zarr_metadata.problems`.""" import zarr_metadata.model import zarr_metadata.problems for name in ("ValidationProblem", "MetadataValidationError", "ProblemKind"): assert name in zarr_metadata.problems.__all__ assert name not in zarr_metadata.model.__all__ for name in ( "read_array_metadata_v3", "read_array_metadata_v2", ): assert name in zarr_metadata.model.__all__ assert hasattr(zarr_metadata.model, name) def test_expect_expectfail_smoke() -> None: """The Expect/ExpectFail test-case dataclasses behave as expected.""" e = Expect(input=1, output=2, id="x") assert (e.input, e.output, e.id) == (1, 2, "x") f = ExpectFail(input=1, exception=ValueError, id="y", msg="boom") with f.raises(): raise ValueError("boom") def test_v3_from_json_error_lists_all_problems() -> None: """A malformed v3 document surfaces every problem via MetadataValidationError.problems.""" doc: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) del doc["shape"] doc["data_type"] = 5 with pytest.raises(MetadataValidationError) as exc_info: ZarrV3ArrayMetadata.from_json(doc) locs = {p.loc for p in exc_info.value.problems} assert ("shape",) in locs assert ("data_type",) in locs # --- JSON type / fill_value contract --------------------------------------- def test_json_value_type_accepts_json_shapes() -> None: # JSONValue is the package's public JSON type alias; assigning JSON-shaped # values to it is valid. """The JSONValue type alias accepts JSON-shaped values.""" value: JSONValue = {"a": [1, 2.0, "x", True, None]} assert value == {"a": [1, 2.0, "x", True, None]} def test_string_nan_fill_value_roundtrips() -> None: # A float's non-finite fill values are the spec strings ("NaN", # "Infinity", "-Infinity"): # https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 # The string form round-trips cleanly under default dataclass equality, # unlike a raw float('nan'), which is not JSON. """A float array's string 'NaN' fill_value round-trips cleanly.""" m = ZarrV3ArrayMetadata.create_default(fill_value="NaN", data_type="float32") assert ZarrV3ArrayMetadata.from_json(m.to_json()) == m assert ZarrV3ArrayMetadata.from_json(m.to_json()).fill_value == "NaN" # --- V3 baseline ----------------------------------------------------------- def test_v3_to_json_emits_canonical_document() -> None: """V3 to_json emits exactly the expected document (which covers every spec-required key by construction).""" out = ZarrV3ArrayMetadata.create_default(shape=(10,), data_type="int32").to_json() assert out == { "zarr_format": 3, "node_type": "array", "shape": (10,), "fill_value": 0, "data_type": "int32", "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (10,)}}, "codecs": ({"name": "bytes", "configuration": {"endian": "little"}},), "chunk_key_encoding": {"name": "default"}, } @pytest.mark.parametrize( "data_type", ["int32", {"name": "int32"}, {"name": "int32", "configuration": {}}], ids=["bare-name", "object", "empty-configuration"], ) def test_v3_a_data_type_with_nothing_to_configure_is_written_by_its_bare_name( data_type: object, ) -> None: # Core data types have been written this way since Zarr v3.0, and # zarr-python reads them this way. Every other extension point is an object. document = { **ZarrV3ArrayMetadata.create_default(shape=(10,)).to_json(), "data_type": data_type, "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], } model = ZarrV3ArrayMetadata.from_json(document) # The document is written back as it was written. The field on its own # is written by its bare name, which every reader accepts. assert model.to_json()["data_type"] == data_type assert model.data_type.to_json() == "int32" def test_v3_dimension_names_included_when_present() -> None: """V3 to_json includes dimension_names when they are set.""" out: dict[str, object] = dict( ZarrV3ArrayMetadata.create_default(shape=(4,), dimension_names=("x",)).to_json() ) assert out["dimension_names"] == ("x",) def test_v3_dimension_names_omitted_when_none() -> None: """V3 to_json omits dimension_names when they are UNSET.""" model = ZarrV3ArrayMetadata.create_default() assert model.dimension_names is UNSET assert "dimension_names" not in model.to_json() # --- BUG 1: attributes gated on dimension_names ---------------------------- def test_v3_attributes_included_when_dimension_names_is_none() -> None: """Attributes must be emitted regardless of dimension_names. Regression: attributes were gated on `dimension_names is not None`, so non-empty attributes were silently dropped when there were no dimension names. """ model = ZarrV3ArrayMetadata.create_default(attributes={"foo": "bar"}) assert model.dimension_names is UNSET out: dict[str, object] = dict(model.to_json()) assert out["attributes"] == {"foo": "bar"} # --- BUG 2: single storage transformer dropped ----------------------------- def test_v3_single_storage_transformer_included() -> None: """A single storage transformer must be emitted. Regression: the guard used `> 1` instead of `> 0`, dropping a lone storage transformer. """ st: ZarrV3NamedConfigJSON = {"name": "some_transformer"} out: dict[str, object] = dict( ZarrV3ArrayMetadata.create_default(storage_transformers=(st,)).to_json() ) assert out["storage_transformers"] == ({"name": "some_transformer"},) def test_v3_no_storage_transformers_omitted() -> None: """V3 to_json omits storage_transformers when the document wrote none, and writes an empty one as written.""" assert "storage_transformers" not in ZarrV3ArrayMetadata.create_default().to_json() written = dict(ZarrV3ArrayMetadata.create_default(storage_transformers=()).to_json()) assert written["storage_transformers"] == () # --- V3 extra fields ------------------------------------------------------- def _written_extra_fields(model: ZarrV3ArrayMetadata) -> dict[str, object]: """Each extra field of `model`, by key, as the document writes the member's value.""" return {key: cast("Any", extra.to_json())[key] for key, extra in model.extra_fields.items()} def test_v3_extra_fields_merged() -> None: """V3 to_json merges extra_fields into the top-level document.""" model = ZarrV3ArrayMetadata.create_default(extra_fields={"my_ext": {"must_understand": False}}) assert _written_extra_fields(model) == {"my_ext": {"must_understand": False}} assert model.to_json()["my_ext"] == {"must_understand": False} # --- V3 key/value ---------------------------------------------------------- def test_v3_to_key_value_is_valid_json_under_zarr_json() -> None: """V3 to_key_value produces valid JSON bytes under the zarr.json key.""" kv = ZarrV3ArrayMetadata.create_default(attributes={"a": 1}).to_key_value() assert set(kv) == {"zarr.json"} parsed = json.loads(kv["zarr.json"].decode("utf-8")) assert parsed["zarr_format"] == 3 assert parsed["attributes"] == {"a": 1} # --- V3 standard-key sets -------------------------------------------------- def test_spec_members_are_the_members_the_spec_defines() -> None: """`ArrayExtraFieldDefinition.spec_members` holds every member of `ZarrV3ArrayMetadataJSON`, required or not, and nothing else: any other member of an array document is an extra field.""" members = ArrayExtraFieldDefinition.spec_members assert ( members == ZarrV3ArrayMetadataJSON.__required_keys__ | ZarrV3ArrayMetadataJSON.__optional_keys__ ) assert {"zarr_format", "node_type", "shape", "codecs", "dimension_names"} <= members assert "my_ext" not in members # --- create_default -------------------------------------------------------- def test_v3_create_default_is_valid_empty_array() -> None: """V3 create_default builds a structurally valid empty array that round-trips.""" m = ZarrV3ArrayMetadata.create_default() assert m.shape == () assert m.data_type.to_json() == "uint8" assert m.fill_value == 0 assert m.attributes == {} assert m.extra_fields == {} # the default document is structurally valid and round-trips assert read_array_metadata_v3(m.to_json()).problems == () assert ZarrV3ArrayMetadata.from_json(m.to_json()) == m def test_v3_create_default_applies_overrides() -> None: """V3 create_default applies keyword overrides over the defaults.""" m = ZarrV3ArrayMetadata.create_default(shape=(4, 4), attributes={"a": 1}) assert m.shape == (4, 4) assert m.attributes == {"a": 1} # un-overridden fields keep their defaults assert m.data_type.to_json() == "uint8" @pytest.mark.parametrize( ("model", "members"), [ (ZarrV3ArrayMetadata, {"attributes": {"a": 1}, "fill_value": 3}), (ZarrV3ArrayMetadata, {"dimension_names": UNSET, "attributes": UNSET}), (ZarrV3GroupMetadata, {"attributes": {"a": 1}}), ( ZarrV3GroupMetadata, { "consolidated_metadata": ZarrV3ConsolidatedMetadata( { "kind": "inline", "must_understand": False, "metadata": {"x": ZarrV3ArrayMetadata.create_default()}, } ) }, ), (ZarrV2ArrayMetadata, {"fill_value": 3, "attributes": {"k": "v"}}), (ZarrV2ArrayMetadata, {"dimension_separator": UNSET}), (ZarrV2GroupMetadata, {"attributes": {"k": "v"}}), (ZarrV2GroupMetadata, {"attributes": UNSET}), ], ids=[ "v3-array", "v3-array-unset", "v3-group", "v3-group-consolidated-model", "v2-array", "v2-array-unset", "v2-group", "v2-group-unset", ], ) def test_create_default_takes_the_members_update_takes( model: object, members: dict[str, Any] ) -> None: """Every model's `create_default` takes the same keyword members as its `update`, including `UNSET` to drop a member and, for a v3 group, a `ZarrV3ConsolidatedMetadata` as `consolidated_metadata`. Applying them at creation gives the same model as applying them afterwards.""" kind = cast("Any", model) assert kind.create_default(**members) == kind.create_default().update(**members) def test_v2_create_default_is_valid_empty_array() -> None: """V2 create_default builds a structurally valid empty array that round-trips.""" m = ZarrV2ArrayMetadata.create_default() assert m.shape == () assert m.chunks == () assert m.fill_value == 0 assert m.compressor is None assert m.filters is None assert m.attributes == {} assert "attributes" not in m.to_json() assert read_array_metadata_v2(m.to_json()).problems == () assert ZarrV2ArrayMetadata.from_json(m.to_json()) == m def test_v2_create_default_applies_overrides() -> None: """V2 create_default applies keyword overrides over the defaults.""" m = ZarrV2ArrayMetadata.create_default(shape=(8,), attributes={"k": "v"}) assert m.shape == (8,) assert m.attributes == {"k": "v"} assert m.dtype.to_json() == "|u1" # default dtype unchanged # --- V3 update ------------------------------------------------------------- # Cluster 3: update same-shape pairs across versions — parametrized UPDATE_NEW_INSTANCE_PARAMS = [ pytest.param(ZarrV3ArrayMetadata, id="v3"), pytest.param(ZarrV2ArrayMetadata, id="v2"), ] @pytest.mark.parametrize("model_cls", UPDATE_NEW_INSTANCE_PARAMS) def test_update_returns_new_instance( model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], ) -> None: """update returns a new instance with the field replaced, leaving the original unchanged.""" base = model_cls.create_default(shape=(10,)) updated = base.update(shape=(20,)) assert updated.shape == (20,) assert base.shape == (10,) # original unchanged assert isinstance(updated, model_cls) UPDATE_NO_ARGS_PARAMS = [ pytest.param(ZarrV3ArrayMetadata, id="v3"), pytest.param(ZarrV2ArrayMetadata, id="v2"), ] @pytest.mark.parametrize("model_cls", UPDATE_NO_ARGS_PARAMS) def test_update_no_args_returns_equal_model( model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], ) -> None: """update with no arguments returns a model equal to the original.""" base = model_cls.create_default() updated = base.update() assert updated == base # V3-only update tests — kept direct (extra_fields is v3-specific) def test_update_can_add_an_extension_member() -> None: """update can add a member the spec does not define, which the model holds in extra_fields.""" base = ZarrV3ArrayMetadata.create_default() updated = base.update(extra_fields={"my_ext": {"must_understand": False}}) assert _written_extra_fields(updated) == {"my_ext": {"must_understand": False}} def test_update_reads_every_member_in_the_models_own_scope() -> None: """`update` reads the new document in the scope the model was read in, so a name that scope leaves unclaimed stays unclaimed. Use `with_context` to read it in another scope. `zstd` is an extension, so `CORE` leaves it unclaimed and unchecked. """ little: ZarrV3NamedConfigJSON = {"name": "bytes", "configuration": {"endian": "little"}} zstd: ZarrV3NamedConfigJSON = {"name": "zstd", "configuration": {"level": 3, "checksum": False}} base = ZarrV3ArrayMetadata.create_default(context=CORE, codecs=(little, zstd)) assert isinstance(base.codecs[1], UnclaimedField) kept = base.update(attributes={"k": 1}) assert kept.codecs[1] == base.codecs[1] assert kept.context == CORE given = base.with_context(CORE_AND_EXTENSIONS).update(codecs=(little, zstd)) assert isinstance(given.codecs[1], AcceptedField) def test_update_leaves_out_a_member_given_as_unset() -> None: """`update` removes each member that a document may omit when it is given as `UNSET`: `dimension_names`, `attributes`, and a member the spec does not define.""" base = ZarrV3ArrayMetadata.create_default( shape=(2,), dimension_names=("x",), attributes={"a": 1}, extra_fields={"my_ext": {"must_understand": False}}, ) for updated, member in ( (base.update(dimension_names=UNSET), "dimension_names"), (base.update(attributes=UNSET), "attributes"), (base.update(extra_fields={"my_ext": UNSET}), "my_ext"), ): written = updated.to_json() assert member not in written assert {key: value for key, value in base.to_json().items() if key != member} == written @pytest.mark.parametrize( "members", [ {"data_type": {"name": "uint8"}}, {"codecs": ("bytes",)}, {"codecs": ({"name": "bytes", "configuration": {}, "must_understand": True},)}, {"chunk_key_encoding": {"name": "default", "configuration": {}}}, {"codecs": ({"name": "bytes"}, {"name": "acme.codec", "configuration": {}})}, { "shape": (4,), "codecs": ( { "name": "sharding_indexed", "configuration": { "chunk_shape": (2,), "codecs": ({"name": "bytes"},), "index_codecs": ( {"name": "bytes", "configuration": {"endian": "little"}}, "crc32c", ), }, }, ), }, ], ids=[ "data-type-object", "codec-bare", "codec-verbose", "empty-configuration", "unclaimed", "a-field-a-codec-holds-bare", ], ) def test_a_model_is_what_its_document_says_not_how_it_is_spelled( members: ZarrV3ArrayMetadataUpdate, ) -> None: """A model read back from its own document equals itself, and it equals the model read from any other spelling of the same document.""" model = ZarrV3ArrayMetadata.create_default(**members) assert ZarrV3ArrayMetadata.from_json(model.to_json()) == model written = {**model.to_json(), **members} assert ZarrV3ArrayMetadata.from_json(written) == model _DATETIME = {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}} _LITTLE = {"name": "bytes", "configuration": {"endian": "little"}} @pytest.mark.parametrize( ("data_type", "left", "right", "same"), [ ("float32", "NaN", "0x7fc00000", True), ("float32", 1, 1.0, True), ("float32", 0.1, 0.10000000149011612, True), ("float32", 0.0, -0.0, False), ("float32", "NaN", "0xffc00000", False), (_DATETIME, "NaT", -(2**63), True), ("bytes", [65], "QR==", True), # The fill value of a data type that nothing in the scope claims is # not interpreted, and is compared as JSON text. ("acme.decimal", {"a": 1, "b": 2}, {"b": 2, "a": 1}, True), ("acme.decimal", [1, 2], [2, 1], False), ("acme.decimal", True, 1, False), ("acme.decimal", 0.0, -0.0, False), ], ) def test_two_models_are_one_array_when_their_fill_values_are_one_value( data_type: object, left: object, right: object, same: bool ) -> None: """Two models are equal when their fill values are the same value, however each is spelled, just as two fields are equal when they read the same.""" codecs = [{"name": "vlen-bytes"} if data_type == "bytes" else _LITTLE] document = {**ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json(), "codecs": codecs} models = [ ZarrV3ArrayMetadata.from_json({**document, "data_type": data_type, "fill_value": value}) for value in (left, right) ] assert (models[0] == models[1]) is same assert (models[1] == models[0]) is same assert models[0] == ZarrV3ArrayMetadata.from_json(models[0].to_json()) # Equal models have equal hashes. if same: assert hash(models[0]) == hash(models[1]) # Consolidated metadata holding the arrays compares the same way. held = [ ZarrV3ConsolidatedMetadata({**_INLINE, "metadata": {"a": model.to_json()}}) for model in models ] assert (held[0] == held[1]) is same _INLINE: dict[str, Any] = {"kind": "inline", "must_understand": False} _NOSHUFFLE = {"cname": "lz4", "clevel": 5, "shuffle": "noshuffle", "blocksize": 0} @pytest.mark.parametrize( ("member", "left", "right"), [ ( "codecs", [_LITTLE, {"name": "blosc", "configuration": _NOSHUFFLE}], [_LITTLE, {"name": "blosc", "configuration": {**_NOSHUFFLE, "typesize": 4}}], ), ( "codecs", [_LITTLE, {"name": "zstd", "configuration": {"level": 1}}], [_LITTLE, {"name": "zstd", "configuration": {"level": 1, "checksum": False}}], ), ( "chunk_key_encoding", "default", {"name": "default", "configuration": {"separator": "/"}}, ), ], ids=["blosc-typesize-noshuffle", "zstd-checksum-false", "default-separator"], ) def test_two_models_are_one_array_when_their_fields_read_the_same( member: str, left: object, right: object ) -> None: """The spec's equivalences, which each definition's `canonical` normalizes, hold for a model's `==`, even though `to_json` writes each field as given.""" document = ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json() models = [ZarrV3ArrayMetadata.from_json({**document, member: value}) for value in (left, right)] assert models[0] == models[1] assert hash(models[0]) == hash(models[1]) assert models[0].to_json() != models[1].to_json() @pytest.mark.parametrize( ("left", "right", "same"), [ ({"a": math.nan}, {"a": math.nan}, True), ({"a": 1, "b": 2}, {"b": 2, "a": 1}, True), ({"a": True}, {"a": 1}, False), ({"a": 0.0}, {"a": -0.0}, False), ({"a": 1}, {"a": 1.0}, False), ], ids=["nan", "key-order", "bool-vs-int", "signed-zero", "int-vs-float"], ) def test_user_json_compares_as_text( left: "dict[str, JSONValue]", right: "dict[str, JSONValue]", same: bool ) -> None: """Attributes are not interpreted, so they compare as the document writes them: `NaN` equals `NaN`, `true` is not `1`, and `-0.0` is not `0.0`.""" arrays = [ZarrV3ArrayMetadata.create_default(attributes=held) for held in (left, right)] groups = [ZarrV3GroupMetadata.create_default(attributes=held) for held in (left, right)] v2 = [ZarrV2ArrayMetadata.create_default(attributes=held) for held in (left, right)] for models in (arrays, groups, v2): assert (models[0] == models[1]) is same if same: assert hash(models[0]) == hash(models[1]) def test_a_model_holding_nan_user_data_equals_its_copies() -> None: # Python's `==` on the value alone would say otherwise, since `nan != nan`. model = ZarrV3ArrayMetadata.create_default(attributes={"_FillValue": math.nan}) assert ZarrV3ArrayMetadata.from_key_value(model.to_key_value()) == model assert pickle.loads(pickle.dumps(model)) == model assert ZarrV3ArrayMetadata.from_json(json.loads(json.dumps(model.to_json()))) == model @pytest.mark.parametrize( ("left", "right", "same"), [ (0.0, 0.0, True), ("NaN", "NaN", True), (0.0, -0.0, False), (1, 1.0, True), ("Infinity", "NaN", False), ], ) def test_two_v2_models_are_one_array_when_their_documents_are_written_alike( left: object, right: object, same: bool ) -> None: """A v2 model compares by what its document means. For a float type, a fill value written as an integer or as a float is the same value, `0.0` and `-0.0` are different values, and `NaN` equals `NaN`.""" model = ZarrV2ArrayMetadata.create_default(shape=(2,), chunks=(2,), dtype=" None: model = ZarrV3ArrayMetadata.create_default( codecs=( {"name": "bytes", "configuration": {"endian": "little"}}, {"name": "gzip", "configuration": {"level": 1}}, ), context=CORE, ) for again in (pickle.loads(pickle.dumps(model)), copy.copy(model), copy.deepcopy(model)): assert again == model assert configuration_of(again.codecs[1], GZIP_CODEC) == {"level": 1} def test_update_replaces_a_member_rather_than_merging_into_it() -> None: """update replaces each member it is given whole, and keeps the others.""" base = ZarrV3ArrayMetadata.create_default( extra_fields={"a": {"must_understand": False, "x": 1}, "b": {"must_understand": False}} ) updated = base.update(extra_fields={"a": {"must_understand": False}}) assert _written_extra_fields(updated) == { "a": {"must_understand": False}, "b": {"must_understand": False}, } # --- V2 model -------------------------------------------------------------- def test_v2_partial_keys_match_settable_model_fields() -> None: """The v2 partial TypedDict must list exactly the settable fields.""" assert set(ZarrV2ArrayMetadataUpdate.__annotations__) == { "shape", "dtype", "chunks", "fill_value", "order", "compressor", "filters", "dimension_separator", "attributes", "extra_fields", } def test_v2_to_key_value_splits_zarray_and_zattrs() -> None: """V2 to_key_value splits the document into .zarray and .zattrs.""" kv = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_key_value() assert set(kv) == {".zarray", ".zattrs"} zarray = json.loads(kv[".zarray"].decode("utf-8")) zattrs = json.loads(kv[".zattrs"].decode("utf-8")) assert zarray["zarr_format"] == 2 assert zattrs == {"a": 1} def test_v2_zarray_excludes_attributes() -> None: """The on-disk `.zarray` document must not contain user attributes. In v2, attributes live only in the sibling `.zattrs` file. The bundled `ZarrV2ArrayMetadataJSON` / `to_json()` carry attributes for convenience, but `to_key_value()` must split them out. """ kv = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_key_value() zarray = json.loads(kv[".zarray"].decode("utf-8")) assert "attributes" not in zarray def test_v2_to_json_still_includes_attributes() -> None: """`to_json()` is the bundled in-memory form and keeps attributes.""" out: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_json()) assert out["attributes"] == {"a": 1} # --- arrays_to_tuples helper ---------------------------------------------- ARRAYS_TO_TUPLES_CASES = [ Expect([1, 2, 3], (1, 2, 3), id="top-level-list"), Expect({"a": [1, [2, 3]], "b": "x"}, {"a": (1, (2, 3)), "b": "x"}, id="nested-in-dict"), Expect(5, 5, id="scalar-int"), Expect("s", "s", id="scalar-str"), Expect(None, None, id="scalar-none"), Expect( {"name": "bytes", "configuration": {"nums": [1, 2]}}, {"name": "bytes", "configuration": {"nums": (1, 2)}}, id="dict-keys-preserved", ), ] @pytest.mark.parametrize("case", ARRAYS_TO_TUPLES_CASES, ids=lambda c: c.id) def test_arrays_to_tuples(case: Expect[object, object]) -> None: """arrays_to_tuples recursively converts JSON arrays to tuples.""" assert arrays_to_tuples(case.input) == case.output # --- ZarrV3ArrayMetadata.from_json ---------------------------------------- def test_v3_from_json_reconstructs_required_fields() -> None: """V3 from_json reconstructs the required fields from a document.""" doc = ZarrV3ArrayMetadata.create_default( shape=(7,), attributes={"a": 1}, data_type="int32" ).to_json() model = ZarrV3ArrayMetadata.from_json(doc) assert model.shape == (7,) assert model.data_type.to_json() == "int32" assert model.attributes == {"a": 1} def test_v3_from_json_defaults_for_omitted_optionals() -> None: """V3 from_json supplies defaults for omitted optional fields.""" doc = ZarrV3ArrayMetadata.create_default(attributes={}, storage_transformers=()).to_json() # to_json omits these entirely; from_json must restore defaults model = ZarrV3ArrayMetadata.from_json(doc) assert model.attributes == {} assert model.storage_transformers == () assert model.dimension_names is UNSET def test_v3_from_json_routes_unknown_keys_to_extra_fields() -> None: """V3 from_json routes unknown top-level keys into extra_fields. No definition in the default scope claims `my_ext`, so it is an `UnclaimedField`, written as the document wrote it.""" doc = ZarrV3ArrayMetadata.create_default( extra_fields={"my_ext": {"must_understand": False}} ).to_json() model = ZarrV3ArrayMetadata.from_json(doc) assert isinstance(model.extra_fields["my_ext"], UnclaimedField) assert _written_extra_fields(model) == {"my_ext": {"must_understand": False}} def test_v3_from_json_standard_keys_not_in_extra_fields() -> None: """V3 from_json keeps standard keys out of extra_fields.""" doc = ZarrV3ArrayMetadata.create_default( shape=(10,), attributes={"a": 1}, dimension_names=("x",) ).to_json() model = ZarrV3ArrayMetadata.from_json(doc) assert model.extra_fields == {} def test_v3_from_json_nested_arrays_in_attributes_become_tuples() -> None: """V3 from_json converts nested arrays in attributes into tuples.""" doc = ZarrV3ArrayMetadata.create_default(attributes={"scale": [[1, 2], [3, 4]]}).to_json() model = ZarrV3ArrayMetadata.from_json(doc) assert model.attributes == {"scale": ((1, 2), (3, 4))} # --- ZarrV3ArrayMetadata.from_key_value ---------------------------------- def test_v3_from_key_value_parses_zarr_json() -> None: """V3 from_key_value parses the zarr.json entry into a model.""" kv = ZarrV3ArrayMetadata.create_default(shape=(3,)).to_key_value() model = ZarrV3ArrayMetadata.from_key_value(kv) assert model.shape == (3,) # --- Cluster 2: from_key_value missing-key raises (parametrized) ----------- FROM_KEY_VALUE_MISSING_PARAMS = [ pytest.param( ZarrV3ArrayMetadata, ExpectFail({}, MetadataValidationError, id="v3-missing-zarr-json", msg="missing store key"), id="v3-missing-zarr-json", ), pytest.param( ZarrV2ArrayMetadata, ExpectFail({}, MetadataValidationError, id="v2-missing-zarray", msg="missing store key"), id="v2-missing-zarray", ), ] @pytest.mark.parametrize(("model_cls", "case"), FROM_KEY_VALUE_MISSING_PARAMS) def test_from_key_value_missing_key_raises( model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], case: ExpectFail[dict[str, bytes]], ) -> None: """from_key_value raises MetadataValidationError when the required store key is absent.""" with case.raises(): model_cls.from_key_value(case.input) # --- Cluster 1: round-trips (model → json → model, parametrized) ----------- ROUNDTRIP_MODEL_JSON_PARAMS = [ pytest.param( ZarrV3ArrayMetadata, ZarrV3ArrayMetadata.create_default( shape=(10,), attributes={"a": 1}, dimension_names=("x",), storage_transformers=({"name": "acme.t"},), extra_fields={"ext": {"must_understand": False}}, ), id="v3-full", ), pytest.param( ZarrV3ArrayMetadata, ZarrV3ArrayMetadata.create_default(attributes={}, storage_transformers=()), id="v3-empty-optionals", ), pytest.param( ZarrV2ArrayMetadata, ZarrV2ArrayMetadata.create_default(attributes={"a": 1}, filters=None, compressor=None), id="v2-basic", ), ] @pytest.mark.parametrize(("model_cls", "model"), ROUNDTRIP_MODEL_JSON_PARAMS) def test_roundtrip_model_json_model( model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], model: ZarrV3ArrayMetadata | ZarrV2ArrayMetadata, ) -> None: """A model round-trips through to_json/from_json back to an equal model.""" assert model_cls.from_json(model.to_json()) == model # --- Round-trips (model → key_value → model) -------------------------------- def test_roundtrip_via_key_value() -> None: """A model round-trips through to_key_value/from_key_value back to an equal model.""" # Not parametrized over the two classes: a mapping's key type is # invariant, so one reader signature cannot accept the union of what # the two classes write. v3 = ZarrV3ArrayMetadata.create_default(attributes={"a": 1}) v2 = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}) assert ZarrV3ArrayMetadata.from_key_value(v3.to_key_value()) == v3 assert ZarrV2ArrayMetadata.from_key_value(v2.to_key_value()) == v2 # --- Round-trips (json → model → json, direction distinct — kept direct) --- def test_v3_roundtrip_json_model_json() -> None: """A v3 document round-trips through from_json/to_json back to an equal document.""" doc = ZarrV3ArrayMetadata.create_default( shape=(10,), attributes={"a": 1}, dimension_names=("x",) ).to_json() assert ZarrV3ArrayMetadata.from_json(doc).to_json() == doc def test_v2_roundtrip_json_model_json() -> None: """A v2 document round-trips through from_json/to_json back to an equal document.""" doc = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_json() assert ZarrV2ArrayMetadata.from_json(doc).to_json() == doc # --- to_json shares no mutable state with the model ------------------------ TO_JSON_NO_ALIASING_PARAMS = [ pytest.param( ZarrV3ArrayMetadata.create_default( shape=(2,), attributes={"a": {"b": [1]}}, # A name that nothing in the scope claims, so reading it checks # only the document structure, and its configuration can nest. codecs=({"name": "acme.nested", "configuration": {"opts": {"level": 1}}},), extra_fields={"ext": {"must_understand": False, "cfg": {"x": [1]}}}, ), id="v3", ), pytest.param( ZarrV2ArrayMetadata.create_default( attributes={"a": {"b": [1]}}, # Ids that nothing in the scope claims, so their parameters can # nest. A complex type, whose fill value is a pair. dtype=" None: """Mutating a document returned by to_json leaves the model unchanged.""" baseline = copy.deepcopy(model.to_json()) mutate_nested_containers(model.to_json()) assert model.to_json() == baseline @pytest.mark.parametrize("model", TO_JSON_NO_ALIASING_PARAMS) def test_from_json_shares_no_mutable_state_with_its_input( model: ZarrV3ArrayMetadata | ZarrV2ArrayMetadata, ) -> None: """Mutating the document a model was read from leaves the model unchanged.""" # With arrays already as tuples, the reader has nothing to rebuild, so # only a copy keeps the model separate from its input. document = arrays_to_tuples(model.to_json()) read = type(model).from_json(document) baseline = copy.deepcopy(read.to_json()) mutate_nested_containers(document) assert read.to_json() == baseline def test_v3_parser_accepts_bare_string_data_type() -> None: """V3 from_json accepts a bare-string data_type and re-serializes it canonically.""" doc = ZarrV3ArrayMetadata.create_default().to_json() doc["data_type"] = "int32" doc["codecs"] = ({"name": "bytes", "configuration": {"endian": "little"}},) model = ZarrV3ArrayMetadata.from_json(doc) assert (model.data_type.json, model.data_type.name) == ("int32", "int32") assert model.to_json()["data_type"] == "int32" @pytest.mark.parametrize("name", ["bytes", "acme.codec", "urn:example:codec"]) def test_metadata_field_accepts_a_name_as_the_spec_names_one(name: str) -> None: """The envelope check accepts a name in a form the spec allows for an extension name, whether or not a definition in scope claims it.""" assert resolve({"name": name}, CodecDefinition, CORE_AND_EXTENSIONS)[1] == () @pytest.mark.parametrize("value", [0, 1, "false", None]) def test_metadata_field_must_understand_must_be_boolean(value: object) -> None: """must_understand is a JSON boolean, not a truthy scalar.""" _, problems = resolve( {"name": "acme.x", "must_understand": value}, CodecDefinition, CORE_AND_EXTENSIONS ) assert [(problem.loc, problem.kind) for problem in problems] == [ (("must_understand",), "invalid_type") ] def test_metadata_field_rejects_unknown_envelope_member() -> None: """Unknown envelope keys cannot be silently discarded during normalization.""" _, problems = resolve({"name": "acme.x", "typo": 1}, CodecDefinition, CORE_AND_EXTENSIONS) assert [(problem.loc, problem.kind) for problem in problems] == [(("typo",), "unknown_key")] @pytest.mark.parametrize("field", ["codecs", "storage_transformers"]) def test_every_extension_point_rejects_must_understand_false(field: str) -> None: """No extension point may be declared ignorable. Ignoring a codec gives wrong bytes as surely as ignoring a data type gives wrong values, so `must_understand` is a property of the kind of metadata rather than a per-occurrence choice. The spec names only the three required points; this package reads that as an oversight. """ doc: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) doc[field] = ({"name": "optional", "must_understand": False},) assert [problem.loc for problem in read_array_metadata_v3(doc).problems] == [ (field, 0, "must_understand") ] @pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) def test_required_extension_points_reject_must_understand_false(field: str) -> None: """Core extension points needed to locate or decode chunks cannot be ignored.""" doc: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) doc[field] = {"name": "optional", "must_understand": False} assert [(problem.loc, problem.kind) for problem in read_array_metadata_v3(doc).problems] == [ ((field, "must_understand"), "invalid_value") ] def test_v3_codecs_cannot_be_empty() -> None: """The core document requires at least one array-to-bytes codec.""" doc: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) doc["codecs"] = () assert [(problem.loc, problem.kind) for problem in read_array_metadata_v3(doc).problems] == [ (("codecs",), "invalid_value") ] def test_v2_roundtrip_with_compressor_and_filters() -> None: # Non-None compressor/filters must round-trip; extra assertion on .compressor. """A v2 model with non-None compressor and filters round-trips.""" compressor: ZarrV2CodecMetadata = {"id": "blosc", "clevel": 5} filters: tuple[ZarrV2CodecMetadata, ...] = ({"id": "delta", "dtype": " None: """V2 from_json reconstructs the fields from a document.""" doc = ZarrV2ArrayMetadata.create_default(shape=(4,), attributes={"a": 1}, dtype=" None: """V2 from_json reads an absent attributes key as empty attributes. The document still differs from one with an explicit empty mapping, but both mean no attributes, so the models are equal, as in v3.""" absent = ZarrV2ArrayMetadata.from_json(ZarrV2ArrayMetadata.create_default().to_json()) explicit = ZarrV2ArrayMetadata.from_json( ZarrV2ArrayMetadata.create_default(attributes={}).to_json() ) assert absent.attributes == {} assert "attributes" not in absent.to_json() assert explicit.attributes == {} assert explicit.to_json().get("attributes") == {} assert absent == explicit # --- ZarrV2ArrayMetadata.from_key_value -------------------------------- def test_v2_from_key_value_remerges_zattrs() -> None: """V2 from_key_value re-merges .zattrs back into attributes.""" kv = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}, shape=(10,)).to_key_value() model = ZarrV2ArrayMetadata.from_key_value(kv) assert model.attributes == {"a": 1} assert model.shape == (10,) def test_v2_from_key_value_rejects_zarray_attributes() -> None: """A raw `.zarray` document must not carry `attributes`: they live in `.zattrs`.""" doc: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default().to_json()) doc.pop("attributes", None) doc["attributes"] = {} with pytest.raises(MetadataValidationError) as exc_info: ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ (("attributes",), "invalid_value") ] def test_v2_from_key_value_keeps_zarray_extra_members() -> None: """Other raw `.zarray` members "SHOULD be ignored by implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92). They are not checked, and they are kept as written.""" doc: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default().to_json()) doc.pop("attributes", None) doc["vendor_extension"] = {} model = ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) assert model.to_json()["vendor_extension"] == {} assert model.extra_fields == {"vendor_extension": {}} def test_v2_zattrs_presence_round_trips() -> None: """The .zattrs file's presence is part of the store: an absent file reads as empty attributes and emits no .zattrs; an explicit empty file reads as {} and emits .zattrs — the two stores stay distinct through a round-trip.""" explicit_kv = dict(ZarrV2ArrayMetadata.create_default(attributes={}).to_key_value()) assert ".zattrs" in explicit_kv absent_kv = dict(explicit_kv) del absent_kv[".zattrs"] absent = ZarrV2ArrayMetadata.from_key_value(absent_kv) explicit = ZarrV2ArrayMetadata.from_key_value(explicit_kv) assert absent.attributes == {} assert explicit.attributes == {} assert ".zattrs" not in absent.to_key_value() assert ".zattrs" in explicit.to_key_value() def test_v2_from_json_nested_arrays_in_attributes_become_tuples() -> None: """V2 from_json converts nested arrays in attributes into tuples.""" doc = ZarrV2ArrayMetadata.create_default(attributes={"axes": [[0, 1], [2, 3]]}).to_json() model = ZarrV2ArrayMetadata.from_json(doc) assert model.attributes == {"axes": ((0, 1), (2, 3))} # --- scalar wire-type guards (is_/validate_/parse_) ------------------------ # # Each value is modelled once as Expect[object, frozenset[tuple[str | int, ...]]] # where `output` is the set of expected problem locs validate_* must report — # frozenset() means VALID. Valid iff output == frozenset(). JSON_VALIDATE_CASES: list[Expect[object, frozenset[tuple[str | int, ...]]]] = [ Expect("s", frozenset(), id="str"), Expect(1, frozenset(), id="int"), Expect(1.5, frozenset(), id="float"), Expect(True, frozenset(), id="bool"), Expect(None, frozenset(), id="none"), Expect({"a": [1, {"b": None}], "c": "x"}, frozenset(), id="nested-containers"), Expect((1, 2, 3), frozenset(), id="tuple-array"), Expect(float("nan"), frozenset({()}), id="nan"), Expect(float("inf"), frozenset({()}), id="inf"), Expect(float("-inf"), frozenset({()}), id="negative-inf"), Expect(object(), frozenset({()}), id="object"), Expect(b"abc", frozenset({()}), id="bytes"), Expect(bytearray(b"abc"), frozenset({()}), id="bytearray"), Expect({1: "x"}, frozenset({()}), id="non-str-key"), Expect([1, object()], frozenset({(1,)}), id="non-json-list-item"), Expect({"ok": object()}, frozenset({("ok",)}), id="non-json-value"), ] @pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) def test_is_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: """is_json reports whether a value is JSON-serializable.""" assert is_json(case.input) is (case.output == frozenset()) @pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) def test_validate_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: """validate_json reports the problems (and their locs) for a value.""" problems = validate_json(case.input) assert (problems == ()) is (case.output == frozenset()) assert {p.loc for p in problems} >= case.output @pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) def test_parse_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: """parse_json returns valid JSON values and raises on invalid ones.""" if case.output == frozenset(): parsed = parse_json(case.input) assert arrays_to_tuples(parsed) == arrays_to_tuples(case.input) else: with pytest.raises(MetadataValidationError): parse_json(case.input) def test_parse_json_materializes_abstract_containers() -> None: """Accepted Mapping and Sequence values normalize to JSON encoder containers.""" value = UserDict({"values": range(3)}) parsed = parse_json(value) assert parsed == {"values": (0, 1, 2)} assert type(parsed) is dict assert type(parsed["values"]) is tuple json.dumps(parsed, allow_nan=False) @pytest.mark.parametrize( "guard", [ is_json, is_node_name_v3, is_node_path_v3, ], ids=lambda guard: guard.__name__, ) def test_a_guard_narrows_only_when_it_says_yes(guard: Callable[[object], bool]) -> None: """Each guard returns False for some values of its type (`is_json(math.nan)` is False, and a NaN is a `float`), so it is a `TypeGuard`. A `TypeIs` would tell a type checker to exclude such a type when the guard returns False.""" assert get_origin(get_type_hints(guard)["return"]) is TypeGuard def test_json_type_guard_rejects_abstract_sequence() -> None: """A guard cannot narrow an abstract sequence that only the parser materializes.""" assert not is_json(range(3)) assert parse_json(range(3)) == (0, 1, 2) def test_a_field_read_from_abstract_containers_writes_canonical_containers() -> None: """`resolve` reads a field given as abstract containers, and the field writes canonical containers at every nesting level.""" value = UserDict({"name": "example", "configuration": UserDict({"values": range(2)})}) field, problems = resolve(value, CodecDefinition, CORE_AND_EXTENSIONS) assert isinstance(field, UnclaimedField) written = field.to_json() assert problems == () assert isinstance(written, dict) assert written == {"name": "example", "configuration": {"values": (0, 1)}} assert type(written["configuration"]) is dict def test_validate_json_reports_json_in_message() -> None: """validate_json's message for a non-JSON value mentions JSON.""" problems = validate_json(object()) assert problems[0].loc == () assert "JSON" in problems[0].message METADATA_FIELD_VALIDATE_CASES: list[Expect[object, frozenset[tuple[str | int, ...]]]] = [ Expect("bytes", frozenset(), id="bare-string"), Expect({"name": "acme.x", "configuration": {"a": 1}}, frozenset(), id="named-config"), Expect({"name": "bytes"}, frozenset(), id="name-only"), Expect(5, frozenset({()}), id="not-str-or-mapping"), Expect({"configuration": {}}, frozenset({("name",)}), id="missing-name"), Expect({"name": 3}, frozenset({("name",)}), id="non-str-name"), Expect( {"name": "x", "configuration": [1]}, frozenset({("configuration",)}), id="config-not-mapping", ), Expect( {"name": "x", "configuration": {1: "y"}}, frozenset({("configuration",)}), id="config-non-str-key", ), ] @pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) def test_resolve_reports_the_problems_of_a_metadata_field( case: Expect[object, frozenset[tuple[str | int, ...]]], ) -> None: """`resolve` reports the problems of a metadata-field value, read as a codec.""" _, problems = resolve(case.input, CodecDefinition, CORE_AND_EXTENSIONS) assert (problems == ()) is (case.output == frozenset()) assert {p.loc for p in problems} >= case.output # --- array-document readers and models -------------------------------------- # # Each case starts from a valid document (built by `make`) and applies a # mutation. `expected_locs` are loc paths the reader must report for the # invalid cases (a subset check, so accumulation of OTHER problems is allowed). def _build_v3(**overrides: Unpack[ZarrV3ArrayMetadataUpdate]) -> dict[str, object]: return dict(ZarrV3ArrayMetadata.create_default(**overrides).to_json()) def _build_v2(**overrides: Unpack[ZarrV2ArrayMetadataUpdate]) -> dict[str, object]: return dict(ZarrV2ArrayMetadata.create_default(**overrides).to_json()) def _mutate(build: Callable[[], dict], mutate: Callable[[dict], object]) -> Callable[[], dict]: def _factory() -> dict: doc = build() mutate(doc) return doc return _factory def _del(key: str) -> Callable[[dict], object]: return lambda doc: doc.pop(key) def _set(key: str, value: object) -> Callable[[dict], object]: return lambda doc: doc.__setitem__(key, value) V3_DOC_CASES: list[Expect[Callable[[], object], frozenset[tuple[str | int, ...]]]] = [ Expect(_build_v3, frozenset(), id="valid"), Expect( lambda: _build_v3(shape=(10,), attributes={"a": 1}, dimension_names=("x",)), frozenset(), id="valid-with-attributes-and-dim-names", ), Expect( lambda: dict( ZarrV3ArrayMetadata.create_default( extra_fields={"my_ext": {"must_understand": False}} ).to_json() ), frozenset(), id="valid-with-extra-fields", ), Expect(_mutate(_build_v3, _del("shape")), frozenset({("shape",)}), id="missing-shape"), Expect( _mutate(_build_v3, _set("data_type", 5)), frozenset({("data_type",)}), id="bad-data-type", ), Expect( _mutate(_build_v3, _set("shape", "not-a-shape")), frozenset({("shape",)}), id="shape-not-sequence", ), Expect( _mutate(_build_v3, _set("shape", [1, "x"])), frozenset({("shape",)}), id="shape-non-int-item", ), Expect( _mutate(_build_v3, _set("codecs", (5,))), frozenset({("codecs", 0)}), id="bad-codec-entry", ), Expect(lambda: [1, 2, 3], frozenset({()}), id="non-mapping-list"), Expect(lambda: "nope", frozenset({()}), id="non-mapping-str"), Expect( _mutate(_mutate(_build_v3, _del("shape")), _set("data_type", 5)), frozenset({("shape",), ("data_type",)}), id="missing-shape-and-bad-data-type", ), ] V2_DOC_CASES: list[Expect[Callable[[], object], frozenset[tuple[str | int, ...]]]] = [ Expect(_build_v2, frozenset(), id="valid"), Expect(lambda: _build_v2(attributes={"a": 1}), frozenset(), id="valid-with-attributes"), Expect( lambda: _build_v2(compressor=None, filters=None), frozenset(), id="valid-none-compressor-filters", ), Expect( _mutate(_build_v2, _del("chunks")), frozenset({("chunks",)}), id="missing-chunks", ), Expect( _mutate(_build_v2, _set("shape", [1, "x"])), frozenset({("shape",)}), id="bad-shape", ), Expect( _mutate(_mutate(_build_v2, _del("chunks")), _set("shape", [1, "x"])), frozenset({("chunks",), ("shape",)}), id="missing-chunks-and-bad-shape", ), ] ALL_DOC_CASES = [ *( pytest.param(read_array_metadata_v3, ZarrV3ArrayMetadata, c, id=f"v3-{c.id}") for c in V3_DOC_CASES ), *( pytest.param(read_array_metadata_v2, ZarrV2ArrayMetadata, c, id=f"v2-{c.id}") for c in V2_DOC_CASES ), ] @pytest.mark.parametrize(("read", "model", "case"), ALL_DOC_CASES) def test_array_metadata_reader_and_model_agree( read: Callable[[object], ZarrV2ArrayMetadataReading | ZarrV3ArrayMetadataReading], model: type[ZarrV2ArrayMetadata | ZarrV3ArrayMetadata], case: Expect[Callable[[], object], frozenset[tuple[str | int, ...]]], ) -> None: """The reader and the model's `from_json` agree on validity for each case, and the reader reports the expected locs.""" doc = case.input() valid = case.output == frozenset() problems = read(doc).problems assert (problems == ()) is valid assert {p.loc for p in problems} >= case.output if valid: assert model.from_json(doc).to_json() == doc else: with pytest.raises(MetadataValidationError): model.from_json(doc) # --- strict from_json validation ------------------------------------------- FROM_JSON_REJECT_PARAMS = [ pytest.param( ZarrV3ArrayMetadata, ExpectFail(lambda: {"zarr_format": 3}, MetadataValidationError, id="x"), id="v3-missing-required", ), pytest.param( ZarrV3ArrayMetadata, ExpectFail(_mutate(_build_v3, _set("data_type", 5)), MetadataValidationError, id="x"), id="v3-bad-field-type", ), pytest.param( ZarrV2ArrayMetadata, ExpectFail(lambda: {"zarr_format": 2}, MetadataValidationError, id="x"), id="v2-missing-required", ), ] @pytest.mark.parametrize(("model", "case"), FROM_JSON_REJECT_PARAMS) def test_from_json_rejects_malformed( model: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], case: ExpectFail[Callable[[], object]], ) -> None: """from_json raises MetadataValidationError on a malformed document.""" with case.raises(): model.from_json(case.input()) # --- ValidationProblem / MetadataValidationError / _prefix ----------------- # Small structural tests — not "parametrize over inputs" shaped, kept direct. def test_validation_problem_str_with_loc() -> None: """ValidationProblem.__str__ renders a non-empty loc as a dotted path.""" p = ValidationProblem(loc=("codecs", 0, "name"), message="expected str", kind="invalid_type") assert str(p) == "codecs.0.name: expected str" def test_validation_problem_str_empty_loc() -> None: """ValidationProblem.__str__ renders an empty loc as .""" p = ValidationProblem(loc=(), message="not a mapping", kind="invalid_type") assert str(p) == ": not a mapping" def test_validation_problem_is_frozen() -> None: """ValidationProblem is immutable (frozen dataclass).""" p = ValidationProblem(loc=("shape",), message="x", kind="invalid_type") with pytest.raises(dataclasses.FrozenInstanceError): # setattr: assigning to a frozen field is an intentional runtime error, # spelled dynamically so it is not also a static type error. setattr(p, "message", "y") # noqa: B010 def test_metadata_validation_error_holds_problems() -> None: """MetadataValidationError carries its problem list and renders them in its message.""" problems = [ ValidationProblem(loc=("shape",), message="missing required key", kind="missing_key"), ValidationProblem( loc=("data_type",), message="expected a metadata field", kind="invalid_type" ), ] err = MetadataValidationError(problems) assert err.problems == tuple(problems) assert "shape: missing required key" in str(err) assert "data_type: expected a metadata field" in str(err) def test_the_error_pickles_and_copies_as_its_problems() -> None: error = MetadataValidationError([ValidationProblem(("a",), "bad a", "invalid_value")]) error.add_note("while reading a") for again in (pickle.loads(pickle.dumps(error)), copy.copy(error), copy.deepcopy(error)): assert type(again) is MetadataValidationError assert again.problems == error.problems assert str(again) == str(error) assert again.__notes__ == ["while reading a"] def test_error_a_problem_refuses_a_loc_that_is_not_a_tuple() -> None: # The missing comma: `("level")` is a string. Read as a location, it # would be a path through each of its characters. with pytest.raises(TypeError, match="loc is a tuple of keys and indices"): ValidationProblem(("level"), "bad level", "invalid_value") # pyright: ignore[reportArgumentType] @pytest.mark.parametrize("part", [True, 1.5], ids=["bool", "float"]) def test_error_a_problem_refuses_a_loc_part_that_is_not_a_key_or_an_index(part: object) -> None: # `True` is an `int` in Python, and would index a sequence as 1. with pytest.raises(TypeError, match="loc is a tuple of keys and indices"): ValidationProblem(("a", part), "bad a", "invalid_value") # pyright: ignore[reportArgumentType] def test_error_a_problem_refuses_a_message_that_is_not_a_string() -> None: with pytest.raises(TypeError, match="message is a string"): ValidationProblem(("level",), 7, "invalid_value") # pyright: ignore[reportArgumentType] def test_error_a_problem_refuses_a_kind_that_is_not_one() -> None: # A consumer that dispatches on kinds would never handle a kind # outside the set. with pytest.raises(TypeError, match="kind is one of"): ValidationProblem(("level",), "bad level", "invalid") # pyright: ignore[reportArgumentType] class _EqualToEveryKind: """An object that is not a kind but compares equal to every kind.""" def __eq__(self, other: object) -> bool: return True def __hash__(self) -> int: return 0 def test_error_a_problem_refuses_a_kind_that_only_compares_equal_to_one() -> None: # `in` tests equality, and any object can claim to be equal. with pytest.raises(TypeError, match="kind is one of"): ValidationProblem(("level",), "bad level", _EqualToEveryKind()) # pyright: ignore[reportArgumentType] def test_error_the_error_refuses_what_is_not_a_problem() -> None: # A list of one-element tuples of problems is a likely mistake. Without # this check it would fail later, far away, where a `loc` is read from # an entry. with pytest.raises(TypeError, match="takes ValidationProblem values, got tuple"): MetadataValidationError([(ValidationProblem(("a",), "bad a", "invalid_value"),)]) # pyright: ignore[reportArgumentType] def test_prefix_prepends_loc_head() -> None: """`prefixed` prepends a loc head to each problem's loc.""" problems = [ValidationProblem(loc=("name",), message="expected str", kind="invalid_type")] located = prefixed(0, problems) assert located == ( ValidationProblem(loc=(0, "name"), message="expected str", kind="invalid_type"), ) # --- Stricter v2/v3 field validation and error kinds ------------------------- def test_v2_dtype_must_be_string_or_records() -> None: """A non-string, non-records v2 dtype is rejected with an invalid_type problem.""" doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dtype": 42} problems = read_array_metadata_v2(doc).problems assert [(p.loc, p.kind) for p in problems] == [(("dtype",), "invalid_type")] @pytest.mark.parametrize( ("read", "document", "loc"), [ ( read_array_metadata_v3, {**ZarrV3ArrayMetadata.create_default().to_json(), "codecs": b""}, ("codecs",), ), ( read_array_metadata_v3, {**ZarrV3ArrayMetadata.create_default().to_json(), "storage_transformers": bytearray()}, ("storage_transformers",), ), ( read_array_metadata_v3, {**ZarrV3ArrayMetadata.create_default(shape=()).to_json(), "dimension_names": b""}, ("dimension_names",), ), ( read_array_metadata_v2, {**ZarrV2ArrayMetadata.create_default().to_json(), "dtype": b""}, ("dtype",), ), ( read_array_metadata_v2, {**ZarrV2ArrayMetadata.create_default().to_json(), "dtype": (("f0", b""),)}, ("dtype", 0, 1), ), ( read_array_metadata_v2, {**ZarrV2ArrayMetadata.create_default().to_json(), "filters": b""}, ("filters",), ), ], ids=["codecs", "storage-transformers", "dimension-names", "dtype", "dtype-record", "filters"], ) def test_error_bytes_are_not_an_array( read: Callable[[object], ZarrV2ArrayMetadataReading | ZarrV3ArrayMetadataReading], document: dict[str, object], loc: tuple[str | int, ...], ) -> None: """`bytes` is a sequence to Python but not an array to JSON. Where a document expects an array, an empty `bytes` no longer passes as an empty array.""" assert [(p.loc, p.kind) for p in read(document).problems] == [(loc, "invalid_type")] def test_v2_structured_dtype_records_accepted() -> None: """A structured v2 dtype (field records, optionally nested/shaped) validates.""" dtype = (("a", " None: """A field record with the wrong arity is rejected, at the record.""" doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dtype": (("a",),)} problems = read_array_metadata_v2(doc).problems assert [p.loc for p in problems] == [("dtype", "fields", 0)] def test_v2_order_literal_enforced() -> None: """An order other than 'C' or 'F' is rejected with an invalid_value problem.""" doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"order": "Q"} problems = read_array_metadata_v2(doc).problems assert [(p.loc, p.kind) for p in problems] == [(("order",), "invalid_value")] def test_v2_compressor_must_be_codec_or_none() -> None: """A compressor that is not null or a codec config mapping is rejected.""" doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"compressor": "zlib"} problems = read_array_metadata_v2(doc).problems assert [(p.loc, p.kind) for p in problems] == [(("compressor",), "invalid_type")] def test_v2_compressor_requires_string_id() -> None: """A compressor mapping without a string id is rejected, at the id.""" doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"compressor": {"level": 3}} problems = read_array_metadata_v2(doc).problems assert [p.loc for p in problems] == [("compressor", "id")] def test_v2_filters_must_be_codec_sequence_or_none() -> None: """Filters that are not null or a sequence of codec configs are rejected. An item that is not a codec is reported at the item, and anything else at the field.""" for bad, at in ((7, ("filters",)), ((5,), ("filters", 0)), ("gzip", ("filters",))): doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"filters": bad} problems = read_array_metadata_v2(doc).problems assert [(p.loc, p.kind) for p in problems] == [(at, "invalid_type")], bad def test_v2_shape_and_chunks_must_have_equal_rank() -> None: """Raw v2 metadata requires one chunk length per array dimension.""" doc = dict(ZarrV2ArrayMetadata.create_default(shape=(2, 3)).to_json()) doc["chunks"] = (1,) assert [(p.loc, p.kind) for p in read_array_metadata_v2(doc).problems] == [ (("chunks",), "invalid_value") ] with pytest.raises(MetadataValidationError, match="same number of dimensions"): ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) def test_v2_filters_may_be_empty() -> None: """An empty filter list is a list: the spec says "a list ... or null", with no minimum. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79 """ doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) doc["filters"] = () assert read_array_metadata_v2(doc).problems == () assert ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}).filters == () def test_v2_dimension_separator_literal_enforced() -> None: """A dimension_separator other than '.' or '/' is rejected.""" doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dimension_separator": "-"} problems = read_array_metadata_v2(doc).problems assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] def test_v2_zarr_format_literal_enforced() -> None: """A v2 document claiming zarr_format 3 is rejected with an invalid_value problem.""" doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"zarr_format": 3} problems = read_array_metadata_v2(doc).problems assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] def test_v3_zarr_format_literal_enforced() -> None: """A v3 document claiming zarr_format 2 is rejected with an invalid_value problem.""" doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"zarr_format": 2} problems = read_array_metadata_v3(doc).problems assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] @pytest.mark.parametrize( ("document", "read"), [ pytest.param( dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"zarr_format": 2.0}, read_array_metadata_v2, id="v2", ), pytest.param( dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"zarr_format": 3.0}, read_array_metadata_v3, id="v3", ), ], ) def test_array_zarr_format_rejects_float( document: object, read: Callable[[object], ZarrV2ArrayMetadataReading | ZarrV3ArrayMetadataReading], ) -> None: """Integer-valued floats do not satisfy integer format literals.""" problems = read(document).problems assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] def test_array_v2_keeps_an_unknown_document_member() -> None: """Other .zarray keys "SHOULD NOT be present ... and SHOULD be ignored": they are tolerated, not checked, and kept as written in `extra_fields`. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92 """ doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"unexpected": 1} assert read_array_metadata_v2(doc).problems == () model = ZarrV2ArrayMetadata.from_json(doc) assert model.to_json()["unexpected"] == 1 assert model.extra_fields == {"unexpected": 1} @pytest.mark.parametrize( ("member", "kind"), [ (object(), "invalid_type"), ({1, 2}, "invalid_type"), (b"\x00", "invalid_type"), (math.nan, "invalid_value"), ], ids=["object", "set", "bytes", "nan"], ) def test_error_array_v2_unknown_member_that_is_not_json(member: object, kind: str) -> None: """Ignored does not mean unchecked: an unknown member must be a JSON value, as in v3.""" doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"unexpected": member} assert [(p.loc, p.kind) for p in read_array_metadata_v2(doc).problems] == [ (("unexpected",), kind) ] @pytest.mark.parametrize("key", [7, None, True, (1, 2)], ids=["int", "none", "bool", "tuple"]) def test_error_array_v2_key_that_is_not_a_string(key: object) -> None: """A document's keys must be strings. Previously, the v2 parser returned a document like this one.""" doc: dict[object, object] = {**ZarrV2ArrayMetadata.create_default().to_json(), key: "x"} assert [(p.loc, p.kind) for p in read_array_metadata_v2(doc).problems] == [((), "invalid_type")] def test_array_v3_from_json_materializes_abstract_containers() -> None: """A flexible input mapping becomes the canonical dict/tuple model shape.""" # `range(2)` gives the shape (0, 1), which the default grid for that shape fits. doc = UserDict(dict(ZarrV3ArrayMetadata.create_default(shape=(0, 1)).to_json())) doc["shape"] = range(2) model = ZarrV3ArrayMetadata.from_json(doc) assert model.shape == (0, 1) assert type(model.shape) is tuple def test_from_key_value_rejects_non_standard_json_constant() -> None: """A JavaScript NaN constant outside the attributes is located, not decoded as a fill value.""" doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) doc["fill_value"] = float("nan") raw = json.dumps(doc) with pytest.raises(MetadataValidationError, match="fill_value: non-finite float nan"): ZarrV3ArrayMetadata.from_key_value({"zarr.json": raw.encode()}) @pytest.mark.parametrize( ("members", "problems"), [ ({"fill_value": math.nan}, [(("fill_value",), "invalid_value")]), # The default fill value, 0, is not a boolean. A data type requires # a fill value of that type. ({"data_type": "bool"}, [(("fill_value",), "invalid_type")]), # Values of more than one byte require an `endian`. ( {"data_type": "int16", "codecs": ({"name": "bytes"},)}, [(("codecs", 0, "configuration", "endian"), "missing_key")], ), ({"shape": (2,), "dimension_names": ("x", "y")}, [(("dimension_names",), "invalid_value")]), # The shape is not derived from the grid, and the scalar default # shape does not fit the grid. A grid must fit its shape. ( {"chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (10, 10)}}}, [(("chunk_grid", "configuration", "chunk_shape"), "invalid_value")], ), ], ids=[ "non-finite-fill-value", "fill-value-of-another-type", "no-endian", "names-past-shape", "grid-of-another-rank", ], ) def test_error_create_default_refuses_a_document_with_a_problem( members: ZarrV3ArrayMetadataUpdate, problems: list[tuple[tuple[str | int, ...], str]] ) -> None: """A model is built by a read, so a document that the read refuses produces no model, and nothing is written.""" with pytest.raises(MetadataValidationError) as raised: ZarrV3ArrayMetadata.create_default(**members) assert [(p.loc, p.kind) for p in raised.value.problems] == problems @pytest.mark.parametrize( ("members", "problems"), [ ({"fill_value": 300}, [(("fill_value",), "invalid_value")]), # A shape requires a grid that fits it. ({"shape": (4, 4)}, [(("chunk_grid", "configuration", "chunk_shape"), "invalid_value")]), ], ids=["fill-value-out-of-range", "shape-without-its-grid"], ) def test_error_update_refuses_a_document_with_a_problem( members: ZarrV3ArrayMetadataUpdate, problems: list[tuple[tuple[str | int, ...], str]] ) -> None: base = ZarrV3ArrayMetadata.create_default(shape=(4,)) with pytest.raises(MetadataValidationError) as raised: base.update(**members) assert [(p.loc, p.kind) for p in raised.value.problems] == problems def test_error_update_refuses_to_leave_out_a_member_a_document_holds() -> None: base = ZarrV3ArrayMetadata.create_default(shape=(4,)) with pytest.raises(MetadataValidationError) as raised: base.update(shape=UNSET) # pyright: ignore[reportArgumentType] assert [(p.loc, p.kind) for p in raised.value.problems] == [(("shape",), "missing_key")] @pytest.mark.parametrize( ("shape", "kind"), [ (5, "invalid_type"), (None, "invalid_type"), (("a",), "invalid_type"), ((-1,), "invalid_value"), ], ids=["a-number", "null", "not-integers", "negative"], ) def test_error_create_default_reports_a_shape_it_cannot_read(shape: object, kind: str) -> None: """A shape that cannot be read is reported the way the read reports it, rather than as a failure to derive a grid from it.""" with pytest.raises(MetadataValidationError) as raised: ZarrV3ArrayMetadata.create_default(shape=shape) # pyright: ignore[reportArgumentType] assert [(p.loc, p.kind) for p in raised.value.problems] == [(("shape",), kind)] def test_a_codec_that_is_not_json_is_placed_by_the_definition_that_claims_its_name() -> None: """A codec whose configuration is not JSON takes the kind of the definition that claims its name, as a codec refused for a JSON configuration does. `gzip` is a bytes -> bytes codec, so the pipeline still lacks an array -> bytes codec.""" document = { **ZarrV3ArrayMetadata.create_default().to_json(), "codecs": [{"name": "gzip", "configuration": {"level": math.nan}}], } assert [(p.loc, p.kind) for p in read_array_metadata_v3(document).problems] == [ (("codecs", 0, "configuration", "level"), "invalid_value"), (("codecs",), "invalid_value"), ] def test_a_shard_nested_as_deep_as_a_reader_walks_is_read_and_written() -> None: """Every walker of fields uses more than one stack frame per shard, so the deepest nesting the cap allows is where the interpreter's recursion limit would show. One shard deeper is reported as the depth problem.""" little = {"name": "bytes", "configuration": {"endian": "little"}} def nested(shards: int) -> list[object]: codecs: list[object] = [little] for _ in range(shards): codecs = [ { "name": "sharding_indexed", "configuration": { "chunk_shape": [1], "codecs": codecs, "index_codecs": [little], }, } ] return codecs # Each shard adds three levels (its object, its configuration, and the # `codecs` in it), and the innermost codec's configuration is the last # container a reader walks. deepest = (JSON_DEPTH - 4) // 3 codecs = nested(deepest) document = {**ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json(), "codecs": codecs} assert read_array_metadata_v3(document).problems == () model = ZarrV3ArrayMetadata.from_json(document) assert json_text(model.to_json()) == json_text(cast("JSONValue", document)) assert ZarrV3ArrayMetadata.from_key_value(model.to_key_value()) == model assert hash(model) == hash(ZarrV3ArrayMetadata.from_json(document)) assert pickle.loads(pickle.dumps(model)) == model assert copy.deepcopy(model) == model (shard,) = model.codecs assert json_text(canonical_of(shard, ())) == json_text(cast("JSONValue", codecs[0])) # Each level has a shard and its index codec, plus the innermost codec. assert len(list(with_problems(fields_of(shard), ()))) == 2 * deepest + 1 problems = read_array_metadata_v3({**document, "codecs": nested(deepest + 1)}).problems assert {(problem.kind, len(problem.loc), problem.message) for problem in problems} == { ("invalid_value", JSON_DEPTH, f"nested deeper than the {JSON_DEPTH} levels a reader walks") } def test_a_model_is_written_as_deep_as_it_is_read() -> None: """A fill value nested as deep as a reader walks is written back. Its data type is claimed by nothing in the scope, so it accepts any JSON. The model also survives pickling and deep-copying, which use two stack frames per level.""" fill_value: dict[str, object] = {} for _ in range(JSON_DEPTH - 2): fill_value = {"x": fill_value} document = { **ZarrV3ArrayMetadata.create_default().to_json(), "data_type": "acme.deep", "fill_value": fill_value, } model = ZarrV3ArrayMetadata.from_json(document) assert model.to_json()["fill_value"] == fill_value assert ZarrV3ArrayMetadata.from_key_value(model.to_key_value()) == model assert pickle.loads(pickle.dumps(model)) == model assert copy.deepcopy(model) == model def test_a_field_s_configuration_member_is_counted_from_the_field_s_root() -> None: # `resolve` reads a field on its own, from the field's root. A member # of its configuration is two levels down, and the cap counts from the # root, not from the member. nested: dict[str, object] = {} for _ in range(JSON_DEPTH - 2): nested = {"x": nested} field = {"name": "acme.x", "configuration": {"y": nested}} _, problems = resolve(field, CodecDefinition, CORE_AND_EXTENSIONS) assert [(len(p.loc), p.kind) for p in problems] == [(JSON_DEPTH, "invalid_value")] assert problems[0].loc[:2] == ("configuration", "y") shallower = {"name": "acme.x", "configuration": {"y": nested["x"]}} assert resolve(shallower, CodecDefinition, CORE_AND_EXTENSIONS)[1] == () def test_a_v2_dtype_of_nested_records_is_read_to_the_levels_a_reader_walks() -> None: """Field records nest a dtype two levels per record. Previously the shape check recursed one record at a time with no cap, and a thousand records overflowed.""" def records(levels: int) -> object: dtype: object = " None: """Previously the v2 models were copied with `copy.deepcopy`, which uses two stack frames per level, and they overflowed on documents that their validators accept.""" fill_value: dict[str, object] = {} for _ in range(JSON_DEPTH - 2): fill_value = {"x": fill_value} # An object type, whose fill value is any JSON. document = { **ZarrV2ArrayMetadata.create_default(shape=(2,), dtype="|O").to_json(), "fill_value": fill_value, } assert read_array_metadata_v2(document).problems == () model = ZarrV2ArrayMetadata.from_json(document) assert json_text(model.to_json()) == json_text(cast("JSONValue", document)) assert ZarrV2ArrayMetadata.from_key_value(model.to_key_value()) == model assert hash(model) == hash(ZarrV2ArrayMetadata.from_json(document)) assert pickle.loads(pickle.dumps(model)) == model assert copy.deepcopy(model) == model problems = read_array_metadata_v2({**document, "fill_value": {"x": fill_value}}).problems assert [(problem.kind, len(problem.loc)) for problem in problems] == [ ("invalid_value", JSON_DEPTH) ] def test_v3_node_type_literal_enforced() -> None: """A v3 array document claiming node_type 'group' is rejected.""" doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"node_type": "group"} problems = read_array_metadata_v3(doc).problems assert [(p.loc, p.kind) for p in problems] == [(("node_type",), "invalid_value")] def test_missing_key_kind_is_machine_readable() -> None: """A missing required key is distinguishable by kind, without message matching.""" doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) del doc["chunk_key_encoding"] problems = read_array_metadata_v3(doc).problems assert problems == ( ValidationProblem(("chunk_key_encoding",), "missing required key", "missing_key"), ) # --- Unified error channels --------------------------------------------------- def test_from_key_value_invalid_json_raises_metadata_error() -> None: """Undecodable store bytes raise MetadataValidationError (kind invalid_json), not JSONDecodeError.""" with pytest.raises(MetadataValidationError) as exc_info: ZarrV3ArrayMetadata.from_key_value({"zarr.json": b"{not json"}) assert [p.kind for p in exc_info.value.problems] == ["invalid_json"] def test_from_key_value_invalid_utf8_raises_metadata_error() -> None: """Invalid UTF-8 store bytes use the same invalid_json error channel.""" with pytest.raises(MetadataValidationError) as exc_info: ZarrV3ArrayMetadata.from_key_value({"zarr.json": b"\x80"}) assert [p.kind for p in exc_info.value.problems] == ["invalid_json"] def test_v2_from_key_value_scalar_root_raises_metadata_error() -> None: """A scalar .zarray document fails through the unified metadata error channel.""" with pytest.raises(MetadataValidationError) as exc_info: ZarrV2ArrayMetadata.from_key_value({".zarray": b"null"}) assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ ((), "invalid_type") ] def test_from_key_value_missing_key_kind() -> None: """A missing store key surfaces as a missing_key problem at the store-key loc.""" with pytest.raises(MetadataValidationError) as exc_info: ZarrV2ArrayMetadata.from_key_value({}) assert exc_info.value.problems == ( ValidationProblem((".zarray",), "missing store key", "missing_key"), ) # --- Adversarial-probe fixes: documents that used to pass validation --------- def test_shape_rejects_json_booleans() -> None: """JSON booleans are not integers: shape/chunks containing true/false are rejected (bool is an int subclass in Python, so isinstance alone passes).""" v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"shape": (True, True)} assert [p.loc for p in read_array_metadata_v3(v3).problems] == [("shape",)] v2 = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"chunks": (True,)} assert [p.loc for p in read_array_metadata_v2(v2).problems] == [("chunks",)] def test_shape_rejects_negative_dimensions() -> None: """Dimension lengths must be non-negative; a negative entry is invalid_value.""" v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"shape": (-1,)} assert [(p.loc, p.kind) for p in read_array_metadata_v3(v3).problems] == [ (("shape",), "invalid_value") ] v2 = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"chunks": (-5,)} assert [(p.loc, p.kind) for p in read_array_metadata_v2(v2).problems] == [ (("chunks",), "invalid_value") ] def test_dimension_names_length_must_match_shape() -> None: """dimension_names must have one entry per dimension of shape.""" doc = dict(ZarrV3ArrayMetadata.create_default(shape=(10,)).to_json()) | { "dimension_names": ("x", "y", "z") } assert [(p.loc, p.kind) for p in read_array_metadata_v3(doc).problems] == [ (("dimension_names",), "invalid_value") ] def test_attributes_values_must_be_json() -> None: """Attribute values are JSON-checked recursively (like fill_value), so a non-serializable value is a validation problem, not a later TypeError.""" doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"attributes": {"a": {1, 2}}} problems = read_array_metadata_v3(doc).problems assert [(p.loc, p.kind) for p in problems] == [(("attributes", "a"), "invalid_type")] def test_configuration_values_must_be_json() -> None: """Configuration values are JSON-checked recursively, so an int-keyed dict cannot pass validation and be silently rewritten by json.dumps.""" doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | { "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": {1: 2}}} } problems = read_array_metadata_v3(doc).problems assert [(p.loc, p.kind) for p in problems] == [ (("chunk_grid", "configuration", "chunk_shape"), "invalid_type") ] def test_v3_extension_keys_must_be_strings() -> None: """A non-string top-level key cannot be represented by a v3 document type.""" doc: dict[object, object] = {**ZarrV3ArrayMetadata.create_default().to_json()} doc[1] = {"must_understand": False} assert [(problem.loc, problem.kind) for problem in read_array_metadata_v3(doc).problems] == [ ((), "invalid_type") ] def test_v3_extension_values_must_be_json() -> None: """Extension payloads are JSON-checked before a model is constructed.""" doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) doc["ext"] = {"must_understand": False, "payload": object()} assert [(problem.loc, problem.kind) for problem in read_array_metadata_v3(doc).problems] == [ (("ext", "payload"), "invalid_type") ] def test_v3_json_extension_without_waiver_is_preserved_as_must_understand() -> None: """A JSON extension without an explicit false waiver remains must-understand.""" doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) doc["ext"] = 1 assert read_array_metadata_v3(doc).problems == () model = ZarrV3ArrayMetadata.from_json(doc) assert model.extra_fields["ext"].to_json() == {"ext": 1} assert model.must_understand_fields == {"ext": 1} def test_v2_codec_configuration_values_must_be_json() -> None: """Non-JSON codec parameters are rejected for compressors and filters.""" for field, value, expected_loc in ( ("compressor", {"id": "x", "payload": object()}, ("compressor", "payload")), ("filters", ({"id": "x", "payload": object()},), ("filters", 0, "payload")), ): doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) doc[field] = value assert [ (problem.loc, problem.kind) for problem in read_array_metadata_v2(doc).problems ] == [(expected_loc, "invalid_type")] def test_dimension_sequences_reject_binary_values() -> None: """Binary buffers are not JSON arrays even though they are integer sequences.""" v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"shape": b"\x02"} v2 = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"chunks": b"\x02"} assert [(problem.loc, problem.kind) for problem in read_array_metadata_v3(v3).problems] == [ (("shape",), "invalid_type") ] assert [(problem.loc, problem.kind) for problem in read_array_metadata_v2(v2).problems] == [ (("chunks",), "invalid_type") ] def test_array_models_write_json_lists_as_tuples() -> None: """A model read from a document with lists, as `json.loads` returns it, writes the document with tuples.""" v3_raw = json.loads(json.dumps(ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json())) v2_raw = json.loads(json.dumps(ZarrV2ArrayMetadata.create_default(shape=(2,)).to_json())) assert read_array_metadata_v3(v3_raw).problems == () assert read_array_metadata_v2(v2_raw).problems == () v3_written = ZarrV3ArrayMetadata.from_json(v3_raw).to_json() v2_written = ZarrV2ArrayMetadata.from_json(v2_raw).to_json() assert isinstance(v3_written["shape"], tuple) assert isinstance(v3_written["codecs"], tuple) assert isinstance(v2_written["shape"], tuple) assert isinstance(v2_written["chunks"], tuple) def test_array_models_materialize_abstract_nested_json() -> None: """A model reads an abstract sequence nested in its document, and writes it as a tuple.""" # A 16-bit raw data type, whose fill value is two byte values. v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"data_type": "r16"} v3["fill_value"] = range(2) # A complex type, whose fill value is a pair. v2 = dict(ZarrV2ArrayMetadata.create_default(dtype=" None: """must_understand_fields contains every extra field not explicitly waived with must_understand: false, including implicitly-true and non-mapping fields, so a reader can discharge the spec's fail-to-open duty by subtracting the extensions it recognizes. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1575-L1578 """ model = ZarrV3ArrayMetadata.create_default( extra_fields={ "ext_a": {"name": "a", "must_understand": False}, "ext_b": {"name": "b"}, "ext_c": {"name": "c", "must_understand": True}, "ext_d": 123, }, ) assert set(model.must_understand_fields) == {"ext_b", "ext_c", "ext_d"} recognized = {"ext_b"} assert model.must_understand_fields.keys() - recognized == {"ext_c", "ext_d"} def test_must_understand_fields_empty_when_all_waived() -> None: """must_understand_fields is empty when every extra field is explicitly waived.""" model = ZarrV3ArrayMetadata.create_default( extra_fields={"ext_a": {"name": "a", "must_understand": False}} ) assert model.must_understand_fields == {} def test_dimension_names_null_field_rejected() -> None: """A dimension_names field whose VALUE is null is invalid: the spec permits null as an element (an unnamed dimension), never as the field value — "not specified" is spelled by omitting the key. Consumers bridging from an in-memory None sentinel must drop the key, not write null. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L635-L638 """ doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"dimension_names": None} problems = read_array_metadata_v3(doc).problems assert [(p.loc, p.kind) for p in problems] == [(("dimension_names",), "invalid_type")] # and the model's own UNSET spelling correctly maps to key absence model = ZarrV3ArrayMetadata.create_default() assert model.dimension_names is UNSET assert "dimension_names" not in model.to_json() # --- create_default derives the chunk grid from shape ------------------------ def test_v3_create_default_chunk_grid_follows_shape() -> None: """Overriding shape without chunk_grid derives a consistent default grid: one chunk covering the array (chunk_shape == shape), instead of silently keeping the scalar default's 0-d grid.""" model = ZarrV3ArrayMetadata.create_default(shape=(100, 100)) assert model.chunk_grid.to_json() == { "name": "regular", "configuration": {"chunk_shape": (100, 100)}, } def test_v3_create_default_explicit_chunk_grid_respected() -> None: """An explicit chunk_grid override wins over the shape-derived default.""" grid: ZarrV3NamedConfigJSON = {"name": "regular", "configuration": {"chunk_shape": (10, 10)}} model = ZarrV3ArrayMetadata.create_default(shape=(100, 100), chunk_grid=grid) assert model.chunk_grid.to_json() == grid def test_v2_create_default_chunks_follow_shape() -> None: """Overriding shape without chunks derives chunks == shape.""" model = ZarrV2ArrayMetadata.create_default(shape=(100, 100)) assert model.chunks == (100, 100) def test_v2_create_default_explicit_chunks_respected() -> None: """An explicit chunks override wins over the shape-derived default.""" model = ZarrV2ArrayMetadata.create_default(shape=(100, 100), chunks=(10, 10)) assert model.chunks == (10, 10) def test_v3_create_default_zero_length_dimensions() -> None: """The derived grid gives a dimension of length 0 a chunk length of 1: the regular grid requires chunk sizes greater than zero, so every reader accepts the written grid, and it fits the shape it chunks.""" model = ZarrV3ArrayMetadata.create_default(shape=(0, 3)) assert model.chunk_grid.to_json() == { "name": "regular", "configuration": {"chunk_shape": (1, 3)}, } assert read_array_metadata_v3(model.to_json()).problems == () # --- v2 dimension_separator default (roborev job 426) ------------------------- def test_v2_absent_dimension_separator_means_dot() -> None: """A .zarray that omits dimension_separator uses the v2 convention default '.', not '/': chunk keys of real-world default-separator v2 arrays look like '0.0'. The model reports '.' and compares equal to a document that writes '.', but `to_json` and `to_key_value` write the document as it was read, without the key. `update` with `UNSET` removes the key.""" explicit = ZarrV2ArrayMetadata.create_default() doc = dict(explicit.to_json()) del doc["dimension_separator"] model = ZarrV2ArrayMetadata.from_json(doc) assert model.dimension_separator == "." assert "dimension_separator" not in model.to_json() assert b"dimension_separator" not in model.to_key_value()[".zarray"] assert model == explicit assert hash(model) == hash(explicit) removed = explicit.update(dimension_separator=UNSET) assert "dimension_separator" not in removed.to_json() assert removed.dimension_separator == "." def test_v2_from_key_value_without_separator_means_dot() -> None: """The .zarray store-file path applies the same '.' default for an absent dimension_separator key.""" doc = { k: v for k, v in ZarrV2ArrayMetadata.create_default().to_json().items() if k not in ("dimension_separator", "attributes") } import json as _json model = ZarrV2ArrayMetadata.from_key_value({".zarray": _json.dumps(doc).encode()}) assert model.dimension_separator == "." def test_v2_null_dimension_separator_rejected() -> None: """dimension_separator may be absent, '.', or '/' — never null: the document grammar has no null spelling for this field.""" doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dimension_separator": None} problems = read_array_metadata_v2(doc).problems assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_type")] def test_dimension_names_absent_and_all_null_are_distinct() -> None: """An absent dimension_names field and an explicit all-null one are semantically different documents: the explicit form says every dimension has a name, which is null; absence says there are no dimension names. The model preserves the distinction (UNSET vs a tuple of Nones), and both spellings round-trip faithfully.""" absent_doc = dict(ZarrV3ArrayMetadata.create_default(shape=(2, 3)).to_json()) explicit_doc = absent_doc | {"dimension_names": (None, None)} absent = ZarrV3ArrayMetadata.from_json(absent_doc) explicit = ZarrV3ArrayMetadata.from_json(explicit_doc) assert absent.dimension_names is UNSET assert explicit.dimension_names == (None, None) assert absent != explicit assert "dimension_names" not in absent.to_json() assert absent.to_json() == absent_doc assert explicit.to_json() == explicit_doc @pytest.mark.parametrize( ("read", "document", "member", "value", "kind"), [ (read_array_metadata_v3, ZarrV3ArrayMetadata, "zarr_format", "3", "invalid_type"), (read_array_metadata_v3, ZarrV3ArrayMetadata, "zarr_format", 2, "invalid_value"), (read_array_metadata_v3, ZarrV3ArrayMetadata, "node_type", 5, "invalid_type"), (read_array_metadata_v3, ZarrV3ArrayMetadata, "node_type", "group", "invalid_value"), (read_array_metadata_v2, ZarrV2ArrayMetadata, "order", 1, "invalid_type"), (read_array_metadata_v2, ZarrV2ArrayMetadata, "order", "Q", "invalid_value"), ( read_array_metadata_v2, ZarrV2ArrayMetadata, "dimension_separator", ":", "invalid_value", ), ], ) def test_error_a_member_outside_the_values_it_takes( read: Callable[[object], ZarrV2ArrayMetadataReading | ZarrV3ArrayMetadataReading], document: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], member: str, value: object, kind: str, ) -> None: # A value is the wrong type when none of the allowed values has its JSON # type (a string where a number belongs). Otherwise it is the wrong value. written = {**document.create_default().to_json(), member: value} assert [(p.loc, p.kind) for p in read(written).problems] == [((member,), kind)] def test_error_a_document_nested_deeper_than_a_reader_walks_is_a_problem() -> None: # Not a `RecursionError`: a 10 KB document from an untrusted writer, # nested past the interpreter's limit, is refused at the first level past # the ones a reader walks. Every reader refuses it, and so does the model. deep: dict[str, object] = {} for _ in range(2_000): deep = {"a": deep} document = {**ZarrV3ArrayMetadata.create_default().to_json(), "attributes": deep} problems = read_array_metadata_v3(document).problems assert [(len(p.loc), p.kind) for p in problems] == [(JSON_DEPTH, "invalid_value")] assert problems[0].loc[:2] == ("attributes", "a") assert not is_json(document) assert read_group_metadata_v3(document).problems != () assert read_array_metadata_v2(document).problems != () assert read_group_metadata_v2(document).problems != () with pytest.raises(MetadataValidationError): ZarrV3ArrayMetadata.from_json(document) def test_error_a_field_object_in_a_document_is_not_json() -> None: # An `AcceptedField` built by hand, with a configuration its definition # refuses, and inserted into a document is refused as a non-JSON value. # So nothing built by hand passes as read. A model holds its own fields # as read. smuggled = AcceptedField( json="gzip", name="gzip", definition=GZIP_CODEC, configuration={"level": 99, "window": 1} ) document = { **ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json(), "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}, smuggled], } assert [(p.loc, p.kind) for p in read_array_metadata_v3(document).problems] == [ (("codecs", 1), "invalid_type") ] with pytest.raises(MetadataValidationError): ZarrV3ArrayMetadata.from_json(document) # `update` refuses a field among the members it is given with a # `TypeError`, as `test_error_update_takes_fields_as_json` shows. model = ZarrV3ArrayMetadata.create_default(shape=(2,)) with pytest.raises(TypeError, match="to_json"): model.update(codecs=cast("Any", (model.codecs[0].to_json(), smuggled))) def test_a_problem_shows_an_integer_too_long_to_write_by_its_size( interpreter_writes_4300_digits: None, ) -> None: # `int` refuses to write more than 4,300 digits. The message gives the # size instead of raising. document = {**ZarrV3ArrayMetadata.create_default().to_json(), "zarr_format": 10**5000} (problem,) = read_array_metadata_v3(document).problems assert problem.message == f"expected 3, got an integer of {(10**5000).bit_length()} bits" (problem,) = resolve({"name": "gzip", 10**5000: 1}, CodecDefinition, CORE_AND_EXTENSIONS)[1] assert problem.message == ( f"non-string key an integer of {(10**5000).bit_length()} bits in JSON object" ) # When such an integer is inside a container (JSON or not) or a key, the # message says the interpreter will not write the value. for held in ([10**5000], [10**5000, object()]): document = {**ZarrV3ArrayMetadata.create_default().to_json(), "zarr_format": held} (problem,) = read_array_metadata_v3(document).problems assert ( problem.message == "expected 3, got a value of type list the interpreter will not write" ) (problem,) = resolve({"name": "gzip", (10**5000,): 1}, CodecDefinition, CORE_AND_EXTENSIONS)[1] assert problem.message == ( "non-string key a value of type tuple the interpreter will not write in JSON object" ) # The same applies to a value that `refine_json` reports as not JSON at # all: a set holding such an integer. document = {**ZarrV3ArrayMetadata.create_default().to_json(), "attributes": {"a": {10**5000}}} (problem,) = read_array_metadata_v3(document).problems assert (problem.loc, problem.message) == ( ("attributes", "a"), "not a JSON-serializable value: a value of type set the interpreter will not write", ) def test_a_problem_shows_a_value_nested_too_deep_to_write_by_saying_so() -> None: # `_refine` stops at `JSON_DEPTH`. A value nested past it is described as # too deep instead of shown by its repr, because repr has its own limit # that depends on the interpreter and platform. One level past the cap, # which any repr would write, is enough to test this. deep: list[object] = [] innermost = deep for _ in range(JSON_DEPTH + 1): nested: list[object] = [] innermost.append(nested) innermost = nested document = {**ZarrV3ArrayMetadata.create_default().to_json(), "zarr_format": deep} # Only one problem: a value that the literal check refuses is not walked, # so the depth rule does not also check it. assert [ (problem.loc, problem.kind, problem.message) for problem in read_array_metadata_v3(document).problems ] == [(("zarr_format",), "invalid_type", "expected 3, got a value nested too deep to show")] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_consolidating.py000066400000000000000000000474671527014137600325310ustar00rootroot00000000000000"""Tests for building consolidated metadata from node models. A model is accepted when it reads the same in the scope of the consolidated metadata, or gains information there. It is refused when it conflicts or would lose information. A group takes the result through `update`, and writes its document.""" from __future__ import annotations import dataclasses import pickle from typing import Any, cast, get_type_hints import pytest from zarr_metadata._json import JSON_DEPTH, is_canonical_json from zarr_metadata.definition import EmptyConfiguration from zarr_metadata.model import ( ZarrV3ArrayMetadata, ZarrV3ConsolidatedMetadata, ZarrV3ConsolidatedMetadataInput, ZarrV3GroupMetadata, ZarrV3NodeMetadataInput, read_consolidated_metadata_v3, read_group_metadata_v3, ) from zarr_metadata.problems import MetadataValidationError from zarr_metadata.v3.codec.gzip import GZIP_CODEC from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE from zarr_metadata.v3.definition import ( CORE, CORE_AND_EXTENSIONS, CodecDefinition, ZarrV3Context, ) ARRAY: dict[str, Any] = { **ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json(), "codecs": ("bytes",), } """A `uint8` array whose `bytes` codec has no configuration, so a private `bytes` definition with no configuration also reads it.""" ZSTD = {"name": "zstd", "configuration": {"level": 3, "checksum": False}} LOOSE_ZSTD = {"name": "zstd", "configuration": {"level": 30}} WITH_ZSTD = {**ARRAY, "codecs": (*ARRAY["codecs"], ZSTD)} WITH_LOOSE_ZSTD = {**ARRAY, "codecs": (*ARRAY["codecs"], LOOSE_ZSTD)} MY_BYTES = CodecDefinition( name="bytes", configuration=EmptyConfiguration, kind="array_bytes", size="static" ) PRIVATE = CORE.update(codecs={**CORE.codecs, "bytes": MY_BYTES}) INLINE: dict[str, Any] = {"kind": "inline", "must_understand": False} def _consolidated( context: ZarrV3Context | None = None, **entries: object ) -> ZarrV3ConsolidatedMetadata: """The consolidated metadata that lists `entries`, read in `context`.""" return ZarrV3ConsolidatedMetadata({**INLINE, "metadata": entries}, context=context) def _group_document(**entries: object) -> dict[str, Any]: """A group document whose consolidated metadata lists `entries`.""" return { "zarr_format": 3, "node_type": "group", "consolidated_metadata": {**INLINE, "metadata": entries}, } @pytest.mark.parametrize( ("context", "child", "beside", "gained"), [ (CORE_AND_EXTENSIONS, ZarrV3ArrayMetadata(ARRAY), {}, False), (CORE_AND_EXTENSIONS, ZarrV3ArrayMetadata(ARRAY, context=CORE), {}, False), (CORE_AND_EXTENSIONS, ZarrV3ArrayMetadata(WITH_ZSTD, context=CORE), {}, True), ( CORE, ZarrV3GroupMetadata(_group_document(x=ARRAY), context=ZarrV3Context()), {"a/x": ZarrV3ArrayMetadata(ARRAY, context=ZarrV3Context())}, True, ), ], ids=["same-scope", "unused-definitions", "gain", "listed-group-and-its-child"], ) def test_a_model_is_accepted_as_a_consolidated_entry_when_it_refines_into_the_scope( context: ZarrV3Context, child: ZarrV3ArrayMetadata | ZarrV3GroupMetadata, beside: dict[str, Any], gained: bool, ) -> None: """A node model can be given wherever consolidated metadata lists a document. The model is accepted in two cases. If the scope of the consolidated metadata reads every claim of the model identically, the model's reading is kept and not read again. If that scope claims something that the model's scope left unclaimed, the model's document is read again in that scope. In both cases the consolidated metadata holds the model in its own scope, equals the one built from the documents, and a group it is given to writes plain JSON. A group model's fields are its extra fields: read in a scope that claims nothing, its `consolidated_metadata` is unclaimed, and the scope of the consolidated metadata gains it. A listed group's own listing is listed flat next to it, as the convention asks.""" held = _consolidated(context, a=child, **beside) node = held.metadata["a"] assert node.context is held.context assert cast("Any", node).refines(child) assert (node == child) is not gained if not gained and isinstance(child, ZarrV3ArrayMetadata): assert isinstance(node, ZarrV3ArrayMetadata) assert node.reading.pipeline is child.reading.pipeline written = {path: _documents(entry) for path, entry in beside.items()} assert held == _consolidated(context, a=child.to_json(), **written) group = ZarrV3GroupMetadata.create_default(context=context, consolidated_metadata=held) document = group.to_json() assert is_canonical_json(document, finite=False) assert read_group_metadata_v3(document).problems == () def _documents(value: object) -> object: """Return `value` with every node model in it, at any depth, replaced by its document.""" if isinstance(value, (ZarrV3ArrayMetadata, ZarrV3GroupMetadata)): return value.to_json() if isinstance(value, dict): return {key: _documents(item) for key, item in cast("dict[str, object]", value).items()} return value def test_documents_as_written_and_pickle() -> None: """Consolidated metadata built from models writes each child's document as the child wrote it, and so does a group it is given to. Both pickle like any other model.""" child = ZarrV3ArrayMetadata({**ARRAY, "data_type": {"name": "uint8"}}) held = _consolidated(None, a=child) assert held.to_json()["metadata"]["a"] == child.to_json() group = ZarrV3GroupMetadata.create_default(consolidated_metadata=held) written = cast("Any", group.to_json())["consolidated_metadata"] assert written["metadata"]["a"] == child.to_json() assert pickle.loads(pickle.dumps(held)) == held assert pickle.loads(pickle.dumps(group)) == group def test_error_a_model_that_conflicts_with_the_scope_is_refused_at_its_path() -> None: """A model read with a private `bytes`, given to consolidated metadata whose scope reads the core `bytes`, is a conflict. The problem is at the entry's path, names the kind and the name, and tells the two definitions apart. The model is not silently read again.""" child = ZarrV3ArrayMetadata(ARRAY, context=PRIVATE) with pytest.raises(MetadataValidationError) as raised: _consolidated(CORE_AND_EXTENSIONS, a=child) (problem,) = raised.value.problems assert problem.loc == ("metadata", "a", "codecs", 0) assert problem.kind == "invalid_value" assert problem.message == ( "expected a document read in the scope of the consolidated metadata, got a model that " "reads the codec 'bytes' by CodecDefinition(name='bytes') of EmptyConfiguration, which " "that scope reads by CodecDefinition(name='bytes') of BytesCodecConfiguration" ) def test_error_a_model_that_would_lose_a_meaning_is_refused_at_its_path() -> None: """A model read in a scope that claims `zstd`, given to consolidated metadata whose scope leaves `zstd` unclaimed, would lose information. It is refused, the same way a conflict is.""" child = ZarrV3ArrayMetadata(WITH_ZSTD, context=CORE_AND_EXTENSIONS) with pytest.raises(MetadataValidationError) as raised: _consolidated(CORE, a=child) (problem,) = raised.value.problems assert problem.loc == ("metadata", "a", "codecs", 1) assert problem.message.endswith("which that scope leaves unclaimed") def test_error_a_gain_that_surfaces_a_problem_is_reported_at_the_problem() -> None: """A model whose scope left `zstd` unclaimed, given to consolidated metadata whose scope claims it, is read again in that scope. A level that the `zstd` definition refuses is reported as a problem at its location.""" child = ZarrV3ArrayMetadata(WITH_LOOSE_ZSTD, context=CORE) with pytest.raises(MetadataValidationError) as raised: _consolidated(CORE_AND_EXTENSIONS, a=child) assert [problem.loc for problem in raised.value.problems] == [ ("metadata", "a", "codecs", 1, "configuration", "level") ] def test_the_reader_sees_models_as_the_constructor_does() -> None: """Given a member that lists models, `read_consolidated_metadata_v3` reads them the same way the constructor does, and holds the adopted model.""" reading = read_consolidated_metadata_v3( {**INLINE, "metadata": {"a": ZarrV3ArrayMetadata(ARRAY)}} ) assert reading.problems == () assert reading.metadata is not None assert isinstance(reading.metadata.metadata["a"], ZarrV3ArrayMetadata) def test_error_the_reader_reports_a_conflicting_model_as_the_constructor_does() -> None: """Given a member that lists a model which conflicts with the scope, `read_consolidated_metadata_v3` reports the conflict at the entry's path, as the constructor refuses it.""" member = {**INLINE, "metadata": {"a": ZarrV3ArrayMetadata(ARRAY, context=PRIVATE)}} reading = read_consolidated_metadata_v3(member) assert [problem.loc for problem in reading.problems] == [("metadata", "a", "codecs", 0)] assert reading.metadata is None def test_a_group_writes_the_consolidated_metadata_it_is_given_in_its_own_scope() -> None: """A `ZarrV3ConsolidatedMetadata` given to `update` is written as its document. It does not bring its own scope into the group: the group's `context` stays its own, and reading the written member in that scope gives the models in it.""" held = _consolidated(CORE_AND_EXTENSIONS, a=ZarrV3ArrayMetadata(ARRAY)) group = ZarrV3GroupMetadata.create_default(context=CORE, attributes={"t": 1}) updated = group.update(consolidated_metadata=held) assert updated.context == CORE assert updated.attributes == {"t": 1} assert updated.consolidated_metadata == held.to_json() again = ZarrV3ConsolidatedMetadata(updated.consolidated_metadata, context=updated.context) assert again.metadata["a"].context == CORE assert again == held def test_error_a_group_refuses_consolidated_metadata_that_conflicts_with_its_scope() -> None: """A `ZarrV3ConsolidatedMetadata` read in another scope is read again in the group's scope before the group writes it, as a listing of node models is. A model read with a private `bytes` conflicts with the core `bytes` the group reads, so `update` and `create_default` refuse it at the child's path.""" held = _consolidated(PRIVATE, a=ZarrV3ArrayMetadata(ARRAY, context=PRIVATE)) group = ZarrV3GroupMetadata.create_default(context=CORE_AND_EXTENSIONS) for write in ( lambda: group.update(consolidated_metadata=held), lambda: ZarrV3GroupMetadata.create_default( context=CORE_AND_EXTENSIONS, consolidated_metadata=held ), ): with pytest.raises(MetadataValidationError) as raised: write() assert [problem.loc for problem in raised.value.problems] == [ ("consolidated_metadata", "metadata", "a", "codecs", 0) ] def test_error_a_group_refuses_consolidated_metadata_that_would_lose_a_meaning() -> None: """A listing read in a scope that claims `zstd`, given to a group whose scope leaves `zstd` unclaimed, would lose information when the group's scope reads it, so the group refuses it at the child's path.""" held = _consolidated(CORE_AND_EXTENSIONS, a=ZarrV3ArrayMetadata(WITH_ZSTD)) with pytest.raises(MetadataValidationError) as raised: ZarrV3GroupMetadata.create_default(context=CORE).update(consolidated_metadata=held) assert [problem.loc for problem in raised.value.problems] == [ ("consolidated_metadata", "metadata", "a", "codecs", 1) ] @pytest.mark.parametrize( ("context", "found"), [ (ZarrV3Context(), []), (CORE, [("metadata", "g", "consolidated_metadata", "garbage")]), ], ids=["unclaimed", "claimed"], ) def test_a_listed_group_s_listing_is_read_only_where_the_scope_claims_it( context: ZarrV3Context, found: list[tuple[str | int, ...]] ) -> None: """A listed group's own `consolidated_metadata` is a listing only in a scope that claims the member. In a scope that leaves it unclaimed, the listed group keeps it as written, so the reader does not read it either, whatever it holds.""" group = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": {"garbage": 1}} reading = read_consolidated_metadata_v3({**INLINE, "metadata": {"g": group}}, context=context) assert [problem.loc for problem in reading.problems if problem.kind == "unknown_key"] == found def test_a_group_reads_its_listing_in_its_own_scope() -> None: """`consolidated()` reads the documents a group's `consolidated_metadata` lists in the group's own scope, so a child reads as the group's reader would read it, and returns None for a group that lists none.""" group = ZarrV3GroupMetadata(_group_document(a=WITH_ZSTD), context=CORE) listing = group.consolidated() assert listing is not None assert listing.context == CORE assert listing.metadata["a"].context == CORE assert listing == ZarrV3ConsolidatedMetadata(group.consolidated_metadata, context=CORE) assert ZarrV3GroupMetadata.create_default().consolidated() is None def test_error_a_group_s_listing_with_a_problem_is_located_in_the_group() -> None: """A group opens whatever its listing holds, but `consolidated()` reads every listed document, and raises their problems located in the group's document, under `consolidated_metadata`.""" group = ZarrV3GroupMetadata(_group_document(a=WITH_LOOSE_ZSTD)) with pytest.raises(MetadataValidationError) as raised: group.consolidated() assert [problem.loc for problem in raised.value.problems] == [ ("consolidated_metadata", "metadata", "a", "codecs", 1, "configuration", "level") ] @pytest.mark.parametrize("listing", [5, None], ids=["a-number", "null"]) def test_error_a_group_s_listing_that_is_not_an_object(listing: object) -> None: """In a scope that leaves `consolidated_metadata` unclaimed, a group holding one that is not an object, a number or null, opens. The group still writes a listing, so `consolidated()` refuses it with an `invalid_type` problem located at the member, rather than answering that the group lists none.""" group = ZarrV3GroupMetadata( {"zarr_format": 3, "node_type": "group", "consolidated_metadata": listing}, context=ZarrV3Context(), ) with pytest.raises(MetadataValidationError) as raised: group.consolidated() assert [(problem.loc, problem.kind, problem.message) for problem in raised.value.problems] == [ (("consolidated_metadata",), "invalid_type", "expected an object") ] def test_children_of_different_scopes_consolidate_in_their_union() -> None: """Children read in different scopes are consolidated in the union of their scopes. Each child refines into that scope, and the consolidated metadata reads in it.""" a = ZarrV3ArrayMetadata(ARRAY, context=CORE) b = ZarrV3ArrayMetadata(WITH_ZSTD, context=CORE_AND_EXTENSIONS) context = a.context.union(b.context) held = _consolidated(context, a=a, b=b) assert held.context == CORE_AND_EXTENSIONS assert held.metadata["a"] == a assert held.metadata["b"] == b def test_error_a_model_inside_a_listed_document_is_not_json() -> None: """Node models are taken only as the entries of consolidated metadata. A model inside a listed group's own listing is a value that is not JSON, and is refused there.""" listed = _group_document(x=ZarrV3ArrayMetadata(ARRAY)) reading = read_consolidated_metadata_v3({**INLINE, "metadata": {"a": listed, "a/x": ARRAY}}) assert [(problem.loc, problem.kind) for problem in reading.problems] == [ (("metadata", "a", "consolidated_metadata", "metadata", "x"), "invalid_type") ] def test_error_a_model_listed_too_deep_is_refused_where_it_sits() -> None: """A model that is valid on its own, with attributes nested to the last level a reader walks, goes past the depth cap when listed as a document in consolidated metadata, which sits under a group's key. The reader and the constructor refuse it there, as a group refuses its document.""" deep: list[object] = [] for _ in range(JSON_DEPTH - 3): deep = [deep] child = ZarrV3ArrayMetadata({**ARRAY, "attributes": {"x": deep}}) member = {**INLINE, "metadata": {"a": child}} reading = read_consolidated_metadata_v3(member) assert [problem.loc[:3] for problem in reading.problems] == [("metadata", "a", "attributes")] assert reading.metadata is None with pytest.raises(MetadataValidationError): ZarrV3ConsolidatedMetadata(member) assert [ problem.loc[:4] for problem in read_group_metadata_v3(_group_document(a=child.to_json())).problems ] == [("consolidated_metadata", "metadata", "a", "attributes")] def test_error_a_data_type_conflict_names_the_kind_and_the_written_name() -> None: """A conflict over a data type names the kind in words and the name as the document writes it (`r16`, which the `r*` definition claims). When the two definitions print the same, the message says that they differ.""" strict = dataclasses.replace( RAW_BYTES_DATA_TYPE, fill_value_rules=lambda configuration, nested, value: iter(()) ) child = ZarrV3ArrayMetadata( {**ARRAY, "data_type": "r16", "fill_value": [0, 0], "codecs": ("bytes",)}, context=CORE.update(data_types={**CORE.data_types, "r*": strict}), ) with pytest.raises(MetadataValidationError) as raised: _consolidated(CORE, a=child) (problem,) = raised.value.problems assert problem.loc == ("metadata", "a", "data_type") assert problem.message == ( "expected a document read in the scope of the consolidated metadata, got a model that " "reads the data type 'r16' by another definition than the one that scope reads it by, " "DataTypeDefinition(name='r*') of RawBytesConfiguration" ) def test_error_a_conflict_between_definitions_alike_says_they_differ() -> None: """Two definitions of the same name that read the same TypedDict but have different rules are distinguished in the message by stating that they differ, since nothing else shows the difference.""" strict = dataclasses.replace(GZIP_CODEC, rules=lambda configuration, nested: iter(())) child = ZarrV3ArrayMetadata( {**ARRAY, "codecs": ("bytes", {"name": "gzip", "configuration": {"level": 1}})}, context=CORE.update(codecs={**CORE.codecs, "gzip": strict}), ) with pytest.raises(MetadataValidationError) as raised: _consolidated(CORE, a=child) (problem,) = raised.value.problems assert problem.message == ( "expected a document read in the scope of the consolidated metadata, got a model that " "reads the codec 'gzip' by another definition than the one that scope reads it by, " "CodecDefinition(name='gzip') of GzipCodecConfiguration" ) def test_the_input_types_are_types_a_signature_can_hold() -> None: """`ZarrV3NodeMetadataInput` and `ZarrV3ConsolidatedMetadataInput` resolve to types, not strings, when a caller's `get_type_hints` reads them as annotations at run time.""" def take(entry: ZarrV3NodeMetadataInput, member: ZarrV3ConsolidatedMetadataInput) -> None: pass hints = get_type_hints(take) assert {"entry", "member"} <= set(hints) assert not isinstance(hints["entry"], str) def test_error_a_conflict_is_reported_before_what_the_re_read_finds() -> None: """A model read with the core `bytes` and an `endian`, given to consolidated metadata whose private `bytes` takes no configuration, is refused first for the conflict and then for the `endian` key that the private `bytes` does not accept. The cause is reported before its symptom.""" child = ZarrV3ArrayMetadata( {**ARRAY, "codecs": ({"name": "bytes", "configuration": {"endian": "little"}},)} ) with pytest.raises(MetadataValidationError) as raised: _consolidated(PRIVATE, a=child) assert [problem.loc for problem in raised.value.problems] == [ ("metadata", "a", "codecs", 0), ("metadata", "a", "codecs", 0, "configuration", "endian"), ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_construction.py000066400000000000000000000555211527014137600324140ustar00rootroot00000000000000"""A model checks itself when it is built, as pydantic's `__init__` does. A v3 model is built only by reading its document. It is changed only by `update`, which reads the new document. A document with a problem is refused, with every problem reported, so no model is ever invalid. A v2 model is still a dataclass, and a `dataclasses.replace` that would make it invalid is refused. `to_key_value` writes a model as it is, without reading anything. """ from __future__ import annotations import dataclasses import math from collections import UserDict from typing import TYPE_CHECKING, Any, cast import pytest from zarr_metadata.definition import RefusedField, resolve from zarr_metadata.model import ( UNSET, ZarrV2ArrayMetadata, ZarrV2ConsolidatedMetadata, ZarrV2GroupMetadata, ZarrV3ArrayMetadata, ZarrV3ConsolidatedMetadata, ZarrV3GroupMetadata, ) from zarr_metadata.problems import MetadataValidationError, ValidationProblem from zarr_metadata.v3.codec.gzip import GZIP_CODEC, GzipCodecConfiguration from zarr_metadata.v3.data_type.int8 import INT8_DATA_TYPE from zarr_metadata.v3.definition import ( CORE, CORE_AND_EXTENSIONS, CodecDefinition, ) if TYPE_CHECKING: from collections.abc import Iterator from zarr_metadata.definition import Nested ARRAY = ZarrV3ArrayMetadata.create_default( shape=(4,), attributes={"a": [1, None]}, dimension_names=("x",), extra_fields={"acme": {"must_understand": False}}, ) GROUP = ZarrV3GroupMetadata.from_json( { "zarr_format": 3, "node_type": "group", "attributes": {"a": 1}, "consolidated_metadata": { "kind": "inline", "must_understand": False, "metadata": { "x": ARRAY.to_json(), "y": {"zarr_format": 3, "node_type": "group"}, "y/z": ARRAY.to_json(), }, }, } ) CONSOLIDATED = ZarrV3ConsolidatedMetadata(GROUP.consolidated_metadata) V2_ARRAY = ZarrV2ArrayMetadata.create_default(shape=(4,), attributes={"a": 1}) V2_GROUP = ZarrV2GroupMetadata.create_default(attributes={}) V2_CONSOLIDATED = ZarrV2ConsolidatedMetadata.from_json( {"zarr_consolidated_format": 1, "metadata": {"a/.zattrs": {"x": 1}}} ) def _rebuilt(model: object, changes: dict[str, object]) -> Any: # noqa: ANN401 - the tests give the model as object """Rebuild the v2 model `model` from its document, with `changes` replacing members, in the model's own scope. This is what `update` does, and it also works for the consolidated model.""" held = cast("Any", model) return type(held)({**held.to_json(), **changes}, context=held.context) _INLINE: dict[str, Any] = {"kind": "inline", "must_understand": False} @pytest.mark.parametrize( "model", [ARRAY, GROUP, CONSOLIDATED], ids=["array", "group", "consolidated"], ) def test_a_v3_model_is_built_of_its_document_in_its_scope(model: object) -> None: """A v3 model is its document read in its scope. Rebuilding it from its document and scope gives the same model.""" held = cast("Any", model) assert type(held)(held.to_json(), context=held.context) == model @pytest.mark.parametrize( "model", [ V2_ARRAY, V2_GROUP, V2_CONSOLIDATED, ], ids=["v2-array", "v2-group", "v2-consolidated"], ) def test_a_v2_model_is_built_as_a_read_builds_it(model: object) -> None: """A v2 model is built from its document in its scope, the same way a read builds it. The constructor, given the model's own document and scope, builds an equal model.""" held = cast("Any", model) assert type(held)(held.to_json(), context=held.context) == model @pytest.mark.parametrize( ("model", "changes"), [ (ARRAY, {"shape": [4], "dimension_names": ["x"]}), (ARRAY, {"attributes": UserDict({"a": [1, None]})}), (GROUP, {"attributes": UserDict({"a": 1})}), ], ids=["array-lists", "array-mapping", "group-mapping"], ) def test_a_v3_model_holds_its_members_as_a_read_refines_them( model: object, changes: dict[str, object] ) -> None: """A model holds arrays as tuples and objects as dicts, as a read holds them. So a model updated with other container types equals the model a read builds, and it round-trips through its store.""" changed = cast("Any", model).update(**changes) assert changed == model assert type(changed).from_key_value(changed.to_key_value()) == changed @pytest.mark.parametrize( ("model", "changes"), [ (V2_ARRAY, {"shape": range(4, 5), "chunks": [4], "attributes": {"a": 1}}), (V2_CONSOLIDATED, {"metadata": {"a/.zattrs": UserDict({"x": 1})}}), ], ids=["v2-sequences", "v2-consolidated-mapping"], ) def test_a_v2_model_holds_its_members_as_a_read_refines_them( model: object, changes: dict[str, object] ) -> None: """A v2 model holds arrays as tuples and objects as dicts, as a read holds them. So a v2 model rebuilt with other container types equals the model a read builds.""" changed = _rebuilt(model, changes) assert changed == model assert type(changed).from_key_value(changed.to_key_value()) == changed @pytest.mark.parametrize( ("model", "member"), [(ARRAY, "acme.x"), (GROUP, "attributes")], ids=["array-extra-field", "group-attributes"], ) def test_a_v3_model_shares_no_container_with_what_it_was_built_of( model: object, member: str ) -> None: """A model holds copies of the containers passed to `update`. Changing those containers afterwards does not change the model.""" held: dict[str, object] = {"must_understand": False, "z": {"y": 1}} changes = {"attributes": held} if member == "attributes" else {"extra_fields": {member: held}} built = cast("Any", model).update(**changes) held["w"] = math.nan cast("dict[str, object]", held["z"])["y"] = math.nan expected = {"must_understand": False, "z": {"y": 1}} if member == "attributes": assert built.attributes == expected else: assert built.extra_fields[member].to_json() == {member: expected} assert type(built).from_key_value(built.to_key_value()) == built @pytest.mark.parametrize( ("model", "member"), [ (V2_ARRAY, "attributes"), (V2_GROUP, "attributes"), (V2_CONSOLIDATED, "metadata"), ], ids=["v2-array-attributes", "v2-group-attributes", "v2-consolidated"], ) def test_a_v2_model_shares_no_container_with_what_it_was_built_of( model: object, member: str ) -> None: """A v2 model holds copies of the containers it is built from: changing them afterwards changes nothing in the document the model writes.""" held: dict[str, object] = {"acme.x": {"must_understand": False}} built = _rebuilt(model, {member: held}) held["acme.y"] = math.nan cast("dict[str, object]", held["acme.x"])["z"] = math.nan assert built.to_json()[member] == {"acme.x": {"must_understand": False}} assert type(built).from_key_value(built.to_key_value()) == built def test_a_model_is_read_when_built_and_written_as_it_is() -> None: """A model is read once, when it is built. `to_key_value` reads nothing. `update` reads the new document. A group does not read the documents its consolidated metadata lists; `ZarrV3ConsolidatedMetadata` reads each of them once.""" values: list[object] = [] def counted( configuration: object, nested: Nested, value: object ) -> Iterator[ValidationProblem]: values.append(value) yield from () counting = dataclasses.replace(INT8_DATA_TYPE, fill_value_rules=counted) context = CORE_AND_EXTENSIONS.update( data_types={**CORE_AND_EXTENSIONS.data_types, "int8": counting} ) model = ZarrV3ArrayMetadata.create_default(context=context, data_type="int8", fill_value=3) assert values == [3] model.to_key_value() assert values == [3] changed = model.update(fill_value=4) assert values == [3, 4] member: dict[str, Any] = {**_INLINE, "metadata": {"a": changed.to_json()}} group = ZarrV3GroupMetadata.create_default(context=context, consolidated_metadata=member) assert values == [3, 4] group.to_key_value() assert values == [3, 4] ZarrV3ConsolidatedMetadata(group.consolidated_metadata, context=context).to_json() assert values == [3, 4, 4] @pytest.mark.parametrize( ("model", "changes", "problems"), [ (ARRAY, {"fill_value": math.nan}, [(("fill_value",), "invalid_value")]), (ARRAY, {"dimension_names": ("x", "y")}, [(("dimension_names",), "invalid_value")]), # Both problems are reported: the grid and the names each cover one dimension. ( ARRAY, {"shape": (4, 4)}, [ (("chunk_grid", "configuration", "chunk_shape"), "invalid_value"), (("dimension_names",), "invalid_value"), ], ), (ARRAY, {"attributes": {1: "a"}}, [(("attributes",), "invalid_type")]), # A value that is not an object is reported as the wrong type, even when empty. (ARRAY, {"attributes": []}, [(("attributes",), "invalid_type")]), (ARRAY, {"attributes": None}, [(("attributes",), "invalid_type")]), (GROUP, {"attributes": {1: "a"}}, [(("attributes",), "invalid_type")]), (GROUP, {"attributes": ()}, [(("attributes",), "invalid_type")]), (GROUP, {"extra_fields": {"acme": {1: "a"}}}, [(("acme",), "invalid_type")]), ], ids=[ "fill-value-not-json", "names-for-another-rank", "shape-without-its-grid", "attribute-key", "attributes-empty-and-no-object", "attributes-null", "group-attribute-key", "group-attributes-empty-and-no-object", "group-extension-not-json", ], ) def test_error_a_v3_model_updated_into_an_invalid_one_is_refused_at_the_change( model: object, changes: dict[str, object], problems: list[tuple[tuple[str | int, ...], str]] ) -> None: """`update` reads the new document, so a change that makes the document invalid is refused, with every problem reported. No model is invalid, however it was made.""" with pytest.raises(MetadataValidationError) as raised: cast("Any", model).update(**changes) assert [(found.loc, found.kind) for found in raised.value.problems] == problems @pytest.mark.parametrize( "kind", [ZarrV3ArrayMetadata, ZarrV3GroupMetadata, ZarrV2ArrayMetadata], ids=["v3-array", "v3-group", "v2-array"], ) def test_a_member_named_extra_fields_is_set_and_removed_through_extra_fields( kind: Any, # noqa: ANN401 - a model class ) -> None: """A document may hold a member that is itself named `extra_fields`. The spec does not define it, so `update` and `create_default` take it in the `extra_fields` mapping, as any other such member.""" added = kind.create_default(extra_fields={"extra_fields": {"must_understand": False}}) assert added.to_json()["extra_fields"] == {"must_understand": False} assert "extra_fields" not in added.update(extra_fields={"extra_fields": UNSET}).to_json() @pytest.mark.parametrize( "kind", [ZarrV3ArrayMetadata, ZarrV3GroupMetadata, ZarrV2ArrayMetadata], ids=["v3-array", "v3-group", "v2-array"], ) def test_members_the_spec_does_not_define_are_set_and_removed_through_extra_fields( kind: Any, # noqa: ANN401 - a model class ) -> None: """`create_default` and `update` take the members the spec does not define in `extra_fields`. Each member given there is set, each given as `UNSET` is removed, and the other members are kept.""" added = kind.create_default( extra_fields={"acme.flag": 1, "acme.other": {"must_understand": False}} ) assert added.to_json()["acme.flag"] == 1 removed = added.update(extra_fields={"acme.flag": UNSET}) assert "acme.flag" not in removed.to_json() assert removed.to_json()["acme.other"] == {"must_understand": False} @pytest.mark.parametrize( "kind", [ZarrV3ArrayMetadata, ZarrV3GroupMetadata, ZarrV2ArrayMetadata, ZarrV2GroupMetadata], ids=["v3-array", "v3-group", "v2-array", "v2-group"], ) def test_error_a_member_update_does_not_take_is_refused(kind: Any) -> None: # noqa: ANN401 - a model class """A misspelled member is a `TypeError` from `create_default` and from `update`. It is never written as a member the spec does not define, which a reader that must understand every member would refuse.""" with pytest.raises(TypeError, match="'attribute'"): kind.create_default(attribute={}) with pytest.raises(TypeError, match="'attribute'"): kind.create_default().update(attribute={}) @pytest.mark.parametrize( ("kind", "member"), [ (ZarrV3ArrayMetadata, "shape"), (ZarrV3GroupMetadata, "node_type"), (ZarrV2ArrayMetadata, "dtype"), ], ids=["v3-array", "v3-group", "v2-array"], ) def test_error_extra_fields_refuses_a_member_the_spec_defines(kind: Any, member: str) -> None: # noqa: ANN401 - a model class """`extra_fields` holds only the members the spec does not define. A member the spec defines is given by its own name, so giving it in `extra_fields` is a `TypeError`.""" with pytest.raises(TypeError, match=repr(member)): kind.create_default(extra_fields={member: 1}) def test_error_update_takes_fields_as_json() -> None: """`update` takes each member as a document writes it. A field the model read, such as each of `model.codecs`, is refused with a `TypeError` that says to write it with `to_json()`, rather than reported as a value that is not JSON.""" with pytest.raises(TypeError, match=r"codecs: .*to_json\(\)"): ARRAY.update(codecs=ARRAY.codecs) # pyright: ignore[reportArgumentType] def test_error_update_takes_no_refused_field() -> None: """A field refused by its definition holds JSON its definition does not read, so it has no JSON for `update` to write: given in a member, it is refused with a `TypeError` that says to give the member as JSON.""" refused, _ = resolve({"name": "gzip", "configuration": {"level": 99}}, CodecDefinition, CORE) assert isinstance(refused, RefusedField) with pytest.raises(TypeError, match="a refused field has no JSON"): ARRAY.update(codecs=(ARRAY.to_json()["codecs"][0], refused)) # pyright: ignore[reportArgumentType] def test_error_extra_fields_takes_values_as_json() -> None: """A field given as the value of a member in `extra_fields`, or an extra field given as a member's value, is refused with a `TypeError` that names `to_json()[key]`, which is the member's value. Writing a whole field there would nest the member inside itself and lose its waiver.""" source = ZarrV3ArrayMetadata.create_default(extra_fields={"foo": {"must_understand": False}}) with pytest.raises(TypeError, match=r"to_json\(\)\['foo'\]"): ARRAY.update(extra_fields=dict(source.extra_fields)) # pyright: ignore[reportArgumentType] member = GROUP.extra_fields["consolidated_metadata"] with pytest.raises(TypeError, match=r"to_json\(\)\['consolidated_metadata'\]"): GROUP.update(consolidated_metadata=member) # pyright: ignore[reportArgumentType] # The advice follows the field's kind, not the key it was given under. with pytest.raises(TypeError, match=r"to_json\(\)\['foo'\]"): ARRAY.update(extra_fields={"bar": source.extra_fields["foo"]}) # pyright: ignore[reportArgumentType] with pytest.raises(TypeError, match=r"to_json\(\)$"): ARRAY.update(extra_fields={"bytes": ARRAY.codecs[0]}) # pyright: ignore[reportArgumentType] def test_error_extra_fields_refuses_a_member_update_takes_by_name() -> None: """A member that `update` takes by name, such as a group's `consolidated_metadata`, is refused in `extra_fields`, so one member is never given twice, and a listing given by name is never replaced unchecked.""" with pytest.raises(TypeError, match="give it by name"): GROUP.update(extra_fields={"consolidated_metadata": UNSET}) @pytest.mark.parametrize( "kind", [ZarrV3ArrayMetadata, ZarrV3GroupMetadata, ZarrV2ArrayMetadata], ids=["v3-array", "v3-group", "v2-array"], ) @pytest.mark.parametrize("extra_fields", [UNSET, [("acme", 1)]], ids=["unset", "list"]) def test_error_extra_fields_that_is_not_a_mapping( kind: Any, # noqa: ANN401 - a model class extra_fields: object, ) -> None: """`extra_fields` maps members to their values, so `update` refuses `UNSET` or a list of pairs given in its place with a `TypeError`, rather than removing every member or reading the pairs as members.""" with pytest.raises(TypeError, match="extra_fields maps members to their values"): kind.create_default().update(extra_fields=extra_fields) @pytest.mark.parametrize( "kind", [ZarrV3ArrayMetadata, ZarrV3GroupMetadata, ZarrV2ArrayMetadata], ids=["v3-array", "v3-group", "v2-array"], ) def test_error_extra_fields_with_a_key_that_is_not_a_string(kind: Any) -> None: # noqa: ANN401 - a model class """A document's members are keyed by strings, so `update` refuses an `extra_fields` with any other key with a `TypeError` that names the key.""" with pytest.raises(TypeError, match="extra_fields maps each member's key, a string"): kind.create_default().update(extra_fields={1: 1}) def test_error_a_v2_group_takes_no_extra_fields() -> None: """A v2 group's `.zgroup` holds only `zarr_format`, so its `update` takes no `extra_fields`, and refuses it as an unknown member.""" with pytest.raises(TypeError, match="unknown member 'extra_fields'"): V2_GROUP.update(extra_fields={"acme": 1}) # pyright: ignore[reportCallIssue] def test_error_consolidated_metadata_of_documents_at_bad_paths_is_refused() -> None: """The consolidated member, read on its own, refuses documents at paths that no node can have. It also reports a missing parent group, as the group's read does.""" documents = { "x": ARRAY.to_json(), "x/a": ARRAY.to_json(), "__b": ARRAY.to_json(), "c/d": ARRAY.to_json(), } with pytest.raises(MetadataValidationError) as raised: ZarrV3ConsolidatedMetadata({**_INLINE, "metadata": documents}) assert [(found.loc, found.kind) for found in raised.value.problems] == [ (("metadata", "__b"), "invalid_value"), (("metadata", "x/a"), "invalid_value"), (("metadata", "c"), "missing_key"), ] @pytest.mark.parametrize( ("model", "changes", "problems"), [ (V2_ARRAY, {"order": "Q"}, [(("order",), "invalid_value")]), (V2_ARRAY, {"chunks": (4, 4)}, [(("chunks",), "invalid_value")]), (V2_GROUP, {"attributes": {1: "a"}}, [(("attributes",), "invalid_type")]), ( V2_CONSOLIDATED, {"metadata": {"a/.zarray": {"x": math.nan}}}, [(("metadata", "a/.zarray", "x"), "invalid_value")], ), ], ids=[ "v2-order", "v2-chunks-for-another-rank", "v2-group-attribute-key", "v2-consolidated-entry-not-json", ], ) def test_error_a_v2_model_changed_by_hand_into_an_invalid_one_is_refused_at_the_change( model: object, changes: dict[str, object], problems: list[tuple[tuple[str | int, ...], str]] ) -> None: """The constructor reads the document the same way a read does, so no v2 model is invalid, however it was made.""" with pytest.raises(MetadataValidationError) as raised: _rebuilt(model, changes) assert [(found.loc, found.kind) for found in raised.value.problems] == problems def test_error_v2_create_default_refuses_chunks_its_default_shape_does_not_take() -> None: # Overriding chunks without shape keeps the scalar default shape (), as # the v3 model keeps its default shape. A chunk grid that does not fit # that shape is refused. with pytest.raises(MetadataValidationError) as raised: ZarrV2ArrayMetadata.create_default(chunks=(10, 10)) assert [(found.loc, found.kind) for found in raised.value.problems] == [ (("chunks",), "invalid_value") ] def test_error_consolidated_metadata_paths_are_strings() -> None: """A `metadata` member with a key that is not a string is reported as a problem of the member, as the read reports it, and does not raise `TypeError`.""" with pytest.raises(MetadataValidationError) as raised: ZarrV3ConsolidatedMetadata({**_INLINE, "metadata": {1: ARRAY.to_json()}}) assert [(found.loc, found.kind) for found in raised.value.problems] == [ (("metadata",), "invalid_type") ] def test_a_model_s_key_is_computed_when_it_is_first_compared() -> None: """`==` and `hash` use a key that represents what the document means. The key is computed the first time either is called, then cached. A model that is only read never calls a definition's `canonical` spelling.""" asked: list[int] = [] def counting(configuration: GzipCodecConfiguration) -> GzipCodecConfiguration: asked.append(1) return configuration context = CORE_AND_EXTENSIONS.update( codecs={ **CORE_AND_EXTENSIONS.codecs, "gzip": dataclasses.replace(GZIP_CODEC, canonical=counting), } ) document = { "zarr_format": 3, "node_type": "array", "shape": [4], "data_type": "int8", "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, "fill_value": 0, "codecs": [{"name": "bytes"}, {"name": "gzip", "configuration": {"level": 1}}], } model = ZarrV3ArrayMetadata(document, context) assert model.to_json()["codecs"][1] == {"name": "gzip", "configuration": {"level": 1}} assert asked == [] other = ZarrV3ArrayMetadata(document, context) assert model == other assert len(asked) == 2 assert hash(model) == hash(other) assert model != ZarrV3ArrayMetadata({**document, "fill_value": 1}, context) assert len(asked) == 3 def test_error_a_canonical_that_raises_is_raised_when_the_model_is_first_compared() -> None: """A definition's `canonical` is code written by the extension author, and it is first called when a model is first compared. The model builds, and `hash` raises the error that `canonical` raised, with a note naming the definition.""" def refuses(configuration: GzipCodecConfiguration) -> GzipCodecConfiguration: raise ValueError("no") context = CORE_AND_EXTENSIONS.update( codecs={ **CORE_AND_EXTENSIONS.codecs, "gzip": dataclasses.replace(GZIP_CODEC, canonical=refuses), } ) document = { "zarr_format": 3, "node_type": "array", "shape": [4], "data_type": "int8", "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, "fill_value": 0, "codecs": [{"name": "bytes"}, {"name": "gzip", "configuration": {"level": 1}}], } model = ZarrV3ArrayMetadata(document, context) assert model.to_json()["codecs"][1] == {"name": "gzip", "configuration": {"level": 1}} with pytest.raises(ValueError, match="no") as raised: hash(model) assert raised.value.__notes__ == ["raised by the canonical of 'gzip'"] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_extension_points.py000066400000000000000000000200031527014137600332550ustar00rootroot00000000000000"""The model's v3 validators read each extension point through its definition. The model checks a document's structure itself. The configuration of a codec, data type, chunk grid or chunk key encoding is checked by its definition. The definition is found in a scope (`CORE_AND_EXTENSIONS` unless a caller passes another), and problems are reported at the field's location in the document. A name that no definition in the scope claims is not checked. """ from __future__ import annotations import math from typing import TYPE_CHECKING, Any, cast import pytest from typing_extensions import TypedDict from zarr_metadata._json import arrays_to_tuples from zarr_metadata.model import ( ZarrV3ArrayMetadata, ZarrV3ConsolidatedMetadata, read_array_metadata_v3, read_consolidated_metadata_v3, ) from zarr_metadata.problems import MetadataValidationError from zarr_metadata.v3.definition import ( CORE, CORE_AND_EXTENSIONS, CodecDefinition, ) if TYPE_CHECKING: from zarr_metadata.v3.definition import ZarrV3Context BYTES = {"name": "bytes", "configuration": {"endian": "little"}} def _document(**fields: object) -> dict[str, Any]: """Return a default v3 array document with `fields` added, with arrays as tuples as a reader holds them.""" document = {**ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json(), **fields} return cast("dict[str, Any]", arrays_to_tuples(document)) class LenientGzipConfiguration(TypedDict, closed=True): """A gzip configuration whose `level` is any integer. The bound on `level` is part of the configuration type, so accepting any level needs a separate type.""" level: int LENIENT_GZIP = CodecDefinition( name="gzip", configuration=LenientGzipConfiguration, kind="bytes_bytes", size="dynamic", ) """A reader's own gzip definition, which accepts any level. A scope can be extended, and an extension can replace a definition.""" BAD_GZIP_ARRAY = _document(codecs=[BYTES, {"name": "gzip", "configuration": {"level": 99}}]) """An array whose gzip level the core `gzip` definition refuses and `LENIENT_GZIP` accepts.""" @pytest.mark.parametrize( ("document", "context"), [ (_document(), CORE_AND_EXTENSIONS), ( _document(codecs=[BYTES, {"name": "gzip", "configuration": {"level": 5}}]), CORE_AND_EXTENSIONS, ), ( _document(codecs=[BYTES, {"name": "acme.lz9", "configuration": {"anything": 1}}]), CORE_AND_EXTENSIONS, ), (_document(codecs=[BYTES, {"name": "zstd", "configuration": {"level": 99}}]), CORE), ( _document(codecs=[BYTES, {"name": "gzip", "configuration": {"level": 99}}]), CORE.update(codecs={**CORE.codecs, "gzip": LENIENT_GZIP}), ), (dict(ZarrV3ArrayMetadata.create_default(shape=(0, 4)).to_json()), CORE_AND_EXTENSIONS), ], ids=[ "default", "a-codec-its-definition-allows", "a-name-nothing-claims", "an-extension-outside-the-scope", "a-reader-s-own-definition", "a-zero-length-array", ], ) def test_a_document_reads_through_the_definitions_in_its_scope( document: dict[str, Any], context: ZarrV3Context ) -> None: # Definitions in the scope check their fields. Fields with no definition # in the scope are left to the reader. The model reads in the same scope # as the reader, and its output reads back in that scope as the same # model. reading = read_array_metadata_v3(document, context=context) assert reading.problems == () model = ZarrV3ArrayMetadata.from_json(document, context=context) assert reading.metadata == model assert ZarrV3ArrayMetadata.from_key_value(model.to_key_value(), context=context) == model @pytest.mark.parametrize( ("fields", "loc", "kind"), [ ( {"codecs": [BYTES, {"name": "gzip", "configuration": {"level": 99}}]}, ("codecs", 1, "configuration", "level"), "invalid_value", ), ( {"codecs": [BYTES, {"name": "gzip", "configuration": {"level": 5, "clevel": 9}}]}, ("codecs", 1, "configuration", "clevel"), "unknown_key", ), ( { "data_type": { "name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 0}, } }, ("data_type", "configuration", "scale_factor"), "invalid_value", ), ( {"chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [-1]}}}, ("chunk_grid", "configuration", "chunk_shape", 0), "invalid_value", ), ( { "codecs": [ { "name": "sharding_indexed", "configuration": { "chunk_shape": [2], "codecs": [BYTES, {"name": "gzip", "configuration": {"level": 99}}], "index_codecs": [BYTES], }, } ] }, ("codecs", 0, "configuration", "codecs", 1, "configuration", "level"), "invalid_value", ), ], ids=["codec", "unknown-key-in-a-configuration", "data-type", "chunk-grid", "nested-in-a-shard"], ) def test_error_a_configuration_its_definition_refuses( fields: dict[str, Any], loc: tuple[str | int, ...], kind: str ) -> None: # The problem is located at the field's position in the document. The # reader reports it, and the model refuses the document. document = _document(**fields) assert [ (problem.loc, problem.kind) for problem in read_array_metadata_v3(document).problems ] == [(loc, kind)] with pytest.raises(MetadataValidationError): ZarrV3ArrayMetadata.from_json(document) def test_error_an_array_in_consolidated_metadata() -> None: """A codec configuration that the codec's definition refuses, in an array listed in consolidated metadata, is reported at its location in the member.""" member = {"kind": "inline", "must_understand": False, "metadata": {"a": BAD_GZIP_ARRAY}} assert [ (problem.loc, problem.kind) for problem in read_consolidated_metadata_v3(member).problems ] == [(("metadata", "a", "codecs", 1, "configuration", "level"), "invalid_value")] def test_an_array_in_consolidated_metadata_reads_in_the_scope_it_is_read_in() -> None: """An array listed in consolidated metadata is read in the scope the member is read in: a lenient `gzip` accepts the level the core `gzip` refuses, and the model round-trips in that scope.""" member = {"kind": "inline", "must_understand": False, "metadata": {"a": BAD_GZIP_ARRAY}} lenient = CORE.update(codecs={**CORE.codecs, "gzip": LENIENT_GZIP}) reading = read_consolidated_metadata_v3(member, context=lenient) assert reading.problems == () assert reading.metadata is not None assert ( ZarrV3ConsolidatedMetadata(reading.metadata.to_json(), context=lenient) == reading.metadata ) def test_error_an_envelope_is_judged_once() -> None: # `resolve` checks the envelope and reads the configuration. The # envelope problem is reported once, and the configuration is still # checked. document = _document( codecs=[BYTES, {"name": "gzip", "configuration": {"level": 5}, "must_understand": False}] ) assert [ (problem.loc, problem.kind) for problem in read_array_metadata_v3(document).problems ] == [(("codecs", 1, "must_understand"), "invalid_value")] def test_error_a_field_that_is_not_json_is_not_read() -> None: # The JSON check runs first. The NaN is reported, and the configuration # next to it is not read. document = _document( codecs=[ BYTES, {"name": "gzip", "configuration": {"level": 99}, "must_understand": math.nan}, ] ) assert [ (problem.loc, problem.kind) for problem in read_array_metadata_v3(document).problems ] == [(("codecs", 1, "must_understand"), "invalid_value")] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_extra_fields.py000066400000000000000000000405201527014137600323240ustar00rootroot00000000000000"""Members of a v3 document that the spec does not define: extra fields, read by the definitions a scope holds for them.""" from __future__ import annotations import json import math from typing import Annotated, Any, Literal, NewType, NotRequired import pytest from annotated_types import Ge from jsonschema import Draft202012Validator from typing_extensions import TypeAliasType, TypedDict from zarr_metadata.definition import ( AcceptedField, RefusedField, UnclaimedField, canonical_of, resolve, ) from zarr_metadata.json_schema import field_json_schema_v3 from zarr_metadata.model import ZarrV3ArrayMetadata, ZarrV3GroupMetadata from zarr_metadata.problems import MetadataValidationError from zarr_metadata.typed_json import UserJSONValue from zarr_metadata.v3.definition import ( CORE, ArrayExtraFieldDefinition, GroupExtraFieldDefinition, ZarrV3Context, ) class AcmeTilingConfiguration(TypedDict, closed=True): """The value of an `acme.tiling` member: a tile edge, and the waiver that lets other readers skip it.""" edge: Annotated[int, Ge(1)] must_understand: bool ACME_TILING = ArrayExtraFieldDefinition(name="acme.tiling", configuration=AcmeTilingConfiguration) """A reader's own definition of the `acme.tiling` member of an array.""" class AcmeNoteConfiguration(TypedDict, closed=True): """The value of an `acme.note` member: any JSON, and the waiver.""" note: UserJSONValue must_understand: NotRequired[bool] ACME_NOTE = ArrayExtraFieldDefinition(name="acme.note", configuration=AcmeNoteConfiguration) """A reader's own definition of the `acme.note` member of an array, whose numbers must be finite.""" CONTEXT = CORE.update( array_extra_fields=[*CORE.array_extra_fields.values(), ACME_TILING, ACME_NOTE] ) ARRAY: dict[str, Any] = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) GROUP: dict[str, Any] = {"zarr_format": 3, "node_type": "group"} TILING = {"edge": 2, "must_understand": True} INLINE = {"kind": "inline", "must_understand": False, "metadata": {}} NAN_LISTING = {**INLINE, "metadata": {"a": {**ARRAY, "attributes": {"x": math.nan}}}} @pytest.mark.parametrize( ("document", "context", "claimed", "must_understand"), [ ({**ARRAY, "acme.tiling": TILING}, CONTEXT, {"acme.tiling"}, {}), ({**ARRAY, "acme.tiling": TILING}, CORE, set(), {"acme.tiling": TILING}), ({**ARRAY, "acme.other": {"must_understand": False}}, CONTEXT, set(), {}), ({**ARRAY, "acme.other": 5}, CONTEXT, set(), {"acme.other": 5}), ({**GROUP, "consolidated_metadata": INLINE}, CORE, {"consolidated_metadata"}, {}), ({**GROUP, "consolidated_metadata": INLINE}, ZarrV3Context(), set(), {}), ( {**GROUP, "consolidated_metadata": None}, ZarrV3Context(), set(), {"consolidated_metadata": None}, ), ({**ARRAY, "consolidated_metadata": INLINE}, CORE, set(), {}), ({**GROUP, "consolidated_metadata": NAN_LISTING}, CORE, {"consolidated_metadata"}, {}), ({**GROUP, "consolidated_metadata": NAN_LISTING}, ZarrV3Context(), set(), {}), ({**ARRAY, "acme.other": {"x": math.inf}}, CONTEXT, set(), {"acme.other": {"x": math.inf}}), ], ids=[ "claimed", "unclaimed", "unclaimed-waived", "unclaimed-not-an-object", "consolidated-in-core", "consolidated-unclaimed", "null-consolidated-unclaimed", "a-group-field-on-an-array", "non-finite-listing-claimed", "non-finite-listing-unclaimed", "non-finite-unclaimed", ], ) def test_a_scope_reads_the_extra_fields_it_claims( document: dict[str, Any], context: ZarrV3Context, claimed: set[str], must_understand: dict[str, object], ) -> None: """Each member the spec does not define is an extra field. A member whose key a definition in scope claims is an `AcceptedField`, and a reader understands it. Any other member is an `UnclaimedField`, kept as written, and `must_understand_fields` lists it unless it says `"must_understand": false`. Nothing reads an unclaimed member, so like attributes it may hold any number, finite or not. A definition of a group's extra field does not read a member of an array, and `CORE` reads a group's `consolidated_metadata`. The document is written back as it was read.""" model: ZarrV3ArrayMetadata | ZarrV3GroupMetadata if document["node_type"] == "array": model = ZarrV3ArrayMetadata(document, context=context) else: model = ZarrV3GroupMetadata(document, context=context) accepted = { key for key, extra in model.extra_fields.items() if isinstance(extra, AcceptedField) } assert accepted == claimed assert all( isinstance(extra, UnclaimedField) for key, extra in model.extra_fields.items() if key not in claimed ) assert _same_json(model.must_understand_fields, must_understand) assert _same_json(model.to_json(), type(model)(model.to_json(), context=context).to_json()) for key, extra in model.extra_fields.items(): assert _same_json(extra.to_json(), {key: model.to_json()[key]}) read, problems = resolve({key: model.to_json()[key]}, extra.read_as, context) assert problems == () assert type(read) is type(extra) def _same_json(a: object, b: object) -> bool: """Whether `a` and `b` are the same JSON, counting NaN as equal to itself and a read-only view as the object it shows.""" return json.dumps(a, default=dict, sort_keys=True) == json.dumps( b, default=dict, sort_keys=True ) @pytest.mark.parametrize( ("first", "second"), [(1, 2), ("a", "b"), (None, 0), ((1,), (2,)), ({"x": 1}, {"x": 2}), ({}, None)], ids=["numbers", "strings", "null", "arrays", "objects", "empty-object-or-null"], ) def test_unclaimed_extra_fields_compare_as_written(first: object, second: object) -> None: """Nothing reads an unclaimed member, so documents that write it differently are different documents: they compare unequal, and neither refines the other, whatever JSON the member holds.""" a = ZarrV3ArrayMetadata({**ARRAY, "acme.flag": first}) b = ZarrV3ArrayMetadata({**ARRAY, "acme.flag": second}) assert a != b assert not a.refines(b) assert not b.refines(a) assert a == ZarrV3ArrayMetadata({**ARRAY, "acme.flag": first}) def test_an_extra_field_is_a_field_of_the_reading() -> None: """A claimed extra field takes part in what the model's reading holds: it is a field at its key, read by the definition of its kind and name, and `resolve` reads it as the member alone, `{key: value}`. The JSON Schema of the kind accepts the member, and, like `resolve`, refuses a member the spec defines.""" model = ZarrV3ArrayMetadata({**ARRAY, "acme.tiling": TILING}, context=CONTEXT) assert dict(model.reading.fields())[("acme.tiling",)] == model.extra_fields["acme.tiling"] tiling = model.extra_fields["acme.tiling"] assert isinstance(tiling, AcceptedField) assert tiling.definition is ACME_TILING read, problems = resolve({"acme.tiling": TILING}, ArrayExtraFieldDefinition, CONTEXT) assert isinstance(read, AcceptedField) assert problems == () schema = field_json_schema_v3(ArrayExtraFieldDefinition, context=CONTEXT) assert Draft202012Validator(schema).is_valid({"acme.tiling": TILING}) assert not Draft202012Validator(schema).is_valid({"acme.tiling": {"edge": 0}}) # A member the spec defines is not an extra field, as `resolve` says. assert not Draft202012Validator(schema).is_valid({"shape": 5}) assert resolve({"shape": 5}, ArrayExtraFieldDefinition, CONTEXT)[1] != () def test_the_simplest_spelling_of_an_extra_field_is_plain_json() -> None: """`canonical_of` returns plain JSON for an extra field, with no read-only views inside, so `json.dumps` writes it. `CORE` claims a group's `consolidated_metadata`, whose configuration holds objects with no nested field.""" model = ZarrV3GroupMetadata({**GROUP, "consolidated_metadata": INLINE}, context=CORE) simplest = canonical_of(model.extra_fields["consolidated_metadata"], ()) assert json.loads(json.dumps(simplest)) == {"consolidated_metadata": INLINE} def test_error_a_claimed_extra_field_that_is_not_an_object() -> None: """A definition reads a member whose value is an object. Any other value of a claimed member is a problem at the member's key, as `CORE` refuses a `consolidated_metadata` of null.""" with pytest.raises(MetadataValidationError) as raised: ZarrV3GroupMetadata({**GROUP, "consolidated_metadata": None}, context=CORE) assert [(p.loc, p.kind) for p in raised.value.problems] == [ (("consolidated_metadata",), "invalid_type") ] def test_error_a_claimed_extra_field_with_a_bad_configuration() -> None: """A claimed member's value is checked against its definition's configuration, and each problem is located in the member, under its key.""" with pytest.raises(MetadataValidationError) as raised: ZarrV3ArrayMetadata( {**ARRAY, "acme.tiling": {"edge": 0, "must_understand": True}}, context=CONTEXT ) assert [(p.loc, p.kind) for p in raised.value.problems] == [ (("acme.tiling", "edge"), "invalid_value") ] def test_error_a_non_finite_number_in_an_extra_field_that_holds_no_user_data() -> None: """Every number in a claimed extra field must be finite unless its definition reads user data, as `CONSOLIDATED_METADATA_EXTRA_FIELD` does for the attributes of the documents it lists.""" with pytest.raises(MetadataValidationError) as raised: ZarrV3ArrayMetadata({**ARRAY, "acme.note": {"note": [math.nan]}}, context=CONTEXT) assert [(p.loc, p.kind) for p in raised.value.problems] == [ (("acme.note", "note", 0), "invalid_value") ] @pytest.mark.parametrize( ("kind", "name"), [ (ArrayExtraFieldDefinition, "shape"), (ArrayExtraFieldDefinition, "attributes"), (GroupExtraFieldDefinition, "attributes"), (GroupExtraFieldDefinition, "node_type"), ], ) def test_error_an_extra_field_named_after_a_member_the_spec_defines(kind: Any, name: str) -> None: # noqa: ANN401 - either extra-field kind """A member the spec defines is never an extra field, so a definition named after one would never read anything: it is refused when it is built, as a data type named like raw bits is.""" with pytest.raises(TypeError, match="the spec defines"): kind(name=name, configuration=AcmeTilingConfiguration) @pytest.mark.parametrize( ("kind", "member"), [ (ArrayExtraFieldDefinition, {"shape": (4,)}), (GroupExtraFieldDefinition, {"node_type": "group"}), ], ids=["array-shape", "group-node-type"], ) def test_error_resolve_a_member_the_spec_defines_as_an_extra_field( kind: Any, # noqa: ANN401 - either extra-field kind member: dict[str, Any], ) -> None: """`resolve` reads a member the spec defines, read as an extra field, as a `RefusedField` with one `invalid_value` problem that says the spec defines it, for an array and for a group.""" read, problems = resolve(member, kind, CORE) assert isinstance(read, RefusedField) assert [(p.loc, p.kind) for p in problems] == [((), "invalid_value")] assert "the spec defines" in problems[0].message class _Bare(TypedDict, closed=True): tool: str class _OnlyTrue(TypedDict, closed=True): must_understand: NotRequired[Literal[True]] class _AnyInt(TypedDict, closed=True): must_understand: NotRequired[int] class _Open(TypedDict, closed=False): tool: str class _FalseOrNo(TypedDict, closed=True): must_understand: NotRequired[Literal[False, "no"]] class _BoolOrArray(TypedDict, closed=True): must_understand: NotRequired[bool | tuple[int, ...]] class _ExtraBooleans(TypedDict, extra_items=bool): tool: str class _OnlyFalse(TypedDict, closed=True): must_understand: NotRequired[Literal[False]] _Waiver = NewType("_Waiver", bool) class _NewTypeOfBool(TypedDict, closed=True): must_understand: NotRequired[_Waiver | Literal[False] | None] class _NewTypeOfBoolOnly(TypedDict, closed=True): must_understand: NotRequired[_Waiver] # A list is not a type: the checker cannot read it as a waiver. class _NotATypeWaiver(TypedDict, closed=True): must_understand: NotRequired[[True]] # pyright: ignore[reportGeneralTypeIssues] # `_Missing` is not defined on purpose: the alias cannot be resolved. _UndefinedWaiver = TypeAliasType("_UndefinedWaiver", "_Missing | bool") # noqa: F821 # pyright: ignore[reportUndefinedVariable] class _WaiverOfAnUndefinedAlias(TypedDict, closed=True): must_understand: NotRequired[_UndefinedWaiver] class _ExtraItemsOfAnUndefinedAlias(TypedDict, extra_items=_UndefinedWaiver): pass @pytest.mark.parametrize( ("configuration", "match"), [ ( _WaiverOfAnUndefinedAlias, r"'acme.x': _WaiverOfAnUndefinedAlias.must_understand: _UndefinedWaiver: name '_Missing'", ), ( _ExtraItemsOfAnUndefinedAlias, r"'acme.x': _ExtraItemsOfAnUndefinedAlias.extra_items: _UndefinedWaiver: name '_Missing'", ), ], ids=["member", "extra-items"], ) def test_error_an_extra_field_waiver_typed_by_an_alias_that_does_not_resolve( configuration: Any, # noqa: ANN401 - a TypedDict match: str, ) -> None: """An alias that names what is not defined is refused when the definition is built, with a `TypeError` that names the definition, the TypedDict and where the alias sits, as for any other member the checker cannot read. The waiver may be typed by the `must_understand` member, or, when the TypedDict declares no such member, by its `extra_items`.""" with pytest.raises(TypeError, match=match): ArrayExtraFieldDefinition(name="acme.x", configuration=configuration) @pytest.mark.parametrize( "configuration", [_ExtraBooleans, _OnlyFalse, _NewTypeOfBoolOnly, AcmeTilingConfiguration] ) def test_an_extra_field_configuration_takes_the_waiver(configuration: Any) -> None: # noqa: ANN401 - a TypedDict """A configuration takes the spec's `"must_understand": false` when the type it gives the key admits `false` and only booleans: a declared `bool`, `Literal[False]` or `NewType` of `bool`, or other keys typed `bool`.""" definition = ArrayExtraFieldDefinition(name="acme.ok", configuration=configuration) assert definition.configuration is configuration @pytest.mark.parametrize( "configuration", [ _Bare, _OnlyTrue, _AnyInt, _Open, _FalseOrNo, _BoolOrArray, _NewTypeOfBool, _NotATypeWaiver, ], ids=[ "undeclared", "only-true", "any-int", "open", "false-or-a-string", "bool-or-an-array", "a-new-type-or-null", "not-a-type", ], ) def test_error_an_extra_field_configuration_that_refuses_the_waiver(configuration: Any) -> None: # noqa: ANN401 - a TypedDict """The spec lets every extension member carry `"must_understand": false`, so the configuration of an extra field takes it: the type it gives `must_understand` admits `false`, and only booleans. A definition whose configuration refuses the waiver, or takes values that are not booleans, or types the waiver with an annotation that is not a type, such as a list, is refused when it is built, rather than misreading the documents that carry it.""" with pytest.raises( TypeError, match="expected a configuration that takes must_understand as a boolean" ): ArrayExtraFieldDefinition(name="acme.bad", configuration=configuration) def test_error_finite_is_a_boolean() -> None: """`finite` decides whether a member may hold non-finite numbers, so it must be a boolean: anything else is refused when the definition is built.""" with pytest.raises(TypeError, match="finite"): ArrayExtraFieldDefinition( name="acme.tiling", configuration=AcmeTilingConfiguration, finite=0, # pyright: ignore[reportArgumentType] ) def test_error_an_extra_field_of_a_group_is_not_an_extra_field_of_an_array() -> None: """The slots of a scope hold extra fields by node type, so a group's extra field given in the array slot is refused, as any definition of another kind is.""" tiling = GroupExtraFieldDefinition(name="acme.tiling", configuration=AcmeTilingConfiguration) with pytest.raises(TypeError, match="array extra field definitions"): ZarrV3Context(array_extra_fields=[tiling]) # pyright: ignore[reportArgumentType] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_group.py000066400000000000000000001724151527014137600310200ustar00rootroot00000000000000"""Tests for the group and consolidated metadata models in `zarr_metadata.model`.""" import copy import dataclasses import json import pickle from collections import UserDict from collections.abc import Callable, Iterator from typing import TYPE_CHECKING, Any, cast import pytest from tests.model._cases import mutate_nested_containers from zarr_metadata._json import JSON_DEPTH, arrays_to_tuples, json_text from zarr_metadata.definition import ( EmptyConfiguration, Nested, UnclaimedField, resolve, ) from zarr_metadata.model import ( UNSET, ZarrV2ConsolidatedMetadata, ZarrV2GroupMetadata, ZarrV2GroupMetadataReading, ZarrV2GroupMetadataUpdate, ZarrV3ArrayMetadata, ZarrV3ArrayMetadataReading, ZarrV3ArrayMetadataUpdate, ZarrV3ConsolidatedMetadata, ZarrV3GroupMetadata, ZarrV3GroupMetadataReading, ZarrV3GroupMetadataUpdate, ZarrV3UnknownNodeMetadataReading, node_metadata_from_json_v3, node_metadata_from_key_value_v3, read_consolidated_metadata_v3, read_group_metadata_v2, read_group_metadata_v3, read_node_metadata_v3, ) from zarr_metadata.model._group_v3 import GroupMembersV3 from zarr_metadata.problems import MetadataValidationError, ValidationProblem from zarr_metadata.v2.group import ( ZarrV2GroupMetadataJSON, ZarrV2ZGroupJSON, ) from zarr_metadata.v3 import ZarrV3NamedConfigJSON from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.codec.gzip import GZIP_CODEC, GzipCodecConfiguration from zarr_metadata.v3.definition import ( CORE, CORE_AND_EXTENSIONS, CodecDefinition, ) from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON if TYPE_CHECKING: from zarr_metadata.typed_json import JSONValue _INLINE_ENVELOPE: dict[str, Any] = {"kind": "inline", "must_understand": False} # --- ZarrV3GroupMetadata --------------------------------------------------- def test_group_v3_roundtrip() -> None: """A v3 group document round-trips through the model unchanged.""" doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": (1, 2)}} model = ZarrV3GroupMetadata.from_json(doc) assert model.to_json() == doc def test_group_v3_omits_empty_attributes() -> None: """to_json omits the attributes key when attributes is empty.""" model = ZarrV3GroupMetadata.create_default() assert "attributes" not in model.to_json() def test_group_v3_lists_become_tuples() -> None: """from_json converts JSON arrays in attributes to tuples.""" doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": [1, 2]}} model = ZarrV3GroupMetadata.from_json(doc) assert model.attributes == {"a": (1, 2)} def test_group_v3_extra_fields_roundtrip() -> None: """Unknown top-level keys land in extra_fields and reappear in to_json.""" doc = { "zarr_format": 3, "node_type": "group", "my_extension": {"name": "thing", "must_understand": False}, } model = ZarrV3GroupMetadata.from_json(doc) assert model.extra_fields["my_extension"].to_json() == { "my_extension": {"name": "thing", "must_understand": False} } assert model.to_json() == doc def test_group_v3_json_extra_field_roundtrips_as_must_understand() -> None: """A non-object extra field is preserved and implicitly requires understanding.""" doc = {"zarr_format": 3, "node_type": "group", "ext": [1, 2]} model = ZarrV3GroupMetadata.from_json(doc) assert model.to_json()["ext"] == (1, 2) assert model.must_understand_fields == {"ext": (1, 2)} def test_group_v3_missing_required_key() -> None: """`ZarrV3GroupMetadata.from_json` reports each missing required key.""" with pytest.raises(MetadataValidationError, match="node_type"): ZarrV3GroupMetadata.from_json({"zarr_format": 3}) def test_group_v3_bad_attributes() -> None: """`ZarrV3GroupMetadata.from_json` rejects a non-mapping attributes value.""" with pytest.raises(MetadataValidationError, match="attributes"): ZarrV3GroupMetadata.from_json({"zarr_format": 3, "node_type": "group", "attributes": 5}) @pytest.mark.parametrize( ("document", "read"), [ pytest.param( {"zarr_format": 2.0}, read_group_metadata_v2, id="v2", ), pytest.param( {"zarr_format": 3.0, "node_type": "group"}, read_group_metadata_v3, id="v3", ), ], ) def test_group_zarr_format_rejects_float( document: object, read: Callable[[object], ZarrV2GroupMetadataReading | ZarrV3GroupMetadataReading], ) -> None: """Integer-valued floats do not satisfy integer format literals.""" problems = read(document).problems assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] def _v2_consolidated_problems(document: object) -> list[tuple[tuple[str | int, ...], str]]: with pytest.raises(MetadataValidationError) as raised: ZarrV2ConsolidatedMetadata.from_json(document) return [(p.loc, p.kind) for p in raised.value.problems] def _v3_field_problems(field: object) -> list[tuple[tuple[str | int, ...], str]]: return [(p.loc, p.kind) for p in resolve(field, CodecDefinition, CORE_AND_EXTENSIONS)[1]] @pytest.mark.parametrize( ("problems", "expected"), [ ( lambda: [ (p.loc, p.kind) for p in read_group_metadata_v2({"zarr_format": 2, "x": 1}).problems ], [(("x",), "unknown_key")], ), ( lambda: _v2_consolidated_problems( {"zarr_consolidated_format": 1, "metadata": {}, "x": 1} ), [(("x",), "unknown_key")], ), ( lambda: [ (p.loc, p.kind) for p in read_group_metadata_v3( _group(consolidated_metadata={**_inline(), "x": 1}) ).problems ], [(("consolidated_metadata", "x"), "unknown_key")], ), ( lambda: _v3_field_problems({"name": "gzip", "configuration": {"level": 1}, "x": 1}), [(("x",), "unknown_key")], ), ( lambda: _v3_field_problems({"name": "gzip", "configuration": {"level": 1, "x": 1}}), [(("configuration", "x"), "unknown_key")], ), ], ids=["v2-group", "v2-consolidated", "v3-consolidated", "v3-field", "v3-configuration"], ) def test_a_member_a_closed_object_does_not_declare_is_an_unknown_key( problems: Callable[[], list[tuple[tuple[str | int, ...], str]]], expected: list[tuple[tuple[str | int, ...], str]], ) -> None: # This holds at any location, so a reader that tolerates keys another # writer added (such as NCZarr's `_nczarr_*` keys) can filter by kind. assert problems() == expected @pytest.mark.parametrize( "document_type", [ZarrV2ZGroupJSON, ZarrV2GroupMetadataJSON], ids=lambda document_type: document_type.__name__, ) def test_v2_group_document_types_are_closed(document_type: type) -> None: """`.zgroup` "Other keys MUST NOT be present": the validator refuses them, and the types are closed to match.""" assert getattr(document_type, "__closed__", None) is True @pytest.mark.parametrize( ("model", "document"), [ pytest.param(ZarrV2GroupMetadata, {"zarr_format": 2}, id="v2"), pytest.param( ZarrV3GroupMetadata, {"zarr_format": 3, "node_type": "group"}, id="v3", ), ], ) def test_group_model_materializes_abstract_mapping( model: type[ZarrV2GroupMetadata | ZarrV3GroupMetadata], document: dict[str, object] ) -> None: """A group model read from an abstract mapping writes the declared concrete TypedDict shape.""" written = model.from_json(UserDict(document)).to_json() assert type(written) is dict assert written == document def test_group_model_materializes_abstract_nested_json() -> None: """A group model reads an abstract sequence nested in its document, and writes it as a tuple.""" v3 = {"zarr_format": 3, "node_type": "group", "extension": range(2)} v2 = {"zarr_format": 2, "attributes": {"values": range(2)}} assert ZarrV3GroupMetadata.from_json(v3).to_json()["extension"] == (0, 1) assert ZarrV2GroupMetadata.from_json(v2).to_json().get("attributes") == {"values": (0, 1)} def test_group_v3_extension_fields_are_validated() -> None: """Group extension payloads must be JSON values with a must-understand flag.""" doc = { "zarr_format": 3, "node_type": "group", "ext": {"must_understand": False, "payload": object()}, } assert [(problem.loc, problem.kind) for problem in read_group_metadata_v3(doc).problems] == [ (("ext", "payload"), "invalid_type") ] def test_group_v3_key_value_roundtrip() -> None: """from_key_value(to_key_value()) is the identity for v3 groups.""" model = ZarrV3GroupMetadata.create_default(attributes={"a": 1}) assert ZarrV3GroupMetadata.from_key_value(model.to_key_value()) == model def test_group_v3_update() -> None: """update replaces the given fields and returns a new instance.""" base = ZarrV3GroupMetadata.create_default() updated = base.update(attributes={"a": 1}) assert updated.attributes == {"a": 1} assert base.attributes == {} # --- ZarrV2GroupMetadata --------------------------------------------------- def test_a_v2_group_nested_as_deep_as_a_reader_walks_is_read_and_written() -> None: """Previously, the v2 group was copied with `copy.deepcopy`, which uses two stack frames per level, and it overflowed on documents that its validator accepts.""" attributes: dict[str, object] = {} for _ in range(JSON_DEPTH - 2): attributes = {"x": attributes} document = {"zarr_format": 2, "attributes": attributes} assert read_group_metadata_v2(document).problems == () model = ZarrV2GroupMetadata.from_json(document) assert json_text(model.to_json()) == json_text(cast("JSONValue", document)) assert ZarrV2GroupMetadata.from_key_value(model.to_key_value()) == model assert pickle.loads(pickle.dumps(model)) == model assert copy.deepcopy(model) == model problems = read_group_metadata_v2({**document, "attributes": {"x": attributes}}).problems assert [(problem.kind, len(problem.loc)) for problem in problems] == [ ("invalid_value", JSON_DEPTH) ] def _chain_of_groups(documents: int) -> dict[str, object]: """Return a group whose consolidated metadata holds a group, which holds a group, and so on, with `documents` groups below the root. Each group lists only the next one. Each document is three levels below the document that holds it.""" document: dict[str, object] = {"zarr_format": 3, "node_type": "group"} for _ in range(documents): document = { "zarr_format": 3, "node_type": "group", "consolidated_metadata": { "kind": "inline", "must_understand": False, "metadata": {"a": document}, }, } return document def test_a_chain_of_consolidated_groups_is_bounded_by_the_levels_a_reader_walks() -> None: """Each document is read from its location in the input document, so a chain is bounded like any other nesting: by a group, which refines its consolidated metadata without reading the documents, and by the consolidated metadata, which reads each document. Previously the reader used three stack frames per document with no bound, and a 20 KB chain overflowed.""" depth = f"nested deeper than the {JSON_DEPTH} levels a reader walks" deepest = (JSON_DEPTH - 1) // 3 assert read_group_metadata_v3(_chain_of_groups(deepest)).problems == () member = _chain_of_groups(deepest)["consolidated_metadata"] problems = read_consolidated_metadata_v3(member).problems assert all(len(problem.loc) < JSON_DEPTH - 1 for problem in problems) assert {problem.message for problem in problems} == { 'expected a node the group lists, got "/a/a", which "/a" lists alone' } for documents in (deepest + 1, 350): chain = _chain_of_groups(documents) assert [ (len(problem.loc), problem.message) for problem in read_group_metadata_v3(chain).problems ] == [(JSON_DEPTH, depth)] problems = read_consolidated_metadata_v3(chain["consolidated_metadata"]).problems # Located in the member, one level below the group document. assert [ (len(problem.loc) + 1, problem.message) for problem in problems if len(problem.loc) + 1 >= JSON_DEPTH ] == [(JSON_DEPTH, depth)] def test_a_consolidated_document_is_read_from_where_it_sits() -> None: """A consolidated document's levels are counted from the root of the input document. So a document that `read_node_metadata_v3` accepts on its own can be too deep when nested. A group refuses each container past the cap, and so does its consolidated metadata, read on its own, at the same place in each document.""" depth = f"nested deeper than the {JSON_DEPTH} levels a reader walks" def nested(levels: int) -> dict[str, object]: value: dict[str, object] = {} for _ in range(levels): value = {"x": value} return value # A document is three levels down. Its attribute, or a member of an # extra field, is two more levels down. group = { "zarr_format": 3, "node_type": "group", "attributes": {"a": nested(JSON_DEPTH - 4)}, "acme_extra": {"must_understand": False, "a": nested(JSON_DEPTH - 4)}, } array = { **ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json(), "data_type": "acme.deep", "fill_value": {"a": nested(JSON_DEPTH - 4)}, "codecs": [ {"name": "bytes", "configuration": {"endian": "little"}}, {"name": "acme.x", "configuration": {"y": nested(JSON_DEPTH - 6)}}, ], } assert read_node_metadata_v3(group).problems == () assert read_node_metadata_v3(array).problems == () member = {"kind": "inline", "must_understand": False, "metadata": {"g": group, "h": array}} document = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": member} assert [ (len(problem.loc), problem.message) for problem in read_group_metadata_v3(document).problems ] == [(JSON_DEPTH, depth)] * 4 # Located in the member, one level below the group document. assert [ (problem.loc[:4], len(problem.loc) + 1) for problem in read_consolidated_metadata_v3(member).problems ] == [ (("metadata", "g", "acme_extra", "a"), JSON_DEPTH), (("metadata", "g", "attributes", "a"), JSON_DEPTH), (("metadata", "h", "fill_value", "a"), JSON_DEPTH), (("metadata", "h", "codecs", 1), JSON_DEPTH), ] def test_a_document_a_level_before_the_cap_holds_its_scalars_and_nothing_else() -> None: """Each container member is past the cap. It is checked at its location before it is walked, as `refine_json` of the whole document would check it. The scalars are read.""" depth = f"nested deeper than the {JSON_DEPTH} levels a reader walks" def a_level_before_the_cap(document: dict[str, object]) -> dict[str, object]: for _ in range((JSON_DEPTH - 1) // 3): document = { "zarr_format": 3, "node_type": "group", "consolidated_metadata": { "kind": "inline", "must_understand": False, "metadata": {"a": document}, }, } return document scalars: dict[str, object] = {"zarr_format": 3, "node_type": "group"} problems = read_group_metadata_v3(a_level_before_the_cap(scalars)).problems assert all(len(problem.loc) < JSON_DEPTH for problem in problems) array = ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json() for document in ({**scalars, "attributes": {"a": 1}}, dict(array)): containers = sorted( key for key, item in document.items() if isinstance(item, (dict, list, tuple)) ) assert len(containers) != 0 problems = read_group_metadata_v3(a_level_before_the_cap(document)).problems assert sorted( (problem.loc[-1], len(problem.loc), problem.message) for problem in problems if len(problem.loc) >= JSON_DEPTH ) == [(key, JSON_DEPTH, depth) for key in containers] def test_a_listing_key_too_long_to_write_is_shown_by_its_size( interpreter_writes_4300_digits: None, ) -> None: member = {"kind": "inline", "must_understand": False, "metadata": {10**5000: 1}} (problem,) = read_consolidated_metadata_v3(member).problems assert problem.message == f"non-string key an integer of {(10**5000).bit_length()} bits" def test_a_group_reads_the_documents_it_holds_from_where_they_sit() -> None: """A child that is valid on its own, nested to the last level a reader walks, is three levels deeper when a group holds it as a document. The group's read refuses it at the cap, as the validator does. The consolidated member, read on its own, also refuses it, counting from its location under the group's key.""" # On its own, the innermost object is at the last level a reader walks. # As a document held by a group, it is three levels deeper. nested: dict[str, object] = {} for _ in range(JSON_DEPTH - 3): nested = {"x": nested} child = ZarrV3GroupMetadata.from_json( {"zarr_format": 3, "node_type": "group", "attributes": {"a": nested}} ) written = { **ZarrV3GroupMetadata.create_default().to_json(), "consolidated_metadata": {**_INLINE_ENVELOPE, "metadata": {"a": child.to_json()}}, } depth = f"nested deeper than the {JSON_DEPTH} levels a reader walks" assert [(len(p.loc), p.message) for p in read_group_metadata_v3(written).problems] == [ (JSON_DEPTH, depth) ] with pytest.raises(MetadataValidationError): ZarrV3GroupMetadata.from_json(written) with pytest.raises(MetadataValidationError): ZarrV3ConsolidatedMetadata.from_json(written["consolidated_metadata"]) def test_a_v2_consolidated_document_is_read_to_the_levels_a_reader_walks() -> None: """An entry past the cap is reported as the depth problem, not as `RecursionError`. Previously, the whole document was normalized first, using one stack frame per level with no bound, before any entry was refined.""" def nested(levels: int) -> dict[str, object]: value: dict[str, object] = {} for _ in range(levels): value = {"x": value} return value # An entry is two levels down, and its innermost object is at the last # level a reader walks. document = {"zarr_consolidated_format": 1, "metadata": {"a/.zattrs": nested(JSON_DEPTH - 3)}} model = ZarrV2ConsolidatedMetadata.from_json(document) assert json_text(model.to_json()) == json_text(cast("JSONValue", document)) for levels in (JSON_DEPTH - 2, 2000): deeper = {**document, "metadata": {"a/.zattrs": nested(levels)}} with pytest.raises(MetadataValidationError) as raised: ZarrV2ConsolidatedMetadata.from_json(deeper) assert [(len(p.loc), p.kind) for p in raised.value.problems] == [ (JSON_DEPTH, "invalid_value") ] def test_group_v2_key_value_split() -> None: """v2 to_key_value writes .zgroup and .zattrs; from_key_value merges them.""" model = ZarrV2GroupMetadata.create_default(attributes={"a": 1}) kv = model.to_key_value() assert set(kv) == {".zgroup", ".zattrs"} assert json.loads(kv[".zgroup"]) == {"zarr_format": 2} assert ZarrV2GroupMetadata.from_key_value(kv) == model @pytest.mark.parametrize("extra_key", ["attributes", "vendor_extension"]) def test_v2_group_from_key_value_rejects_zgroup_extra_members(extra_key: str) -> None: """Raw `.zgroup` documents reject every non-spec member.""" doc: dict[str, object] = {"zarr_format": 2, extra_key: {}} with pytest.raises(MetadataValidationError) as exc_info: ZarrV2GroupMetadata.from_key_value({".zgroup": json.dumps(doc).encode()}) assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ ((extra_key,), "unknown_key") ] def test_group_v2_zattrs_presence_round_trips() -> None: """A v2 group with no .zattrs file has empty attributes and emits no .zattrs; an explicit empty .zattrs stays a file, so the stores remain distinct through a round-trip. Both mean no attributes, so the models are equal, with equal hashes, and refine each other, as a v3 group with and without an empty `attributes` do.""" absent = ZarrV2GroupMetadata.from_key_value({".zgroup": b'{"zarr_format": 2}'}) assert absent.attributes == {} assert ".zattrs" not in absent.to_key_value() explicit = ZarrV2GroupMetadata.from_key_value( {".zgroup": b'{"zarr_format": 2}', ".zattrs": b"{}"} ) assert explicit.attributes == {} assert ".zattrs" in explicit.to_key_value() assert absent == explicit assert hash(absent) == hash(explicit) assert absent.refines(explicit) assert explicit.refines(absent) def test_group_v2_json_roundtrip() -> None: """A merged-form v2 group document round-trips through the model unchanged.""" doc = {"zarr_format": 2, "attributes": {"a": 1}} model = ZarrV2GroupMetadata.from_json(doc) assert model.to_json() == doc def test_group_v2_omits_empty_attributes() -> None: """to_json omits the attributes key when attributes is empty.""" assert "attributes" not in ZarrV2GroupMetadata.create_default().to_json() def test_group_v2_not_a_mapping() -> None: """`ZarrV2GroupMetadata.from_json` rejects a non-mapping document.""" with pytest.raises(MetadataValidationError, match="expected an object"): ZarrV2GroupMetadata.from_json([1, 2, 3]) def test_group_v2_missing_required_key() -> None: """`ZarrV2GroupMetadata.from_json` reports a missing zarr_format key.""" with pytest.raises(MetadataValidationError, match="zarr_format"): ZarrV2GroupMetadata.from_json({}) # --- Update TypedDict drift guards ------------------------------------------ def test_group_update_keys_match_settable_model_fields() -> None: """The v2 group `Update` TypedDict lists exactly the settable model fields. Guards against drift: adding/removing a settable field on the model without updating its `Update` TypedDict fails here. """ assert set(ZarrV2GroupMetadataUpdate.__annotations__) == {"attributes"} def test_update_takes_every_member_of_the_document_it_may_change() -> None: """Each v3 model's `update` takes every member of its document except `zarr_format` and `node_type`, which it cannot change, and `extra_fields`, which holds the members the spec does not define. A group's `update` also declares `consolidated_metadata`, which the document type leaves to its extra items, because `update` accepts node models there.""" fixed = {"zarr_format", "node_type"} for update, document, declared in ( (ZarrV3ArrayMetadataUpdate, ZarrV3ArrayMetadataJSON, {"extra_fields"}), ( ZarrV3GroupMetadataUpdate, ZarrV3GroupMetadataJSON, {"consolidated_metadata", "extra_fields"}, ), ): assert set(update.__annotations__) == (set(document.__annotations__) - fixed) | declared # --- ZarrV3ConsolidatedMetadata -------------------------------------------- def test_consolidated_v3_roundtrip() -> None: """A v3 group with inline consolidated metadata round-trips, holding the member as written. Read on its own, the member holds an array model and a group model.""" child = ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json() doc = { "zarr_format": 3, "node_type": "group", "consolidated_metadata": { "kind": "inline", "must_understand": False, "metadata": {"a": child, "g": {"zarr_format": 3, "node_type": "group"}}, }, } model = ZarrV3GroupMetadata.from_json(doc) assert model.to_json() == doc assert model.consolidated_metadata == doc["consolidated_metadata"] held = ZarrV3ConsolidatedMetadata(model.consolidated_metadata) assert isinstance(held.metadata["a"], ZarrV3ArrayMetadata) assert isinstance(held.metadata["g"], ZarrV3GroupMetadata) def test_error_consolidated_v3_must_understand_is_not_a_member_to_set() -> None: """`must_understand` is always `False`, as `kind` is always `"inline"`, so the constructor does not take it as an argument.""" with pytest.raises(TypeError, match="must_understand"): cast("Any", ZarrV3ConsolidatedMetadata)(must_understand=True, metadata={}) def test_consolidated_v3_from_json_must_understand_true_rejected() -> None: """from_json rejects a consolidated document carrying must_understand=true.""" with pytest.raises(MetadataValidationError, match="must_understand"): ZarrV3ConsolidatedMetadata.from_json( {"kind": "inline", "must_understand": True, "metadata": {}} ) def test_consolidated_v3_entry_without_node_type_rejected() -> None: """from_json rejects a consolidated entry lacking a recognizable node_type.""" with pytest.raises(MetadataValidationError, match="node_type"): ZarrV3ConsolidatedMetadata.from_json( {"kind": "inline", "must_understand": False, "metadata": {"a": {"zarr_format": 3}}} ) def test_consolidated_v3_not_a_mapping() -> None: """from_json rejects a non-mapping consolidated document.""" with pytest.raises(MetadataValidationError, match="expected an object"): ZarrV3ConsolidatedMetadata.from_json(5) # --- read_group_metadata_v3 ------------------------------------------------ LITTLE: ZarrV3NamedConfigJSON = {"name": "bytes", "configuration": {"endian": "little"}} GZIP_99 = {"name": "gzip", "configuration": {"level": 99}} A: tuple[str, ...] = ("metadata", "a") """The location, in a `consolidated_metadata` member, of the document at path `a`.""" def _array(**members: object) -> dict[str, object]: return {**ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json(), **members} def _inline(**documents: object) -> dict[str, object]: return {"kind": "inline", "must_understand": False, "metadata": documents} def _group(**members: object) -> dict[str, object]: return {"zarr_format": 3, "node_type": "group", **members} def test_error_a_consolidated_envelope_reports_the_members_it_lacks_in_its_order() -> None: document = _group(consolidated_metadata={"kind": "inline"}) assert [(p.loc, p.kind) for p in read_group_metadata_v3(document).problems] == [ (("consolidated_metadata", "must_understand"), "missing_key"), (("consolidated_metadata", "metadata"), "missing_key"), ] ACME_X = CodecDefinition( name="acme.x", configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic" ) def test_group_update_keeps_the_documents_it_holds() -> None: """`update` does not read the consolidated documents again: a child read in an extended scope keeps each field as it was read, and the group keeps its scope.""" context = CORE_AND_EXTENSIONS.update(codecs=[*CORE_AND_EXTENSIONS.codecs.values(), ACME_X]) child = ZarrV3ArrayMetadata.create_default( context=context, shape=(4,), codecs=(LITTLE, {"name": "acme.x"}) ) group = ZarrV3GroupMetadata.create_default( context=context, consolidated_metadata={**_INLINE_ENVELOPE, "metadata": {"a": child.to_json()}}, ) updated = group.update(attributes={"k": 1}) assert updated.attributes == {"k": 1} assert updated.context == context assert updated.consolidated_metadata == group.consolidated_metadata def test_group_update_writes_the_consolidated_metadata_it_is_given() -> None: """`update` takes consolidated metadata as a document writes it, or as a `ZarrV3ConsolidatedMetadata`, whose document it writes; the group holds the same member either way. `UNSET` removes it. `zstd` is an extension, so `CORE` leaves it unclaimed, and the member read in the group's scope holds it as unclaimed.""" zstd = {"name": "zstd", "configuration": {"level": 3, "checksum": False}} member = cast("Any", _inline(a=_array(codecs=[LITTLE, zstd]))) held = ZarrV3ConsolidatedMetadata(member, context=CORE) child = held.metadata["a"] assert isinstance(child, ZarrV3ArrayMetadata) assert isinstance(child.codecs[1], UnclaimedField) group = ZarrV3GroupMetadata.create_default(context=CORE) updated = group.update(consolidated_metadata=held) assert updated.consolidated_metadata == held.to_json() assert updated == group.update(consolidated_metadata=member) assert ZarrV3ConsolidatedMetadata(updated.consolidated_metadata, context=CORE) == held removed = updated.update(consolidated_metadata=UNSET) assert removed.consolidated_metadata is UNSET def _fields_of_an_array(*at: str | int) -> list[tuple[str | int, ...]]: """Return the locations of a default array's fields, under `at`.""" points = ("data_type", "chunk_grid", "chunk_key_encoding") return [*((*at, point) for point in points), (*at, "codecs", 0)] @pytest.mark.parametrize( ("member", "paths", "locs"), [ (_inline(a=_array(), g=_group()), ["a", "g"], _fields_of_an_array(*A)), # Every node below the group, including one below another group, at its path. ( _inline(g=_group(), **{"g/b": _array()}), ["g", "g/b"], _fields_of_an_array("metadata", "g/b"), ), # A group's consolidated metadata inside another group's, listing a node # the outer group also lists. The listed group's own listing is # checked against the outer one, and its documents are not models # of the outer member. The listed group's member is its extra field, # which `CORE` reads as an envelope. ( _inline(g=_group(consolidated_metadata=_inline(b=_array())), **{"g/b": _array()}), ["g", "g/b"], [("metadata", "g", "consolidated_metadata"), *_fields_of_an_array("metadata", "g/b")], ), ], ids=["an-array-and-a-group", "paths", "nested"], ) def test_consolidated_metadata_reads_each_document_it_lists( member: dict[str, object], paths: list[str], locs: list[tuple[str | int, ...]] ) -> None: """`read_consolidated_metadata_v3` reads each document a member lists as the node its `node_type` declares, and the reading's fields are each listed array's fields, located in the member. The model equals the one the constructor builds.""" reading = read_consolidated_metadata_v3(member) assert reading.problems == () assert reading.metadata is not None assert list(reading.metadata.metadata) == paths assert [loc for loc, _ in reading.fields()] == locs assert reading.metadata == ZarrV3ConsolidatedMetadata(member) @pytest.mark.parametrize( ("path", "fault"), [ ("", "is the group's own"), ("/a", 'starts with "/"'), ("a/", 'ends with "/"'), ("a//b", 'holds an empty name between two "/"'), (".", 'holds ".", a name that is periods alone'), ("a/../b", 'holds "..", a name that is periods alone'), ("__a", 'holds "__a", a name that starts with the reserved "__"'), ("zarr.json", 'holds "zarr.json", a name that is the reserved "zarr.json"'), ], ) def test_error_a_document_in_consolidated_metadata_is_at_a_node_s_path_below_the_group( path: str, fault: str ) -> None: # A path is node names joined by "/", as the reference implementation # stores it. The group's own path, "/", and this path make the node's path. member = _inline(**{path: _group()}) message = f"expected the path of a node below the group, got {json.dumps(path)}, which {fault}" assert read_consolidated_metadata_v3(member).problems == ( ValidationProblem(("metadata", path), message, "invalid_value"), ) @pytest.mark.parametrize( ("documents", "path"), [ ({"a": _array(), "a/b": _group()}, "a/b"), # No number of missing groups between them would make this valid. ({"a": _array(), "a/b/c": _array()}, "a/b/c"), ], ids=["child", "descendant"], ) def test_error_no_document_in_consolidated_metadata_is_below_an_array( documents: dict[str, object], path: str ) -> None: # "Group nodes may have children but array nodes may not." The message # names each node by its path in the hierarchy below the group. member = _inline(**documents) message = f'expected a node below a group, got "/{path}", below the array "/a"' assert read_consolidated_metadata_v3(member).problems == ( ValidationProblem(("metadata", path), message, "invalid_value"), ) def test_a_document_of_no_node_type_is_taken_as_a_group_s() -> None: # The document's own problem is reported at its location, and the nodes # below it are not refused because of it. member = _inline(a={"zarr_format": 3, "node_type": "x"}, **{"a/b": _array()}) assert [(p.loc, p.kind) for p in read_consolidated_metadata_v3(member).problems] == [ ((*A, "node_type"), "invalid_value") ] def test_error_consolidated_metadata_holds_the_group_holding_each_document() -> None: # The nearest missing group above a node is reported once, with a count # of the missing groups above it, up to the group that holds the documents. member = _inline(**{"a/b/c": _array(), "a/b/d": _array()}) assert [(p.loc, p.message, p.kind) for p in read_consolidated_metadata_v3(member).problems] == [ ( ("metadata", "a/b"), 'missing the group holding "/a/b/c", and 1 group above it', "missing_key", ), ] def test_error_a_consolidated_metadata_key_that_is_not_a_string() -> None: listing: dict[object, object] = {"a": _array(), 1: _array()} document = _group(consolidated_metadata={**_inline(), "metadata": listing}) assert [(p.loc, p.kind) for p in read_group_metadata_v3(document).problems] == [ (("consolidated_metadata", "metadata"), "invalid_type") ] NESTED = ("metadata", "g", "consolidated_metadata", "metadata") """The location, in the outer member, of the own listing of the group at `g`.""" @pytest.mark.parametrize( ("listed", "flat", "expected"), [ # What a listed group lists itself is also listed by the outer group. ({"b": _array()}, {"g/b": _array()}, []), # The reference reader keeps the flat listing, so it would drop a # node that only the listed group lists. ( {"b": _array()}, {}, [ ( (*NESTED, "b"), 'expected a node the group lists, got "/g/b", which "/g" lists alone', "invalid_value", ) ], ), # The two listings also must not disagree on a node's type. ( {"b": _array()}, {"g/b": _group()}, [ ( (*NESTED, "b"), 'expected a group, as the group lists "/g/b", got an array', "invalid_value", ) ], ), # A deeper listing is checked against the listing that holds it, when # that listing is read. The problem is located in that listing. ( {"h": _group(consolidated_metadata=_inline(x=_array()))}, {"g/h": _group()}, [ ( (*NESTED, "h", "consolidated_metadata", "metadata", "x"), 'expected a node the group lists, got "/h/x", which "/h" lists alone', "invalid_value", ) ], ), ], ids=["agreeing", "listed-alone", "contradicting", "deeper"], ) def test_a_listed_group_s_own_listing_lists_what_the_group_lists( listed: dict[str, object], flat: dict[str, object], expected: list[tuple[tuple[str, ...], str, str]], ) -> None: """A group listed in consolidated metadata may list nodes of its own, and each must be listed by the outer member too, with the same node type. A group holding the member does not read the listings, so it reads without problems.""" member = _inline(g=_group(consolidated_metadata=_inline(**listed)), **flat) problems = read_consolidated_metadata_v3(member).problems assert [(p.loc, p.message, p.kind) for p in problems] == expected assert read_group_metadata_v3(_group(consolidated_metadata=member)).problems == () def test_each_document_its_consolidated_metadata_holds_is_read_once() -> None: """A group does not read the documents its consolidated metadata lists. Reading the member reads each of them once.""" reads: list[GzipCodecConfiguration] = [] def counted( configuration: GzipCodecConfiguration, nested: Nested ) -> Iterator[ValidationProblem]: reads.append(configuration) yield from () context = CORE_AND_EXTENSIONS.update( codecs={ **CORE_AND_EXTENSIONS.codecs, "gzip": dataclasses.replace(GZIP_CODEC, rules=counted), } ) array = _array(codecs=[LITTLE, {"name": "gzip", "configuration": {"level": 5}}]) member = _inline(a=array) assert ( read_group_metadata_v3(_group(consolidated_metadata=member), context=context).problems == () ) assert reads == [] assert read_consolidated_metadata_v3(member, context=context).problems == () assert reads == [{"level": 5}] ZarrV3ConsolidatedMetadata(member, context=context) assert len(reads) == 2 def test_error_a_group_document_that_is_not_an_object_reads_as_nothing() -> None: not_an_object = ValidationProblem((), "expected an object", "invalid_type") assert read_group_metadata_v3([1]) == ZarrV3GroupMetadataReading(problems=(not_an_object,)) def test_error_a_group_document_with_a_problem_reads_as_no_model() -> None: """A group document with a problem of its own reads as its problems and no model.""" reading = read_group_metadata_v3( _group(attributes=5, consolidated_metadata=_inline(a=_array())) ) assert [(p.loc, p.kind) for p in reading.problems] == [(("attributes",), "invalid_type")] assert reading.metadata is None @pytest.mark.parametrize( ("member", "problems"), [ ( _inline(a=_array(codecs=[LITTLE, GZIP_99]), b=_array()), [((*A, "codecs", 1, "configuration", "level"), "invalid_value")], ), # A document with no node type is read as no node, and nothing else in it is read. (_inline(a={"zarr_format": 3}), [((*A, "node_type"), "missing_key")]), ], ids=["a-document", "no-node-type"], ) def test_error_consolidated_metadata_with_a_problem_reads_as_no_model( member: dict[str, object], problems: list[tuple[tuple[str | int, ...], str]] ) -> None: """A problem in a listed document is a problem of the member, located at the document's path, and the reading holds no model. The group holding the member has no problem.""" reading = read_consolidated_metadata_v3(member) assert [(p.loc, p.kind) for p in reading.problems] == problems assert reading.metadata is None assert read_group_metadata_v3(_group(consolidated_metadata=member)).problems == () # --- read_node_metadata_v3 ------------------------------------------------- @pytest.mark.parametrize( ("document", "reading", "problems"), [ (_array(), ZarrV3ArrayMetadataReading, []), (_group(attributes={"a": 1}), ZarrV3GroupMetadataReading, []), (_array(fill_value=300), ZarrV3ArrayMetadataReading, [(("fill_value",), "invalid_value")]), (_group(attributes=5), ZarrV3GroupMetadataReading, [(("attributes",), "invalid_type")]), ], ids=["array", "group", "array-with-a-problem", "group-with-a-problem"], ) def test_a_node_is_read_as_the_node_its_node_type_says( document: dict[str, object], reading: type[ZarrV3ArrayMetadataReading | ZarrV3GroupMetadataReading], problems: list[tuple[tuple[str | int, ...], str]], ) -> None: """The node is read the same way its own read would read it, and it has a model only when it has no problem.""" read = read_node_metadata_v3(document) assert type(read) is reading assert [(p.loc, p.kind) for p in read.problems] == problems assert [(p.loc, p.kind) for p in read_node_metadata_v3(document).problems] == problems assert (read.metadata is not None) is (len(problems) == 0) @pytest.mark.parametrize( ("node_type", "kind"), [("dataset", "invalid_value"), (5, "invalid_type"), (None, "invalid_type")], ids=["another-kind", "a-number", "null"], ) def test_error_a_node_type_the_spec_does_not_define(node_type: object, kind: str) -> None: """Only the document's `zarr_format` (3 here) is read, so nothing else is checked.""" read = read_node_metadata_v3({**_array(), "node_type": node_type, "shape": "not a shape"}) assert isinstance(read, ZarrV3UnknownNodeMetadataReading) assert [(p.loc, p.kind) for p in read.problems] == [(("node_type",), kind)] assert read.metadata is None assert list(read.fields()) == [] def test_error_a_document_without_a_node_type() -> None: document = {key: value for key, value in _array().items() if key != "node_type"} read = read_node_metadata_v3(document) assert isinstance(read, ZarrV3UnknownNodeMetadataReading) assert [(p.loc, p.kind) for p in read.problems] == [(("node_type",), "missing_key")] @pytest.mark.parametrize( ("document", "problems"), [ # zarr-python 2's draft of v3 (zarr-python#2982): a root `zarr.json` # that names its format by URL and has no node type. ( { "zarr_format": "https://purl.org/zarr/spec/protocol/core/3.0", "metadata_encoding": "https://purl.org/zarr/spec/protocol/core/3.0", "metadata_key_suffix": ".json", "extensions": [], }, [(("zarr_format",), "invalid_type"), (("node_type",), "missing_key")], ), ( {"zarr_format": 2, "shape": [4], "chunks": [4], "dtype": "|u1"}, [(("zarr_format",), "invalid_value"), (("node_type",), "missing_key")], ), ( {"node_type": "dataset"}, [(("zarr_format",), "missing_key"), (("node_type",), "invalid_value")], ), ], ids=["v3-draft", "v2", "no-format"], ) def test_error_a_document_of_another_format_says_so( document: dict[str, object], problems: list[tuple[tuple[str | int, ...], str]] ) -> None: read = read_node_metadata_v3(document) assert isinstance(read, ZarrV3UnknownNodeMetadataReading) assert [(p.loc, p.kind) for p in read.problems] == problems # The validator reads a node's type as the reader does. assert [(p.loc, p.kind) for p in read_node_metadata_v3(document).problems] == problems def test_error_a_node_that_is_not_an_object() -> None: read = read_node_metadata_v3([_array()]) assert isinstance(read, ZarrV3UnknownNodeMetadataReading) assert [(p.loc, p.kind) for p in read.problems] == [((), "invalid_type")] @pytest.mark.parametrize( "model", [ ZarrV3ArrayMetadata.create_default(shape=(4,)), ZarrV3GroupMetadata.create_default(attributes={"a": 1}), ], ids=["array", "group"], ) def test_a_node_is_built_as_the_model_its_node_type_says( model: ZarrV3ArrayMetadata | ZarrV3GroupMetadata, ) -> None: """A node is built from its JSON and from a store's bytes the same way the model's own class builds it.""" built = node_metadata_from_key_value_v3(model.to_key_value()) assert type(built) is type(model) assert built == model assert node_metadata_from_json_v3(model.to_json()) == model def test_error_a_node_built_from_a_store_without_its_document() -> None: with pytest.raises(MetadataValidationError) as raised: node_metadata_from_key_value_v3({}) assert [(p.loc, p.kind) for p in raised.value.problems] == [(("zarr.json",), "missing_key")] def test_error_a_node_built_from_bytes_that_are_not_json() -> None: with pytest.raises(MetadataValidationError) as raised: node_metadata_from_key_value_v3({"zarr.json": b"{"}) assert [(p.loc, p.kind) for p in raised.value.problems] == [(("zarr.json",), "invalid_json")] def test_error_a_node_built_from_a_document_of_no_node_type() -> None: document = json.dumps({**_array(), "node_type": "dataset"}).encode() with pytest.raises(MetadataValidationError) as raised: node_metadata_from_key_value_v3({"zarr.json": document}) assert [(p.loc, p.kind) for p in raised.value.problems] == [(("node_type",), "invalid_value")] def test_error_a_node_built_from_a_document_with_a_problem() -> None: """The error holds the problems of the node type that `node_type` names.""" with pytest.raises(MetadataValidationError) as raised: node_metadata_from_json_v3(_array(fill_value=300)) assert [(p.loc, p.kind) for p in raised.value.problems] == [(("fill_value",), "invalid_value")] # --- ZarrV2ConsolidatedMetadata -------------------------------------------- def test_consolidated_v2_verbatim_roundtrip() -> None: """The v2 .zmetadata model writes the flat file-keyed map verbatim, including nodes that have no .zattrs entry.""" doc = { "zarr_consolidated_format": 1, "metadata": { ".zgroup": {"zarr_format": 2}, "a/.zarray": { "zarr_format": 2, "shape": (2,), "chunks": (2,), "dtype": "|u1", "fill_value": 0, "order": "C", "compressor": None, "filters": None, }, }, } model = ZarrV2ConsolidatedMetadata.from_json(doc) assert model.to_json() == doc @pytest.mark.parametrize(("value", "kind"), [("1", "invalid_type"), (2, "invalid_value")]) def test_error_consolidated_v2_format_other_than_1(value: object, kind: str) -> None: document = {"zarr_consolidated_format": value, "metadata": {}} with pytest.raises(MetadataValidationError) as raised: ZarrV2ConsolidatedMetadata.from_json(document) assert [(p.loc, p.kind) for p in raised.value.problems] == [ (("zarr_consolidated_format",), kind) ] def test_consolidated_v2_key_value_roundtrip() -> None: """from_key_value(to_key_value()) is the identity for .zmetadata documents.""" model = ZarrV2ConsolidatedMetadata.from_json( {"zarr_consolidated_format": 1, "metadata": {".zgroup": {"zarr_format": 2}}} ) assert ZarrV2ConsolidatedMetadata.from_key_value(model.to_key_value()) == model def test_consolidated_v2_lists_become_tuples() -> None: """from_json converts JSON arrays inside entries to tuples.""" doc = { "zarr_consolidated_format": 1, "metadata": {"a/.zattrs": {"shape": [2, 3]}}, } model = ZarrV2ConsolidatedMetadata.from_json(doc) assert model.to_json()["metadata"] == {"a/.zattrs": {"shape": (2, 3)}} def test_consolidated_v2_envelope_validation() -> None: """from_json rejects a .zmetadata document missing the metadata key.""" with pytest.raises(MetadataValidationError, match="metadata"): ZarrV2ConsolidatedMetadata.from_json({"zarr_consolidated_format": 1}) def test_consolidated_v2_not_a_mapping() -> None: """from_json rejects a non-mapping .zmetadata document.""" with pytest.raises(MetadataValidationError, match="expected an object"): ZarrV2ConsolidatedMetadata.from_json([1]) def test_consolidated_v2_format_literal_enforced() -> None: """A .zmetadata document must declare consolidated format 1.""" with pytest.raises(MetadataValidationError) as exc_info: ZarrV2ConsolidatedMetadata.from_json({"zarr_consolidated_format": 2, "metadata": {}}) assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ (("zarr_consolidated_format",), "invalid_value") ] def test_consolidated_v2_metadata_values_must_be_json() -> None: """Non-JSON values in the flat metadata map are rejected during ingestion.""" with pytest.raises(MetadataValidationError) as exc_info: ZarrV2ConsolidatedMetadata.from_json( {"zarr_consolidated_format": 1, "metadata": {".zgroup": object()}} ) assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ (("metadata", ".zgroup"), "invalid_type") ] @pytest.mark.parametrize( "stored", [ '{"zarr_format": 2}', None, memoryview(b'{"zarr_format": 2}'), bytearray(b'{"zarr_format": 2}'), ], ids=["str", "none", "memoryview", "bytearray"], ) def test_error_a_store_value_that_is_not_bytes_is_refused(stored: object) -> None: """A store maps keys to bytes, and a reader checks that it read bytes.""" with pytest.raises(MetadataValidationError) as exc_info: ZarrV2GroupMetadata.from_key_value({".zgroup": stored}) # pyright: ignore[reportArgumentType] assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ ((".zgroup",), "invalid_type") ] def test_group_v2_from_key_value_scalar_root_raises_metadata_error() -> None: """A scalar .zgroup document fails through the unified metadata error channel.""" with pytest.raises(MetadataValidationError) as exc_info: ZarrV2GroupMetadata.from_key_value({".zgroup": b"null"}) assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ ((), "invalid_type") ] # --- Literal-value enforcement ----------------------------------------------- def test_group_v3_literals_enforced() -> None: """A v3 group doc with wrong zarr_format or node_type is rejected with invalid_value.""" base = ZarrV3GroupMetadata.create_default().to_json() for key, bad in (("zarr_format", 2), ("node_type", "array")): problems = read_group_metadata_v3(dict(base) | {key: bad}).problems assert [(p.loc, p.kind) for p in problems] == [((key,), "invalid_value")], key def test_group_v2_zarr_format_literal_enforced() -> None: """A v2 group doc claiming zarr_format 3 is rejected with invalid_value.""" problems = read_group_metadata_v2({"zarr_format": 3}).problems assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] # --- Consolidated envelope validated by the group reader --------------------- def test_group_v3_reader_agrees_with_from_json_on_consolidated() -> None: """The group reader checks the consolidated envelope, as `from_json` does, so the reader never finds no problem with a document `from_json` would reject.""" bad_docs = ( # empty envelope: missing kind/must_understand/metadata {"zarr_format": 3, "node_type": "group", "consolidated_metadata": {}}, # must_understand: true { "zarr_format": 3, "node_type": "group", "consolidated_metadata": { "kind": "inline", "must_understand": True, "metadata": {}, }, }, # metadata that is not an object { "zarr_format": 3, "node_type": "group", "consolidated_metadata": {"kind": "inline", "must_understand": False, "metadata": []}, }, ) for doc in bad_docs: assert read_group_metadata_v3(doc).problems != (), doc with pytest.raises(MetadataValidationError): ZarrV3GroupMetadata.from_json(doc) @pytest.mark.parametrize( "listed", [ ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json(), {**ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json(), "shape": [-1]}, {"zarr_format": 3}, ], ids=["an-array", "an-array-with-a-problem", "no-node-type"], ) def test_group_v3_valid_consolidated_passes_the_reader(listed: dict[str, object]) -> None: """A group whose consolidated metadata has the envelope the core spec fixes validates cleanly, whatever the documents it lists hold: the group does not read them.""" doc = { "zarr_format": 3, "node_type": "group", "consolidated_metadata": { "kind": "inline", "must_understand": False, "metadata": {"a": listed, "g": {"zarr_format": 3, "node_type": "group"}}, }, } assert read_group_metadata_v3(doc).problems == () assert ZarrV3GroupMetadata(doc).to_json() == arrays_to_tuples(doc) def test_v3_consolidated_rejects_unknown_envelope_member() -> None: """The inline consolidated envelope is closed and never drops accepted members.""" doc = { "kind": "inline", "must_understand": False, "metadata": {}, "unexpected": 1, } with pytest.raises(MetadataValidationError, match="unexpected"): ZarrV3ConsolidatedMetadata.from_json(doc) def test_v2_consolidated_rejects_unknown_document_member() -> None: """The v2 consolidated document is closed and never drops accepted members.""" doc = {"zarr_consolidated_format": 1, "metadata": {}, "unexpected": 1} with pytest.raises(MetadataValidationError, match="unexpected"): ZarrV2ConsolidatedMetadata.from_json(doc) @pytest.mark.parametrize("key", [1, None], ids=["int", "none"]) def test_v2_consolidated_rejects_non_string_document_key(key: object) -> None: """A non-string key is reported as a problem at the document root. It is not reported as a member at a location that looks like an index, and it does not raise `TypeError`.""" doc = {"zarr_consolidated_format": 1, "metadata": {}, key: "x"} with pytest.raises(MetadataValidationError) as exc_info: ZarrV2ConsolidatedMetadata.from_json(doc) assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ ((), "invalid_type") ] # --- must_understand partition ------------------------------------------------ def test_group_must_understand_fields_partition() -> None: """The group model partitions extra fields by the spec's implicit-true rule, like the array model. https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1573 """ model = ZarrV3GroupMetadata.create_default( extra_fields={"waived": {"name": "w", "must_understand": False}, "implicit": {"name": "i"}} ) assert set(model.must_understand_fields) == {"implicit"} def test_error_a_null_consolidated_metadata_is_a_value_the_document_wrote() -> None: """zarr-python 3.0 and 3.1 wrote `consolidated_metadata: null`. The spec requires an object, and the package treats nothing else as valid. A reader of those stores must remove the key first.""" null_doc = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": None} assert [(p.loc, p.kind) for p in read_group_metadata_v3(null_doc).problems] == [ (("consolidated_metadata",), "invalid_type") ] with pytest.raises(MetadataValidationError): ZarrV3GroupMetadata.from_json(null_doc) del null_doc["consolidated_metadata"] assert ZarrV3GroupMetadata.from_json(null_doc).consolidated_metadata is UNSET # --- to_json shares no mutable state with the model ------------------------ TO_JSON_NO_ALIASING_PARAMS = [ pytest.param( ZarrV3GroupMetadata.create_default( attributes={"a": {"b": [1]}}, consolidated_metadata={ **_INLINE_ENVELOPE, "metadata": { "child": ZarrV3ArrayMetadata.create_default( attributes={"x": {"y": 1}} ).to_json(), "grp": ZarrV3GroupMetadata.create_default(attributes={"x": {"y": 1}}).to_json(), }, }, extra_fields={"ext": {"must_understand": False, "cfg": {"x": [1]}}}, ), id="v3-group", ), pytest.param( ZarrV2GroupMetadata.create_default(attributes={"a": {"b": [1]}}), id="v2-group", ), pytest.param( ZarrV3ConsolidatedMetadata( { **_INLINE_ENVELOPE, "metadata": { "child": ZarrV3ArrayMetadata.create_default( attributes={"x": {"y": 1}} ).to_json() }, } ), id="v3-consolidated", ), pytest.param( # A `.zattrs` entry: user data, whose containers can nest. ZarrV2ConsolidatedMetadata( {"zarr_consolidated_format": 1, "metadata": {"a/.zattrs": {"nested": {"x": [1]}}}} ), id="v2-consolidated", ), ] @pytest.mark.parametrize("model", TO_JSON_NO_ALIASING_PARAMS) def test_to_json_shares_no_mutable_state_with_model( model: ZarrV3GroupMetadata | ZarrV2GroupMetadata | ZarrV3ConsolidatedMetadata | ZarrV2ConsolidatedMetadata, ) -> None: """Mutating a document returned by to_json leaves the model unchanged.""" baseline = copy.deepcopy(model.to_json()) mutate_nested_containers(model.to_json()) assert model.to_json() == baseline @pytest.mark.parametrize("model", TO_JSON_NO_ALIASING_PARAMS) def test_from_json_shares_no_mutable_state_with_its_input( model: ZarrV3GroupMetadata | ZarrV2GroupMetadata | ZarrV3ConsolidatedMetadata | ZarrV2ConsolidatedMetadata, ) -> None: """Mutating the document a model was read from leaves the model unchanged.""" # With arrays already as tuples, the reader has nothing to rebuild, so # only a copy keeps the model separate from its input. document = arrays_to_tuples(model.to_json()) read = type(model).from_json(document) baseline = copy.deepcopy(read.to_json()) mutate_nested_containers(document) assert read.to_json() == baseline def test_a_group_writes_no_empty_attributes() -> None: model = ZarrV3GroupMetadata.create_default() assert model.attributes == {} assert "attributes" not in model.to_json() assert json.loads(model.to_key_value()["zarr.json"]) == {"zarr_format": 3, "node_type": "group"} def test_error_a_listed_group_s_own_listing_holds_the_documents_the_group_lists() -> None: """Each node has one document. A group listed in consolidated metadata may list a node that the outer member also lists, but only with the same document (as read) that the outer member holds at the joined key. A document that reads differently is a problem at the nested entry. The reader, the constructor, and models given as entries all report it the same way.""" same = _array(shape=(4,), attributes={"a": 1}) other = _array(shape=(9,), attributes={"a": 1}) agreed = _inline(b=_group(consolidated_metadata=_inline(c=same)), **{"b/c": same}) assert read_consolidated_metadata_v3(agreed).problems == () contradicted = _inline(b=_group(consolidated_metadata=_inline(c=other)), **{"b/c": same}) found = read_consolidated_metadata_v3(contradicted).problems at = ("metadata", "b", "consolidated_metadata", "metadata", "c") assert [(p.loc, p.kind) for p in found] == [(at, "invalid_value")] assert "/b/c" in found[0].message with pytest.raises(MetadataValidationError): ZarrV3ConsolidatedMetadata(contradicted) inner = ZarrV3GroupMetadata(_group(consolidated_metadata=_inline(c=other))) with pytest.raises(MetadataValidationError) as raised: ZarrV3ConsolidatedMetadata(_inline(b=inner, **{"b/c": ZarrV3ArrayMetadata(same)})) assert [(p.loc, p.kind) for p in raised.value.problems] == [(at, "invalid_value")] def test_error_a_model_inside_consolidated_metadata_is_not_json() -> None: """A group holds its consolidated metadata as JSON, so a node model inside the member is a value that is not JSON. `ZarrV3ConsolidatedMetadata` is the place that takes node models as entries.""" member = _inline(a=ZarrV3ArrayMetadata.create_default()) problems = read_group_metadata_v3(_group(consolidated_metadata=member)).problems assert [(p.loc, p.kind) for p in problems] == [(("consolidated_metadata", *A), "invalid_type")] with pytest.raises(MetadataValidationError): ZarrV3GroupMetadata.create_default(consolidated_metadata=cast("Any", member)) def test_error_a_model_is_built_only_of_members_a_read_found_nothing_wrong_with() -> None: """The members of a group read with a problem hold no document. Building a model from them raises `TypeError` instead of producing an empty model.""" from zarr_metadata.model._group_v3 import _document_of with pytest.raises(TypeError, match="nothing wrong with"): _document_of(GroupMembersV3(None, {}, UNSET, None)) def test_error_the_unset_sentinel_at_consolidated_metadata_is_not_an_absent_member() -> None: """A reader returns `UNSET` for a key that a document does not hold. If `UNSET` is written as the value of `consolidated_metadata`, it is treated like any other value that is not an object, and it is never written back as null.""" document = { "zarr_format": 3, "node_type": "group", "attributes": {}, "consolidated_metadata": UNSET, } assert [(found.loc, found.kind) for found in read_group_metadata_v3(document).problems] == [ (("consolidated_metadata",), "invalid_type") ] with pytest.raises(MetadataValidationError): ZarrV3GroupMetadata(document) def test_a_consolidated_model_given_as_the_member_keeps_the_order_its_document_writes() -> None: """A `ZarrV3ConsolidatedMetadata` given to `update` is written as its document, in the document's key order, so the group writes it the same way the member wrote it.""" array = ZarrV3ArrayMetadata.create_default(shape=(2,)) member = ZarrV3ConsolidatedMetadata( {"metadata": {"a": array.to_json()}, "kind": "inline", "must_understand": False} ) group = ZarrV3GroupMetadata.create_default().update(consolidated_metadata=member) written = cast("dict[str, Any]", group.to_json())["consolidated_metadata"] assert list(written) == ["metadata", "kind", "must_understand"] assert group == ZarrV3GroupMetadata( {"zarr_format": 3, "node_type": "group", "consolidated_metadata": member.to_json()} ) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_pair.py000066400000000000000000000761221527014137600306150ustar00rootroot00000000000000"""Tests that a v3 model is a document plus the scope it was read in, and for the behavior that follows from that.""" from __future__ import annotations import copy import dataclasses import operator import pickle from collections.abc import Callable, Iterator, Mapping from typing import TYPE_CHECKING, Any, cast import pytest from zarr_metadata._json import refine_json from zarr_metadata.definition import ( AcceptedField, EmptyConfiguration, Nested, ScopeConflictError, UnclaimedField, ) from zarr_metadata.model import ( UNSET, ZarrV3ArrayMetadata, ZarrV3ConsolidatedMetadata, ZarrV3ConsolidatedMetadataReading, ZarrV3GroupMetadata, ZarrV3GroupMetadataReading, read_array_metadata_v3, read_consolidated_metadata_v3, read_group_metadata_v3, ) from zarr_metadata.problems import MetadataValidationError from zarr_metadata.v3.codec.bytes import BYTES_CODEC from zarr_metadata.v3.codec.zstd import ZSTD_CODEC from zarr_metadata.v3.definition import ( CORE, CORE_AND_EXTENSIONS, CodecDefinition, ZarrV3Context, ) if TYPE_CHECKING: from zarr_metadata import ValidationProblem ARRAY: dict[str, Any] = { "zarr_format": 3, "node_type": "array", "shape": [4], "data_type": "uint8", "fill_value": 0, "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [2]}}, "chunk_key_encoding": {"name": "default"}, "codecs": ["bytes"], } MY_BYTES = CodecDefinition( name="bytes", configuration=EmptyConfiguration, kind="array_bytes", size="static" ) """A private `bytes` definition: a different meaning under the core name, with no configuration.""" PRIVATE = CORE.update(codecs={**CORE.codecs, "bytes": MY_BYTES}) SPELLED_OUT: dict[str, Any] = { **ARRAY, "data_type": {"name": "uint8"}, "codecs": [{"name": "bytes", "configuration": {}}], "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, } @pytest.mark.parametrize( ("document", "context", "expected_context"), [ (ARRAY, None, CORE_AND_EXTENSIONS), (ARRAY, CORE, CORE), (SPELLED_OUT, ZarrV3Context(), ZarrV3Context()), ], ids=["default-scope", "core", "empty-scope"], ) def test_a_model_is_its_document_read_in_its_scope( document: dict[str, Any], context: ZarrV3Context | None, expected_context: ZarrV3Context ) -> None: """A model built from a document holds that document, as written and refined, and the scope it was read in. The scope is `CORE_AND_EXTENSIONS` when none is given.""" model = ZarrV3ArrayMetadata(document, context=context) assert model.context == expected_context assert model.to_json() == refine_json(document)[0] @pytest.mark.parametrize( ("document", "key", "written"), [ (ARRAY, "data_type", "uint8"), (SPELLED_OUT, "data_type", {"name": "uint8"}), (ARRAY, "codecs", ("bytes",)), (SPELLED_OUT, "codecs", ({"name": "bytes", "configuration": {}},)), ], ids=["bare-data-type", "object-data-type", "bare-codec", "object-codec"], ) def test_to_json_keeps_the_spelling_the_document_was_written_in( document: dict[str, Any], key: str, written: object ) -> None: """`to_json` writes each field as the document wrote it, not in a reader's preferred spelling, because the model is the document.""" assert ZarrV3ArrayMetadata(document).to_json()[key] == written def test_properties_are_what_the_reading_holds() -> None: """The typed members (fields as the scope read them, shape, fill value, attributes) are read-only views of the reading.""" model = ZarrV3ArrayMetadata({**ARRAY, "attributes": {"a": [1]}, "acme": 1}) assert isinstance(model.data_type, AcceptedField) assert isinstance(model.codecs[0], AcceptedField) assert model.shape == (4,) assert model.fill_value == 0 assert model.dimension_names is UNSET assert isinstance(model.attributes, Mapping) assert model.attributes == {"a": (1,)} assert model.extra_fields["acme"].to_json() == {"acme": 1} assert (model.zarr_format, model.node_type) == (3, "array") with pytest.raises(TypeError): model.attributes["b"] = 1 # pyright: ignore[reportIndexIssue] with pytest.raises(AttributeError): model.shape = (5,) # pyright: ignore[reportAttributeAccessIssue] assert isinstance(ZarrV3ArrayMetadata(ARRAY, context=ZarrV3Context()).data_type, UnclaimedField) def test_a_reading_without_problems_builds_the_model_without_reading_again() -> None: """`read_array_metadata_v3` passes its reading to the model it builds, so the model's `reading` is that same reading, not a second one.""" reading = read_array_metadata_v3(ARRAY) assert reading.metadata is not None assert reading.metadata.reading.pipeline is reading.pipeline def test_error_a_document_with_a_problem_is_refused_at_construction() -> None: """The constructor raises `MetadataValidationError` with every problem, as `from_json` does.""" with pytest.raises(MetadataValidationError) as raised: ZarrV3ArrayMetadata({**ARRAY, "fill_value": 300, "shape": [-1]}) assert sorted(problem.loc for problem in raised.value.problems) == [ ("fill_value",), ("shape",), ] @pytest.mark.parametrize( ("left", "right", "equal"), [ (ZarrV3ArrayMetadata(ARRAY), ZarrV3ArrayMetadata(SPELLED_OUT), True), (ZarrV3ArrayMetadata(ARRAY, context=CORE), ZarrV3ArrayMetadata(ARRAY), True), (ZarrV3ArrayMetadata(ARRAY), ZarrV3ArrayMetadata(ARRAY, context=PRIVATE), False), (ZarrV3ArrayMetadata(ARRAY), ZarrV3ArrayMetadata(ARRAY, context=ZarrV3Context()), False), (ZarrV3ArrayMetadata(ARRAY), ZarrV3ArrayMetadata({**ARRAY, "attributes": {}}), True), ], ids=["spellings", "unused-definitions", "private-bytes", "unclaimed", "empty-attributes"], ) def test_models_are_equal_by_what_their_documents_mean( left: ZarrV3ArrayMetadata, right: ZarrV3ArrayMetadata, equal: bool ) -> None: """Two models are equal when every field reads the same with the same definition and the rest is the same JSON. Spelling and unused definitions do not matter. A private definition under a core name does matter, and so does a name that one model reads and the other leaves unclaimed.""" assert (left == right) is equal if equal: assert hash(left) == hash(right) @pytest.mark.parametrize( "context", [None, CORE, PRIVATE, ZarrV3Context()], ids=["default", "core", "private", "empty"] ) def test_a_model_round_trips_through_its_document_in_its_scope( context: ZarrV3Context | None, ) -> None: """`from_json(m.to_json(), context=m.context) == m`. The document and the scope determine the model, and pickle preserves both.""" model = ZarrV3ArrayMetadata(ARRAY, context=context) assert ZarrV3ArrayMetadata(model.to_json(), context=model.context) == model loaded = pickle.loads(pickle.dumps(model)) assert loaded == model assert loaded.context == model.context assert loaded.to_json() == model.to_json() def test_error_a_model_whose_scope_does_not_pickle_says_so() -> None: """A scope that holds a definition with a local function cannot be pickled. Pickling the model raises the same error as pickling the `Context`.""" local = CodecDefinition( name="acme.c", configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic", rules=lambda configuration, nested: iter(()), ) model = ZarrV3ArrayMetadata( {**ARRAY, "codecs": ["bytes", "acme.c"]}, context=CORE.update(codecs=[*CORE.codecs.values(), local]), ) with pytest.raises((pickle.PicklingError, AttributeError)): pickle.dumps(model) FLOAT: dict[str, Any] = { **ARRAY, "data_type": "float32", "fill_value": "0x7fc00000", "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], } SHARDED: dict[str, Any] = { **ARRAY, "codecs": [ { "name": "sharding_indexed", "configuration": { "chunk_shape": [1], "codecs": ["bytes"], "index_codecs": [ {"name": "bytes", "configuration": {"endian": "little"}}, "crc32c", ], }, } ], } @pytest.mark.parametrize( ("model", "members", "expected"), [ ( ZarrV3ArrayMetadata(ARRAY), {"attributes": {"a": 1}}, ZarrV3ArrayMetadata({**ARRAY, "attributes": {"a": 1}}), ), ( ZarrV3ArrayMetadata({**ARRAY, "attributes": {"a": 1}}), {"attributes": UNSET}, ZarrV3ArrayMetadata(ARRAY), ), ( ZarrV3ArrayMetadata(ARRAY, context=PRIVATE), {"attributes": {"a": 1}}, ZarrV3ArrayMetadata({**ARRAY, "attributes": {"a": 1}}, context=PRIVATE), ), ( ZarrV3ArrayMetadata(ARRAY), { "shape": (6,), "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (3,)}}, }, ZarrV3ArrayMetadata( { **ARRAY, "shape": [6], "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [3]}}, } ), ), ], ids=["set", "unset", "update-keeps-scope", "members-together"], ) def test_update_reads_new_members_in_the_models_own_scope( model: ZarrV3ArrayMetadata, members: dict[str, Any], expected: ZarrV3ArrayMetadata ) -> None: """`update` replaces the document's members with the given JSON members, removes a member given as `UNSET`, and reads the result in the model's own scope. So a model read in a private scope stays in that scope.""" updated = model.update(**members) assert updated == expected assert updated.context == model.context def test_error_update_refuses_a_document_with_a_problem() -> None: """`update` raises `MetadataValidationError` when the members make an invalid document, as the constructor does. Here the invalid document has two dimension names for one dimension.""" with pytest.raises(MetadataValidationError): ZarrV3ArrayMetadata(ARRAY).update(dimension_names=("x", "y")) @pytest.mark.parametrize( ("model", "context", "gained"), [ (ZarrV3ArrayMetadata(ARRAY, context=ZarrV3Context()), CORE, True), (ZarrV3ArrayMetadata(ARRAY, context=CORE), CORE_AND_EXTENSIONS, False), (ZarrV3ArrayMetadata(FLOAT, context=ZarrV3Context()), CORE, True), (ZarrV3ArrayMetadata(SHARDED, context=ZarrV3Context()), CORE, True), ], ids=["gain", "nothing-to-gain", "gain-data-type-with-fill-value", "gain-of-a-shard"], ) def test_refined_in_moves_a_model_up_the_order( model: ZarrV3ArrayMetadata, context: ZarrV3Context, gained: bool ) -> None: """`refined_in` reads the document in a scope that claims what the current scope left unclaimed and contradicts nothing. The result refines the model and keeps its document. When the new scope reads nothing differently, the result equals the original model.""" refined = model.refined_in(context) assert refined.context == context assert refined.to_json() == model.to_json() assert refined.refines(model) assert (refined == model) is not gained if not gained: # Not read again: the reading's pipeline is the original one. assert refined.reading.pipeline is model.reading.pipeline @pytest.mark.parametrize( ("model", "context"), [ (ZarrV3ArrayMetadata(ARRAY), PRIVATE), (ZarrV3ArrayMetadata(ARRAY, context=CORE), ZarrV3Context()), ], ids=["conflict", "loss"], ) def test_error_refined_in_refuses_a_conflict_or_a_loss( model: ZarrV3ArrayMetadata, context: ZarrV3Context ) -> None: """`refined_in` raises `ScopeConflictError` that names each name the new scope reads with another definition, or with no definition.""" with pytest.raises(ScopeConflictError) as raised: model.refined_in(context) assert "bytes" in [conflict.key[1] for conflict in raised.value.conflicts] def test_with_context_reads_the_document_in_any_scope() -> None: """`with_context` reads the same document in another scope, whatever that changes. A private `bytes` is read with the private definition, and a scope that claims nothing leaves every name unclaimed.""" model = ZarrV3ArrayMetadata(ARRAY) private = model.with_context(PRIVATE) assert private.context == PRIVATE assert private.codecs[0].definition == MY_BYTES assert isinstance(model.with_context(ZarrV3Context()).codecs[0], UnclaimedField) assert model.with_context(None).context == CORE_AND_EXTENSIONS def test_error_with_context_refuses_a_document_the_scope_reads_with_a_problem() -> None: """`with_context` raises `MetadataValidationError` when the document has a problem in the new scope. Here the problem is a gzip `level` that the core definition refuses.""" loose = ZarrV3ArrayMetadata( {**ARRAY, "codecs": ["bytes", {"name": "gzip", "configuration": {"level": 12}}]}, context=ZarrV3Context(), ) with pytest.raises(MetadataValidationError): loose.with_context(CORE) @pytest.mark.parametrize( ("upper", "lower", "expected"), [ (ZarrV3ArrayMetadata(ARRAY), ZarrV3ArrayMetadata(ARRAY, context=ZarrV3Context()), True), (ZarrV3ArrayMetadata(ARRAY, context=ZarrV3Context()), ZarrV3ArrayMetadata(ARRAY), False), (ZarrV3ArrayMetadata(ARRAY), ZarrV3ArrayMetadata(SPELLED_OUT), True), (ZarrV3ArrayMetadata(ARRAY, context=PRIVATE), ZarrV3ArrayMetadata(ARRAY), False), (ZarrV3ArrayMetadata({**ARRAY, "attributes": {"a": 1}}), ZarrV3ArrayMetadata(ARRAY), False), ( ZarrV3ArrayMetadata(FLOAT, context=CORE), ZarrV3ArrayMetadata(FLOAT, context=ZarrV3Context()), True, ), ( ZarrV3ArrayMetadata(SHARDED, context=CORE), ZarrV3ArrayMetadata(SHARDED, context=ZarrV3Context()), True, ), ( ZarrV3ArrayMetadata(FLOAT, context=CORE), ZarrV3ArrayMetadata({**FLOAT, "fill_value": "banana"}, context=ZarrV3Context()), False, ), ], ids=[ "gain", "loss", "equal", "conflict", "other-members-differ", "fill-value-spelled-by-the-informed-side", "gain-of-a-field-holding-fields", "fill-value-the-gained-definition-refuses", ], ) def test_refines_orders_models_by_information( upper: ZarrV3ArrayMetadata, lower: ZarrV3ArrayMetadata, expected: bool ) -> None: """A model refines another when every field refines its counterpart, and every other member is the same. Fields held inside a field count too, so gaining a shard's inner fields is a gain. The fill value is compared as the more informed data type spells it. A fill value that this data type's definition refuses is not a refinement, and does not raise an error.""" assert upper.refines(lower) is expected GROUP: dict[str, Any] = {"zarr_format": 3, "node_type": "group", "attributes": {"g": 1}} CONSOLIDATED: dict[str, Any] = { **GROUP, "consolidated_metadata": { "kind": "inline", "must_understand": False, "metadata": { "a": ARRAY, "b": { **GROUP, "consolidated_metadata": { "kind": "inline", "must_understand": False, "metadata": {"c": SPELLED_OUT}, }, }, "b/c": SPELLED_OUT, }, }, } MEMBER: dict[str, Any] = CONSOLIDATED["consolidated_metadata"] def test_a_group_holds_its_consolidated_metadata_as_written() -> None: """A group model holds its `consolidated_metadata` member as written, refined and read-only, and does not read the documents it lists. `ZarrV3ConsolidatedMetadata` reads them: each becomes a model in its scope, and writes its document as written.""" group = ZarrV3GroupMetadata(CONSOLIDATED, context=CORE) member = group.consolidated_metadata assert member == refine_json(MEMBER)[0] with pytest.raises(TypeError): operator.setitem(cast("Any", member)["metadata"], "a", ARRAY) assert group.to_json() == refine_json(CONSOLIDATED)[0] assert ZarrV3GroupMetadata(GROUP).consolidated_metadata is UNSET held = ZarrV3ConsolidatedMetadata(member, context=group.context) assert set(held.metadata) == {"a", "b", "b/c"} for node in held.metadata.values(): assert node.context is held.context assert held.metadata["b/c"].to_json()["data_type"] == {"name": "uint8"} def test_consolidated_metadata_reads_each_nested_field_once() -> None: """Reading a group calls no definition's rules for the documents its consolidated metadata lists. Reading the consolidated metadata calls them once per field, and builds the models from that read.""" calls: list[int] = [] def counted(configuration: object, nested: Nested) -> Iterator[ValidationProblem]: calls.append(1) return iter(()) context = CORE.update( codecs={**CORE.codecs, "bytes": dataclasses.replace(BYTES_CODEC, rules=counted)} ) ZarrV3GroupMetadata(CONSOLIDATED, context=context) assert calls == [] ZarrV3ConsolidatedMetadata(MEMBER, context=context) assert len(calls) == 3 # `a`, `b/c`, and `c` in `b`'s own listing, each one bytes codec def _listing(a: dict[str, Any]) -> dict[str, Any]: """`MEMBER` with `a` listed at `a`.""" return {**MEMBER, "metadata": {**MEMBER["metadata"], "a": a}} @pytest.mark.parametrize( ("left", "right", "equal"), [ (ZarrV3GroupMetadata(CONSOLIDATED), ZarrV3GroupMetadata(CONSOLIDATED, context=CORE), True), ( ZarrV3GroupMetadata(CONSOLIDATED), ZarrV3GroupMetadata(CONSOLIDATED, context=PRIVATE), True, ), (ZarrV3GroupMetadata(GROUP), ZarrV3GroupMetadata({**GROUP, "attributes": {"g": 2}}), False), ( ZarrV3GroupMetadata({**GROUP, "consolidated_metadata": _listing(ARRAY)}), ZarrV3GroupMetadata({**GROUP, "consolidated_metadata": _listing(SPELLED_OUT)}), False, ), ( ZarrV3ConsolidatedMetadata(MEMBER), ZarrV3ConsolidatedMetadata(MEMBER, context=CORE), True, ), ( ZarrV3ConsolidatedMetadata(MEMBER), ZarrV3ConsolidatedMetadata(MEMBER, context=PRIVATE), False, ), ( ZarrV3ConsolidatedMetadata(_listing(ARRAY)), ZarrV3ConsolidatedMetadata(_listing(SPELLED_OUT)), True, ), ], ids=[ "group-unused-definitions", "group-private-bytes-inside", "group-attributes", "group-listing-spelled-otherwise", "consolidated-unused-definitions", "consolidated-private-bytes-inside", "consolidated-listing-spelled-otherwise", ], ) def test_groups_and_consolidated_metadata_are_equal_by_what_they_read( left: ZarrV3GroupMetadata | ZarrV3ConsolidatedMetadata, right: ZarrV3GroupMetadata | ZarrV3ConsolidatedMetadata, equal: bool, ) -> None: """A group is compared by its attributes, consolidated metadata and extra fields as JSON text, because it reads no field: no scope changes it, and a document listed in another spelling makes another group. Consolidated metadata is compared by what each document it lists means in its scope. Equal models have equal hashes.""" assert (left == right) is equal if equal: assert hash(left) == hash(right) @pytest.mark.parametrize("document", [GROUP, CONSOLIDATED], ids=["group", "consolidated"]) def test_a_group_round_trips_through_its_document_in_its_scope(document: dict[str, Any]) -> None: """`from_json(g.to_json(), context=g.context) == g`, and pickle preserves both the document and the scope.""" group = ZarrV3GroupMetadata(document, context=CORE) assert ZarrV3GroupMetadata(group.to_json(), context=group.context) == group assert pickle.loads(pickle.dumps(group)) == group def test_group_update_with_context_and_refined_in_behave_as_the_arrays_do() -> None: """`update` reads in the group's scope and keeps the consolidated metadata unless a new one is given. A group's fields are its extra fields: in a scope that claims nothing, `consolidated_metadata` is unclaimed, and `refined_in` and `with_context` move the group to a scope that reads it as the envelope, which is a gain.""" group = ZarrV3GroupMetadata(CONSOLIDATED, context=ZarrV3Context()) assert group.update(attributes={"g": 2}).consolidated_metadata == group.consolidated_metadata assert isinstance(group.extra_fields["consolidated_metadata"], UnclaimedField) for moved in (group.refined_in(CORE), group.with_context(PRIVATE)): assert isinstance(moved.extra_fields["consolidated_metadata"], AcceptedField) assert moved.consolidated_metadata == group.consolidated_metadata assert moved.refines(group) assert not group.refines(moved) assert moved != group assert group.refined_in(CORE).context == CORE def test_error_group_refined_in_refuses_a_scope_that_loses_an_extra_field() -> None: """A group read in `CORE` reads its `consolidated_metadata` as the envelope. A scope that leaves the member unclaimed would lose that reading, so `refined_in` raises `ScopeConflictError`, located at the member.""" group = ZarrV3GroupMetadata(CONSOLIDATED, context=CORE) with pytest.raises(ScopeConflictError) as raised: group.refined_in(ZarrV3Context()) assert [conflict.loc for conflict in raised.value.conflicts] == [("consolidated_metadata",)] def test_consolidated_metadata_with_context_and_refined_in_behave_as_the_arrays_do() -> None: """`refined_in` moves every document up the order. `with_context` reads every document in another scope.""" held = ZarrV3ConsolidatedMetadata(MEMBER, context=ZarrV3Context()) refined = held.refined_in(CORE) assert refined.refines(held) assert refined.context == CORE array = refined.metadata["a"] assert isinstance(array, ZarrV3ArrayMetadata) assert isinstance(array.codecs[0], AcceptedField) private_array = held.with_context(PRIVATE).metadata["a"] assert isinstance(private_array, ZarrV3ArrayMetadata) assert private_array.codecs[0].definition == MY_BYTES def test_error_consolidated_metadata_refined_in_refuses_a_conflict() -> None: """`refined_in` raises `ScopeConflictError` for a scope that reads a name of a listed document by another definition, and locates each conflict under that document's path.""" held = ZarrV3ConsolidatedMetadata(MEMBER, context=CORE) with pytest.raises(ScopeConflictError) as raised: held.refined_in(PRIVATE) assert {conflict.loc for conflict in raised.value.conflicts} == { ("metadata", "a", "codecs", 0), ("metadata", "b/c", "codecs", 0), } def test_consolidated_metadata_reads_on_its_own() -> None: """`ZarrV3ConsolidatedMetadata(member, context)` reads a member as written, or as a group holds it, with each document as a model in that scope. `read_consolidated_metadata_v3` returns the reading of the same model.""" held = ZarrV3ConsolidatedMetadata(MEMBER, context=CORE) group = ZarrV3GroupMetadata(CONSOLIDATED, context=CORE) assert held == ZarrV3ConsolidatedMetadata(group.consolidated_metadata, context=CORE) assert held.to_json() == refine_json(MEMBER)[0] assert ZarrV3ConsolidatedMetadata.from_json(MEMBER) == ZarrV3ConsolidatedMetadata(MEMBER) reading = read_consolidated_metadata_v3(MEMBER, context=CORE) assert reading.problems == () assert reading.metadata == held assert held.reading.metadata is held def test_error_consolidated_metadata_with_a_problem_is_refused_at_the_documents_path() -> None: """A problem in a listed document is reported as a problem of the consolidated metadata, located under `metadata.`. `read_consolidated_metadata_v3` returns the same problems and no model.""" member = {**MEMBER, "metadata": {"a": {**ARRAY, "shape": [-1]}}} with pytest.raises(MetadataValidationError) as raised: ZarrV3ConsolidatedMetadata(member) assert [problem.loc for problem in raised.value.problems] == [("metadata", "a", "shape")] reading = read_consolidated_metadata_v3(member) assert reading.problems == raised.value.problems assert reading.metadata is None def test_the_held_field_machinery_is_gone() -> None: """A v3 model is built only by reading its document. So nothing in the package accepts fields that were already read, or re-reads a model in an empty scope. `NO_SCOPE`, `overlapping` and `held` are removed.""" import inspect import zarr_metadata.model._array_v3 as validation assert not hasattr(validation, "NO_SCOPE") assert not hasattr(validation, "overlapping") assert "held" not in inspect.signature(validation.read_array_v3).parameters def test_error_refined_in_refuses_a_gain_that_surfaces_a_problem() -> None: """A scope that claims a name the current scope left unclaimed may refuse what was written under that name. Then `refined_in` raises `MetadataValidationError`, because the document has a problem in that scope.""" loose = ZarrV3ArrayMetadata({**ARRAY, "fill_value": "banana"}, context=ZarrV3Context()) with pytest.raises(MetadataValidationError) as raised: loose.refined_in(CORE) assert [problem.loc for problem in raised.value.problems] == [("fill_value",)] def test_a_scope_conflict_says_where_each_conflict_sits() -> None: """`refined_in` reports each conflict with the field's location in the document, the same way a problem is located. For example, a loss of `bytes` is reported at `codecs.0`, including inside the shard.""" model = ZarrV3ArrayMetadata(SHARDED, context=CORE) with pytest.raises(ScopeConflictError) as raised: model.refined_in(ZarrV3Context()) assert sorted((conflict.key[1], conflict.loc) for conflict in raised.value.conflicts) == [ ("bytes", ("codecs", 0, "configuration", "codecs", 0)), ("bytes", ("codecs", 0, "configuration", "index_codecs", 0)), ("crc32c", ("codecs", 0, "configuration", "index_codecs", 1)), ("default", ("chunk_key_encoding",)), ("regular", ("chunk_grid",)), ("sharding_indexed", ("codecs", 0)), ("uint8", ("data_type",)), ] @pytest.mark.parametrize( "build", [ lambda: ZarrV3ConsolidatedMetadata(MEMBER, context=CORE_AND_EXTENSIONS), lambda: read_consolidated_metadata_v3(MEMBER, context=CORE_AND_EXTENSIONS).metadata, ], ids=["constructor", "reader"], ) def test_with_context_moves_every_listed_model_without_reading_again( build: Callable[[], ZarrV3ConsolidatedMetadata | None], ) -> None: """However consolidated metadata was built, its reading holds it, and each listed model's reading holds that model. `with_context` into a scope that reads every claim identically moves every listed model to the new scope without reading again.""" held = build() assert held is not None assert held.reading.metadata is held for node in held.metadata.values(): assert node.reading.metadata is node context = CORE.update(codecs=[*CORE.codecs.values(), ZSTD_CODEC]) moved = held.with_context(context) assert moved.context is context before = held.metadata["a"] after = moved.metadata["a"] assert isinstance(before, ZarrV3ArrayMetadata) assert isinstance(after, ZarrV3ArrayMetadata) assert after.reading.pipeline is before.reading.pipeline # not read again for node in moved.metadata.values(): assert node.context is context assert node.reading.metadata is node def test_consolidated_metadata_refines_nothing_of_another_type() -> None: """`refines` on consolidated metadata returns False for an object that is not consolidated metadata, as it does for arrays and groups, rather than raising.""" held = ZarrV3ConsolidatedMetadata(MEMBER) assert held.refines(cast("Any", ZarrV3GroupMetadata(GROUP))) is False @pytest.mark.parametrize( ("document", "change"), [ ( {**ARRAY, "attributes": {"a": {"b": 1}}}, lambda model: operator.setitem(model.attributes["a"], "b", 2), ), ( {**ARRAY, "acme": {"x": 1, "must_understand": False}}, lambda model: operator.setitem(model.extra_fields["acme"], "x", 2), ), ( { **ARRAY, "data_type": { "name": "struct", "configuration": {"fields": [{"name": "a", "data_type": "uint8"}]}, }, "fill_value": {"a": 0}, }, lambda model: operator.setitem(model.fill_value, "a", 7), ), ], ids=["attributes", "extra-fields", "fill-value"], ) def test_a_models_views_cannot_be_changed_in_place( document: dict[str, Any], change: Callable[[ZarrV3ArrayMetadata], None] ) -> None: """What a model exposes (attributes, extra fields, a fill value) is read-only at every level. So a model cannot reach a state where its document, its comparison key and `refines` disagree.""" model = ZarrV3ArrayMetadata(document) same = ZarrV3ArrayMetadata(document) with pytest.raises(TypeError): change(model) assert model == same assert model.to_json() == same.to_json() assert model.refines(same) @pytest.mark.parametrize( "reading", [ read_group_metadata_v3(GROUP), read_group_metadata_v3(CONSOLIDATED), read_group_metadata_v3({**GROUP, "attributes": 5}), read_consolidated_metadata_v3(MEMBER), read_consolidated_metadata_v3({**MEMBER, "kind": "sidecar"}), ], ids=["group", "consolidated", "group-problem", "member", "member-problem"], ) def test_a_group_or_consolidated_reading_pickles_and_copies( reading: ZarrV3GroupMetadataReading | ZarrV3ConsolidatedMetadataReading, ) -> None: """A group's reading and a consolidated-metadata reading pickle and deep-copy, with the model they hold, and compare equal afterwards, as an array's reading does. A reading that holds a model is restored through that model, which reads once.""" for again in (pickle.loads(pickle.dumps(reading)), copy.deepcopy(reading)): assert again == reading assert again.metadata == reading.metadata if again.metadata is not None: assert again.metadata.reading == again def test_an_array_reading_pickles_through_its_model() -> None: """An array's reading that holds a model pickles and deep-copies the same way that model does, and is restored as the model's own reading.""" reading = read_array_metadata_v3(ARRAY) for again in (pickle.loads(pickle.dumps(reading)), copy.deepcopy(reading)): assert again == reading assert again.metadata is not None assert again.metadata.reading is again def test_error_a_models_fields_cannot_be_changed_in_place() -> None: """The fields a model returns (a codec's configuration, the fields a shard holds) are read-only, so `codecs ==` and `refines` cannot become inconsistent with `==`.""" model = ZarrV3ArrayMetadata(ARRAY) same = ZarrV3ArrayMetadata(ARRAY) codec = model.codecs[0] assert isinstance(codec, AcceptedField) with pytest.raises(TypeError): codec.configuration["endian"] = "big" # pyright: ignore[reportIndexIssue] assert model.codecs == same.codecs assert model.refines(same) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_pair_v2.py000066400000000000000000000712521527014137600312230ustar00rootroot00000000000000"""Tests that a v2 model is its document plus the scope it was read in, as the v3 models are.""" from __future__ import annotations import copy import dataclasses import json import pickle from collections.abc import Mapping from typing import Any, cast, get_args import pytest from zarr_metadata.definition import ( AcceptedField, EmptyConfiguration, ScopeConflictError, UnclaimedField, ) from zarr_metadata.model import ( UNSET, ZarrV2ArrayMetadata, ZarrV2ConsolidatedMetadata, ZarrV2GroupMetadata, read_array_metadata_v2, read_consolidated_metadata_v2, ) from zarr_metadata.problems import MetadataValidationError from zarr_metadata.v2.codec.compression import ZLIB_CODEC_V2 from zarr_metadata.v2.data_type.scalar import FLOAT_DATA_TYPE_V2, UINT_DATA_TYPE_V2 from zarr_metadata.v2.definition import ( CORE_V2, ZarrV2CodecDefinition, ZarrV2Context, ) Loc = tuple[str | int, ...] ARRAY: dict[str, Any] = { "zarr_format": 2, "shape": [4], "chunks": [2], "dtype": " dict[str, Any]: """Return `ARRAY` with `changes` applied, leaving out any member given as `UNSET`.""" return {key: value for key, value in {**ARRAY, **changes}.items() if value is not UNSET} def test_a_model_is_its_document_read_in_its_scope() -> None: """The constructor reads the document in the given scope (`CORE_V2` by default). `to_json` returns the refined document (arrays as tuples), which shares nothing with the model. A missing `dimension_separator` stays missing. `context` is the scope, and the reading holds the model.""" model = ZarrV2ArrayMetadata(ARRAY) assert model.context is CORE_V2 document = model.to_json() assert document["shape"] == (4,) assert document.get("attributes") == {"a": (1, 2)} without = {k: v for k, v in ARRAY.items() if k != "dimension_separator"} assert "dimension_separator" not in ZarrV2ArrayMetadata(without).to_json() assert model.reading.metadata is model assert read_array_metadata_v2(ARRAY).metadata == model assert ZarrV2ArrayMetadata(ARRAY, SMALL).context is SMALL def test_properties_are_what_the_reading_holds() -> None: """`dtype`, `compressor` and `filters` are the fields as the scope read them, or None where the document has `null`. `shape`, `chunks`, `fill_value`, `order`, `dimension_separator`, `attributes` and `extra_fields` are the members as the read refined them, and they are read-only. `zarr_format` and `node_type` are 2 and `"array"`, as a v3 array model has 3 and `"array"`.""" model = ZarrV2ArrayMetadata({**ARRAY, "filters": [{"id": "x"}], "extra": [1]}) assert isinstance(model.dtype, AcceptedField) assert model.dtype.definition is FLOAT_DATA_TYPE_V2 assert isinstance(model.compressor, AcceptedField) assert model.compressor.definition is ZLIB_CODEC_V2 assert model.filters is not None assert isinstance(model.filters[0], UnclaimedField) members = (model.shape, model.chunks, model.fill_value, model.order, model.dimension_separator) assert members == ((4,), (2,), 0, "C", ".") assert model.attributes == {"a": (1, 2)} assert model.extra_fields == {"extra": (1,)} assert (model.zarr_format, model.node_type) == (2, "array") with pytest.raises(TypeError): cast("dict[str, object]", model.attributes)["b"] = 1 assert ZarrV2ArrayMetadata({**ARRAY, "compressor": None}).compressor is None without = ZarrV2ArrayMetadata(_document({"attributes": UNSET})) assert without.attributes == {} assert isinstance(without.attributes, Mapping) assert "attributes" not in without.to_json() @pytest.mark.parametrize( ("left", "right", "same"), [ ({}, {"shape": (4,), "attributes": {"a": (1, 2)}}, True), ({"dtype": "f4"}, False), ({"compressor": {"id": "zstd", "level": 0}}, {"compressor": {"id": "zstd"}}, True), ({"compressor": {"id": "zstd", "level": 1}}, {"compressor": {"id": "zstd"}}, False), ({"attributes": {"a": [1, 2]}}, {"attributes": {"a": [2, 1]}}, False), ({"attributes": UNSET}, {"attributes": {}}, True), ({"extra": 1}, {}, False), ], ids=[ "same", "int-for-float", "nan", "spelling", "key-order", "numcodecs-default", "numcodecs-default-filter", "signed-zero", "byte-order", "numcodecs-default-omitted", "codec-parameter", "attributes", "zattrs-presence", "extra", ], ) def test_models_are_equal_by_what_their_documents_mean( left: dict[str, Any], right: dict[str, Any], same: bool ) -> None: """Two models are equal when their documents mean the same in their scopes. A typestr with two spellings, a fill value written as an integer or a float, or a codec parameter written or left to its numcodecs default, gives equal models. No `.zattrs` and an empty one give equal models, since both mean no attributes, as in v3. A different byte order or a codec parameter away from its default gives different models. Equal models have equal hashes, and refine each other: read in one scope, `refines` is that equality.""" one, other = ZarrV2ArrayMetadata(_document(left)), ZarrV2ArrayMetadata(_document(right)) assert (one == other) is same assert one.refines(other) is same assert other.refines(one) is same if same: assert hash(one) == hash(other) def test_a_model_read_in_two_scopes_that_read_it_alike_is_one_model() -> None: """Equality depends on interpretation. The same document read with a private `zlib` that takes no parameters and with the core `zlib` gives two different models. Read in two scopes that hold the same `zlib`, it gives equal models.""" document = {**ARRAY, "compressor": {"id": "zlib"}} assert ZarrV2ArrayMetadata(document) != ZarrV2ArrayMetadata(document, PRIVATE) same = ZarrV2Context(data_types=CORE_V2.data_types.values(), codecs=CORE_V2.codecs.values()) assert ZarrV2ArrayMetadata(document) == ZarrV2ArrayMetadata(document, same) @pytest.mark.parametrize("context", [None, SMALL, PRIVATE], ids=["core", "small", "private"]) def test_a_model_round_trips_through_its_document_pickle_and_copy( context: ZarrV2Context | None, ) -> None: """`from_json(m.to_json(), context=m.context)`, `from_key_value(m.to_key_value())`, `pickle` and `copy.deepcopy` each give an equal model in the same scope. A pickled reading unpickles as its model's own reading.""" model = ZarrV2ArrayMetadata(ARRAY, context) assert ZarrV2ArrayMetadata.from_json(model.to_json(), context=model.context) == model restored = ZarrV2ArrayMetadata.from_key_value(model.to_key_value(), context=model.context) assert restored == model loaded = pickle.loads(pickle.dumps(model)) assert loaded == model assert loaded.context == model.context assert copy.deepcopy(model) == model assert pickle.loads(pickle.dumps(model.reading)).metadata == model @pytest.mark.parametrize( ("changes", "expect", "absent"), [ ({"shape": [8], "chunks": [4]}, {"shape": (8,), "chunks": (4,)}, ()), ({"dtype": " None: """`update` replaces the document's members with the given JSON members, takes a member the spec does not define in `extra_fields`, leaves out any member given as `UNSET`, and reads the result in the model's own scope. The original model is unchanged.""" model = ZarrV2ArrayMetadata({**ARRAY, "extra": 0}, PRIVATE) changed = model.update(**changes) assert changed.context is PRIVATE document = changed.to_json() for key, value in expect.items(): assert document[key] == value for key in absent: assert key not in document assert model.to_json()["extra"] == 0 @pytest.mark.parametrize( ("changes", "at"), [ ({"dtype": "float32"}, ("dtype",)), ({"chunks": [1, 1]}, ("chunks",)), ({"dtype": "|b1"}, ("fill_value",)), ], ids=["dtype", "rank", "fill-no-longer-fits"], ) def test_error_update_refuses_a_document_with_a_problem(changes: dict[str, Any], at: Loc) -> None: """A change that produces a document with a problem is refused when the change is made, and the problem is reported at its location. For example, a dtype change that makes the fill value invalid is reported at the fill value.""" with pytest.raises(MetadataValidationError) as raised: ZarrV2ArrayMetadata(ARRAY).update(**changes) assert raised.value.problems[0].loc == at def test_with_context_and_refined_in_read_the_document_in_another_scope() -> None: """`with_context` reads the document in any scope, and losing information is allowed. `refined_in` only moves up the order. A scope that claims what the current scope left unclaimed is a gain. A scope that reads a name with another definition, or with no definition, raises a `ScopeConflictError` that gives the name's location. A gain that reveals a problem raises a `MetadataValidationError`.""" unclaimed = ZarrV2ArrayMetadata({**ARRAY, "compressor": {"id": "zlib"}}, SMALL) assert isinstance(unclaimed.compressor, UnclaimedField) gained = unclaimed.refined_in(PRIVATE) assert isinstance(gained.compressor, AcceptedField) assert gained.compressor.definition is BARE_ZLIB assert unclaimed.refines(unclaimed) assert gained.refines(unclaimed) assert not unclaimed.refines(gained) with pytest.raises(ScopeConflictError) as conflict: gained.refined_in(CORE_V2) assert [c.loc for c in conflict.value.conflicts] == [("compressor",)] lost = gained.with_context(SMALL) assert isinstance(lost.compressor, UnclaimedField) assert lost == unclaimed assert gained.with_context(PRIVATE) == gained with pytest.raises(MetadataValidationError): ZarrV2ArrayMetadata({**ARRAY, "compressor": {"id": "zlib", "level": 1}}, SMALL).refined_in( PRIVATE ) def test_refines_orders_models_by_information() -> None: """`refines` is true when each field refines its counterpart and every other member is the same, with the fill value compared as the more informed dtype spells it. A fill value that this dtype refuses is not a refinement. A value of another type refines nothing.""" core = ZarrV2ArrayMetadata(ARRAY) assert core.refines(ZarrV2ArrayMetadata({**ARRAY, "fill_value": 0.0})) filled = ZarrV2ArrayMetadata({**ARRAY, "compressor": {"id": "zlib", "level": 1}}) assert core.refines(filled) assert filled.refines(core) assert not core.refines(ZarrV2ArrayMetadata({**ARRAY, "shape": [8], "chunks": [2]})) small = ZarrV2ArrayMetadata({**ARRAY, "dtype": " None: """No model is invalid: the constructor raises with every problem the read finds.""" with pytest.raises(MetadataValidationError) as raised: ZarrV2ArrayMetadata({**ARRAY, "dtype": "float32", "order": "Q"}) assert [p.loc for p in raised.value.problems] == [("dtype",), ("order",)] def test_the_dataclass_machinery_is_gone() -> None: """A v2 model is built only from a document. `dataclasses.replace` and `dataclasses.fields` do not apply, and the old `...Partial` names are removed.""" import zarr_metadata assert not dataclasses.is_dataclass(ZarrV2ArrayMetadata) assert not hasattr(zarr_metadata, "ZarrV2ArrayMetadataPartial") assert "ZarrV2ArrayMetadataUpdate" in zarr_metadata.__all__ GROUP: dict[str, Any] = {"zarr_format": 2, "attributes": {"g": [1]}} @pytest.mark.parametrize( ("document", "attributes"), [ (GROUP, {"g": (1,)}), ({"zarr_format": 2}, {}), ({"zarr_format": 2, "attributes": {}}, {}), ], ids=["attributes", "no-zattrs", "empty-zattrs"], ) def test_a_group_is_its_document_read_in_its_scope( document: dict[str, Any], attributes: object ) -> None: """A v2 group model is its document plus its scope. `attributes` holds the attributes as the read refined them, and is empty when no `.zattrs` exists, as a v3 model's `attributes` is empty when the document writes none. `zarr_format` and `node_type` are 2 and `"group"`, as a v3 group model has 3 and `"group"`. The model round-trips through its document, its store keys, pickle and copy, in any scope.""" model = ZarrV2GroupMetadata(document, SMALL) assert model.context is SMALL assert (model.zarr_format, model.node_type) == (2, "group") assert model.attributes == attributes assert model.to_json() == ZarrV2GroupMetadata.from_json(document).to_json() assert ZarrV2GroupMetadata.from_key_value(model.to_key_value(), context=SMALL) == model assert pickle.loads(pickle.dumps(model)) == model assert copy.deepcopy(model) == model def test_a_group_updates_moves_scope_and_refines_as_the_arrays_do() -> None: """`update` reads the new attributes in the model's own scope, and `UNSET` removes them. `with_context` and `refined_in` read the document in another scope and never conflict, since a group holds no field. Two groups are equal when their attributes are written the same way, and `refines` is that equality. No `.zattrs` counts as empty attributes, so a group with none and a group with an empty one are equal and refine each other.""" model = ZarrV2GroupMetadata(GROUP) assert model.update(attributes={"h": 2}).attributes == {"h": 2} removed = model.update(attributes=UNSET) assert removed.attributes == {} assert ".zattrs" not in removed.to_key_value() assert model.refined_in(SMALL).context is SMALL assert model.with_context(PRIVATE) == model assert model == ZarrV2GroupMetadata({"zarr_format": 2, "attributes": {"g": (1,)}}) assert model != ZarrV2GroupMetadata({"zarr_format": 2}) assert model.refines(model) assert not model.refines(ZarrV2GroupMetadata({"zarr_format": 2})) absent = ZarrV2GroupMetadata({"zarr_format": 2}) empty = ZarrV2GroupMetadata({"zarr_format": 2, "attributes": {}}) assert absent == empty assert absent.refines(empty) assert empty.refines(absent) assert not dataclasses.is_dataclass(ZarrV2GroupMetadata) def test_error_a_group_document_with_a_problem_is_refused_at_construction() -> None: """A group document with a problem -- a member the spec does not define, an attribute key that is not a string -- is refused with every problem.""" with pytest.raises(MetadataValidationError) as raised: ZarrV2GroupMetadata({"zarr_format": 2, "attributes": {1: "a"}, "extra": 1}) assert sorted((p.loc, p.kind) for p in raised.value.problems) == [ (("attributes",), "invalid_type"), (("extra",), "unknown_key"), ] ZARRAY = {k: v for k, v in ARRAY.items() if k != "attributes"} CONSOLIDATED: dict[str, Any] = { "zarr_consolidated_format": 1, "metadata": { ".zgroup": {"zarr_format": 2}, ".zattrs": {"root": True}, "a/.zarray": ZARRAY, "a/.zattrs": {"a": [1, 2]}, "b/.zgroup": {"zarr_format": 2}, "orphan/.zattrs": {"o": 1}, }, } def _consolidated(**entries: object) -> dict[str, Any]: return {**CONSOLIDATED, "metadata": {**CONSOLIDATED["metadata"], **entries}} def test_consolidated_metadata_maps_each_node_path_to_its_model() -> None: """`metadata` maps the path of each node, without the leading `/` (`""` for the root), to the model of its `.zarray` or `.zgroup` entry merged with its sibling `.zattrs`, read in the consolidated scope, as `ZarrV3ConsolidatedMetadata.metadata` maps each path to its model. It is read-only. The entries stay in `to_json()["metadata"]`, keyed by file, as written and refined. A `.zattrs` with no sibling is kept there but makes no node. The whole model round-trips through its document, store keys, pickle and copy.""" model = ZarrV2ConsolidatedMetadata(CONSOLIDATED, PRIVATE) assert model.context is PRIVATE assert set(model.metadata) == {"", "a", "b"} assert model.metadata["a"] == ZarrV2ArrayMetadata(ARRAY, PRIVATE) root = ZarrV2GroupMetadata({"zarr_format": 2, "attributes": {"root": True}}, PRIVATE) assert model.metadata[""] == root assert model.metadata["b"].attributes == {} assert not hasattr(model, "nodes") entries = cast("dict[str, Any]", model.to_json()["metadata"]) assert entries["a/.zattrs"] == {"a": (1, 2)} assert entries["orphan/.zattrs"] == {"o": 1} assert ZarrV2ConsolidatedMetadata.from_key_value(model.to_key_value(), context=PRIVATE) == model assert pickle.loads(pickle.dumps(model)) == model assert copy.deepcopy(model) == model with pytest.raises(TypeError): cast("dict[str, object]", model.metadata)["x"] = 1 assert not dataclasses.is_dataclass(ZarrV2ConsolidatedMetadata) @pytest.mark.parametrize( "document", [CONSOLIDATED, {"metadata": CONSOLIDATED["metadata"], "zarr_consolidated_format": 1}], ids=["format-first", "metadata-first"], ) def test_consolidated_metadata_is_written_in_the_order_it_was_read( document: dict[str, Any], ) -> None: """`to_json` and `to_key_value` write the top-level keys of a `.zmetadata` in the order the document writes them, as the v3 models do. zarr-python 3 writes `metadata` first. A model built by `read_consolidated_metadata_v2`, unpickled, or moved to another scope by `with_context` keeps the order too. The order means nothing, so the models of a document and of its keys in the other order are equal, with equal hashes.""" model = ZarrV2ConsolidatedMetadata(document) assert list(model.to_json()) == list(document) assert list(json.loads(model.to_key_value()[".zmetadata"])) == list(document) reading = read_consolidated_metadata_v2(document) assert reading.metadata is not None assert list(reading.metadata.to_json()) == list(document) assert list(pickle.loads(pickle.dumps(model)).to_json()) == list(document) assert list(model.with_context(PRIVATE).to_json()) == list(document) reordered = ZarrV2ConsolidatedMetadata(dict(reversed(document.items()))) assert list(reordered.to_json()) != list(document) assert model == reordered assert hash(model) == hash(reordered) def test_consolidated_metadata_is_equal_by_its_nodes_and_moves_scope_with_them() -> None: """Two consolidated documents are equal when each node means the same and the other entries are written the same way. A node with no `.zattrs` means the same as one with an empty `.zattrs`, as a node read on its own does, so the documents are equal, with equal hashes, and refine each other. `with_context` and `refined_in` read every node in the new scope, and a conflict is located at the node's entry. `refines` is true when each node refines its counterpart.""" spelled = _consolidated(**{"a/.zarray": {**ZARRAY, "fill_value": 0.0}}) assert ZarrV2ConsolidatedMetadata(CONSOLIDATED) == ZarrV2ConsolidatedMetadata(spelled) other = _consolidated(**{"orphan/.zattrs": {"o": 2}}) assert ZarrV2ConsolidatedMetadata(CONSOLIDATED) != ZarrV2ConsolidatedMetadata(other) empty = ZarrV2ConsolidatedMetadata(_consolidated(**{"b/.zattrs": {}})) assert ZarrV2ConsolidatedMetadata(CONSOLIDATED) == empty assert hash(ZarrV2ConsolidatedMetadata(CONSOLIDATED)) == hash(empty) assert ZarrV2ConsolidatedMetadata(CONSOLIDATED).refines(empty) assert empty.refines(ZarrV2ConsolidatedMetadata(CONSOLIDATED)) small = ZarrV2ConsolidatedMetadata(CONSOLIDATED, SMALL) assert isinstance(small.metadata["a"], ZarrV2ArrayMetadata) assert isinstance(small.metadata["a"].compressor, UnclaimedField) gained = small.refined_in(PRIVATE) assert isinstance(gained.metadata["a"], ZarrV2ArrayMetadata) assert isinstance(gained.metadata["a"].compressor, AcceptedField) assert gained.refines(small) assert not small.refines(gained) with pytest.raises(ScopeConflictError) as conflict: gained.refined_in(CORE_V2) assert [c.loc for c in conflict.value.conflicts] == [("metadata", "a/.zarray", "compressor")] assert gained.with_context(SMALL) == small @pytest.mark.parametrize( ("entries", "at"), [ ({"a/.zarray": {**ZARRAY, "dtype": "float32"}}, ("metadata", "a/.zarray", "dtype")), ({"a/.zarray": ZARRAY, "a/.zattrs": {1: 2}}, ("metadata", "a/.zattrs")), ({"a/.zarray": ZARRAY, "a/.zgroup": {"zarr_format": 2}}, ("metadata", "a/.zgroup")), ({"a/.zgroup": {"zarr_format": 3}}, ("metadata", "a/.zgroup", "zarr_format")), ({"a/.zarray": 3}, ("metadata", "a/.zarray")), ], ids=["array-dtype", "zattrs-key", "array-and-group", "group-format", "not-an-object"], ) def test_error_a_node_entry_with_a_problem_is_refused_at_the_entry( entries: dict[str, Any], at: Loc ) -> None: """A `.zarray` or `.zgroup` entry is read as its document type, in the consolidated scope. Its problems are located under the entry, and a `.zattrs` entry's problems under its own entry. A path that is both an array and a group is a problem at the `.zgroup`.""" with pytest.raises(MetadataValidationError) as raised: ZarrV2ConsolidatedMetadata({"zarr_consolidated_format": 1, "metadata": entries}) assert raised.value.problems[0].loc == at def test_error_two_entries_for_one_node_file_are_refused() -> None: """Two keys that name the same file of the same node (`.zarray` and `/.zarray`) are a problem at the second key, since one of them would otherwise not be read. A conflict is located at the key as the document writes it.""" with pytest.raises(MetadataValidationError) as raised: ZarrV2ConsolidatedMetadata( {"zarr_consolidated_format": 1, "metadata": {".zarray": ZARRAY, "/.zarray": ZARRAY}} ) assert [(p.loc, p.kind) for p in raised.value.problems] == [ (("metadata", "/.zarray"), "invalid_value") ] gained = ZarrV2ConsolidatedMetadata( {"zarr_consolidated_format": 1, "metadata": {"/.zarray": ZARRAY}}, PRIVATE ) with pytest.raises(ScopeConflictError) as conflict: gained.refined_in(CORE_V2) assert [c.loc for c in conflict.value.conflicts] == [("metadata", "/.zarray", "compressor")] def test_the_node_type_is_exported() -> None: """`ZarrV2NodeMetadata`, the type of each node in a `.zmetadata`'s `metadata`, is exported alongside the models, as `ZarrV3NodeMetadata` is.""" from zarr_metadata import model assert "ZarrV2NodeMetadata" in model.__all__ def test_each_node_alias_holds_its_union_at_run_time() -> None: """Each alias for a node of either format, v2 and v3, is a type alias at run time, so the aliases of the two formats form a union. Each holds the union of its classes, not the union written as a string, so `get_args` of its value names the classes that a type checker sees.""" from zarr_metadata import model either = model.ZarrV2NodeMetadata | model.ZarrV3NodeMetadata assert get_args(either) == (model.ZarrV2NodeMetadata, model.ZarrV3NodeMetadata) assert get_args(model.ZarrV2NodeMetadata.__value__) == ( model.ZarrV2ArrayMetadata, model.ZarrV2GroupMetadata, ) assert get_args(model.ZarrV2NodeMetadataReading.__value__) == ( model.ZarrV2ArrayMetadataReading, model.ZarrV2GroupMetadataReading, model.ZarrV2UnknownNodeMetadataReading, ) assert get_args(model.ZarrV3NodeMetadata.__value__) == ( model.ZarrV3ArrayMetadata, model.ZarrV3GroupMetadata, ) assert get_args(model.ZarrV3NodeMetadataReading.__value__) == ( model.ZarrV3ArrayMetadataReading, model.ZarrV3GroupMetadataReading, model.ZarrV3UnknownNodeMetadataReading, ) def test_error_a_second_key_for_a_node_file_hides_no_other_problem() -> None: """A second key that names the same file of the same node is one problem among the document's problems. Every node is still read and each node's problems are reported, so a user sees everything at once. A leading or repeated `/` does not make a second node.""" with pytest.raises(MetadataValidationError) as raised: ZarrV2ConsolidatedMetadata( { "zarr_consolidated_format": 1, "metadata": { "a/.zarray": ZARRAY, "/a/.zarray": ZARRAY, "b/.zgroup": {"zarr_format": 3}, }, } ) assert [p.loc for p in raised.value.problems] == [ ("metadata", "/a/.zarray"), ("metadata", "b/.zgroup", "zarr_format"), ] one = ZarrV2ConsolidatedMetadata( {"zarr_consolidated_format": 1, "metadata": {"/a/.zarray": ZARRAY}} ) assert set( ZarrV2ConsolidatedMetadata( { "zarr_consolidated_format": 1, "metadata": {"x/.zgroup": {"zarr_format": 2}, "x//y/.zarray": ZARRAY}, } ).metadata ) == {"x", "x/y"} assert set(one.metadata) == {"a"} assert one == ZarrV2ConsolidatedMetadata( {"zarr_consolidated_format": 1, "metadata": {"a/.zarray": ZARRAY}} ) @pytest.mark.parametrize("name", ["__private", "zarr.json"]) def test_a_v2_node_may_have_a_name_that_only_v3_reserves(name: str) -> None: """Zarr v2 reserves no node names. A `.zmetadata` node named `__private` or `zarr.json`, which v3 reserves, is a valid v2 node.""" model = ZarrV2ConsolidatedMetadata( { "zarr_consolidated_format": 1, "metadata": {".zgroup": {"zarr_format": 2}, f"{name}/.zarray": ZARRAY}, } ) assert set(model.metadata) == {"", name} def test_error_a_key_with_a_dot_segment_is_no_node_s_path() -> None: """A `.zmetadata` key whose path has a `.` or `..` segment names no node, unlike a key with a leading or repeated `/`. It is a problem at the key, and it does not make a second node next to the one it would name.""" with pytest.raises(MetadataValidationError) as raised: ZarrV2ConsolidatedMetadata( { "zarr_consolidated_format": 1, "metadata": { ".zgroup": {"zarr_format": 2}, "./a/.zarray": ZARRAY, "b/../a/.zarray": ZARRAY, }, } ) assert [(p.loc, p.kind) for p in raised.value.problems] == [ (("metadata", "./a/.zarray"), "invalid_value"), (("metadata", "b/../a/.zarray"), "invalid_value"), ] @pytest.mark.parametrize( ("entries", "at", "kind"), [ ( {".zgroup": {"zarr_format": 2}, "a/.zarray": ZARRAY, "a/b/.zarray": ZARRAY}, "a/b/.zarray", "invalid_value", ), ({".zarray": ZARRAY, "x/.zgroup": {"zarr_format": 2}}, "x/.zgroup", "invalid_value"), ({".zgroup": {"zarr_format": 2}, "x/y/.zarray": ZARRAY}, "x/.zgroup", "missing_key"), ], ids=["below-an-array", "below-the-root-array", "missing-parent"], ) def test_error_the_nodes_of_a_zmetadata_make_a_hierarchy( entries: dict[str, Any], at: str, kind: str ) -> None: """The nodes in a `.zmetadata` form a hierarchy, as a v3 group's consolidated metadata does. A node below an array is a problem at its entry. A missing parent group is a `missing_key` problem at the entry that the group would have.""" with pytest.raises(MetadataValidationError) as raised: ZarrV2ConsolidatedMetadata({"zarr_consolidated_format": 1, "metadata": entries}) assert [(p.loc, p.kind) for p in raised.value.problems] == [(("metadata", at), kind)] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_pydantic.py000066400000000000000000000305041527014137600314670ustar00rootroot00000000000000"""Executable example: integrating the metadata models with pydantic (v2). Pydantic's native dataclass introspection CAN be made to work (see `test_native_dataclass_introspection_is_possible_but_diverges`): the models keep their annotation-only imports behind `TYPE_CHECKING`, so a bare `TypeAdapter(ZarrV3ArrayMetadata)` raises `class-not-fully-defined`, but `rebuild(_types_namespace=...)` with the names supplied resolves the schema. It is still the wrong tool: it validates the MODEL SHAPE, not the DOCUMENT — no `from_json` normalization (a bare-string `data_type` is rejected), and pydantic's lax coercion silently re-opens holes the library's validators close (`shape=[True, -5]` coerces to `(1, -5)`; a wrong `dimension_names` count passes). The recommended integration delegates wholesale — treat the model as an opaque value: - `InstanceOf` makes pydantic's core schema an is-instance check (no field introspection), - validation goes through `from_json` (the single source of truth for what a well-formed document is, including normalization: bare-string metadata fields, arrays-to-tuples), - serialization goes through `to_json` (the canonical document form). `MetadataValidationError` subclasses `ValueError`, so pydantic converts a failed parse into its own `ValidationError` with the loc-annotated problem messages intact. """ from collections.abc import Mapping from typing import Annotated, Generic, TypeVar import pytest from pydantic import ( BaseModel, BeforeValidator, ConfigDict, InstanceOf, PlainSerializer, PydanticSchemaGenerationError, PydanticUserError, TypeAdapter, ValidationError, model_validator, ) from zarr_metadata import JSONValue from zarr_metadata.model import ( ZarrV3ArrayMetadata, ) # --- the integration (this is the example) ----------------------------------- def _as_array_metadata_v3(value: object) -> ZarrV3ArrayMetadata: """Accept an existing model instance or a raw metadata document.""" if isinstance(value, ZarrV3ArrayMetadata): return value return ZarrV3ArrayMetadata.from_json(value) # return_type is explicit because to_json's own annotation (`ZarrV3ArrayMetadataJSON`) # is a TYPE_CHECKING-only name pydantic cannot resolve at runtime. ArrayMetadataV3Field = Annotated[ InstanceOf[ZarrV3ArrayMetadata], BeforeValidator(_as_array_metadata_v3), PlainSerializer(ZarrV3ArrayMetadata.to_json, return_type=dict), ] """A pydantic-ready field type for v3 array metadata. Validates raw documents via `from_json`, passes model instances through, and serializes to the canonical document form via `to_json`. """ class ArrayManifest(BaseModel): """Example consumer model: a named array with its metadata document.""" path: str metadata: ArrayMetadataV3Field # --- tests pinning the example ------------------------------------------------ VALID_DOC = { "zarr_format": 3, "node_type": "array", "shape": [10], "data_type": "uint8", "fill_value": 0, "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, "chunk_key_encoding": {"name": "default"}, "codecs": [{"name": "bytes"}], } def test_raw_document_is_validated_into_a_model() -> None: """A raw metadata document on a pydantic field is parsed by from_json, with the library's normalization applied (tuples, canonical field form).""" manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) assert isinstance(manifest.metadata, ZarrV3ArrayMetadata) assert manifest.metadata.shape == (10,) assert manifest.metadata.data_type.name == "uint8" def test_model_instance_passes_through() -> None: """An already-constructed model instance is accepted unchanged.""" model = ZarrV3ArrayMetadata.from_json(VALID_DOC) manifest = ArrayManifest(path="a/b", metadata=model) assert manifest.metadata is model def test_invalid_document_surfaces_problems_in_validation_error() -> None: """A structurally-invalid document fails pydantic validation, carrying the loc-annotated problem messages from MetadataValidationError.""" doc = dict(VALID_DOC) del doc["chunk_key_encoding"] with pytest.raises(ValidationError) as exc_info: ArrayManifest.model_validate({"path": "a/b", "metadata": doc}) assert "chunk_key_encoding: missing required key" in str(exc_info.value) def test_dump_emits_canonical_document() -> None: """model_dump serializes the field via to_json — the canonical document, not pydantic's field-by-field view of the dataclass.""" manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) dumped = manifest.model_dump() assert dumped["metadata"] == manifest.metadata.to_json() # Empty configurations use the extension-definition shorthand form. assert dumped["metadata"]["data_type"] == "uint8" def test_json_roundtrip_through_pydantic() -> None: """model_dump_json output re-validates to an equal manifest (JSON emits tuples as arrays; from_json converts them back).""" manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) revived = ArrayManifest.model_validate_json(manifest.model_dump_json()) assert revived == manifest def test_type_adapter_standalone() -> None: """The annotated alias also works without a BaseModel, via TypeAdapter.""" adapter = TypeAdapter(ArrayMetadataV3Field) model = adapter.validate_python(VALID_DOC) assert isinstance(model, ZarrV3ArrayMetadata) assert adapter.dump_python(model) == model.to_json() # --- the road not taken: native dataclass introspection ---------------------- def test_native_dataclass_introspection_is_not_supported() -> None: """Pydantic cannot field-introspect the model dataclasses: the UNSET sentinel (PEP 661, typing_extensions.Sentinel) in the optional-field annotations has no pydantic schema (as of pydantic 2.13), so even the rebuild-with-namespace recipe fails. Introspection was already the wrong tool before the sentinel existed — it validated the model shape rather than the document, and its lax coercion re-opened validator holes (e.g. shape=[True, -5] coerced to (1, -5)) — so the delegation patterns above are the only supported integrations. If this test ever fails because pydantic learned to handle sentinels, revisit whether the introspection path needs its divergences documented again.""" from zarr_metadata.model import UNSET from zarr_metadata.typed_json import JSONValue from zarr_metadata.v3 import ZarrV3ExtensionField, ZarrV3MetadataFieldJSON from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON def build_and_use() -> None: adapter = TypeAdapter(ZarrV3ArrayMetadata) adapter.rebuild( force=True, _types_namespace={ "JSONValue": JSONValue, "ZarrV3ExtensionField": ZarrV3ExtensionField, "ZarrV3MetadataFieldJSON": ZarrV3MetadataFieldJSON, "ZarrV3ArrayMetadataJSON": ZarrV3ArrayMetadataJSON, "UNSET": UNSET, }, ) adapter.validate_python({}) with pytest.raises((AttributeError, PydanticSchemaGenerationError, PydanticUserError)): build_and_use() # --- a first-class pydantic model, engine-backed (the pydantic-zarr pattern) -- # # When a consumer wants a real BaseModel — JSON schema generation, and # generics for typed attributes, as in pydantic-zarr's ArraySpec — the model # fields are pydantic-native, but validation and serialization still route # through the library: a mode="before" validator canonicalizes every input # document with from_json(...).to_json(), so the structural validators and # normalization run BEFORE pydantic parses fields (no coercion divergence), # and the document form is the bridge in both directions. AttrsT = TypeVar("AttrsT") class NamedConfig(BaseModel): """Pydantic mirror of a normalized metadata extension envelope.""" name: str configuration: dict[str, JSONValue] = {} must_understand: bool = True class ArrayMetadataV3Spec(BaseModel, Generic[AttrsT]): """A pydantic-native, attribute-typed view of a v3 array metadata document. The library is the engine: every input is canonicalized and structurally validated by `ZarrV3ArrayMetadata.from_json` before pydantic sees the fields, and `to_document` / `to_metadata_model` emit through the library. """ model_config = ConfigDict(frozen=True) zarr_format: int = 3 node_type: str = "array" shape: tuple[int, ...] data_type: NamedConfig chunk_grid: NamedConfig chunk_key_encoding: NamedConfig fill_value: JSONValue codecs: tuple[NamedConfig, ...] attributes: AttrsT dimension_names: tuple[str | None, ...] | None = None storage_transformers: tuple[NamedConfig, ...] = () @model_validator(mode="before") @classmethod def _canonicalize(cls, data: object) -> object: """Route every input document through the library's validation and normalization; pydantic then parses only canonical documents.""" if isinstance(data, Mapping): metadata = ZarrV3ArrayMetadata.from_json(data).to_json() doc: dict[str, object] = dict(metadata) for key in ("data_type", "chunk_grid", "chunk_key_encoding"): if isinstance(doc[key], str): doc[key] = {"name": doc[key]} for key, entries in ( ("codecs", metadata["codecs"]), ("storage_transformers", metadata.get("storage_transformers", ())), ): doc[key] = tuple( {"name": item} if isinstance(item, str) else item for item in entries ) doc.setdefault("attributes", {}) return doc return data def to_metadata_model(self) -> ZarrV3ArrayMetadata: """Bridge back to the canonical model, via the document form. In the document, "no dimension names" is key-absence, not null; the pydantic-side None translates to dropping the key. """ doc = self.model_dump() if doc["dimension_names"] is None: del doc["dimension_names"] return ZarrV3ArrayMetadata.from_json(doc) def to_document(self) -> dict[str, object]: """The canonical document (omit-empty conventions applied).""" return dict(self.to_metadata_model().to_json()) class MicroscopyAttrs(BaseModel): """Example of consumer-typed attributes, pydantic-zarr style.""" resolution_um: float def test_spec_typed_attributes() -> None: """The generic parameter types the attributes, so consumers get validated, attribute-level access — the pydantic-zarr ArraySpec pattern.""" doc = dict(VALID_DOC) | {"attributes": {"resolution_um": 0.5}} spec = ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(doc) assert spec.attributes.resolution_um == 0.5 assert spec.data_type == NamedConfig(name="uint8") def test_spec_engine_validates_before_pydantic() -> None: """The library's structural validation runs before pydantic's parsing, so coercion cannot re-open validator holes (contrast with the native introspection test above, where [True, -5] coerced to (1, -5)).""" with pytest.raises(ValidationError, match="shape"): ArrayMetadataV3Spec[MicroscopyAttrs].model_validate( dict(VALID_DOC) | {"shape": [True, -5], "attributes": {"resolution_um": 0.5}} ) def test_spec_bridges_to_canonical_model_and_document() -> None: """to_metadata_model / to_document round-trip through the document form, and the emitted document matches what the library itself would emit.""" doc = dict(VALID_DOC) | {"attributes": {"resolution_um": 0.5}} spec = ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(doc) model = spec.to_metadata_model() assert isinstance(model, ZarrV3ArrayMetadata) assert spec.to_document() == dict(model.to_json()) # and back: the document revalidates to an equal spec assert ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(spec.to_document()) == spec def test_spec_json_schema_generation() -> None: """A real BaseModel means model_json_schema works — the capability the opaque InstanceOf pattern cannot provide.""" schema = ArrayMetadataV3Spec[MicroscopyAttrs].model_json_schema() assert schema["properties"]["shape"]["type"] == "array" assert "MicroscopyAttrs" in schema["$defs"] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_pydantic_module.py000066400000000000000000001030771527014137600330420ustar00rootroot00000000000000"""Tests for `zarr_metadata.pydantic`, the optional pydantic field-type module. The hand-rolled recipes in `test_pydantic.py` document how the integration works; this module ships it. Instances are the CORE model classes (no parallel hierarchy), so values interoperate freely with non-pydantic code. """ import json import math import warnings from collections.abc import Callable, Iterator, Mapping from pathlib import Path from typing import Annotated, Any import pytest from jsonschema import Draft202012Validator from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from pydantic.json_schema import JsonSchemaMode import zarr_metadata.pydantic as zmp from zarr_metadata.definition import AcceptedField, UnclaimedField from zarr_metadata.json_schema import json_schema, node_metadata_json_schema_v3 from zarr_metadata.model import ( ZarrV2ArrayMetadata, ZarrV2ConsolidatedMetadata, ZarrV2GroupMetadata, ZarrV3ArrayMetadata, ZarrV3ConsolidatedMetadata, ZarrV3GroupMetadata, ) from zarr_metadata.problems import ValidationProblem from zarr_metadata.typed_json import JSONValue from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON from zarr_metadata.v2.definition import CORE_V2, ZarrV2Context from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON from zarr_metadata.v3.codec.gzip import GZIP_CODEC from zarr_metadata.v3.definition import CORE, CORE_AND_EXTENSIONS, ZarrV3Context V3_ARRAY_DOC = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) V2_ARRAY_DOC = dict(ZarrV2ArrayMetadata.create_default(shape=(4,), chunks=(2,)).to_json()) V3_GROUP_DOC = {"zarr_format": 3, "node_type": "group", "attributes": {"a": 1}} V2_GROUP_DOC = {"zarr_format": 2, "attributes": {"a": 1}} V3_CONSOLIDATED_DOC = { "kind": "inline", "must_understand": False, "metadata": {"a": dict(V3_ARRAY_DOC)}, } V2_CONSOLIDATED_DOC = { "zarr_consolidated_format": 1, "metadata": {".zgroup": {"zarr_format": 2}}, } FIELD_CASES = [ pytest.param(zmp.ZarrV3ArrayMetadataField, ZarrV3ArrayMetadata, V3_ARRAY_DOC, id="array-v3"), pytest.param(zmp.ZarrV2ArrayMetadataField, ZarrV2ArrayMetadata, V2_ARRAY_DOC, id="array-v2"), pytest.param(zmp.ZarrV3GroupMetadataField, ZarrV3GroupMetadata, V3_GROUP_DOC, id="group-v3"), pytest.param(zmp.ZarrV2GroupMetadataField, ZarrV2GroupMetadata, V2_GROUP_DOC, id="group-v2"), pytest.param( zmp.ZarrV3ConsolidatedMetadataField, ZarrV3ConsolidatedMetadata, V3_CONSOLIDATED_DOC, id="consolidated-v3", ), pytest.param( zmp.ZarrV2ConsolidatedMetadataField, ZarrV2ConsolidatedMetadata, V2_CONSOLIDATED_DOC, id="consolidated-v2", ), ] @pytest.mark.parametrize(("field_type", "model_cls", "doc"), FIELD_CASES) def test_field_type_validates_and_dumps_canonically( field_type: object, model_cls: type, doc: dict[str, object] ) -> None: """Each field type parses its raw document into the CORE model class, passes existing instances through unchanged, and dumps the canonical document via to_json.""" adapter = TypeAdapter(field_type) model = adapter.validate_python(doc) assert type(model) is model_cls assert adapter.validate_python(model) is model assert adapter.dump_python(model) == model.to_json() def test_core_instances_interoperate() -> None: """A core model instance (e.g. returned by zarr-python) drops straight into a pydantic field — the reason the module ships Annotated aliases over the core classes rather than pydantic-aware subclasses.""" class Manifest(BaseModel): metadata: zmp.ZarrV3ArrayMetadataField core = ZarrV3ArrayMetadata.from_json(V3_ARRAY_DOC) manifest = Manifest(metadata=core) assert manifest.metadata is core def test_validation_error_carries_problems() -> None: """A defective document fails with the library's loc-annotated messages.""" class Manifest(BaseModel): metadata: zmp.ZarrV3ArrayMetadataField doc = dict(V3_ARRAY_DOC) del doc["chunk_key_encoding"] with pytest.raises(ValidationError) as raised: Manifest.model_validate({"metadata": doc}) (error,) = raised.value.errors() assert (error["type"], error["loc"], error["msg"]) == ( "missing_key", ("metadata", "chunk_key_encoding"), "missing required key", ) # For a missing key, the input is the object that lacks the key, as in pydantic's own errors. assert error["input"] == doc def test_json_schema_generation() -> None: """model_json_schema works, describing the document form each field accepts.""" class Manifest(BaseModel): metadata: zmp.ZarrV3ArrayMetadataField schema = Manifest.model_json_schema() metadata_schema = schema["$defs"]["ZarrV3ArrayMetadataJSON"] assert schema["properties"]["metadata"]["$ref"] == "#/$defs/ZarrV3ArrayMetadataJSON" assert metadata_schema["required"] == [ "zarr_format", "node_type", "data_type", "shape", "chunk_grid", "chunk_key_encoding", "fill_value", "codecs", ] assert metadata_schema["properties"]["zarr_format"] == { "const": 3, "title": "Zarr Format", "type": "integer", } def test_json_schema_generation_emits_no_warnings() -> None: """Consumers can generate every public integration schema without warning filters.""" field_types = ( zmp.ZarrV3ArrayMetadataField, zmp.ZarrV2ArrayMetadataField, zmp.ZarrV3GroupMetadataField, zmp.ZarrV2GroupMetadataField, zmp.ZarrV3ConsolidatedMetadataField, zmp.ZarrV2ConsolidatedMetadataField, ) with warnings.catch_warnings(): warnings.simplefilter("error") for field_type in field_types: TypeAdapter(field_type).json_schema() def test_v2_recursive_structured_dtype_is_in_pydantic_schema() -> None: """The schema accepts nested structured dtypes supported by the v2 specification.""" doc = json.loads(json.dumps(V2_ARRAY_DOC)) doc["dtype"] = [["outer", [["inner", " None: adapter = TypeAdapter(field_type) with pytest.raises(ValidationError): adapter.validate_python(document) assert list(Draft202012Validator(adapter.json_schema()).iter_errors(document)) def test_v3_array_schema_rejects_empty_codecs() -> None: """The generated schema mirrors the runtime non-empty codec pipeline rule.""" doc = json.loads(json.dumps(V3_ARRAY_DOC)) doc["codecs"] = [] _assert_runtime_and_schema_reject(zmp.ZarrV3ArrayMetadataField, doc) def test_array_schemas_reject_negative_dimensions() -> None: """Both array schemas mirror the runtime non-negative dimension rule.""" for field_type, source in ( (zmp.ZarrV3ArrayMetadataField, V3_ARRAY_DOC), (zmp.ZarrV2ArrayMetadataField, V2_ARRAY_DOC), ): doc = json.loads(json.dumps(source)) doc["shape"] = [-1] _assert_runtime_and_schema_reject(field_type, doc) def test_v2_array_schema_allows_empty_filters() -> None: """The v2 schema, like the runtime, accepts an empty list, because the spec says "a list ... or null" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79).""" doc = json.loads(json.dumps(V2_ARRAY_DOC)) doc["filters"] = [] adapter = TypeAdapter(zmp.ZarrV2ArrayMetadataField) assert adapter.validate_python(doc).filters == () assert Draft202012Validator(adapter.json_schema()).is_valid(doc) @pytest.mark.parametrize( "field", ["data_type", "chunk_grid", "chunk_key_encoding", "codecs", "storage_transformers"] ) def test_v3_array_schema_rejects_false_at_every_extension_point(field: str) -> None: """No extension point of an array document can set `must_understand` to false, in the schema or at runtime.""" doc = json.loads(json.dumps(V3_ARRAY_DOC)) entry = {"name": "example", "must_understand": False} doc[field] = [entry] if field in ("codecs", "storage_transformers") else entry _assert_runtime_and_schema_reject(zmp.ZarrV3ArrayMetadataField, doc) def test_metadata_field_schema_rejects_unknown_members() -> None: """Named-configuration envelopes are closed in both runtime and schema validation.""" doc = json.loads(json.dumps(V3_ARRAY_DOC)) doc["codecs"] = [{"name": "bytes", "unexpected": 1}] _assert_runtime_and_schema_reject(zmp.ZarrV3ArrayMetadataField, doc) @pytest.mark.parametrize( ("field_type", "source"), [ (zmp.ZarrV2GroupMetadataField, V2_GROUP_DOC), (zmp.ZarrV2ConsolidatedMetadataField, V2_CONSOLIDATED_DOC), ], ) def test_v2_schema_rejects_unknown_document_members( field_type: object, source: dict[str, object] ) -> None: """Closed v2 merged documents expose their runtime boundary in JSON Schema.""" doc = json.loads(json.dumps(source)) doc["unexpected"] = 1 _assert_runtime_and_schema_reject(field_type, doc) def test_v2_array_schema_allows_unknown_document_members() -> None: """The v2 array document is open ("SHOULD be ignored", https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92) in both runtime and schema validation. An unknown member is kept as written.""" doc = json.loads(json.dumps(V2_ARRAY_DOC)) doc["unexpected"] = 1 adapter = TypeAdapter(zmp.ZarrV2ArrayMetadataField) assert adapter.validate_python(doc).to_json()["unexpected"] == 1 assert Draft202012Validator(adapter.json_schema()).is_valid(doc) def test_v3_array_schema_allows_unknown_extension_fields() -> None: """Schema constraints do not close the v3 top-level extension namespace.""" doc = json.loads(json.dumps(V3_ARRAY_DOC)) doc["vendor_extension"] = {"anything": [1, 2]} adapter = TypeAdapter(zmp.ZarrV3ArrayMetadataField) held = adapter.validate_python(doc).extra_fields["vendor_extension"] assert held.to_json() == {"vendor_extension": {"anything": (1, 2)}} assert Draft202012Validator(adapter.json_schema()).is_valid(doc) def test_json_roundtrip() -> None: """model_dump_json output re-validates to an equal pydantic model.""" class Manifest(BaseModel): metadata: zmp.ZarrV3ArrayMetadataField manifest = Manifest.model_validate({"metadata": V3_ARRAY_DOC}) assert Manifest.model_validate_json(manifest.model_dump_json()) == manifest def test_core_package_does_not_import_pydantic() -> None: """Importing zarr_metadata (in a fresh interpreter) must not import pydantic: the integration is opt-in via zarr_metadata.pydantic.""" import subprocess import sys code = "import sys, zarr_metadata; assert 'pydantic' not in sys.modules, 'leaked'" subprocess.run([sys.executable, "-c", code], check=True) def test_a_non_finite_attribute_is_kept_by_constants_serialization() -> None: """Pydantic writes a non-finite number as `null` unless the enclosing model writes it the way the store does. The module docstring explains how.""" class Manifest(BaseModel): model_config = ConfigDict(ser_json_inf_nan="constants") metadata: zmp.ZarrV3GroupMetadataField document = b'{"metadata": {"zarr_format": 3, "node_type": "group", "attributes": {"x": NaN}}}' written = Manifest.model_validate_json(document).model_dump_json() held = Manifest.model_validate_json(written).metadata.attributes["x"] assert isinstance(held, float) assert math.isnan(held) # --- the scope a v3 field type reads in -------------------------------------- _WITH_ZSTD = ZarrV3ArrayMetadata.create_default( shape=(4,), codecs=({"name": "bytes"}, {"name": "zstd", "configuration": {"level": 3, "checksum": False}}), ).to_json() @pytest.mark.parametrize( ("context", "read"), [ (None, True), (CORE, False), ({zmp.CONTEXT_KEY: CORE}, False), ({"another validator's": 1}, True), ], ids=["none", "a-scope", "a-mapping-holding-one", "a-mapping-holding-none"], ) def test_a_v3_field_type_reads_in_the_scope_the_validation_context_holds( context: object, read: bool ) -> None: """The field type reads in the scope from the validation context, which pydantic passes to every validator. `zstd` is an extension, so `CORE` leaves it unclaimed.""" model = TypeAdapter(zmp.ZarrV3ArrayMetadataField).validate_python(_WITH_ZSTD, context=context) assert isinstance(model.codecs[1], AcceptedField) is read class _Bound(BaseModel): """A model whose field is read in `CORE`, whatever the validation context holds.""" metadata: Annotated[ZarrV3ArrayMetadata, zmp.InScope(CORE)] def test_a_field_bound_to_a_scope_reads_in_that_scope() -> None: """`Annotated[ZarrV3ArrayMetadata, InScope(CORE)]` reads a document in `CORE`, whatever the validation context holds, and reads an instance again in `CORE`: an instance read in `CORE` with `zstd` unclaimed stays as it is. The field writes the document as written, and a type checker sees the model class.""" for context in (None, CORE_AND_EXTENSIONS, {zmp.CONTEXT_KEY: CORE_AND_EXTENSIONS}): read = _Bound.model_validate({"metadata": _WITH_ZSTD}, context=context).metadata assert read.context == CORE assert isinstance(read.codecs[1], UnclaimedField) held = ZarrV3ArrayMetadata(_WITH_ZSTD, context=CORE) assert _Bound(metadata=held).metadata == held assert _Bound(metadata=held).model_dump()["metadata"] == held.to_json() @pytest.mark.parametrize("mode", ["validation", "serialization"]) @pytest.mark.parametrize( ("bound", "field_type"), [ (Annotated[ZarrV3ArrayMetadata, zmp.InScope(CORE)], zmp.ZarrV3ArrayMetadataField), (Annotated[ZarrV3GroupMetadata, zmp.InScope(CORE)], zmp.ZarrV3GroupMetadataField), ( Annotated[ZarrV3ConsolidatedMetadata, zmp.InScope(CORE)], zmp.ZarrV3ConsolidatedMetadataField, ), (Annotated[ZarrV2ArrayMetadata, zmp.InScope(CORE_V2)], zmp.ZarrV2ArrayMetadataField), (Annotated[ZarrV2GroupMetadata, zmp.InScope(CORE_V2)], zmp.ZarrV2GroupMetadataField), ( Annotated[ZarrV2ConsolidatedMetadata, zmp.InScope(CORE_V2)], zmp.ZarrV2ConsolidatedMetadataField, ), ], ids=[ "array-v3", "group-v3", "consolidated-v3", "array-v2", "group-v2", "consolidated-v2", ], ) def test_a_field_bound_to_a_scope_has_the_schema_of_its_field_type( bound: object, field_type: object, mode: JsonSchemaMode, ) -> None: """A field bound to a scope reads and writes the same documents as the field type of its model class, so its JSON Schema is the field type's, for validation and for serialization.""" assert TypeAdapter(bound).json_schema(mode=mode) == TypeAdapter(field_type).json_schema( mode=mode ) def test_error_a_field_bound_to_a_scope_refuses_an_instance_it_reads_differently() -> None: """An instance read where `zstd` is claimed would lose that meaning in `CORE`, so a field bound to `CORE` refuses it with a `ValidationError`, rather than passing it through in its own scope.""" with pytest.raises(ValidationError): _Bound(metadata=ZarrV3ArrayMetadata(_WITH_ZSTD, context=CORE_AND_EXTENSIONS)) def test_error_a_field_bound_to_a_scope_of_the_other_format() -> None: """A field of a v3 model bound to a v2 scope is refused when pydantic builds its schema, as every v3 reader refuses a v2 scope.""" with pytest.raises(TypeError, match="v3 document is read in a scope of that format"): TypeAdapter(Annotated[ZarrV3ArrayMetadata, zmp.InScope(CORE_V2)]) def test_error_a_scope_given_to_validation_reads_an_instance_in_it() -> None: """A scope given in the validation context reads an instance again in that scope, as a bound field does, so an instance that would lose a meaning there is refused. With no scope given, an instance passes through unchanged.""" adapter = TypeAdapter(zmp.ZarrV3ArrayMetadataField) instance = ZarrV3ArrayMetadata(_WITH_ZSTD, context=CORE_AND_EXTENSIONS) assert adapter.validate_python(instance) is instance with pytest.raises(ValidationError): adapter.validate_python(instance, context=CORE) _READ_IN_SCOPE = [ pytest.param( lambda value, context: TypeAdapter(zmp.ZarrV3ArrayMetadataField).validate_python( value, context=context ), id="validation-context", ), pytest.param( lambda value, context: TypeAdapter( Annotated[ZarrV3ArrayMetadata, zmp.InScope(context)] ).validate_python(value), id="in-scope", ), ] @pytest.mark.parametrize("validate", _READ_IN_SCOPE) def test_error_an_instance_a_scope_finds_problems_in_reports_each_one(validate: Any) -> None: # noqa: ANN401 - a validating function """An instance read again in a scope that claims more can show problems its own scope did not check. pydantic gets one line error per problem, with the problem's kind as its type, at its location, and the value found there as its input, as for a document.""" document: dict[str, Any] = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) document["codecs"] = ( *document["codecs"], {"name": "gzip", "configuration": {"level": 99}}, {"name": "zstd", "configuration": {"level": "x", "checksum": False}}, ) instance = ZarrV3ArrayMetadata(document, context=ZarrV3Context()) with pytest.raises(ValidationError) as raised: validate(instance, CORE_AND_EXTENSIONS) assert [(line["type"], line["loc"], line["input"]) for line in raised.value.errors()] == [ ("invalid_value", ("codecs", 1, "configuration", "level"), 99), ("invalid_type", ("codecs", 2, "configuration", "level"), "x"), ] @pytest.mark.parametrize("validate", _READ_IN_SCOPE) def test_error_an_instance_a_scope_reads_differently_reports_each_conflict(validate: Any) -> None: # noqa: ANN401 - a validating function """An instance read where `zstd` is claimed would lose that meaning in `CORE`. pydantic gets one `scope_conflict` line error per conflict, at the field it is in, with that field as written as its input, and a message that names the codec.""" instance = ZarrV3ArrayMetadata(_WITH_ZSTD, context=CORE_AND_EXTENSIONS) with pytest.raises(ValidationError) as raised: validate(instance, CORE) (line,) = raised.value.errors() assert (line["type"], line["loc"]) == ("scope_conflict", ("codecs", 1)) assert line["input"] == instance.to_json()["codecs"][1] assert "zstd" in line["msg"] _ZLIB_V2 = {"id": "zlib", "level": 1} _V3_LISTING_WITH_ZSTD = ZarrV3ConsolidatedMetadata( {**V3_CONSOLIDATED_DOC, "metadata": {"a": _WITH_ZSTD}}, context=CORE_AND_EXTENSIONS ) _V2_LISTING_WITH_ZLIB = ZarrV2ConsolidatedMetadata( { "zarr_consolidated_format": 1, "metadata": { ".zgroup": {"zarr_format": 2}, "a/.zarray": {**V2_ARRAY_DOC, "compressor": _ZLIB_V2}, }, }, context=CORE_V2, ) _V2_WITHOUT_CODECS = ZarrV2Context(data_types=CORE_V2.data_types.values()) @pytest.mark.parametrize( ("validate", "instance", "loc", "found", "named"), [ pytest.param( lambda value: TypeAdapter(zmp.ZarrV3ConsolidatedMetadataField).validate_python( value, context=CORE ), _V3_LISTING_WITH_ZSTD, ("metadata", "a", "codecs", 1), _WITH_ZSTD["codecs"][1], "zstd", id="v3-validation-context", ), pytest.param( lambda value: TypeAdapter( Annotated[ZarrV3ConsolidatedMetadata, zmp.InScope(CORE)] ).validate_python(value), _V3_LISTING_WITH_ZSTD, ("metadata", "a", "codecs", 1), _WITH_ZSTD["codecs"][1], "zstd", id="v3-in-scope", ), pytest.param( lambda value: TypeAdapter(zmp.ZarrV2ConsolidatedMetadataField).validate_python( value, context={zmp.CONTEXT_KEY_V2: _V2_WITHOUT_CODECS} ), _V2_LISTING_WITH_ZLIB, ("metadata", "a/.zarray", "compressor"), _ZLIB_V2, "zlib", id="v2-validation-context", ), pytest.param( lambda value: TypeAdapter( Annotated[ZarrV2ConsolidatedMetadata, zmp.InScope(_V2_WITHOUT_CODECS)] ).validate_python(value), _V2_LISTING_WITH_ZLIB, ("metadata", "a/.zarray", "compressor"), _ZLIB_V2, "zlib", id="v2-in-scope", ), ], ) def test_error_a_consolidated_instance_a_scope_reads_differently_reports_each_conflict( validate: Callable[[object], object], instance: object, loc: tuple[str | int, ...], found: object, named: str, ) -> None: """A consolidated instance read again in a scope that claims less would lose the meaning of a listed node's field, through a validation context or a field bound with `InScope`. pydantic gets one `scope_conflict` line error, located at the field inside the listed node, a node's path for v3 and its file for v2, with that field as written as its input, and a message that names the codec.""" with pytest.raises(ValidationError) as raised: validate(instance) (line,) = raised.value.errors() assert (line["type"], line["loc"]) == ("scope_conflict", loc) assert line["input"] == found assert named in line["msg"] @pytest.mark.parametrize( "annotation", [ Annotated[ZarrV3ArrayMetadata | None, zmp.InScope(CORE)], Annotated[dict[str, Any], zmp.InScope(CORE)], ], ids=["an-optional-model", "a-dict"], ) def test_error_in_scope_annotates_a_model_class(annotation: object) -> None: """`InScope` reads a document as one model class of this package, so annotating anything else, such as an optional model or a `dict`, is a `TypeError` when pydantic builds the schema.""" with pytest.raises(TypeError, match="InScope annotates a model class of zarr_metadata.model"): TypeAdapter(annotation) def test_error_in_scope_takes_a_scope() -> None: """`InScope` fixes a scope, so `None`, which elsewhere means the default scope, is a `TypeError` when it is built.""" with pytest.raises(TypeError, match="InScope takes a ZarrV3Context or a ZarrV2Context"): zmp.InScope(None) # pyright: ignore[reportArgumentType] def test_error_a_validation_context_names_a_key_of_this_package_it_does_not_have() -> None: """A mapping validation context with a key that starts like this package's keys, but is not one of them, is a `TypeError`, so a misspelled key is not read as no scope at all.""" with pytest.raises(TypeError, match="zarr_metadata_contxt"): TypeAdapter(zmp.ZarrV3ArrayMetadataField).validate_python( _WITH_ZSTD, context={"zarr_metadata_contxt": CORE} ) def test_error_a_validation_context_holds_a_scope_that_is_not_one() -> None: with pytest.raises(TypeError, match=zmp.CONTEXT_KEY): TypeAdapter(zmp.ZarrV3ArrayMetadataField).validate_python( _WITH_ZSTD, context={zmp.CONTEXT_KEY: "CORE"} ) def test_each_problem_is_a_line_error_of_pydantic_s() -> None: # Problems are reported the way pydantic reports its own errors: one line # error per problem, with the problem's kind as its type, at the problem's # loc under the field's loc, with the input found there and the expected # values. A message that contains JSON is kept unchanged. class Manifest(BaseModel): metadata: zmp.ZarrV3ArrayMetadataField doc = { **V3_ARRAY_DOC, "fill_value": {"a": 1}, "codecs": [ {"name": "bytes", "configuration": {"endian": "little"}}, {"name": "gzip", "configuration": {"level": 12}}, ], } with pytest.raises(ValidationError) as raised: Manifest.model_validate({"metadata": doc}) errors = raised.value.errors() assert [(e["type"], e["loc"], e["input"]) for e in errors] == [ ("invalid_type", ("metadata", "fill_value"), {"a": 1}), ("invalid_value", ("metadata", "codecs", 1, "configuration", "level"), 12), ] assert errors[0]["msg"] == 'expected an integer, got {"a": 1}' assert errors[1].get("ctx") == {"ge": 0, "le": 9} def test_a_message_holding_a_ctx_placeholder_is_reported_as_it_is() -> None: # pydantic renders a message as a template over its ctx, with no way to # escape a placeholder. A document value written as `{expected}` would be # shown as the ctx's `expected`. So such a message is passed whole in the # ctx instead. doc = {**V3_ARRAY_DOC, "zarr_format": "{expected}"} with pytest.raises(ValidationError) as raised: TypeAdapter(zmp.ZarrV3ArrayMetadataField).validate_python(doc) (error,) = raised.value.errors() assert (error["type"], error["loc"], error["msg"]) == ( "invalid_type", ("zarr_format",), 'expected 3, got "{expected}"', ) assert error.get("ctx") == {"expected": (3,), "message": 'expected 3, got "{expected}"'} def test_a_ctx_member_named_message_yields_to_a_message_holding_a_placeholder() -> None: # The `message` key that carries the message goes last in the ctx, so no # ctx key after it is substituted into the carried text. When no message # needs to be carried, a ctx member named `message` is kept. ctx = {"message": "theirs", "expected": 3} problem = ValidationProblem(("a",), 'got "{expected}"', "invalid_value", ctx=ctx) error = ValidationError.from_exception_data("T", [zmp._line_error(problem, {"a": 1})]).errors()[ 0 ] assert error["msg"] == 'got "{expected}"' assert error.get("ctx") == {"expected": 3, "message": 'got "{expected}"'} assert list(error.get("ctx", {})) == ["expected", "message"] plain = ValidationProblem(("a",), "got 1", "invalid_value", ctx=ctx) error = ValidationError.from_exception_data("T", [zmp._line_error(plain, {"a": 1})]).errors()[0] assert (error["msg"], error.get("ctx")) == ("got 1", ctx) def test_a_line_error_for_what_a_problem_could_not_hold_reports_what_sits_there() -> None: # A problem holds no input when the value is not JSON that a reader walks # (here, an `AcceptedField` built by hand and placed among the codecs). The # line error then reports that object. For a missing key, the line error # reports the object that lacks the key. smuggled = AcceptedField( json="gzip", name="gzip", definition=GZIP_CODEC, configuration={"level": 1} ) doc = { **V3_ARRAY_DOC, "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}, smuggled], } with pytest.raises(ValidationError) as raised: TypeAdapter(zmp.ZarrV3ArrayMetadataField).validate_python(doc) (error,) = raised.value.errors() assert (error["type"], error["loc"]) == ("invalid_type", ("codecs", 1)) assert error["input"] is smuggled def test_the_pydantic_schema_refuses_a_null_consolidated_metadata_as_the_reader_does() -> None: # The schema and the reader give the same result for `null` here: both refuse it. schema = TypeAdapter(zmp.ZarrV3GroupMetadataField).json_schema() document = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": None} assert not Draft202012Validator(schema).is_valid(document) with pytest.raises(ValidationError): TypeAdapter(zmp.ZarrV3GroupMetadataField).validate_python(document) def test_a_line_error_for_a_document_past_the_cap_renders() -> None: # The depth problem holds no input. The line error reports the subtree # at its loc. pydantic renders it like any other input: truncated in # `str` and whole in `json`. deep: dict[str, object] = {} for _ in range(2_000): deep = {"a": deep} with pytest.raises(ValidationError) as raised: TypeAdapter(zmp.ZarrV3ArrayMetadataField).validate_python( {**V3_ARRAY_DOC, "attributes": deep} ) (error,) = raised.value.errors() assert (error["type"], len(error["loc"])) == ("invalid_value", 256) assert "nested deeper" in str(raised.value) assert json.loads(raised.value.json())[0]["type"] == "invalid_value" @pytest.mark.parametrize("field", ["Foo/bar", "", "r*", {"name": "Int8"}]) def test_the_pydantic_schema_names_an_extension_as_the_reader_does(field: object) -> None: # The schema that pydantic generates checks a name the same way the reader does. schema = TypeAdapter(zmp.ZarrV3ArrayMetadataField).json_schema() # Use arrays as lists, as JSON parses them, because `jsonschema` does not accept a tuple. document: dict[str, Any] = json.loads(json.dumps(V3_ARRAY_DOC)) assert Draft202012Validator(schema).is_valid(document) named: dict[str, Any] = {**document, "data_type": field} assert not Draft202012Validator(schema).is_valid(named) with pytest.raises(ValidationError): TypeAdapter(zmp.ZarrV3ArrayMetadataField).validate_python(named) def test_v2_field_types_read_in_the_validation_contexts_scope() -> None: """The v2 field types read a document in the scope that the validation context holds. The context is either a `Context` itself or holds one in its `zarr_metadata_context_v2` item. When it holds none, they read in `CORE_V2`. The v3 field types work the same way.""" from zarr_metadata.definition import UnclaimedField from zarr_metadata.v2.data_type.scalar import UINT_DATA_TYPE_V2 from zarr_metadata.v2.definition import CORE_V2, ZarrV2Context adapter = TypeAdapter(zmp.ZarrV2ArrayMetadataField) doc = json.loads(json.dumps(V2_ARRAY_DOC)) assert adapter.validate_python(doc).context is CORE_V2 small = ZarrV2Context(data_types=[UINT_DATA_TYPE_V2]) read = adapter.validate_python({**doc, "compressor": {"id": "zlib"}}, context=small) assert read.context is small assert isinstance(read.compressor, UnclaimedField) held = adapter.validate_python(doc, context={"zarr_metadata_context_v2": small}) assert held.context is small group = TypeAdapter(zmp.ZarrV2GroupMetadataField).validate_python(V2_GROUP_DOC, context=small) assert group.context is small def test_each_format_reads_in_its_own_context_key() -> None: """A mapping context gives each format's scope under its own key: `zarr_metadata_context` for v3 and `zarr_metadata_context_v2` for v2. So a v3 scope given for the v3 fields leaves the v2 fields in `CORE_V2`. A bare `Context` is the scope for every field type.""" from zarr_metadata.definition import UnclaimedField from zarr_metadata.v2.data_type.scalar import UINT_DATA_TYPE_V2 from zarr_metadata.v2.definition import CORE_V2, ZarrV2Context from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS adapter = TypeAdapter(zmp.ZarrV2ArrayMetadataField) doc = json.loads(json.dumps(V2_ARRAY_DOC)) small = ZarrV2Context(data_types=[UINT_DATA_TYPE_V2]) v3_only = adapter.validate_python(doc, context={"zarr_metadata_context": CORE_AND_EXTENSIONS}) assert v3_only.context is CORE_V2 with pytest.raises(ValidationError): adapter.validate_python( {**doc, "fill_value": "garbage"}, context={"zarr_metadata_context": CORE_AND_EXTENSIONS} ) own = adapter.validate_python(doc, context={"zarr_metadata_context_v2": small}) assert own.context is small bare = adapter.validate_python({**doc, "compressor": {"id": "zlib"}}, context=small) assert bare.context is small assert isinstance(bare.compressor, UnclaimedField) assert zmp.CONTEXT_KEY_V2 == "zarr_metadata_context_v2" _TESTS = Path(__file__).parent.parent _NODE_SCHEMA_V3 = node_metadata_json_schema_v3(context=ZarrV3Context()) _SCHEMA_PAIRS: dict[str, tuple[object, Mapping[str, Any], str]] = { "v3-array": (zmp.ZarrV3ArrayMetadataField, _NODE_SCHEMA_V3, "v3/array"), "v3-group": (zmp.ZarrV3GroupMetadataField, _NODE_SCHEMA_V3, "v3/group"), "v2-array": (zmp.ZarrV2ArrayMetadataField, json_schema(ZarrV2ArrayMetadataJSON), "v2/array"), "v2-group": (zmp.ZarrV2GroupMetadataField, json_schema(ZarrV2GroupMetadataJSON), "v2/group"), } def _variants(document: dict[str, Any]) -> Iterator[dict[str, Any]]: """The document, and copies of it that break one rule each.""" yield document for key in ("data_type", "chunk_grid", "chunk_key_encoding"): if isinstance(document.get(key), dict): yield {**document, key: {**document[key], "name": "Not A Name"}} yield {**document, key: {**document[key], "must_understand": False}} if "codecs" in document: yield {**document, "codecs": []} for key in ("shape", "chunks"): if key in document: yield {**document, key: [-1] * len(document[key])} yield {**document, "zarr_format": 7} @pytest.mark.parametrize( ("field_type", "schema", "corpus"), list(_SCHEMA_PAIRS.values()), ids=list(_SCHEMA_PAIRS) ) def test_the_pydantic_schema_agrees_with_the_package_schema( field_type: object, schema: Mapping[str, Any], corpus: str ) -> None: """Pydantic needs its own copies of the document types to write a JSON Schema, because it cannot embed a schema with its own `$defs`. Each fixture document, and each copy of it that breaks one rule, is accepted by the pydantic field type's schema exactly when `zarr_metadata.json_schema` accepts it, so the copies cannot drift.""" ours = Draft202012Validator(schema) theirs = Draft202012Validator(TypeAdapter(field_type).json_schema()) documents = sorted((_TESTS / corpus).glob("*.json")) assert documents != [] for path in documents: for document in _variants(json.loads(path.read_text())): assert theirs.is_valid(document) == ours.is_valid(document), (path.name, document) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_read_array_metadata.py000066400000000000000000000344031527014137600336270ustar00rootroot00000000000000"""Tests for reading a v3 array document: each field as a scope read it, and its codecs as a pipeline. `read_array_metadata_v3` returns everything that one read of a document finds: each field with its location, the kind it was read as, and the scope's result for it; the chunk the codecs receive; each codec with the chunk it receives; every problem; and the document's model when there are no problems. """ from __future__ import annotations from typing import TYPE_CHECKING, Any, cast import pytest from zarr_metadata._json import arrays_to_tuples from zarr_metadata.definition import ( AcceptedField, Definition, RefusedField, UnclaimedField, resolve, with_problems, ) from zarr_metadata.definition._definition import fields_of from zarr_metadata.model import ( UNSET, ZarrV3ArrayMetadata, ZarrV3ArrayMetadataReading, read_array_metadata_v3, ) from zarr_metadata.problems import ValidationProblem from zarr_metadata.v3.definition import ( CORE, CORE_AND_EXTENSIONS, ChunkGridDefinition, ChunkKeyEncodingDefinition, CodecDefinition, DataTypeDefinition, StorageTransformerDefinition, ZarrV3Context, ) if TYPE_CHECKING: from collections.abc import Callable, Iterator from zarr_metadata.definition import ResolvedField from zarr_metadata.typed_json import Loc from zarr_metadata.v3.definition import Lengths LITTLE = {"name": "bytes", "configuration": {"endian": "little"}} ZSTD = {"name": "zstd", "configuration": {"level": 1}} POINTS: list[tuple[Loc, type[Definition[Any]], type]] = [ (("data_type",), DataTypeDefinition, AcceptedField), (("chunk_grid",), ChunkGridDefinition, AcceptedField), (("chunk_key_encoding",), ChunkKeyEncodingDefinition, AcceptedField), ] """The single-valued extension points of a default document, each read at its location.""" def _document(shape: tuple[int, ...] = (4,), **fields: object) -> dict[str, Any]: """Return a default v3 array document of `shape` with `fields` added, with arrays as tuples as a reader holds them.""" document = {**ZarrV3ArrayMetadata.create_default(shape=shape).to_json(), **fields} return cast("dict[str, Any]", arrays_to_tuples(document)) def _codec(loc: Loc, variant: type = AcceptedField) -> tuple[Loc, type[Definition[Any]], type]: return (loc, CodecDefinition, variant) def _shard(chunk_shape: list[int], codecs: list[object] | None = None, **more: object) -> object: configuration = { "chunk_shape": chunk_shape, "codecs": [LITTLE] if codecs is None else codecs, "index_codecs": [LITTLE], **more, } return {"name": "sharding_indexed", "configuration": configuration} @pytest.mark.parametrize( ("document", "context", "fields", "handed"), [ (_document(), CORE_AND_EXTENSIONS, [*POINTS, _codec(("codecs", 0))], [(frozenset({4}),)]), # Each codec receives the chunk that the previous codec outputs. ( _document( (4, 6), codecs=[{"name": "transpose", "configuration": {"order": [1, 0]}}, "bytes"] ), CORE_AND_EXTENSIONS, [*POINTS, _codec(("codecs", 0)), _codec(("codecs", 1))], [(frozenset({4}), frozenset({6})), (frozenset({6}), frozenset({4}))], ), # The fields a shard holds come after it, at their locations in its # configuration. In the core spec's scope, an extension inside the # shard is unclaimed at its location. ( _document( (16, 16), chunk_grid={"name": "regular", "configuration": {"chunk_shape": [8, 8]}}, codecs=[_shard([4, 4], codecs=[LITTLE, ZSTD], index_codecs=[LITTLE, "crc32c"])], ), CORE, [ *POINTS, _codec(("codecs", 0)), _codec(("codecs", 0, "configuration", "codecs", 0)), _codec(("codecs", 0, "configuration", "codecs", 1), UnclaimedField), _codec(("codecs", 0, "configuration", "index_codecs", 0)), _codec(("codecs", 0, "configuration", "index_codecs", 1)), ], [(frozenset({8}), frozenset({8}))], ), # A shard within a shard: each field comes first, then the fields it holds. ( _document( (16, 16), chunk_grid={"name": "regular", "configuration": {"chunk_shape": [8, 8]}}, codecs=[_shard([4, 4], codecs=[_shard([2, 2])])], ), CORE_AND_EXTENSIONS, [ *POINTS, _codec(("codecs", 0)), _codec(("codecs", 0, "configuration", "codecs", 0)), _codec(("codecs", 0, "configuration", "codecs", 0, "configuration", "codecs", 0)), _codec( ("codecs", 0, "configuration", "codecs", 0, "configuration", "index_codecs", 0) ), _codec(("codecs", 0, "configuration", "index_codecs", 0)), ], [(frozenset({8}), frozenset({8}))], ), # A shard that its check refuses still keeps the fields it read inside it. ( _document( (16, 16), chunk_grid={"name": "regular", "configuration": {"chunk_shape": [8, 8]}}, codecs=[_shard([4, 4], index_location="middle", codecs=[LITTLE, ZSTD])], ), CORE, [ *POINTS, _codec(("codecs", 0), RefusedField), _codec(("codecs", 0, "configuration", "codecs", 0)), _codec(("codecs", 0, "configuration", "codecs", 1), UnclaimedField), _codec(("codecs", 0, "configuration", "index_codecs", 0)), ], [(frozenset({8}), frozenset({8}))], ), # A struct's field types are read as data types, whether or not a # definition in the scope claims them. ( _document( data_type={ "name": "struct", "configuration": { "fields": [ {"name": "a", "data_type": "int8"}, {"name": "b", "data_type": "acme.t"}, ] }, }, fill_value={"a": 0, "b": "anything"}, codecs=[LITTLE], ), CORE_AND_EXTENSIONS, [ POINTS[0], ( ("data_type", "configuration", "fields", 0, "data_type"), DataTypeDefinition, AcceptedField, ), ( ("data_type", "configuration", "fields", 1, "data_type"), DataTypeDefinition, UnclaimedField, ), *POINTS[1:], _codec(("codecs", 0)), ], [(frozenset({4}),)], ), # A field that its definition refuses, or that nothing claims, is # read as the kind of its position. An unclaimed chunk grid gives no # lengths, and a codec after the array -> bytes codec receives bytes. ( _document( chunk_grid={"name": "acme.grid", "configuration": {}}, codecs=["bytes", {"name": "gzip", "configuration": {"level": 99}}], storage_transformers=[{"name": "acme.transformer"}], ), CORE_AND_EXTENSIONS, [ POINTS[0], (("chunk_grid",), ChunkGridDefinition, UnclaimedField), POINTS[2], _codec(("codecs", 0)), _codec(("codecs", 1), RefusedField), (("storage_transformers", 0), StorageTransformerDefinition, UnclaimedField), ], [(None,), None], ), # A data type that is not JSON is refused. ( _document(data_type=float("nan")), CORE_AND_EXTENSIONS, [ (("data_type",), DataTypeDefinition, RefusedField), *POINTS[1:], _codec(("codecs", 0)), ], [(frozenset({4}),)], ), ], ids=[ "default", "transposed", "a-shard-holding-an-extension-in-the-core-scope", "a-shard-within-a-shard", "a-shard-its-check-refuses", "a-struct-s-field-types", "fields-read-as-nothing", "a-data-type-that-is-not-json", ], ) def test_a_document_reads_as_each_field_where_it_sits_and_its_codecs_as_a_pipeline( document: dict[str, Any], context: ZarrV3Context, fields: list[tuple[Loc, type[Definition[Any]], type]], handed: list[Lengths | None], ) -> None: reading = read_array_metadata_v3(document, context=context) # A model exists only for a document with no problem. assert (reading.metadata is None) is (reading.problems != ()) assert [(loc, field.read_as, type(field)) for loc, field in reading.fields()] == fields # The chunk uses None for a property that nothing declares. The reading # uses `UNSET` for a key that the document lacks. if reading.data_type is UNSET: assert reading.chunk.data_type is None else: assert reading.chunk.data_type is reading.data_type assert reading.pipeline[0].incoming == reading.chunk assert [None if s.incoming is None else s.incoming.lengths for s in reading.pipeline] == handed _INNER_GZIP = {"name": "gzip", "configuration": {"level": 12}} _WITH_PROBLEMS = _document( (4, 4), chunk_grid={"name": "regular", "configuration": {"chunk_shape": [4]}}, fill_value="high", codecs=[ _shard([2, 2], [LITTLE, _INNER_GZIP]), ZSTD, {"name": "transpose", "configuration": {"order": [1, 0]}}, ], ) """An array document with a problem in each place where a field can have one, plus one problem outside any field.""" def _array_fields( document: object, ) -> Iterator[tuple[Loc, ResolvedField[Any], tuple[ValidationProblem, ...]]]: reading = read_array_metadata_v3(document) return with_problems(reading.fields(), reading.problems) def _field_fields( document: object, ) -> Iterator[tuple[Loc, ResolvedField[Any], tuple[ValidationProblem, ...]]]: resolved, problems = resolve( cast("dict[str, Any]", document)["codecs"][0], CodecDefinition, CORE_AND_EXTENSIONS, ("codecs", 0), ) return with_problems(fields_of(resolved, ("codecs", 0)), problems) _INNER_LEVEL = ("codecs", 0, "configuration", "codecs", 1, "configuration", "level") _EACH_FIELD_S = { ("data_type",): [], ("chunk_grid",): [("chunk_grid", "configuration", "chunk_shape")], ("chunk_key_encoding",): [], ("codecs", 0): [_INNER_LEVEL], ("codecs", 0, "configuration", "codecs", 0): [], ("codecs", 0, "configuration", "codecs", 1): [_INNER_LEVEL], ("codecs", 0, "configuration", "index_codecs", 0): [], ("codecs", 1): [], ("codecs", 2): [("codecs", 2)], } """Each field of `_WITH_PROBLEMS`, with the location of each of its problems.""" @pytest.mark.parametrize( ("fields", "expected"), [ (_array_fields, _EACH_FIELD_S), ( _field_fields, {loc: problems for loc, problems in _EACH_FIELD_S.items() if loc[:2] == ("codecs", 0)}, ), ], ids=["an-array", "one-field"], ) def test_each_field_comes_with_the_problems_located_in_it( fields: Callable[ [object], Iterator[tuple[Loc, ResolvedField[Any], tuple[ValidationProblem, ...]]] ], expected: dict[Loc, list[Loc]], ) -> None: # Each field gets the problems found while reading it, plus the problems # the document check found at its location: a transpose out of the # pipeline's order, at its own position, and a chunk grid that does not # match the shape. A field also gets the problems of the fields it # holds, so a shard with a bad inner codec has that problem too. A # problem outside any field (the fill value) belongs to no field. assert { loc: [p.loc for p in problems] for loc, _, problems in fields(_WITH_PROBLEMS) } == expected def test_a_document_with_no_problem_reads_as_its_model_holding_the_fields_read() -> None: # The model holds the fields the read produced, not a second reading of them. document = _document(codecs=[{"name": "transpose", "configuration": {"order": [0]}}, LITTLE]) reading = read_array_metadata_v3(document) model = reading.metadata assert reading.problems == () assert model is not None assert model == ZarrV3ArrayMetadata.from_json(document) assert model.data_type is reading.data_type assert model.chunk_grid is reading.chunk_grid assert model.chunk_key_encoding is reading.chunk_key_encoding assert all( codec is stage.codec for codec, stage in zip(model.codecs, reading.pipeline, strict=True) ) def test_error_a_value_that_is_not_a_mapping_reads_as_nothing() -> None: reading = read_array_metadata_v3(["not", "a", "document"]) not_an_object = ValidationProblem((), "expected an object", "invalid_type") assert reading == ZarrV3ArrayMetadataReading(problems=(not_an_object,)) assert list(reading.fields()) == [] @pytest.mark.parametrize( ("member", "attribute"), [("codecs", "pipeline"), ("storage_transformers", "storage_transformers")], ) def test_error_a_list_of_fields_that_is_not_a_list_reads_as_empty( member: str, attribute: str ) -> None: document = _document() document[member] = "bytes" reading = read_array_metadata_v3(document) assert getattr(reading, attribute) == () assert [(p.loc, p.kind) for p in reading.problems] == [((member,), "invalid_type")] @pytest.mark.parametrize("member", ["data_type", "chunk_grid", "chunk_key_encoding"]) def test_error_a_document_without_a_field_reads_it_as_unset(member: str) -> None: document = _document() del document[member] reading = read_array_metadata_v3(document) assert getattr(reading, member) is UNSET assert (member,) not in [loc for loc, _ in reading.fields()] assert ValidationProblem((member,), "missing required key", "missing_key") in reading.problems def test_error_a_document_without_a_shape_hands_its_codecs_chunks_of_no_known_rank() -> None: document = _document() del document["shape"] reading = read_array_metadata_v3(document) assert reading.chunk.lengths is None test_read_array_metadata_v2.py000066400000000000000000000146601527014137600341620ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model"""Tests for reading a v2 array document once. A reading holds each field as the scope read it, every problem, and the model.""" from __future__ import annotations from typing import Any import pytest from zarr_metadata.definition import ( AcceptedField, EmptyConfiguration, RefusedField, UnclaimedField, ) from zarr_metadata.model import ( UNSET, ZarrV2ArrayMetadata, read_array_metadata_v2, ) from zarr_metadata.problems import MetadataValidationError from zarr_metadata.v2.codec.compression import ZLIB_CODEC_V2 from zarr_metadata.v2.data_type.scalar import FLOAT_DATA_TYPE_V2 from zarr_metadata.v2.definition import ( CORE_V2, ZarrV2CodecDefinition, ZarrV2Context, ) Loc = tuple[str | int, ...] BASE: dict[str, Any] = dict(ZarrV2ArrayMetadata.create_default(shape=(4,)).to_json()) PRIVATE = ZarrV2Context(data_types=[FLOAT_DATA_TYPE_V2], codecs=[ZLIB_CODEC_V2]) @pytest.mark.parametrize( ("changes", "context", "dtype", "compressor", "filters", "locs"), [ ({}, None, AcceptedField, None, None, [("dtype",)]), ( { "dtype": " None: """`read_array_metadata_v2` reads the document once in the given scope (`CORE_V2` by default). Each of `dtype`, `compressor` and `filters` is an `AcceptedField`, an `UnclaimedField`, or None, depending on what the document contains. `fields()` returns them in document order, with a struct's record types after the struct. A valid document produces no problems.""" reading = read_array_metadata_v2({**BASE, **changes}, context=context) assert reading.problems == () assert type(reading.dtype) is dtype assert (None if reading.compressor is None else type(reading.compressor)) == compressor assert reading.filters is not UNSET found = None if reading.filters is None else tuple(type(f) for f in reading.filters) assert found == filters assert [loc for loc, _ in reading.fields()] == locs @pytest.mark.parametrize( ("value", "problems"), [ ({**BASE, "dtype": "float32"}, [(("dtype",), "invalid_value")]), ( {**BASE, "compressor": {"id": "zlib", "level": 10}}, [(("compressor", "level"), "invalid_value")], ), ({**BASE, "fill_value": "NaN"}, [(("fill_value",), "invalid_type")]), (3, [((), "invalid_type")]), ], ids=["dtype", "compressor", "fill", "not-an-object"], ) def test_error_a_reading_reports_every_problem( value: object, problems: list[tuple[Loc, str]] ) -> None: """Reading an invalid document reports every problem. A refused field is held as a `RefusedField`, and the reading has no model.""" reading = read_array_metadata_v2(value) assert [(p.loc, p.kind) for p in reading.problems] == problems assert reading.metadata is None if isinstance(value, dict) and "dtype" in problems[0][0]: assert isinstance(reading.dtype, RefusedField) def test_error_filters_that_are_not_an_array_are_not_read_as_null() -> None: """A `filters` member that is neither null nor an array is a problem, and the reading holds no filters for it: `UNSET`, not None, because None means that the document writes `null`.""" reading = read_array_metadata_v2({**BASE, "filters": {"id": "crc32"}}) assert [(p.loc, p.kind) for p in reading.problems] == [(("filters",), "invalid_type")] assert reading.filters is UNSET def test_the_v2_readers_read_in_the_scope_given() -> None: """`read_array_metadata_v2` and the model's `from_json` take a scope. In a scope that does not claim `|u1`, the default document still validates, because an unclaimed dtype is not checked. In a scope where a private `zlib` definition takes no level, a `level` in the document is a problem.""" assert read_array_metadata_v2(BASE, context=PRIVATE).problems == () assert ZarrV2ArrayMetadata.from_json(BASE, context=PRIVATE).to_json()["dtype"] == "|u1" bare = ZarrV2CodecDefinition(name="zlib", configuration=EmptyConfiguration) doc = {**BASE, "compressor": {"id": "zlib", "level": 1}} assert read_array_metadata_v2(doc).problems == () found = read_array_metadata_v2( doc, context=CORE_V2.update(codecs={**CORE_V2.codecs, "zlib": bare}) ).problems assert [p.loc for p in found] == [("compressor", "level")] class _EqualTo: """A value that compares equal to a string without being one.""" def __init__(self, text: str) -> None: self.text = text def __eq__(self, other: object) -> bool: return other == self.text __hash__ = None # pyright: ignore[reportAssignmentType] @pytest.mark.parametrize( ("key", "value"), [("order", _EqualTo("C")), ("dimension_separator", _EqualTo("."))], ids=["order", "dimension_separator"], ) def test_error_a_v2_member_that_only_compares_equal_to_an_allowed_value_is_refused( key: str, value: object ) -> None: """`order` and `dimension_separator` are checked after refinement. A value that is not a string, but compares equal to an allowed string, is outside the allowed set. It is reported as the wrong type, and no model accepts it.""" document = {**BASE, key: value} problems = read_array_metadata_v2(document).problems assert [(found.loc, found.kind) for found in problems] == [((key,), "invalid_type")] assert read_array_metadata_v2(document).metadata is None with pytest.raises(MetadataValidationError): ZarrV2ArrayMetadata(document) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_read_node_metadata_v2.py000066400000000000000000000322431527014137600340450ustar00rootroot00000000000000"""The v2 readers that return problems instead of raising: the group reader, the consolidated-metadata reader, and the node reader, which reads a node's files from a store.""" from __future__ import annotations import json from typing import Any, cast import pytest from zarr_metadata.model import ( ZarrV2ArrayMetadata, ZarrV2ArrayMetadataReading, ZarrV2ConsolidatedMetadata, ZarrV2GroupMetadata, ZarrV2GroupMetadataReading, ZarrV2UnknownNodeMetadataReading, node_metadata_from_key_value_v2, read_consolidated_metadata_v2, read_group_metadata_v2, read_node_metadata_v2, ) from zarr_metadata.problems import MetadataValidationError ZARRAY: dict[str, Any] = { "zarr_format": 2, "shape": [4], "chunks": [2], "dtype": " dict[str, object]: """A node's files as decoded JSON, keyed by file name: `_zarray=...` is `.zarray`.""" return {key.replace("_", "."): value for key, value in files.items()} def _store(**files: object) -> dict[str, bytes]: """The same files as a store holds them: keys and bytes.""" return {key: json.dumps(value).encode() for key, value in _files(**files).items()} def test_read_group_metadata_v2_reads_a_group() -> None: """`read_group_metadata_v2` returns a reading with no problems, whose model equals the group read from the same document, and the model's `reading` is that reading. A group has no fields for a scope to read.""" document = {"zarr_format": 2, "attributes": {"a": 1}} reading = read_group_metadata_v2(document) assert reading.problems == () assert reading.metadata == ZarrV2GroupMetadata(document) assert reading.metadata is not None assert reading.metadata.reading == reading assert list(reading.fields()) == [] def test_error_read_group_metadata_v2_reports_the_problems_of_a_bad_group() -> None: """A group document with a problem gives a reading that holds the problems and no model.""" reading = read_group_metadata_v2({"zarr_format": 2, "extra": 1}) assert [(p.loc, p.kind) for p in reading.problems] == [(("extra",), "unknown_key")] assert reading.metadata is None @pytest.mark.parametrize( ("files", "kind", "model"), [ ({"_zarray": ZARRAY}, ZarrV2ArrayMetadataReading, ZarrV2ArrayMetadata), ( {"_zarray": ZARRAY, "_zattrs": {"a": 1}}, ZarrV2ArrayMetadataReading, ZarrV2ArrayMetadata, ), ({"_zgroup": {"zarr_format": 2}}, ZarrV2GroupMetadataReading, ZarrV2GroupMetadata), ( {"_zgroup": {"zarr_format": 2}, "_zattrs": {"a": 1}}, ZarrV2GroupMetadataReading, ZarrV2GroupMetadata, ), ], ids=["array", "array-with-attributes", "group", "group-with-attributes"], ) def test_a_v2_node_is_read_as_the_file_it_has( files: dict[str, object], kind: type, model: object ) -> None: """A v2 node is an array when its files include `.zarray` and a group when they include `.zgroup`, with the attributes at `.zattrs`. `read_node_metadata_v2` reads the decoded files and gives the reading of that node type, whose model is what the model's `from_key_value` builds from the same files in a store. `node_metadata_from_key_value_v2` returns that model from the store.""" reading = read_node_metadata_v2(_files(**files)) assert type(reading) is kind assert reading.problems == () assert reading.metadata == cast("Any", model).from_key_value(_store(**files)) assert node_metadata_from_key_value_v2(_store(**files)) == reading.metadata def test_a_v2_node_reads_the_same_from_zmetadata_entries_as_from_a_store() -> None: """The entries of a `.zmetadata` hold the same JSON as the node's own files, so the slice of entries for one path is a node's files: `read_node_metadata_v2` reads it as it reads the decoded store files, and builds the same model.""" entries = {"a/.zarray": ZARRAY, "a/.zattrs": {"x": 1}, "b/.zgroup": {"zarr_format": 2}} node_files = { key.removeprefix("a/"): value for key, value in entries.items() if key.startswith("a/") } reading = read_node_metadata_v2(node_files) assert reading.metadata == node_metadata_from_key_value_v2( _store(_zarray=ZARRAY, _zattrs={"x": 1}) ) def test_error_files_with_no_node_file_are_no_node() -> None: """Files with neither `.zarray` nor `.zgroup` describe no node: the reading is a `ZarrV2UnknownNodeMetadataReading` with one `missing_key` problem, and building a model from such a store raises.""" reading = read_node_metadata_v2(_files(_zattrs={"a": 1})) assert type(reading) is ZarrV2UnknownNodeMetadataReading assert [(p.loc, p.kind) for p in reading.problems] == [((), "missing_key")] assert reading.metadata is None with pytest.raises(MetadataValidationError): node_metadata_from_key_value_v2(_store(_zattrs={"a": 1})) def test_error_files_with_both_node_files_are_no_node() -> None: """A node is an array or a group, not both: files with `.zarray` and `.zgroup` give a `ZarrV2UnknownNodeMetadataReading` with an `invalid_value` problem at `.zgroup`.""" reading = read_node_metadata_v2(_files(_zarray=ZARRAY, _zgroup={"zarr_format": 2})) assert type(reading) is ZarrV2UnknownNodeMetadataReading assert [(p.loc, p.kind) for p in reading.problems] == [((".zgroup",), "invalid_value")] @pytest.mark.parametrize( ("name", "document", "kind"), [(".zarray", ZARRAY, "invalid_value"), (".zgroup", {"zarr_format": 2}, "unknown_key")], ids=["array", "group"], ) def test_error_a_node_file_that_holds_attributes_is_refused_alike_in_a_store_and_a_zmetadata( name: str, document: dict[str, Any], kind: str ) -> None: """A `.zarray` or `.zgroup` that holds `attributes` itself is refused, with a message that says the attributes belong in `.zattrs`. `read_node_metadata_v2` and `read_consolidated_metadata_v2` report the same problem, of the same kind, at the file and at its entry. For a `.zgroup`, which declares no other member, the kind is `unknown_key`.""" held = {**document, "attributes": {}} from_files = read_node_metadata_v2({name: held}).problems from_entries = read_consolidated_metadata_v2( { "zarr_consolidated_format": 1, "metadata": {".zgroup": {"zarr_format": 2}, f"a/{name}": held}, } ).problems assert [(p.loc, p.kind) for p in from_files] == [(("attributes",), kind)] assert [(p.loc, p.kind) for p in from_entries] == [ (("metadata", f"a/{name}", "attributes"), kind) ] assert from_files[0].message == from_entries[0].message assert "belong in .zattrs" in from_files[0].message @pytest.mark.parametrize( ("name", "document", "found"), [ ( ".zarray", {**ZARRAY, "dtype": "bogus", "order": "Q"}, [ (("attributes",), "invalid_value"), (("dtype",), "invalid_value"), (("order",), "invalid_value"), ], ), ( ".zgroup", {"zarr_format": 3}, [(("attributes",), "unknown_key"), (("zarr_format",), "invalid_value")], ), ], ids=["array", "group"], ) def test_error_a_node_file_that_holds_attributes_still_reports_its_other_problems( name: str, document: dict[str, Any], found: list[tuple[tuple[str, ...], str]] ) -> None: """A `.zarray` or `.zgroup` that holds `attributes` does not hide its other problems: the rest of the file is still read, and its problems follow the one about `attributes`. `read_node_metadata_v2`, `read_consolidated_metadata_v2` (at the entry) and the model's `from_key_value` report the same problems.""" held = {**document, "attributes": {}} model = ZarrV2ArrayMetadata if name == ".zarray" else ZarrV2GroupMetadata from_entries = read_consolidated_metadata_v2( { "zarr_consolidated_format": 1, "metadata": {".zgroup": {"zarr_format": 2}, f"a/{name}": held}, } ).problems with pytest.raises(MetadataValidationError) as raised: model.from_key_value({name: json.dumps(held).encode()}) assert [(p.loc, p.kind) for p in read_node_metadata_v2({name: held}).problems] == found assert [(p.loc, p.kind) for p in from_entries] == [ (("metadata", f"a/{name}", *loc), kind) for loc, kind in found ] assert [(p.loc, p.kind) for p in raised.value.problems] == found def test_error_a_node_file_that_holds_attributes_next_to_a_bad_zattrs_reports_both() -> None: """A `.zarray` that holds `attributes`, next to a `.zattrs` that is not an object, has two problems, one in each file, and neither hides the other. `read_node_metadata_v2` and the model's `from_key_value` merge the files, so both are at `attributes`. `read_consolidated_metadata_v2` reads the entries apart, so it reports each at its own entry.""" held = {**ZARRAY, "attributes": {}} found = [(("attributes",), "invalid_value"), (("attributes",), "invalid_type")] assert [ (p.loc, p.kind) for p in read_node_metadata_v2(_files(_zarray=held, _zattrs=5)).problems ] == found with pytest.raises(MetadataValidationError) as raised: ZarrV2ArrayMetadata.from_key_value(_store(_zarray=held, _zattrs=5)) assert [(p.loc, p.kind) for p in raised.value.problems] == found from_entries = read_consolidated_metadata_v2( { "zarr_consolidated_format": 1, "metadata": {".zgroup": {"zarr_format": 2}, "a/.zarray": held, "a/.zattrs": 5}, } ).problems assert [(p.loc, p.kind) for p in from_entries] == [ (("metadata", "a/.zarray", "attributes"), "invalid_value"), (("metadata", "a/.zattrs"), "invalid_type"), ] def test_error_a_store_file_that_is_not_json_raises_when_building_a_node() -> None: """Decoding is the store's step: a store file whose bytes are not JSON makes `node_metadata_from_key_value_v2` raise with an `invalid_json` problem at the file.""" with pytest.raises(MetadataValidationError) as raised: node_metadata_from_key_value_v2({".zarray": b"{not json"}) assert [(p.loc, p.kind) for p in raised.value.problems] == [((".zarray",), "invalid_json")] def test_error_a_v2_node_with_a_bad_document_reports_its_problems() -> None: """A node whose document has a problem gives the reading of its node type with the problems, as the model's reader reports them, and no model.""" files = _files(_zarray={**ZARRAY, "shape": [-1]}) reading = read_node_metadata_v2(files) assert type(reading) is ZarrV2ArrayMetadataReading assert [(p.loc, p.kind) for p in reading.problems] == [(("shape",), "invalid_value")] assert reading.metadata is None CONSOLIDATED: dict[str, Any] = { "zarr_consolidated_format": 1, "metadata": { ".zgroup": {"zarr_format": 2}, "a/.zarray": ZARRAY, "a/.zattrs": {"x": 1}, }, } def test_read_consolidated_metadata_v2_reads_a_zmetadata() -> None: """`read_consolidated_metadata_v2` returns a reading with no problems whose model equals the one the constructor builds, and the model's `reading` is that reading. `fields()` gives each node's fields located in the `.zmetadata`.""" reading = read_consolidated_metadata_v2(CONSOLIDATED) assert reading.problems == () assert reading.metadata == ZarrV2ConsolidatedMetadata(CONSOLIDATED) assert reading.metadata is not None assert reading.metadata.reading == reading assert [loc for loc, _ in reading.fields()] == [("metadata", "a/.zarray", "dtype")] @pytest.mark.parametrize( ("entries", "key"), [ ({".zgroup": {"zarr_format": 2}, "a/.zarray": ZARRAY}, "a/.zarray"), ({".zgroup": {"zarr_format": 2}, "/a/.zarray": ZARRAY}, "/a/.zarray"), ( { ".zgroup": {"zarr_format": 2}, "a/.zgroup": {"zarr_format": 2}, "a//b/.zarray": ZARRAY, }, "a//b/.zarray", ), ({".zarray": ZARRAY}, ".zarray"), ({"/.zarray": ZARRAY}, "/.zarray"), ], ids=["path", "leading-slash", "doubled-slash", "root", "root-with-slash"], ) def test_consolidated_fields_are_located_at_the_keys_the_document_writes( entries: dict[str, Any], key: str ) -> None: """`fields()` locates each node's fields under the key that the `.zmetadata` writes for the node's `.zarray`, however that key spells the node's path, so each location leads into the document.""" reading = read_consolidated_metadata_v2({"zarr_consolidated_format": 1, "metadata": entries}) assert reading.problems == () assert [loc for loc, _ in reading.fields()] == [("metadata", key, "dtype")] def test_error_read_consolidated_metadata_v2_reports_problems_instead_of_raising() -> None: """A `.zmetadata` with a problem gives a reading with every problem and no model, where the constructor raises.""" bad = { **CONSOLIDATED, "metadata": {**CONSOLIDATED["metadata"], "a/.zarray": {**ZARRAY, "shape": [-1]}}, } reading = read_consolidated_metadata_v2(bad) assert [(p.loc, p.kind) for p in reading.problems] == [ (("metadata", "a/.zarray", "shape"), "invalid_value") ] assert reading.metadata is None with pytest.raises(MetadataValidationError): ZarrV2ConsolidatedMetadata(bad) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_refine_json.py000066400000000000000000000227421527014137600321620ustar00rootroot00000000000000"""Tests for refining a value to JSON, or reporting why it is not JSON.""" from __future__ import annotations import math from collections import OrderedDict, UserDict from types import MappingProxyType from typing import TYPE_CHECKING, cast import pytest from zarr_metadata._json import JSON_DEPTH, refine_json, refine_user_data, shown if TYPE_CHECKING: from collections.abc import Callable from zarr_metadata.problems import ValidationProblem from zarr_metadata.typed_json import JSONValue _Refiner = Callable[[object], tuple[JSONValue | None, tuple[ValidationProblem, ...]]] @pytest.mark.parametrize( ("value", "refined"), [ (1, 1), (2.5, 2.5), (True, True), ("x", "x"), (None, None), ([1, [2, 3]], (1, (2, 3))), ((1, 2), (1, 2)), ({"a": [1], "b": {"c": None}}, {"a": (1,), "b": {"c": None}}), (OrderedDict(k=[0]), {"k": (0,)}), ], ids=["int", "float", "bool", "str", "null", "nested-lists", "tuple", "object", "any-mapping"], ) def test_json_is_refined_to_tuples_and_dicts(value: object, refined: object) -> None: # A single walk both normalizes and validates. assert refine_json(value) == (refined, ()) def test_user_data_holds_non_finite_numbers() -> None: # User data is JSON as Python's `json` module reads and writes it. The # walk is the same as in `refine_json`, and a non-finite number stays a float. assert refine_user_data({"range": [-math.inf, math.inf]}) == ( {"range": (-math.inf, math.inf)}, (), ) refined, problems = refine_user_data(math.nan) assert problems == () assert isinstance(refined, float) assert math.isnan(refined) # User data, like JSON, is refused at any position where it is not JSON. @pytest.mark.parametrize("refine", [refine_json, refine_user_data], ids=["json", "user-data"]) @pytest.mark.parametrize( ("value", "loc"), # `bytes` is a sequence of integers, but it is not a JSON array. [({"a": object()}, ("a",)), (b"bytes", ())], ids=["object", "bytes"], ) def test_error_a_leaf_that_is_not_json_is_located( refine: _Refiner, value: object, loc: tuple[str | int, ...] ) -> None: refined, problems = refine(value) assert refined is None assert [(problem.loc, problem.kind) for problem in problems] == [(loc, "invalid_type")] @pytest.mark.parametrize("refine", [refine_json, refine_user_data], ids=["json", "user-data"]) def test_error_a_non_string_key_is_located_at_its_object(refine: _Refiner) -> None: refined, problems = refine([1, [2, {3: 4}]]) assert refined is None assert [(problem.loc, problem.kind) for problem in problems] == [((1, 1), "invalid_type")] @pytest.mark.parametrize( ("value", "loc"), [({"x": math.inf}, ("x",)), ({"x": [math.nan]}, ("x", 0))], ids=["infinite", "nan-in-array"], ) def test_error_a_non_finite_number_is_not_json(value: object, loc: tuple[str | int, ...]) -> None: refined, problems = refine_json(value) assert refined is None assert [(problem.loc, problem.kind) for problem in problems] == [(loc, "invalid_value")] def test_error_every_leaf_that_is_not_json_is_reported() -> None: refined, problems = refine_json({"a": object(), "b": [1, object()]}) assert refined is None assert [problem.loc for problem in problems] == [("a",), ("b", 1)] def test_a_value_nested_hundreds_deep_is_read() -> None: # The walk uses one frame per level of nesting, up to `JSON_DEPTH` levels. deep: dict[str, object] = {} for _ in range(JSON_DEPTH - 1): deep = {"k": deep} refined, problems = refine_json(deep) assert problems == () assert refined is not None def test_error_a_value_nested_deeper_than_a_reader_walks() -> None: # The first level past the limit is reported as the problem, at any # position. This keeps any document from pushing a reader past the # interpreter's recursion limit. deep: dict[str, object] = {} for _ in range(JSON_DEPTH + 40): deep = {"k": deep} refined, problems = refine_json({"a": [deep]}) assert refined is None assert [(len(p.loc), p.kind, p.message) for p in problems] == [ (JSON_DEPTH, "invalid_value", f"nested deeper than the {JSON_DEPTH} levels a reader walks") ] assert problems[0].loc[:2] == ("a", 0) def _nested(depth: int, innermost: object) -> list[object]: """Wrap `innermost` in `depth` nested arrays, so `innermost` is `depth` levels down.""" value: object = innermost for _ in range(depth): value = [value] return cast("list[object]", value) @pytest.mark.parametrize( ("value", "text"), [ (None, "null"), (True, "true"), ("C", '"C"'), ((1, (2,)), "[1, [2]]"), ({"a": None}, '{"a": null}'), (float("nan"), "NaN"), ({1: 2}, "{1: 2}"), (_nested(JSON_DEPTH, []), "a value nested too deep to show"), (_nested(JSON_DEPTH, {1}), "[" * JSON_DEPTH + "{1}" + "]" * JSON_DEPTH), ], ids=[ "null", "true", "string", "array", "object", "non-finite", "not-json", "past-the-levels", "not-json-at-the-last-level", ], ) def test_a_value_is_shown_as_the_json_a_document_writes(value: object, text: str) -> None: """A problem's message shows a value as JSON, or as its repr when it is not JSON. When a container is nested past the levels a reader walks, the message says so instead. A non-JSON value on the last level is still shown.""" assert shown(value) == text def test_bytes_past_the_levels_a_reader_walks_are_not_json_rather_than_nested() -> None: # `bytes` is a sequence to Python but not a container to JSON, at any # position. Past the depth cap it is still reported as a non-JSON value, # not as nesting. deep: list[object] = [b""] for _ in range(JSON_DEPTH - 1): deep = [deep] _, problems = refine_json(deep) assert [(len(p.loc), p.kind, p.message[:31]) for p in problems] == [ (JSON_DEPTH, "invalid_type", "not a JSON-serializable value: ") ] def test_error_an_integer_of_more_digits_than_json_text_holds_is_a_problem( interpreter_writes_4300_digits: None, ) -> None: """An integer with more digits than `sys.get_int_max_str_digits` allows cannot be converted to text. It is reported as an `invalid_value` problem at its location, so no reader raises on it. The document could not be written back. `validate_*` and `read_*` report the same problems.""" from zarr_metadata.model import ( ZarrV3ArrayMetadata, read_array_metadata_v3, ) refined, problems = refine_json({"x": [10**5000]}) assert refined is None assert [(p.loc, p.kind) for p in problems] == [(("x", 0), "invalid_value")] assert "digits" in problems[0].message document = {**ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()} document["attributes"] = {"x": 10**5000} found = read_array_metadata_v3(document).problems assert [(p.loc, p.kind) for p in found] == [(("attributes", "x"), "invalid_value")] assert read_array_metadata_v3(document).problems == found shaped = {**document, "attributes": {}, "shape": [10**5000]} assert [p.loc for p in read_array_metadata_v3(shaped).problems] == [("shape", 0)] def test_a_str_or_int_subclass_is_refined_to_the_json_type_it_is() -> None: """A `StrEnum` member, an `IntEnum` member, or any `str`, `int` or `float` subclass is refined to the plain value that JSON writes for it. A `Literal` or a tag then reads it as that value. `bool` stays separate from `int`.""" import enum from zarr_metadata.model import ZarrV3GroupMetadata class NodeType(enum.StrEnum): GROUP = "group" class Format(enum.IntEnum): THREE = 3 class Score(float): pass refined, problems = refine_json( {"a": NodeType.GROUP, "b": Format.THREE, "c": Score(1.5), "d": True} ) assert problems == () assert refined == {"a": "group", "b": 3, "c": 1.5, "d": True} assert isinstance(refined, dict) assert all(type(refined[key]) in (str, int, float, bool) for key in refined) assert type(refined["d"]) is bool model = ZarrV3GroupMetadata.from_json( {"zarr_format": Format.THREE, "node_type": NodeType.GROUP} ) assert model.to_json() == {"zarr_format": 3, "node_type": "group"} def test_a_mapping_or_sequence_of_another_type_is_refined_as_a_dict_or_list_is() -> None: """A mapping proxy, a `UserDict`, a subclass of `dict` or `list`, or a tuple is refined to the same JSON as the equivalent dict or list. It is validated the same way, with each problem at its location.""" class Listed(list[object]): pass class Mapped(dict[str, object]): pass held: dict[str, object] = {"a": [1, {"b": (2, 3)}], "c": "x", "d": [True, None, 2.5]} expected = refine_json(held) assert expected == ({"a": (1, {"b": (2, 3)}), "c": "x", "d": (True, None, 2.5)}, ()) for other in ( MappingProxyType(held), UserDict(held), Mapped(held), {**held, "a": Listed(cast("list[object]", held["a"]))}, ): assert refine_json(other) == expected bad: dict[str, object] = {"a": [1, {"b": math.nan}], "c": {1, 2}} for other in ( bad, MappingProxyType(bad), Mapped(bad), {**bad, "a": Listed([1, {"b": math.nan}])}, ): refined, problems = refine_json(other) assert refined is None assert [(p.loc, p.kind) for p in problems] == [ (("a", 1, "b"), "invalid_value"), (("c",), "invalid_type"), ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_repair.py000066400000000000000000000501351527014137600311400ustar00rootroot00000000000000"""Tests for repairing v2 and v3 metadata that a known writer bug made invalid, before a strict read.""" from __future__ import annotations import copy import json import math from typing import Any import pytest from zarr_metadata.model import ( ZarrV2ArrayMetadata, ZarrV2ConsolidatedMetadata, ZarrV2GroupMetadata, ZarrV3ArrayMetadata, ZarrV3GroupMetadata, read_consolidated_metadata_v2, read_node_metadata_v2, read_node_metadata_v3, read_repaired_consolidated_metadata_v2, read_repaired_node_metadata_v2, read_repaired_node_metadata_v3, repair_consolidated_metadata_v2, repair_node_metadata_v2, repair_node_metadata_v3, ) from zarr_metadata.problems import MetadataValidationError ARRAY: dict[str, Any] = dict(ZarrV3ArrayMetadata.create_default(shape=(0, 3)).to_json()) GROUP: dict[str, Any] = {"zarr_format": 3, "node_type": "group"} def _grid(*chunk_shape: int) -> dict[str, Any]: return {"name": "regular", "configuration": {"chunk_shape": chunk_shape}} def _consolidated(**metadata: object) -> dict[str, Any]: envelope = {"kind": "inline", "must_understand": False, "metadata": metadata} return {**GROUP, "consolidated_metadata": envelope} ZERO_CHUNK = {**ARRAY, "chunk_grid": _grid(0, 3)} NULL_CONSOLIDATED = {**GROUP, "consolidated_metadata": None} FLOAT64: dict[str, Any] = {**ARRAY, "data_type": "float64", "fill_value": 0.0} COMPLEX128: dict[str, Any] = {**ARRAY, "data_type": "complex128", "fill_value": (0.0, 0.0)} NAN_INT: dict[str, Any] = {**ARRAY, "fill_value": math.nan} NAN_FLOAT64: dict[str, Any] = {**FLOAT64, "fill_value": math.nan} @pytest.mark.parametrize( ("value", "repaired", "repairs"), [ ( ZERO_CHUNK, {**ARRAY, "chunk_grid": _grid(1, 3)}, [(("chunk_grid", "configuration", "chunk_shape", 0), "zero_chunk_length")], ), ( {**ARRAY, "shape": (0, 0), "chunk_grid": _grid(0, 0)}, {**ARRAY, "shape": (0, 0), "chunk_grid": _grid(1, 1)}, [ (("chunk_grid", "configuration", "chunk_shape", 0), "zero_chunk_length"), (("chunk_grid", "configuration", "chunk_shape", 1), "zero_chunk_length"), ], ), (NULL_CONSOLIDATED, GROUP, [(("consolidated_metadata",), "null_consolidated_metadata")]), ( _consolidated(a=ZERO_CHUNK, b=NULL_CONSOLIDATED), _consolidated(a={**ARRAY, "chunk_grid": _grid(1, 3)}, b=GROUP), [ ( ("consolidated_metadata", "metadata", "a", "chunk_grid", "configuration") + ("chunk_shape", 0), "zero_chunk_length", ), ( ("consolidated_metadata", "metadata", "b", "consolidated_metadata"), "null_consolidated_metadata", ), ], ), ( _consolidated(a={**FLOAT64, "fill_value": math.nan}), _consolidated(a={**FLOAT64, "fill_value": "NaN"}), [(("consolidated_metadata", "metadata", "a", "fill_value"), "non_finite_fill_value")], ), ( _consolidated(a={**COMPLEX128, "fill_value": (math.inf, 0.0)}), _consolidated(a={**COMPLEX128, "fill_value": ("Infinity", 0.0)}), [ ( ("consolidated_metadata", "metadata", "a", "fill_value", 0), "non_finite_fill_value", ) ], ), ( _consolidated(b={**FLOAT64, "fill_value": -math.inf}), _consolidated(b={**FLOAT64, "fill_value": "-Infinity"}), [ ( ("consolidated_metadata", "metadata", "b", "fill_value"), "non_finite_fill_value", ) ], ), ( _consolidated(a={**COMPLEX128, "fill_value": (math.nan, -math.inf)}), _consolidated(a={**COMPLEX128, "fill_value": ("NaN", "-Infinity")}), [ ( ("consolidated_metadata", "metadata", "a", "fill_value", 0), "non_finite_fill_value", ), ( ("consolidated_metadata", "metadata", "a", "fill_value", 1), "non_finite_fill_value", ), ], ), ( _consolidated(a={**COMPLEX128, "fill_value": [math.nan, 0.0]}), _consolidated(a={**COMPLEX128, "fill_value": ("NaN", 0.0)}), [ ( ("consolidated_metadata", "metadata", "a", "fill_value", 0), "non_finite_fill_value", ) ], ), # Not a known writer bug: left for the strict read to check. (ARRAY, ARRAY, []), ( _consolidated(a={**FLOAT64, "fill_value": "NaN"}), _consolidated(a={**FLOAT64, "fill_value": "NaN"}), [], ), (NAN_INT, NAN_INT, []), (NAN_FLOAT64, NAN_FLOAT64, []), ({**ARRAY, "chunk_grid": _grid(3, 0)}, {**ARRAY, "chunk_grid": _grid(3, 0)}, []), ({**ARRAY, "chunk_grid": _grid(0)}, {**ARRAY, "chunk_grid": _grid(0)}, []), ( {key: item for key, item in ZERO_CHUNK.items() if key != "data_type"}, {key: item for key, item in ARRAY.items() if key != "data_type"} | {"chunk_grid": _grid(1, 3)}, [(("chunk_grid", "configuration", "chunk_shape", 0), "zero_chunk_length")], ), ({**GROUP, "consolidated_metadata": 0}, {**GROUP, "consolidated_metadata": 0}, []), ([ZERO_CHUNK], [ZERO_CHUNK], []), ], ids=[ "zero-chunk", "zero-chunks", "null-consolidated", "inside-consolidated", "non-finite-float-fill", "non-finite-complex-fill", "non-finite-fill-inside-consolidated", "non-finite-complex-fill-in-both-parts", "non-finite-complex-fill-as-a-list", "valid", "named-non-finite-fill", "non-finite-fill-of-an-integer-type", "non-finite-fill-outside-a-listing", "zero-chunk-on-a-full-dimension", "zero-chunk-of-another-rank", "zero-chunk-without-a-data-type", "consolidated-of-another-type", "not-a-document", ], ) def test_repair_undoes_each_known_writer_bug( value: object, repaired: object, repairs: list[tuple[tuple[str | int, ...], str]] ) -> None: """Each known writer bug in a document, including in its consolidated documents, is undone, and each repair reports its location: a zero chunk length along an empty dimension, a null `consolidated_metadata`, and a float or complex fill value that zarr-python 3.0.0 to 3.0.6 wrote as a bare `NaN` or infinity in a listing. Each non-finite part of a complex fill value is its own repair, and the repaired fill value is a tuple, as a JSON array is. Those versions wrote a node's own `zarr.json` correctly, so such a fill value there is no known writer bug, nor is a fill value already written by name, such as `"NaN"`. Anything else is left unchanged for the strict read to report. The input document is not modified.""" before = copy.deepcopy(value) document, made = repair_node_metadata_v3(value) assert document == repaired assert [(repair.loc, repair.kind) for repair in made] == repairs assert value == before if len(repairs) == 0: assert document is value @pytest.mark.parametrize( ("value", "model"), [ (ZERO_CHUNK, ZarrV3ArrayMetadata.from_json({**ARRAY, "chunk_grid": _grid(1, 3)})), (NULL_CONSOLIDATED, ZarrV3GroupMetadata.from_json(GROUP)), (ARRAY, ZarrV3ArrayMetadata.from_json(ARRAY)), ], ids=["zero-chunk", "null-consolidated", "valid"], ) def test_read_repaired_reads_what_the_strict_read_refuses(value: object, model: object) -> None: """A document made invalid by a writer bug is refused by `read_node_metadata_v3`. `read_repaired_node_metadata_v3` reads it, with the same result as a strict read of the repaired document. A valid document reads the same either way.""" repaired = read_repaired_node_metadata_v3(value) assert repaired.reading.problems == () assert repaired.reading.metadata == model assert (len(read_node_metadata_v3(value).problems) == 0) is (len(repaired.repairs) == 0) def test_read_repaired_reports_what_no_repair_applies_to() -> None: """A problem that no repair fixes is reported the same way the strict read reports it, alongside the repairs that were made.""" value = {key: item for key, item in ZERO_CHUNK.items() if key != "data_type"} repaired = read_repaired_node_metadata_v3(value) assert [repair.kind for repair in repaired.repairs] == ["zero_chunk_length"] assert [(problem.loc, problem.kind) for problem in repaired.reading.problems] == [ (("data_type",), "missing_key") ] assert repaired.reading.metadata is None # --- v2 --------------------------------------------------------------------- # The `.zarray` that zarr-python 3.0.0 to 3.0.6 wrote for a ` dict[str, object]: return {".zarray": {**zarray, "fill_value": fill_value}, ".zattrs": {}} @pytest.mark.parametrize( ("files", "repaired", "repairs"), [ ( _with_fill(NAN_ZARRAY, math.nan), _with_fill(NAN_ZARRAY, "NaN"), [((".zarray", "fill_value"), "non_finite_fill_value")], ), ( _with_fill({**NAN_ZARRAY, "dtype": ">f8"}, -math.inf), _with_fill({**NAN_ZARRAY, "dtype": ">f8"}, "-Infinity"), [((".zarray", "fill_value"), "non_finite_fill_value")], ), ( _with_fill(COMPLEX_ZARRAY, [math.nan, math.inf]), _with_fill(COMPLEX_ZARRAY, ("NaN", "Infinity")), [ ((".zarray", "fill_value", 0), "non_finite_fill_value"), ((".zarray", "fill_value", 1), "non_finite_fill_value"), ], ), ( _with_fill(COMPLEX_ZARRAY, [1.5, -math.inf]), _with_fill(COMPLEX_ZARRAY, (1.5, "-Infinity")), [((".zarray", "fill_value", 1), "non_finite_fill_value")], ), # Not a known writer bug: left for the strict read to check. (_with_fill(NAN_ZARRAY, "NaN"), _with_fill(NAN_ZARRAY, "NaN"), []), (_with_fill(NAN_ZARRAY, 0.5), _with_fill(NAN_ZARRAY, 0.5), []), ( _with_fill({**NAN_ZARRAY, "dtype": " None: """zarr-python 3.0.0 to 3.0.6 wrote a non-finite fill value of a float or complex `.zarray` as a bare token, `NaN`, which is not JSON. Python's `json` module reads it as a float. `repair_node_metadata_v2` writes each one as the string the v2 spec uses, and reports its location. Anything else is left unchanged, and the input files are not modified.""" before = copy.deepcopy(files) document, made = repair_node_metadata_v2(files) # `==` cannot compare NaN, which is never equal to itself, so compare the JSON text. assert json.dumps(document) == json.dumps(repaired) assert [(repair.loc, repair.kind) for repair in made] == repairs assert json.dumps(files) == json.dumps(before) if len(repairs) == 0: assert document is files @pytest.mark.parametrize( ("files", "model", "repaired_at"), [ ({".zarray": NAN_ZARRAY}, ZarrV2ArrayMetadata(NAMED_ZARRAY), [("fill_value",)]), ( {".zarray": {**COMPLEX_ZARRAY, "fill_value": [math.nan, math.inf]}}, ZarrV2ArrayMetadata({**COMPLEX_ZARRAY, "fill_value": ("NaN", "Infinity")}), [("fill_value", 0), ("fill_value", 1)], ), ( {".zarray": {**COMPLEX_ZARRAY, "fill_value": [1.5, -math.inf]}}, ZarrV2ArrayMetadata({**COMPLEX_ZARRAY, "fill_value": (1.5, "-Infinity")}), [("fill_value", 1)], ), ({".zgroup": {"zarr_format": 2}}, ZarrV2GroupMetadata({"zarr_format": 2}), []), ], ids=["float", "complex", "complex-one-component", "valid-group"], ) def test_read_repaired_v2_node_reads_what_the_strict_read_refuses( files: dict[str, object], model: object, repaired_at: list[tuple[str | int, ...]] ) -> None: """The strict `read_node_metadata_v2` refuses the `.zarray` that zarr-python 3.0.0 to 3.0.6 wrote for a non-finite float or complex fill value, at each non-finite number. `read_repaired_node_metadata_v2` reads the repaired files, with the same result as a strict read of them, and lists a repair at each place the strict read refused. A valid node reads the same either way.""" assert [p.loc for p in read_node_metadata_v2(files).problems] == repaired_at repaired = read_repaired_node_metadata_v2(files) assert repaired.reading.problems == () assert repaired.reading.metadata == model assert [(r.loc, r.kind) for r in repaired.repairs] == [ ((".zarray", *loc), "non_finite_fill_value") for loc in repaired_at ] def test_read_repaired_v2_node_reports_what_no_repair_applies_to() -> None: """A problem that no repair fixes is reported the same way the strict read reports it, alongside the repairs that were made.""" files = {".zarray": {**NAN_ZARRAY, "order": "X"}} repaired = read_repaired_node_metadata_v2(files) assert [r.kind for r in repaired.repairs] == ["non_finite_fill_value"] assert [(p.loc, p.kind) for p in repaired.reading.problems] == [(("order",), "invalid_value")] assert repaired.reading.metadata is None ZGROUP_FROM_ZARR3: dict[str, Any] = { "zarr_format": 2, "consolidated_metadata": {"metadata": {}, "must_understand": False, "kind": "inline"}, } ZMETADATA_FROM_ZARR3: dict[str, Any] = { "zarr_consolidated_format": 1, "metadata": { ".zgroup": {"zarr_format": 2}, "b/.zgroup": ZGROUP_FROM_ZARR3, "b/.zattrs": {"x": 1}, }, } ZMETADATA_CLEAN: dict[str, Any] = { "zarr_consolidated_format": 1, "metadata": { ".zgroup": {"zarr_format": 2}, "b/.zgroup": {"zarr_format": 2}, "b/.zattrs": {"x": 1}, }, } @pytest.mark.parametrize( ("value", "repaired", "repairs"), [ ( ZMETADATA_FROM_ZARR3, ZMETADATA_CLEAN, [ ( ("metadata", "b/.zgroup", "consolidated_metadata"), "consolidated_metadata_in_zgroup_entry", ) ], ), (ZMETADATA_CLEAN, ZMETADATA_CLEAN, []), # The root's .zgroup: no zarr-python version writes the member there. ( {"zarr_consolidated_format": 1, "metadata": {".zgroup": ZGROUP_FROM_ZARR3}}, {"zarr_consolidated_format": 1, "metadata": {".zgroup": ZGROUP_FROM_ZARR3}}, [], ), ( { "zarr_consolidated_format": 1, "metadata": {"b/.zgroup": {"zarr_format": 2, "consolidated_metadata": 3}}, }, { "zarr_consolidated_format": 1, "metadata": {"b/.zgroup": {"zarr_format": 2, "consolidated_metadata": 3}}, }, [], ), (3, 3, []), ( {"zarr_consolidated_format": 1, "metadata": {"a/.zarray": NAN_ZARRAY}}, {"zarr_consolidated_format": 1, "metadata": {"a/.zarray": NAMED_ZARRAY}}, [(("metadata", "a/.zarray", "fill_value"), "non_finite_fill_value")], ), ], ids=[ "zarr-3-zgroup-entry", "clean", "root", "not-the-bug", "not-a-document", "bare-nan-in-zarray-entry", ], ) def test_repair_v2_undoes_the_consolidated_metadata_zarr_3_writes_into_a_zgroup_entry( value: object, repaired: object, repairs: list[tuple[tuple[str | int, ...], str]] ) -> None: """zarr-python 3.x writes a `consolidated_metadata` member into each non-root `.zgroup` entry of a `.zmetadata`, but the v2 group document does not allow that member. `repair_consolidated_metadata_v2` removes it and reports its location. Anything else is left unchanged, and the input document is not modified.""" before = copy.deepcopy(value) document, made = repair_consolidated_metadata_v2(value) assert document == repaired assert [(repair.loc, repair.kind) for repair in made] == repairs assert value == before if len(repairs) == 0: assert document is value def test_read_repaired_consolidated_v2_reads_what_the_strict_read_refuses() -> None: """The strict `ZarrV2ConsolidatedMetadata` refuses the `.zmetadata` that zarr-python 3.x writes, reporting the problem at the entry's member. `read_repaired_consolidated_metadata_v2` returns the strict reading of the repaired document, as `read_consolidated_metadata_v2` reads it, and the repairs, as the node repair readers do. A problem that no repair fixes is in that reading, which then holds no model.""" with pytest.raises(MetadataValidationError) as raised: ZarrV2ConsolidatedMetadata(ZMETADATA_FROM_ZARR3) assert [p.loc for p in raised.value.problems] == [ ("metadata", "b/.zgroup", "consolidated_metadata") ] repaired = read_repaired_consolidated_metadata_v2(ZMETADATA_FROM_ZARR3) assert repaired.reading == read_consolidated_metadata_v2(ZMETADATA_CLEAN) assert repaired.reading.problems == () assert repaired.reading.metadata == ZarrV2ConsolidatedMetadata(ZMETADATA_CLEAN) assert [r.kind for r in repaired.repairs] == ["consolidated_metadata_in_zgroup_entry"] broken = read_repaired_consolidated_metadata_v2( { **ZMETADATA_FROM_ZARR3, "metadata": {**ZMETADATA_FROM_ZARR3["metadata"], "b/.zattrs": {1: "x"}}, } ) assert broken.reading.metadata is None assert [p.loc for p in broken.reading.problems] == [("metadata", "b/.zattrs")] assert [r.kind for r in broken.repairs] == ["consolidated_metadata_in_zgroup_entry"] def test_read_repaired_consolidated_v2_writes_a_bare_non_finite_fill_value_by_name() -> None: """A `.zarray` entry of a `.zmetadata` with a bare NaN fill value, as zarr-python 3.0.0 to 3.0.6 wrote it, is refused by the strict read and read by `read_repaired_consolidated_metadata_v2`, after the same repair as for a `.zarray` file.""" value = { **ZMETADATA_CLEAN, "metadata": {**ZMETADATA_CLEAN["metadata"], "b/a/.zarray": NAN_ZARRAY}, } with pytest.raises(MetadataValidationError) as raised: ZarrV2ConsolidatedMetadata(value) assert [p.loc for p in raised.value.problems] == [("metadata", "b/a/.zarray", "fill_value")] repaired = read_repaired_consolidated_metadata_v2(value) assert repaired.reading.problems == () assert repaired.reading.metadata is not None assert repaired.reading.metadata.metadata["b/a"] == ZarrV2ArrayMetadata(NAMED_ZARRAY) assert [(r.loc, r.kind) for r in repaired.repairs] == [ (("metadata", "b/a/.zarray", "fill_value"), "non_finite_fill_value") ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_scope_threading.py000066400000000000000000000171411527014137600330140ustar00rootroot00000000000000"""Tests that every entry point reads in the scope it is given, and in `CORE_AND_EXTENSIONS` when given none.""" from __future__ import annotations import json from typing import TYPE_CHECKING, Any import pytest import zarr_metadata.model as zm from zarr_metadata.json_schema import node_metadata_json_schema_v3 from zarr_metadata.model import ( ZarrV3ArrayMetadata, ) from zarr_metadata.problems import MetadataValidationError from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, ZarrV3Context if TYPE_CHECKING: from collections.abc import Callable from zarr_metadata.definition import Context EMPTY = ZarrV3Context() BASE: dict[str, Any] = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) BAD: dict[str, Any] = { **BASE, "codecs": (*BASE["codecs"], {"name": "gzip", "configuration": {"level": 12}}), } """An array that the default scope refuses (a gzip level of 12) and that an empty scope does not check.""" GROUP: dict[str, Any] = { "zarr_format": 3, "node_type": "group", "consolidated_metadata": {"kind": "inline", "must_understand": False, "metadata": {"a": BAD}}, } STORE_A = {"zarr.json": json.dumps(BAD).encode()} STORE_G = {"zarr.json": json.dumps(GROUP).encode()} def _accepts(read: Callable[..., object]) -> Callable[[Context | None], bool]: """Return whether `read`, given `context`, finds nothing wrong. Success is a model, or a reading with a model.""" def accepted(context: Context | None) -> bool: try: found = read(context=context) except MetadataValidationError: return False reading = getattr(found, "reading", found) metadata = getattr(reading, "metadata", found) return metadata is not None return accepted ENTRY_POINTS: dict[str, Callable[..., object]] = { "read_array_metadata_v3": lambda context: zm.read_array_metadata_v3(BAD, context=context), "ZarrV3ArrayMetadata": lambda context: zm.ZarrV3ArrayMetadata(BAD, context=context), "ZarrV3ArrayMetadata.from_json": lambda context: zm.ZarrV3ArrayMetadata.from_json( BAD, context=context ), "ZarrV3ArrayMetadata.from_key_value": lambda context: zm.ZarrV3ArrayMetadata.from_key_value( STORE_A, context=context ), "ZarrV3ArrayMetadata.create_default": lambda context: zm.ZarrV3ArrayMetadata.create_default( shape=(4,), codecs=BAD["codecs"], context=context ), "ZarrV3ConsolidatedMetadata": lambda context: zm.ZarrV3ConsolidatedMetadata( GROUP["consolidated_metadata"], context=context ), "ZarrV3ConsolidatedMetadata.from_json": lambda context: zm.ZarrV3ConsolidatedMetadata.from_json( GROUP["consolidated_metadata"], context=context ), "read_consolidated_metadata_v3": lambda context: zm.read_consolidated_metadata_v3( GROUP["consolidated_metadata"], context=context ), "read_node_metadata_v3": lambda context: zm.read_node_metadata_v3(BAD, context=context), "node_metadata_from_json_v3": lambda context: zm.node_metadata_from_json_v3( BAD, context=context ), "node_metadata_from_key_value_v3": lambda context: zm.node_metadata_from_key_value_v3( STORE_A, context=context ), "read_repaired_node_metadata_v3": lambda context: zm.read_repaired_node_metadata_v3( BAD, context=context ), } """Each entry point that reads `BAD`, as a document or as an entry of consolidated metadata.""" GROUP_ENTRY_POINTS: dict[str, Callable[..., object]] = { "read_group_metadata_v3": lambda context: zm.read_group_metadata_v3(GROUP, context=context), "ZarrV3GroupMetadata": lambda context: zm.ZarrV3GroupMetadata(GROUP, context=context), "ZarrV3GroupMetadata.from_json": lambda context: zm.ZarrV3GroupMetadata.from_json( GROUP, context=context ), "ZarrV3GroupMetadata.from_key_value": lambda context: zm.ZarrV3GroupMetadata.from_key_value( STORE_G, context=context ), "ZarrV3GroupMetadata.create_default": lambda context: zm.ZarrV3GroupMetadata.create_default( consolidated_metadata=GROUP["consolidated_metadata"], context=context ), "read_node_metadata_v3-group": lambda context: zm.read_node_metadata_v3(GROUP, context=context), "node_metadata_from_key_value_v3-group": lambda context: zm.node_metadata_from_key_value_v3( STORE_G, context=context ), } """Each entry point that reads `GROUP`, which lists `BAD` in its consolidated metadata.""" @pytest.mark.parametrize("read", ENTRY_POINTS.values(), ids=ENTRY_POINTS.keys()) def test_every_entry_point_reads_in_the_scope_it_is_given(read: Callable[..., object]) -> None: """Each entry point refuses a gzip level of 12 in the default scope (also selected by `None`), and accepts it in a scope that leaves gzip unclaimed. This shows that each entry point reads in the scope it is given.""" accepted = _accepts(read) assert accepted(EMPTY) is True assert accepted(CORE_AND_EXTENSIONS) is False assert accepted(None) is False @pytest.mark.parametrize("read", GROUP_ENTRY_POINTS.values(), ids=GROUP_ENTRY_POINTS.keys()) def test_a_group_reads_the_same_in_every_scope(read: Callable[..., object]) -> None: """A group document has no fields, and a group does not read the documents its consolidated metadata lists. So each group entry point accepts a group that lists `BAD`, in every scope.""" accepted = _accepts(read) assert accepted(EMPTY) is True assert accepted(CORE_AND_EXTENSIONS) is True assert accepted(None) is True def test_the_json_schema_is_written_in_the_scope_it_is_given() -> None: """`node_metadata_json_schema_v3` takes `None` for the default scope, and writes a different schema for an empty one.""" assert node_metadata_json_schema_v3(context=None) == node_metadata_json_schema_v3() assert node_metadata_json_schema_v3(context=EMPTY) != node_metadata_json_schema_v3() @pytest.mark.parametrize( "read", [*ENTRY_POINTS.values(), *GROUP_ENTRY_POINTS.values()], ids=[*ENTRY_POINTS.keys(), *GROUP_ENTRY_POINTS.keys()], ) def test_error_every_entry_point_refuses_a_scope_of_another_format( read: Callable[..., object], ) -> None: """A v3 entry point given a v2 scope raises `TypeError`. A scope reads documents of one format, and a v2 scope claims nothing that a v3 document contains.""" from zarr_metadata.v2.definition import CORE_V2 with pytest.raises(TypeError, match="format"): read(context=CORE_V2) NOT_JSON = b"{not json" STORE_READERS: dict[str, Callable[..., object]] = { "ZarrV3ArrayMetadata.from_key_value": lambda context: zm.ZarrV3ArrayMetadata.from_key_value( {"zarr.json": NOT_JSON}, context=context ), "ZarrV3GroupMetadata.from_key_value": lambda context: zm.ZarrV3GroupMetadata.from_key_value( {"zarr.json": NOT_JSON}, context=context ), "node_metadata_from_key_value_v3": lambda context: zm.node_metadata_from_key_value_v3( {"zarr.json": NOT_JSON}, context=context ), } """Each v3 entry point that decodes a store's bytes, given bytes that are not JSON.""" @pytest.mark.parametrize("name", STORE_READERS.keys()) def test_error_a_v3_store_reader_refuses_a_scope_of_another_format_before_decoding( name: str, ) -> None: """A v3 entry point that reads a store's bytes raises `TypeError` for a v2 scope before it decodes the bytes, so the wrong scope is reported even when the bytes are not JSON. With a v3 scope, the same bytes raise `MetadataValidationError`.""" from zarr_metadata.v2.definition import CORE_V2 with pytest.raises(TypeError, match="format"): STORE_READERS[name](context=CORE_V2) with pytest.raises(MetadataValidationError): STORE_READERS[name](context=CORE_AND_EXTENSIONS) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_scope_threading_v2.py000066400000000000000000000163701527014137600334260ustar00rootroot00000000000000"""Tests that every v2 entry point reads in the scope it is given, uses `CORE_V2` when given none, and refuses a scope of another format.""" from __future__ import annotations import json from typing import TYPE_CHECKING, Any import pytest import zarr_metadata.model as zm from zarr_metadata.model import ZarrV2ArrayMetadata from zarr_metadata.problems import MetadataValidationError from zarr_metadata.v2.definition import ( CORE_V2, ZarrV2Context, ) from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS if TYPE_CHECKING: from collections.abc import Callable from zarr_metadata.definition import Context EMPTY = ZarrV2Context() BASE: dict[str, Any] = dict(ZarrV2ArrayMetadata.create_default(shape=(4,), chunks=(2,)).to_json()) BAD: dict[str, Any] = {**BASE, "compressor": {"id": "gzip", "level": "x"}} """An array that the v2 scope refuses (a gzip level that is not an integer) and that an empty scope does not check.""" GROUP: dict[str, Any] = {"zarr_format": 2} CONSOLIDATED: dict[str, Any] = { "zarr_consolidated_format": 1, "metadata": {".zgroup": GROUP, "a/.zarray": BAD}, } STORE_A = {".zarray": json.dumps(BAD).encode()} STORE_G = {".zgroup": json.dumps(GROUP).encode()} STORE_C = {".zmetadata": json.dumps(CONSOLIDATED).encode()} def _accepts(read: Callable[..., object]) -> Callable[[Context | None], bool]: """Return whether `read`, given `context`, finds nothing wrong. Success is a model, a reading with a model, or a field that was read or left unclaimed.""" def accepted(context: Context | None) -> bool: try: found = read(context=context) except MetadataValidationError: return False if isinstance(found, tuple) and len(found) == 2 and isinstance(found[1], tuple): return len(found[1]) == 0 # a resolved field and its problems reading = getattr(found, "reading", found) metadata = getattr(reading, "metadata", found) return metadata is not None return accepted ENTRY_POINTS: dict[str, Callable[..., object]] = { "read_array_metadata_v2": lambda context: zm.read_array_metadata_v2(BAD, context=context), "ZarrV2ArrayMetadata": lambda context: zm.ZarrV2ArrayMetadata(BAD, context=context), "ZarrV2ArrayMetadata.from_json": lambda context: zm.ZarrV2ArrayMetadata.from_json( BAD, context=context ), "ZarrV2ArrayMetadata.from_key_value": lambda context: zm.ZarrV2ArrayMetadata.from_key_value( STORE_A, context=context ), "ZarrV2ArrayMetadata.create_default": lambda context: zm.ZarrV2ArrayMetadata.create_default( shape=(4,), chunks=(2,), compressor=BAD["compressor"], context=context ), "ZarrV2ArrayMetadata.with_context": lambda context: zm.ZarrV2ArrayMetadata( BAD, context=EMPTY ).with_context(context), "ZarrV2ArrayMetadata.refined_in": lambda context: zm.ZarrV2ArrayMetadata( BAD, context=EMPTY ).refined_in(context), "ZarrV2GroupMetadata": lambda context: zm.ZarrV2GroupMetadata(GROUP, context=context), "ZarrV2GroupMetadata.from_json": lambda context: zm.ZarrV2GroupMetadata.from_json( GROUP, context=context ), "ZarrV2GroupMetadata.from_key_value": lambda context: zm.ZarrV2GroupMetadata.from_key_value( STORE_G, context=context ), "ZarrV2GroupMetadata.create_default": lambda context: zm.ZarrV2GroupMetadata.create_default( context=context ), "ZarrV2ConsolidatedMetadata": lambda context: zm.ZarrV2ConsolidatedMetadata( CONSOLIDATED, context=context ), "ZarrV2ConsolidatedMetadata.from_json": lambda context: zm.ZarrV2ConsolidatedMetadata.from_json( CONSOLIDATED, context=context ), "ZarrV2ConsolidatedMetadata.from_key_value": ( lambda context: zm.ZarrV2ConsolidatedMetadata.from_key_value(STORE_C, context=context) ), "read_repaired_consolidated_metadata_v2": ( lambda context: zm.read_repaired_consolidated_metadata_v2(CONSOLIDATED, context=context) ), "read_repaired_node_metadata_v2": lambda context: zm.read_repaired_node_metadata_v2( {".zarray": BAD}, context=context ), "read_group_metadata_v2": lambda context: zm.read_group_metadata_v2(GROUP, context=context), "read_consolidated_metadata_v2": lambda context: zm.read_consolidated_metadata_v2( CONSOLIDATED, context=context ), "read_node_metadata_v2": lambda context: zm.read_node_metadata_v2( {".zarray": BAD}, context=context ), "node_metadata_from_key_value_v2": lambda context: zm.node_metadata_from_key_value_v2( STORE_A, context=context ), } """Each v2 entry point, reading a document that contains the refused compressor wherever that document type allows one.""" REFUSES_BAD = frozenset( name for name in ENTRY_POINTS if "Group" not in name and "group" not in name ) """The entry points whose document contains the refused compressor. The other entry points accept their document in every scope.""" @pytest.mark.parametrize("name", ENTRY_POINTS.keys()) def test_every_v2_entry_point_reads_in_the_scope_it_is_given(name: str) -> None: """Each v2 entry point accepts its document in an empty scope. In `CORE_V2` (also selected by `None`), it refuses the non-integer gzip level when the document contains it. This shows that each entry point reads in the scope it is given.""" accepted = _accepts(ENTRY_POINTS[name]) assert accepted(EMPTY) is True assert accepted(CORE_V2) is (name not in REFUSES_BAD) assert accepted(None) is (name not in REFUSES_BAD) @pytest.mark.parametrize("name", ENTRY_POINTS.keys()) def test_error_every_v2_entry_point_refuses_a_scope_of_another_format(name: str) -> None: """A v2 entry point given a v3 scope raises `TypeError`. A scope reads documents of one format, and a v3 scope claims nothing that a v2 document contains.""" with pytest.raises(TypeError, match="format"): ENTRY_POINTS[name](context=CORE_AND_EXTENSIONS) NOT_JSON = b"{not json" STORE_READERS: dict[str, Callable[..., object]] = { "ZarrV2ArrayMetadata.from_key_value": lambda context: zm.ZarrV2ArrayMetadata.from_key_value( {".zarray": NOT_JSON}, context=context ), "ZarrV2GroupMetadata.from_key_value": lambda context: zm.ZarrV2GroupMetadata.from_key_value( {".zgroup": NOT_JSON}, context=context ), "ZarrV2ConsolidatedMetadata.from_key_value": ( lambda context: zm.ZarrV2ConsolidatedMetadata.from_key_value( {".zmetadata": NOT_JSON}, context=context ) ), "node_metadata_from_key_value_v2": lambda context: zm.node_metadata_from_key_value_v2( {".zarray": NOT_JSON}, context=context ), } """Each v2 entry point that decodes a store's bytes, given bytes that are not JSON.""" @pytest.mark.parametrize("name", STORE_READERS.keys()) def test_error_a_v2_store_reader_refuses_a_scope_of_another_format_before_decoding( name: str, ) -> None: """A v2 entry point that reads a store's bytes raises `TypeError` for a v3 scope before it decodes the bytes, so the wrong scope is reported even when the bytes are not JSON. With a v2 scope, the same bytes raise `MetadataValidationError`.""" with pytest.raises(TypeError, match="format"): STORE_READERS[name](context=CORE_AND_EXTENSIONS) with pytest.raises(MetadataValidationError): STORE_READERS[name](context=CORE_V2) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_sentinel.py000066400000000000000000000071321527014137600314760ustar00rootroot00000000000000"""Tests for the pickling and copying behavior of the `UNSET` sentinel. The sentinel's contract is identity, so it must never be reconstructed from state. typing_extensions >= 4.16 pickles sentinels by reference (a lookup of the sentinel's name on its defining module), which preserves the singleton across process boundaries; these tests pin that behavior, since models hold `UNSET` as field values and must survive pickling and deep-copying. The model round-trip tests compare whole structures: a model's equality compares every member, and `UNSET` compares by identity, so an impostor sentinel produced by state-based pickling would fail the equality check. """ from __future__ import annotations import copy import pickle import pytest from typing_extensions import Sentinel from zarr_metadata.model import ( UNSET, ZarrV2ArrayMetadata, ZarrV2GroupMetadata, ZarrV3ArrayMetadata, ZarrV3GroupMetadata, ) # Whole-model cases covering the states we know are problematic for # serialization: every optional-key field in the UNSET (absent) state, the # same fields in the present state (including present-but-empty, which must # stay distinct from absent), and UNSET nested inside consolidated metadata. MODEL_CASES = { "array-v3-dimension-names-unset": ZarrV3ArrayMetadata.create_default(shape=(4,)), "array-v3-dimension-names-set": ZarrV3ArrayMetadata.create_default( shape=(2, 2), dimension_names=("x", None) ), "array-v2-attributes-unset": ZarrV2ArrayMetadata.create_default(shape=(4,)), "array-v2-attributes-empty": ZarrV2ArrayMetadata.create_default(shape=(4,), attributes={}), "group-v2-attributes-unset": ZarrV2GroupMetadata.create_default(), "group-v2-attributes-set": ZarrV2GroupMetadata.create_default(attributes={"a": 1}), "group-v3-consolidated-unset": ZarrV3GroupMetadata.create_default(), "group-v3-consolidated-with-unset-inside": ZarrV3GroupMetadata.create_default( consolidated_metadata={ "kind": "inline", "must_understand": False, "metadata": { "child": ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json(), "subgroup": ZarrV3GroupMetadata.create_default().to_json(), }, }, ), } def test_unset_pickle_round_trip_preserves_identity() -> None: restored = pickle.loads(pickle.dumps(UNSET)) assert restored is UNSET def test_unset_copy_preserves_identity() -> None: assert copy.copy(UNSET) is UNSET assert copy.deepcopy(UNSET) is UNSET @pytest.mark.parametrize("model", MODEL_CASES.values(), ids=MODEL_CASES.keys()) def test_model_pickle_round_trip( model: ZarrV2ArrayMetadata | ZarrV3ArrayMetadata | ZarrV2GroupMetadata | ZarrV3GroupMetadata, ) -> None: restored = pickle.loads(pickle.dumps(model)) assert restored == model @pytest.mark.parametrize("model", MODEL_CASES.values(), ids=MODEL_CASES.keys()) def test_model_deepcopy( model: ZarrV2ArrayMetadata | ZarrV3ArrayMetadata | ZarrV2GroupMetadata | ZarrV3GroupMetadata, ) -> None: assert copy.deepcopy(model) == model def test_non_importable_sentinel_fails_to_pickle() -> None: """Sentinels pickle by reference, never by state. A sentinel that is not an importable attribute of its module has no reference to pickle, so dumping it must fail loudly — a successful dump here would mean the implementation regressed to state-based pickling, which would produce identity-breaking impostor objects on the receiving side.""" local_sentinel = Sentinel("local_sentinel") with pytest.raises((pickle.PicklingError, TypeError)): pickle.dumps(local_sentinel) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_store_json.py000066400000000000000000000310301527014137600320340ustar00rootroot00000000000000"""Tests for where a non-finite number is allowed: in a node's attributes, as user data. zarr-python writes attributes with the defaults of Python's `json` module, so an attribute may hold `NaN`, `Infinity` or `-Infinity`. The model reads such a store, validates it, and writes it back the same way. Wherever the spec interprets a value, a non-finite number is refused when read. A document that the reader would refuse produces no model, so it is not written. """ from __future__ import annotations import json import math from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, Protocol import pytest from zarr_metadata.model import ( ZarrV2ArrayMetadata, ZarrV2ConsolidatedMetadata, ZarrV2GroupMetadata, ZarrV3ArrayMetadata, ZarrV3ConsolidatedMetadata, ZarrV3GroupMetadata, read_array_metadata_v2, read_array_metadata_v3, read_group_metadata_v2, read_group_metadata_v3, ) from zarr_metadata.problems import MetadataValidationError, ValidationProblem if TYPE_CHECKING: from collections.abc import Iterable class _Written(Protocol): """The output of a model write: store keys, each with its bytes.""" def items(self) -> Iterable[tuple[str, bytes]]: ... class _Stored(Protocol): """A model that writes itself to a store.""" def to_json(self) -> Mapping[str, object]: ... def to_key_value(self, *, indent: int | str | None = None) -> _Written: ... class _Reading(Protocol): """What a reader returns: every problem it found.""" @property def problems(self) -> tuple[ValidationProblem, ...]: ... ATTRIBUTES: dict[str, object] = { "_FillValue": math.nan, "valid_range": [-math.inf, math.inf], "cf": {"missing_value": -math.inf}, } def _held(attributes: object) -> tuple[bool, bool, bool]: """Report whether each non-finite number in `ATTRIBUTES` is still present, with arrays as tuples.""" assert isinstance(attributes, Mapping) fill = attributes["_FillValue"] cf = attributes["cf"] assert isinstance(cf, Mapping) return ( isinstance(fill, float) and math.isnan(fill), attributes["valid_range"] == (-math.inf, math.inf), cf["missing_value"] == -math.inf, ) _ARRAY: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) _INLINE: dict[str, object] = {"kind": "inline", "must_understand": False} def _stored(key: str, document: object) -> Mapping[str, bytes]: return {key: json.dumps(document).encode()} def _v3_array() -> Mapping[str, bytes]: return _stored("zarr.json", {**_ARRAY, "attributes": ATTRIBUTES}) def _v3_group() -> Mapping[str, bytes]: return _stored("zarr.json", {"zarr_format": 3, "node_type": "group", "attributes": ATTRIBUTES}) def _v3_consolidated() -> Mapping[str, bytes]: node = {**_ARRAY, "attributes": ATTRIBUTES} document = { "zarr_format": 3, "node_type": "group", "consolidated_metadata": {**_INLINE, "metadata": {"a": node}}, } return _stored("zarr.json", document) def _v2_array() -> Mapping[str, bytes]: zarray = dict(ZarrV2ArrayMetadata.create_default().to_json()) return {".zarray": json.dumps(zarray).encode(), ".zattrs": json.dumps(ATTRIBUTES).encode()} def _v2_group() -> Mapping[str, bytes]: return { ".zgroup": json.dumps({"zarr_format": 2}).encode(), ".zattrs": json.dumps(ATTRIBUTES).encode(), } def _v2_consolidated() -> Mapping[str, bytes]: document = { "zarr_consolidated_format": 1, "metadata": {".zgroup": {"zarr_format": 2}, ".zattrs": ATTRIBUTES}, } return {".zmetadata": json.dumps(document).encode()} def _consolidated_from_key_value(store: Mapping[str, bytes]) -> ZarrV3ConsolidatedMetadata: """The consolidated metadata of the group at `zarr.json` in `store`, with every document it lists read.""" return ZarrV3ConsolidatedMetadata( ZarrV3GroupMetadata.from_key_value(store).consolidated_metadata ) def _consolidated_node_attributes(store: Mapping[str, bytes]) -> object: return _consolidated_from_key_value(store).metadata["a"].attributes # Each case: the store, its reader, a function that reads the attributes # from a store, and the reader of the model's document, which finds no # problem with it. The type is declared so that each reader and lambda is # type-checked against its model. _Case = tuple[ Callable[[], Mapping[str, bytes]], Callable[[Mapping[str, bytes]], _Stored], Callable[[Mapping[str, bytes]], object], Callable[[object], _Reading] | None, ] _CASES: dict[str, _Case] = { "v3-array": ( _v3_array, ZarrV3ArrayMetadata.from_key_value, lambda store: ZarrV3ArrayMetadata.from_key_value(store).attributes, read_array_metadata_v3, ), "v3-group": ( _v3_group, ZarrV3GroupMetadata.from_key_value, lambda store: ZarrV3GroupMetadata.from_key_value(store).attributes, read_group_metadata_v3, ), "v3-consolidated-node": ( _v3_consolidated, ZarrV3GroupMetadata.from_key_value, _consolidated_node_attributes, read_group_metadata_v3, ), "v2-array": ( _v2_array, ZarrV2ArrayMetadata.from_key_value, lambda store: ZarrV2ArrayMetadata.from_key_value(store).attributes, read_array_metadata_v2, ), "v2-group": ( _v2_group, ZarrV2GroupMetadata.from_key_value, lambda store: ZarrV2GroupMetadata.from_key_value(store).attributes, read_group_metadata_v2, ), "v2-consolidated": ( _v2_consolidated, ZarrV2ConsolidatedMetadata.from_key_value, lambda store: ZarrV2ConsolidatedMetadata.from_key_value(store).metadata[""].attributes, None, ), } @pytest.mark.parametrize( ("store", "read", "attributes_in", "reader"), list(_CASES.values()), ids=list(_CASES) ) def test_attributes_holding_non_finite_numbers_round_trip_through_the_store( store: Callable[[], Mapping[str, bytes]], read: Callable[[Mapping[str, bytes]], _Stored], attributes_in: Callable[[Mapping[str, bytes]], object], reader: Callable[[object], _Reading] | None, ) -> None: stored = store() assert _held(attributes_in(stored)) == (True, True, True) model = read(stored) if reader is not None: assert reader(model.to_json()).problems == () written = dict(model.to_key_value().items()) assert b"NaN" in b"".join(written.values()) assert _held(attributes_in(written)) == (True, True, True) @pytest.mark.parametrize( ("key", "member"), [ ("extension", {"must_understand": False, "x": math.nan}), # Only a v3 group holds node documents inline. In an array, a member # with that name is an extra field that nothing in scope claims. ( "consolidated_metadata", {"must_understand": False, "metadata": {"a": {"attributes": {"x": math.nan}}}}, ), ], ids=["extension-field", "consolidated-in-an-array"], ) def test_an_unclaimed_member_holding_non_finite_numbers_round_trips_through_the_store( key: str, member: dict[str, object] ) -> None: """Nothing reads a member that no definition in scope claims, so like attributes it may hold `NaN` and the infinities: the model reads it, and writes it back as bare tokens.""" model = ZarrV3ArrayMetadata.from_key_value(_stored("zarr.json", {**_ARRAY, key: member})) written = model.to_key_value() assert b"NaN" in written["zarr.json"] assert ZarrV3ArrayMetadata.from_key_value(written) == model def test_a_sequence_is_written_as_an_array() -> None: # The writer writes the document that passed the reader's validation, # with arrays normalized the same way the reader normalizes them. model = ZarrV3GroupMetadata.create_default(attributes={"r": range(3)}) # pyright: ignore[reportArgumentType] written = json.loads(model.to_key_value()["zarr.json"]) assert written["attributes"] == {"r": [0, 1, 2]} _V2_ARRAY: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default().to_json()) # Each case: a reader, a store it refuses, and the location of the non-finite number. _Refused = tuple[ Callable[[Mapping[str, bytes]], object], Mapping[str, bytes], tuple[str | int, ...] ] _REFUSED: dict[str, _Refused] = { "fill-value": ( ZarrV3ArrayMetadata.from_key_value, _stored("zarr.json", {**_ARRAY, "fill_value": math.nan}), ("fill_value",), ), "codec-configuration": ( ZarrV3ArrayMetadata.from_key_value, _stored( "zarr.json", { **_ARRAY, "codecs": [ {"name": "bytes", "configuration": {"endian": "little", "scale": math.inf}} ], }, ), ("codecs", 0, "configuration", "scale"), ), # A group does not read the documents its consolidated metadata lists; # reading them finds the number. "consolidated-node": ( _consolidated_from_key_value, _stored( "zarr.json", { "zarr_format": 3, "node_type": "group", "consolidated_metadata": { **_INLINE, "metadata": {"a": {**_ARRAY, "fill_value": math.nan}}, }, }, ), ("metadata", "a", "fill_value"), ), # A v2 array's attributes live in `.zattrs`. Its `.zarray` must be RFC 8259 # JSON throughout, and zarr-python 3.0 once wrote a bare `NaN` there. "zarray": ( ZarrV2ArrayMetadata.from_key_value, _stored(".zarray", {**_V2_ARRAY, "fill_value": math.nan}), ("fill_value",), ), # Each `.zmetadata` entry is validated as the document type its key names. "zmetadata-entry": ( ZarrV2ConsolidatedMetadata.from_key_value, _stored( ".zmetadata", { "zarr_consolidated_format": 1, "metadata": {"a/.zattrs": {"x": math.nan}, "a/.zarray": {"fill_value": math.nan}}, }, ), ("metadata", "a/.zarray", "fill_value"), ), } @pytest.mark.parametrize(("read", "store", "loc"), list(_REFUSED.values()), ids=list(_REFUSED)) def test_error_a_non_finite_number_outside_attributes_is_located_when_read( read: Callable[[Mapping[str, bytes]], object], store: Mapping[str, bytes], loc: tuple[str | int, ...], ) -> None: # Python's decoder reads a bare `NaN` as a float. Where the spec # interprets a value, the document's validator reports its location. with pytest.raises(MetadataValidationError) as raised: read(store) assert [(problem.loc, problem.kind) for problem in raised.value.problems] == [ (loc, "invalid_value") ] @pytest.mark.parametrize( ("build", "problems"), [ ( lambda: ZarrV3ArrayMetadata.create_default( fill_value=math.nan, attributes={"x": math.nan} ), [(("fill_value",), "invalid_value")], ), ( lambda: ZarrV3GroupMetadata.create_default(attributes={"s": {1, 2}}), # pyright: ignore[reportArgumentType] [(("attributes", "s"), "invalid_type")], ), ( lambda: ZarrV3GroupMetadata.create_default(attributes={1: "a"}), # pyright: ignore[reportArgumentType] [(("attributes",), "invalid_type")], ), ( lambda: ZarrV3ArrayMetadata.create_default(shape=(2,), dimension_names=("x", "y")), [(("dimension_names",), "invalid_value")], ), ], ids=["non-finite-fill-value", "not-json", "non-string-key", "dimension-names-past-shape"], ) def test_error_a_document_the_reader_refuses_is_not_written( build: Callable[[], _Stored], problems: list[tuple[tuple[str | int, ...], str]] ) -> None: # A model is built by reading its document, so a document that the reader # refuses produces no model to write. When such documents were written, a # non-string key became a string, a value that is not JSON raised # `TypeError`, and the last case was refused only when read again. with pytest.raises(MetadataValidationError) as raised: build() assert [(problem.loc, problem.kind) for problem in raised.value.problems] == problems def test_error_store_bytes_nested_deeper_than_python_reads_are_invalid_json() -> None: # `json.loads` raises `RecursionError` on a hundred thousand `[`. This is # reported as an ingestion failure like any other. with pytest.raises(MetadataValidationError) as raised: ZarrV3ArrayMetadata.from_key_value({"zarr.json": b"[" * 100_000}) assert [(p.loc, p.kind) for p in raised.value.problems] == [(("zarr.json",), "invalid_json")] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_unclaimed_fields.py000066400000000000000000000165031527014137600331460ustar00rootroot00000000000000"""`unclaimed_fields`: every field of a node that nothing in its scope claims, at any depth, which is what a reader must understand before it opens the node.""" from __future__ import annotations from typing import Any, NotRequired import pytest from typing_extensions import TypedDict from zarr_metadata.definition import UnclaimedField from zarr_metadata.model import ( ZarrV2ArrayMetadata, ZarrV2GroupMetadata, ZarrV2NodeMetadata, ZarrV3ArrayMetadata, ZarrV3GroupMetadata, ZarrV3NodeMetadata, ) from zarr_metadata.v3 import CORE from zarr_metadata.v3.definition import ( ArrayExtraFieldDefinition, GroupExtraFieldDefinition, ZarrV3CodecField, ) ARRAY: dict[str, Any] = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) GROUP: dict[str, Any] = {"zarr_format": 3, "node_type": "group"} V2_ARRAY: dict[str, Any] = dict(ZarrV2ArrayMetadata.create_default(shape=(4,)).to_json()) ZSTD = {"name": "zstd", "configuration": {"level": 1, "checksum": False}} class _MirrorConfiguration(TypedDict, closed=True): """The value of an `acme.mirror` member: a codec, and the waiver.""" codec: ZarrV3CodecField must_understand: NotRequired[bool] MIRRORING = CORE.update( array_extra_fields=[ *CORE.array_extra_fields.values(), ArrayExtraFieldDefinition(name="acme.mirror", configuration=_MirrorConfiguration), ] ) """`CORE`, with a definition of an `acme.mirror` member that holds a codec.""" GROUP_MIRRORING = CORE.update( group_extra_fields=[ *CORE.group_extra_fields.values(), GroupExtraFieldDefinition(name="acme.mirror", configuration=_MirrorConfiguration), ] ) """`CORE`, with a definition of an `acme.mirror` member of a group that holds a codec.""" SHARD = { "name": "sharding_indexed", "configuration": { "chunk_shape": (2,), "codecs": ("bytes", ZSTD), "index_codecs": ({"name": "bytes", "configuration": {"endian": "little"}},), }, } @pytest.mark.parametrize( ("model", "unclaimed"), [ (ZarrV3ArrayMetadata(ARRAY, context=CORE), set()), (ZarrV3ArrayMetadata({**ARRAY, "codecs": ("bytes", ZSTD)}, context=CORE), {("codecs", 1)}), ( ZarrV3ArrayMetadata({**ARRAY, "codecs": (SHARD,)}, context=CORE), {("codecs", 0, "configuration", "codecs", 1)}, ), (ZarrV3ArrayMetadata({**ARRAY, "acme": {"x": 1}}, context=CORE), {("acme",)}), (ZarrV3ArrayMetadata({**ARRAY, "acme": {"must_understand": False}}, context=CORE), set()), (ZarrV3GroupMetadata({**GROUP, "acme": 1}, context=CORE), {("acme",)}), (ZarrV2ArrayMetadata({**V2_ARRAY, "compressor": {"id": "acme"}}), {("compressor",)}), (ZarrV2GroupMetadata.create_default(), set()), ( ZarrV3ArrayMetadata( {**ARRAY, "acme.mirror": {"codec": {"name": "acme.zip"}}}, context=MIRRORING ), {("acme.mirror", "codec")}, ), ( ZarrV3ArrayMetadata( {**ARRAY, "acme.mirror": {"codec": {"name": "acme.zip"}, "must_understand": False}}, context=MIRRORING, ), set(), ), ( ZarrV3GroupMetadata( {**GROUP, "acme.mirror": {"codec": {"name": "acme.zip"}}}, context=GROUP_MIRRORING ), {("acme.mirror", "codec")}, ), ( ZarrV3GroupMetadata( {**GROUP, "acme.mirror": {"codec": {"name": "acme.zip"}, "must_understand": False}}, context=GROUP_MIRRORING, ), set(), ), ], ids=[ "all-claimed", "a-codec", "a-codec-inside-a-shard", "an-extra-member", "a-waived-extra-member", "a-group-member", "a-v2-codec", "a-v2-group", "a-codec-inside-a-claimed-member", "a-codec-inside-a-claimed-member-that-waives", "a-codec-inside-a-claimed-group-member", "a-codec-inside-a-claimed-group-member-that-waives", ], ) def test_unclaimed_fields_are_what_a_reader_must_understand( model: Any, # noqa: ANN401 - a model of either format unclaimed: set[tuple[str | int, ...]], ) -> None: """`unclaimed_fields` maps the location of each field nothing in scope claims to the field, at any depth: a codec inside a shard is located inside the shard's configuration. An extra member that says `"must_understand": false` is left out with every field inside it, claimed or not, as the spec lets a reader ignore it. So a reader can open a node exactly when it is empty.""" found = model.unclaimed_fields assert set(found) == unclaimed assert all(isinstance(field, UnclaimedField) for field in found.values()) @pytest.mark.parametrize( ("kind", "document", "context"), [(ZarrV3ArrayMetadata, ARRAY, MIRRORING), (ZarrV3GroupMetadata, GROUP, GROUP_MIRRORING)], ids=["array", "group"], ) @pytest.mark.parametrize( ("member", "wider", "narrower"), [ ( {"codec": {"name": "gzip", "configuration": {"level": 1}}}, {("acme.mirror",)}, set(), ), ({"codec": {"name": "acme.zip"}}, {("acme.mirror",)}, {("acme.mirror", "codec")}), ({"codec": {"name": "acme.zip"}, "must_understand": False}, set(), set()), ], ids=["a-claimed-codec", "an-unclaimed-codec", "waived"], ) def test_a_model_that_refines_another_has_no_more_to_understand( kind: Any, # noqa: ANN401 - a v3 node model class document: dict[str, Any], context: Any, # noqa: ANN401 - a v3 scope member: dict[str, Any], wider: set[tuple[str | int, ...]], narrower: set[tuple[str | int, ...]], ) -> None: """A scope that claims more of a document reads a model that refines the model of a scope that claims less, and its `unclaimed_fields` lie at or inside those of the model it refines, for an array and for a group. Here `CORE` leaves an unwaived `acme.mirror` member unclaimed as a whole; the scope that claims the member leaves at most the codec inside it, which lies inside the member, and a waived member is left out by both.""" claimed_less = kind({**document, "acme.mirror": member}, context=CORE) claimed_more = kind({**document, "acme.mirror": member}, context=context) assert claimed_more.refines(claimed_less) assert set(claimed_less.unclaimed_fields) == wider assert set(claimed_more.unclaimed_fields) == narrower assert all( any(loc[: len(outer)] == outer for outer in claimed_less.unclaimed_fields) for loc in claimed_more.unclaimed_fields ) def test_node_type_tells_a_type_checker_which_model_a_node_is() -> None: """`node_type` and `zarr_format` are literal types, so branching on them narrows a union of node models for a type checker as well as at run time. pyright checks this test: it reads `shape` only on arrays, and `consolidated()` only on a v3 group.""" nodes: list[ZarrV2NodeMetadata | ZarrV3NodeMetadata] = [ ZarrV3ArrayMetadata.create_default(shape=(4,)), ZarrV3GroupMetadata.create_default(), ZarrV2ArrayMetadata.create_default(shape=(4,)), ZarrV2GroupMetadata.create_default(), ] shapes = [node.shape for node in nodes if node.node_type == "array"] assert shapes == [(4,), (4,)] for node in nodes: if node.zarr_format == 3 and node.node_type == "group": assert node.consolidated() is None zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/model/test_v2_scope_reads.py000066400000000000000000000105731527014137600325560ustar00rootroot00000000000000"""Tests that the v2 array validator reads its dtype, codecs and fill value in the v2 scope.""" from __future__ import annotations from typing import Any import pytest from zarr_metadata.model import ZarrV2ArrayMetadata, read_array_metadata_v2 from zarr_metadata.problems import MetadataValidationError Loc = tuple[str | int, ...] BASE: dict[str, Any] = dict(ZarrV2ArrayMetadata.create_default().to_json()) @pytest.mark.parametrize( "changes", [ {"dtype": " None: """Each of these validates with no problem: a dtype that the scope reads with a fill value that its family allows, a compressor or filters that the scope reads or leaves unclaimed, and a null fill value.""" assert read_array_metadata_v2({**BASE, **changes}).problems == () @pytest.mark.parametrize( ("changes", "at", "kind"), [ ({"dtype": "float32"}, ("dtype",), "invalid_value"), ({"dtype": " None: """A dtype, fill value, compressor or filter that the scope refuses produces one problem in the document, at the wrong field or parameter. Previously, the string content of a dtype and the parameters of a codec were not checked.""" problems = read_array_metadata_v2({**BASE, **changes}).problems assert [(p.loc, p.kind) for p in problems] == [(at, kind)] def test_create_default_keeps_zero_where_the_family_takes_it() -> None: """When `create_default` is given a dtype and no fill value, it uses `0` if the dtype family allows it and `null` otherwise. A given fill value is kept.""" assert ZarrV2ArrayMetadata.create_default(dtype="|b1").fill_value is None assert ZarrV2ArrayMetadata.create_default(dtype=" None: """`ZarrV2ArrayMetadata` validates itself when built, so a dtype that the scope refuses raises like any other problem.""" with pytest.raises(MetadataValidationError, match="typestr"): ZarrV2ArrayMetadata.create_default(dtype="float32") zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/test_contexts.py000066400000000000000000000325521527014137600304300ustar00rootroot00000000000000"""The scope of each format: `ZarrV3Context` and `ZarrV2Context` are separate classes that implement the `Context` protocol, and behave alike.""" from __future__ import annotations import copy import dataclasses import pickle from collections.abc import Callable, Iterable, Mapping from typing import Any import pytest from typing_extensions import TypedDict from zarr_metadata.definition import Context, ScopeConflictError, UnclaimedField, resolve from zarr_metadata.definition._context import slot_kinds from zarr_metadata.model import ( ZarrV2ArrayMetadata, ZarrV3ArrayMetadata, read_array_metadata_v2, read_array_metadata_v3, ) from zarr_metadata.v2 import CORE_V2, GZIP_CODEC_V2, ZarrV2Context from zarr_metadata.v2._context import _KINDS as V2_KINDS from zarr_metadata.v2.definition import ( ZarrV2CodecDefinition, ZarrV2ContextSlots, ZarrV2DataTypeDefinition, ) from zarr_metadata.v3 import CORE, GZIP_CODEC, ZarrV3Context from zarr_metadata.v3._context import _KINDS as V3_KINDS from zarr_metadata.v3._context import is_v3_kind from zarr_metadata.v3.definition import CodecDefinition, DataTypeDefinition, ZarrV3ContextSlots def _other(definition: Any) -> Any: # noqa: ANN401 - a definition of either format """A second definition under the name of `definition`, with other rules.""" return dataclasses.replace(definition, rules=lambda configuration, nested: iter(())) ARRAY = ZarrV3ArrayMetadata.create_default().to_json() V2_ARRAY = ZarrV2ArrayMetadata.create_default().to_json() FORMATS = [ pytest.param(ZarrV3Context, CORE, CodecDefinition, GZIP_CODEC, 3, id="v3"), pytest.param(ZarrV2Context, CORE_V2, ZarrV2CodecDefinition, GZIP_CODEC_V2, 2, id="v2"), ] @pytest.mark.parametrize(("cls", "core", "kind", "gzip", "zarr_format"), FORMATS) def test_the_scope_of_each_format_behaves_alike( cls: Any, # noqa: ANN401 - either scope class core: Context, kind: Any, # noqa: ANN401 - either codec kind gzip: Any, # noqa: ANN401 - either gzip definition zarr_format: int, ) -> None: """Both scope classes implement `Context` in the same way: the format they read, `claimant`, `definitions_of`, a view per slot, `update` replacing a slot, `union`, equality and hashing by what each slot files, a short repr, and pickling and copying. A field is read in either one with `resolve`.""" built = cls(codecs=[gzip]) assert built.zarr_format == zarr_format assert built.claimant(kind, "gzip") is gzip assert dict(built.definitions_of(kind)) == {"gzip": gzip} assert dict(built.codecs) == {"gzip": gzip} assert built == cls(codecs=[gzip, gzip]) assert hash(built) == hash(cls(codecs=[gzip])) assert built != cls() assert repr(built) == f"{cls.__name__}(<1 definitions>)" replaced = built.update(codecs=[_other(gzip)]) assert replaced.claimant(kind, "gzip") is not gzip assert built.union(cls()) == built assert built.union() == built assert pickle.loads(pickle.dumps(core)) == core assert copy.copy(core) is core assert copy.deepcopy(core) is core field, _ = resolve({"id": "acme"} if zarr_format == 2 else "acme", kind, cls()) assert isinstance(field, UnclaimedField) @pytest.mark.parametrize(("cls", "core", "kind", "gzip", "zarr_format"), FORMATS) def test_update_replaces_each_slot_it_is_given_and_keeps_the_rest( cls: Any, # noqa: ANN401 - either scope class core: Any, # noqa: ANN401 - either core scope kind: Any, # noqa: ANN401 - either codec kind gzip: Any, # noqa: ANN401 - either gzip definition zarr_format: int, ) -> None: """`update` returns the scope with each slot it is given replaced whole, and every other slot kept. A slot is given as definitions, or as a mapping from each definition's name to it, so a slot of a scope passes back in as it is, or changed by name. The scope it is called on is unchanged.""" only = core.update(codecs=[gzip]) assert dict(only.codecs) == {"gzip": gzip} assert only.data_types == core.data_types assert core.update() == core assert core.update(codecs=core.codecs) == core assert cls(codecs={"gzip": gzip}) == cls(codecs=[gzip]) other = _other(gzip) replaced = core.update(codecs={**core.codecs, "gzip": other}) assert replaced.codecs["gzip"] is other assert {**replaced.codecs, "gzip": gzip} == dict(core.codecs) assert dict(cls().update(codecs=[*cls().codecs.values(), gzip]).codecs) == {"gzip": gzip} assert core.codecs["gzip"] is gzip @pytest.mark.parametrize(("cls", "core", "kind", "gzip", "zarr_format"), FORMATS) def test_error_a_slot_given_as_a_mapping_files_each_definition_under_its_name( cls: Any, # noqa: ANN401 - either scope class core: Any, # noqa: ANN401 - either core scope kind: Any, # noqa: ANN401 - either codec kind gzip: Any, # noqa: ANN401 - either gzip definition zarr_format: int, ) -> None: """A slot given as a mapping keys each definition by its own name, the name the scope files it under. Another key raises `TypeError` naming both, in the constructor and in `update`, rather than filing the definition under a name it does not read, or ignoring a misspelled key.""" with pytest.raises( TypeError, match="codecs files each definition under its name, got 'gzip' under 'zstd'" ): cls(codecs={"zstd": gzip}) with pytest.raises(TypeError, match="got 'gzip' under 'gzipp'"): core.update(codecs={**core.codecs, "gzipp": _other(gzip)}) @pytest.mark.parametrize(("cls", "core", "kind", "gzip", "zarr_format"), FORMATS) def test_error_a_scope_has_only_the_slots_it_declares( cls: Any, # noqa: ANN401 - either scope class core: Any, # noqa: ANN401 - either core scope kind: Any, # noqa: ANN401 - either codec kind gzip: Any, # noqa: ANN401 - either gzip definition zarr_format: int, ) -> None: """A slot the scope does not declare raises `TypeError` naming it, in the constructor and in `update`, as a type checker reports it.""" with pytest.raises(TypeError, match=f"a v{zarr_format} scope has no slot 'codec'"): cls(codec=[gzip]) with pytest.raises(TypeError, match=f"a v{zarr_format} scope has no slot 'codec'"): core.update(codec=[gzip]) @pytest.mark.parametrize(("cls", "core", "kind", "gzip", "zarr_format"), FORMATS) def test_error_union_refuses_one_name_filed_two_ways( cls: Any, # noqa: ANN401 - either scope class core: Context, kind: Any, # noqa: ANN401 - either codec kind gzip: Any, # noqa: ANN401 - either gzip definition zarr_format: int, ) -> None: """Scopes of either format that file different definitions under one name in a slot have no union: `union` raises `ScopeConflictError` naming the kind and name. The two definitions read the same TypedDict, so the message says they differ rather than showing two identical descriptions.""" with pytest.raises(ScopeConflictError) as raised: cls(codecs=[gzip]).union(cls(codecs=[_other(gzip)])) (conflict,) = raised.value.conflicts assert conflict.key == (kind, "gzip") assert "'gzip': two different definitions, each " in str(raised.value) @pytest.mark.parametrize(("cls", "core", "kind", "gzip", "zarr_format"), FORMATS) def test_error_a_slot_refuses_a_definition_of_the_other_format( cls: Any, # noqa: ANN401 - either scope class core: Context, kind: Any, # noqa: ANN401 - either codec kind gzip: Any, # noqa: ANN401 - either gzip definition zarr_format: int, ) -> None: """A scope's `codecs` slot holds its own format's codecs only: the other format's gzip raises `TypeError`.""" other = GZIP_CODEC_V2 if zarr_format == 3 else GZIP_CODEC with pytest.raises(TypeError, match="codecs holds"): cls(codecs=[other]) @pytest.mark.parametrize(("cls", "core", "kind", "gzip", "zarr_format"), FORMATS) def test_error_a_scope_refuses_a_kind_of_the_other_format( cls: Any, # noqa: ANN401 - either scope class core: Any, # noqa: ANN401 - either core scope kind: Any, # noqa: ANN401 - either codec kind gzip: Any, # noqa: ANN401 - either gzip definition zarr_format: int, ) -> None: """A scope holds the kinds of its own format only, so asking it for the definitions of the other format's codec kind, or for the one that claims a name, raises `TypeError` rather than answering that there are none.""" other = ZarrV2CodecDefinition if zarr_format == 3 else CodecDefinition with pytest.raises(TypeError, match="kinds of its format"): core.definitions_of(other) with pytest.raises(TypeError, match="kinds of its format"): core.claimant(other, "gzip") other_data_type = ZarrV2DataTypeDefinition if zarr_format == 3 else DataTypeDefinition with pytest.raises(TypeError, match="kinds of its format"): core.claimant(other_data_type, "r*") class _LooksLikeAScope: """A value with the `Context` protocol's members that is not one of this package's scope classes.""" zarr_format = 3 def claimant(self, kind: object, name: str) -> None: return None class _BrokenScope: """A value whose `zarr_format` raises.""" @property def zarr_format(self) -> int: raise RuntimeError class _BooleanFormat: """A value whose `zarr_format` is a boolean, which no format is.""" zarr_format = True @pytest.mark.parametrize( ("given", "named"), [ ("nope", "got 'nope'"), (_LooksLikeAScope(), "got a _LooksLikeAScope, a v3 scope"), (_BrokenScope(), "got <"), (_BooleanFormat(), "got <"), ], ids=["not-a-scope", "a-look-alike", "a-broken-format", "a-boolean-format"], ) @pytest.mark.parametrize( ("read", "document", "cls"), [ (read_array_metadata_v3, ARRAY, "ZarrV3Context"), (read_array_metadata_v2, V2_ARRAY, "ZarrV2Context"), ], ids=["v3", "v2"], ) def test_error_a_reader_refuses_what_is_not_a_scope_of_its_format( read: Callable[..., object], document: object, cls: str, given: object, named: str, ) -> None: """A reader of either format takes a scope of its format's class, or None. Anything else raises `TypeError` naming the class the reader takes and what it was given, rather than failing inside the read or contradicting itself.""" with pytest.raises(TypeError, match=f"a {cls}, {named}"): read(document, context=given) @pytest.mark.parametrize( ("core", "given", "zarr_format"), [(CORE, "nope", 3), (CORE, CORE_V2, 3), (CORE_V2, "nope", 2), (CORE_V2, CORE, 2)], ids=["v3-not-a-scope", "v3-a-v2-scope", "v2-not-a-scope", "v2-a-v3-scope"], ) def test_error_union_refuses_what_is_not_a_scope_of_its_format( core: Any, # noqa: ANN401 - either core scope given: object, zarr_format: int, ) -> None: """A scope's `union` takes scopes of its own format only. A scope of the other format, or a value that is not a scope, raises `TypeError` naming what was given, for a v3 and a v2 scope.""" with pytest.raises( TypeError, match=f"a union of v{zarr_format} scopes takes v{zarr_format} scopes, got", ): core.union(given) @pytest.mark.parametrize("cls", [ZarrV3Context, ZarrV2Context], ids=["v3", "v2"]) def test_each_declared_slot_has_a_view(cls: Any) -> None: # noqa: ANN401 - either scope class """Each scope class reads its slots from its `...Slots` TypedDict, so the slots are declared once. The slots hold every kind of the format, as its own check of a kind knows them. Each slot has a view of the same name, which is the slot that holds its kind: a type checker cannot match a view's name to a declared slot, so this test does.""" slots_type = ZarrV3ContextSlots if cls is ZarrV3Context else ZarrV2ContextSlots assert list(cls.kinds) == list(slots_type.__annotations__) assert set(cls.kinds.values()) == (V3_KINDS if cls is ZarrV3Context else V2_KINDS) scope: Any = CORE if cls is ZarrV3Context else CORE_V2 for slot, kind in cls.kinds.items(): assert getattr(scope, slot) == scope.definitions_of(kind) @pytest.mark.parametrize( "annotation", [ list[CodecDefinition[Any]], Iterable[CodecDefinition[Any]], Iterable[CodecDefinition[Any]] | Mapping[str, DataTypeDefinition[Any]], Iterable[CodecDefinition[Any]] | Mapping[int, CodecDefinition[Any]], Iterable[CodecDefinition[Any]] | Mapping[str, CodecDefinition[Any]] | None, ], ids=["a-list", "no-mapping", "two-kinds", "keyed-by-integers", "or-none"], ) def test_error_a_slot_is_declared_as_definitions_of_a_kind_or_a_mapping_of_them( annotation: object, ) -> None: """`slot_kinds` reads a slot's kind from its annotation, `Iterable[K] | Mapping[str, K]`, with one kind `K` in both. Any other annotation raises `TypeError` when the scope class is defined, so a wrong declaration cannot pass unnoticed.""" wrong = type("Wrong", (), {"__annotations__": {"codecs": annotation}}) with pytest.raises( TypeError, match=r"Wrong\.codecs: a slot is annotated Iterable\[K\] \| Mapping\[str, K\]", ): slot_kinds(wrong, is_v3_kind) def test_error_a_slot_holds_a_kind_of_its_format() -> None: """`slot_kinds` checks each slot's kind with the format's own check, so a v3 scope that declared a slot of v2 codecs raises `TypeError` when the scope class is defined.""" class Mixed(TypedDict, total=False): codecs: Iterable[ZarrV2CodecDefinition[Any]] | Mapping[str, ZarrV2CodecDefinition[Any]] with pytest.raises(TypeError, match=r"Mixed\.codecs: ZarrV2CodecDefinition is not a kind"): slot_kinds(Mixed, is_v3_kind) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/test_docs.py000066400000000000000000000055561527014137600275150ustar00rootroot00000000000000"""The python code blocks in the README and the docs run without error. As in zarr-python's docs tests, a block that runs declares `test="true"`, and a `session` name. The blocks of one session in one file run in order, in one namespace, so a block can use what an earlier block of its session defined. A block that cannot run says so with `exec="false"` and a `reason`. """ from __future__ import annotations from collections import defaultdict from pathlib import Path import pytest from pytest_examples import CodeExample, EvalExample, find_examples from zarr_metadata.model import ZarrV3ArrayMetadata PACKAGE_ROOT = Path(__file__).parent.parent SOURCES = (PACKAGE_ROOT / "README.md", PACKAGE_ROOT / "docs") def _examples() -> list[CodeExample]: """The python blocks of the README and the docs, each file's blocks in the order they are written.""" return list(find_examples(*SOURCES)) def _sessions() -> dict[tuple[Path, str], list[CodeExample]]: """The blocks that declare `test="true"`, grouped by file and session.""" sessions: defaultdict[tuple[Path, str], list[CodeExample]] = defaultdict(list) for example in _examples(): settings = example.prefix_settings() if settings.get("test") == "true": sessions[example.path, settings.get("session", "_default")].append(example) return dict(sessions) SESSIONS = _sessions() @pytest.mark.parametrize( "key", list(SESSIONS), ids=[f"{path.relative_to(PACKAGE_ROOT)}:{session}" for path, session in SESSIONS], ) def test_each_session_of_code_blocks_runs( key: tuple[Path, str], eval_example: EvalExample, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: """The blocks of each session run in order, in one namespace, without an error. They run in a temporary working directory, since a block may write a file, such as a JSON Schema. The directory holds the `zarr.json` of an array, which a block may read, as a reader reads one from disk.""" monkeypatch.chdir(tmp_path) (tmp_path / "zarr.json").write_bytes( ZarrV3ArrayMetadata.create_default(shape=(4,)).to_key_value()["zarr.json"] ) module_globals: dict[str, object] = {} for example in SESSIONS[key]: module_globals.update(eval_example.run(example, module_globals=module_globals)) def test_every_python_block_says_whether_it_runs() -> None: """Every python block in the README and the docs declares `test="true"`, or `exec="false"` with a reason, so a block cannot skip these tests by accident.""" unmarked = [ f"{example.path.relative_to(PACKAGE_ROOT)}:{example.start_line}" for example in _examples() if example.prefix_settings().get("test") != "true" and not ( example.prefix_settings().get("exec") == "false" and example.prefix_settings().get("reason", "").strip() != "" ) ] assert unmarked == [] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/test_json_schema.py000066400000000000000000001055701527014137600310530ustar00rootroot00000000000000"""JSON Schemas for what the package reads: a TypedDict, a field in a scope, and a `zarr.json`. Each schema is tested against the reader it is generated from. The schema accepts every value the reader finds no problem with. The schema also expresses what the type declares: a bound, a key that a closed TypedDict does not declare, a name that a scope claims. The schema does not express constraints that only a rule checks. """ from __future__ import annotations import json import types from collections.abc import Callable, Mapping from typing import Annotated, Any, Literal, NewType, NotRequired, cast import pytest from annotated_types import Ge, Gt, Interval, Le, Lt, MinLen from hypothesis import HealthCheck, event, given, settings from hypothesis import strategies as st from jsonschema import Draft202012Validator from typing_extensions import Doc, TypeAliasType, TypedDict from tests.v3.test_every_definition import CASES, KINDS from zarr_metadata._typed_json import Schemas from zarr_metadata.definition import ( Definition, resolve, ) from zarr_metadata.json_schema import ( field_json_schema_v3, json_schema, node_metadata_json_schema_v3, ) from zarr_metadata.model import read_node_metadata_v3 from zarr_metadata.typed_json import JSONValue, check from zarr_metadata.v2.definition import CORE_V2 from zarr_metadata.v3.codec.gzip import GzipCodecConfiguration from zarr_metadata.v3.definition import ( CORE, CORE_AND_EXTENSIONS, ChunkGridDefinition, ChunkKeyEncodingDefinition, CodecDefinition, DataTypeDefinition, StorageTransformerDefinition, ZarrV3Context, ZarrV3Kind, ) DIALECT = "https://json-schema.org/draft/2020-12/schema" # Special forms called at run time. They are typed loosely because a type # checker would check their arguments as it does in an annotation. _annotated: Any = Annotated _new_type: Callable[[str, object], object] = cast("Any", NewType) _alias_type: Callable[[str, object], object] = cast("Any", TypeAliasType) Level = TypeAliasType("Level", Annotated[int, Interval(ge=0, le=9)]) Tree = TypeAliasType("Tree", "int | tuple[Tree, ...]") Name = NewType("Name", str) Count = _new_type("Count", Annotated[int, Ge(0)]) class Inner(TypedDict, closed=True): x: int class Open(TypedDict, closed=False): x: NotRequired[int] class Extra(TypedDict, extra_items=int): x: str class ExtraJSON(TypedDict, extra_items=JSONValue): x: str class Node(TypedDict, closed=True): children: tuple[Node, ...] def _closed(name: str, annotations: dict[str, object]) -> type: """A closed TypedDict with these required keys, created here so its annotations resolve in this module.""" base: object = TypedDict namespace = {"__annotations__": annotations, "__module__": __name__} return types.new_class(name, (base,), {"closed": True}, lambda body: body.update(namespace)) def _holding(annotation: object) -> type: """A closed TypedDict with one required key, `value`, of type `annotation`.""" return _closed("Holder", {"value": annotation}) # A second class named `Inner`. The schema must tell it apart from the first. Both = _closed("Both", {"first": Inner, "second": _closed("Inner", {"y": str})}) def _held(member: dict[str, Any], defs: dict[str, Any] | None = None) -> dict[str, Any]: """The schema of `_holding(annotation)`, where `member` is the schema of `annotation`.""" schema: dict[str, Any] = { "$schema": DIALECT, "type": "object", "properties": {"value": member}, "required": ["value"], "additionalProperties": False, } return schema if defs is None else {**schema, "$defs": defs} INNER = { "type": "object", "properties": {"x": {"type": "integer"}}, "required": ["x"], "additionalProperties": False, } SHAPES: list[tuple[str, type, dict[str, Any]]] = [ ("int", _holding(int), _held({"type": "integer"})), ("float", _holding(float), _held({"type": "number"})), ("bool", _holding(bool), _held({"type": "boolean"})), ("str", _holding(str), _held({"type": "string"})), ("null", _holding(None), _held({"type": "null"})), ("json", _holding(JSONValue), _held({})), ("one-value", _holding(Literal["a"]), _held({"const": "a"})), # Sorted in the checker's order, so the order in the source does not matter. ("values", _holding(Literal["b", "a", 1]), _held({"enum": ["a", "b", 1]})), ("array", _holding(tuple[int, ...]), _held({"type": "array", "items": {"type": "integer"}})), ("array-of-json", _holding(tuple[JSONValue, ...]), _held({"type": "array"})), ( "pair", _holding(tuple[int, str]), _held( { "type": "array", "prefixItems": [{"type": "integer"}, {"type": "string"}], "items": False, "minItems": 2, } ), ), ("empty-array", _holding(tuple[()]), _held({"type": "array", "maxItems": 0})), ( "union", _holding(int | None), _held({"anyOf": [{"type": "integer"}, {"type": "null"}]}), ), ( "mapping", _holding(Mapping[str, int]), _held({"type": "object", "additionalProperties": {"type": "integer"}}), ), ("mapping-of-json", _holding(Mapping[str, JSONValue]), _held({"type": "object"})), ("newtype", _holding(Name), _held({"type": "string"})), ( "interval", _holding(Annotated[int, Interval(ge=0, le=9)]), _held({"type": "integer", "minimum": 0, "maximum": 9}), ), ( "exclusive-bounds", _holding(Annotated[float, Gt(0), Lt(1)]), _held({"type": "number", "exclusiveMinimum": 0, "exclusiveMaximum": 1}), ), ( "doc", _holding(Annotated[int, Ge(0), Doc("a count")]), _held({"type": "integer", "description": "a count", "minimum": 0}), ), # Metadata that is not a `Doc` adds nothing to the schema. ("note", _holding(Annotated[str, "a note"]), _held({"type": "string"})), # A value must meet its type's bounds and its `NewType`'s bounds. When # both set the same keyword, the stricter bound is kept. ( "bounds-on-a-bounded-newtype", _holding(_annotated[Count, Ge(-5), Lt(9)]), _held({"type": "integer", "minimum": 0, "exclusiveMaximum": 9}), ), ( "alias", _holding(Level), _held( {"$ref": "#/$defs/Level"}, {"Level": {"type": "integer", "minimum": 0, "maximum": 9}}, ), ), ( "alias-holding-itself", _holding(Tree), _held( {"$ref": "#/$defs/Tree"}, { "Tree": { "anyOf": [ {"type": "integer"}, {"type": "array", "items": {"$ref": "#/$defs/Tree"}}, ] } }, ), ), ("typeddict", _holding(Inner), _held({"$ref": "#/$defs/Inner"}, {"Inner": INNER})), ( "open", Open, {"$schema": DIALECT, "type": "object", "properties": {"x": {"type": "integer"}}}, ), ( "extra-items", Extra, { "$schema": DIALECT, "type": "object", "properties": {"x": {"type": "string"}}, "required": ["x"], "additionalProperties": {"type": "integer"}, }, ), ( "extra-items-of-json", ExtraJSON, { "$schema": DIALECT, "type": "object", "properties": {"x": {"type": "string"}}, "required": ["x"], }, ), # A recursive TypedDict is referred to with `$ref`, not written inline. ( "holding-itself", Node, { "$schema": DIALECT, "$ref": "#/$defs/Node", "$defs": { "Node": { "type": "object", "properties": { "children": {"type": "array", "items": {"$ref": "#/$defs/Node"}} }, "required": ["children"], "additionalProperties": False, } }, }, ), ( "one-name-two-classes", Both, { "$schema": DIALECT, "type": "object", "properties": { "first": {"$ref": "#/$defs/Inner"}, "second": {"$ref": "#/$defs/Inner2"}, }, "required": ["first", "second"], "additionalProperties": False, "$defs": { "Inner": INNER, "Inner2": { "type": "object", "properties": {"y": {"type": "string"}}, "required": ["y"], "additionalProperties": False, }, }, }, ), ( "a-definition-s-configuration", GzipCodecConfiguration, { "$schema": DIALECT, "type": "object", "properties": {"level": {"type": "integer", "minimum": 0, "maximum": 9}}, "required": ["level"], "additionalProperties": False, }, ), ] @pytest.mark.parametrize( ("shape", "expected"), [case[1:] for case in SHAPES], ids=[case[0] for case in SHAPES] ) def test_json_schema_writes_what_check_reads(shape: type, expected: dict[str, Any]) -> None: schema = json_schema(shape) assert schema == expected Draft202012Validator.check_schema(schema) assert json.loads(json.dumps(schema)) == schema def test_json_schema_refuses_what_is_not_a_typeddict() -> None: with pytest.raises(TypeError, match="is not a TypedDict"): json_schema(int) def test_json_schema_refuses_what_check_cannot_read() -> None: unread = _holding(bytes) with pytest.raises(TypeError) as raised: check({}, unread) with pytest.raises(TypeError, match=str(raised.value)): json_schema(unread) def test_error_a_schema_that_fails_to_write_leaves_nothing_behind() -> None: # A schema is reserved before it is written, so that it can refer to # itself. The reservation is dropped when writing fails, so that nothing # refers to an empty schema, which would accept anything. schemas = Schemas() unread = _closed("Unread", {"value": bytes}) with pytest.raises(TypeError, match="is not a shape JSON takes"): schemas.of(unread) assert schemas.document({}) == {"$schema": DIALECT} with pytest.raises(TypeError, match="is not a shape JSON takes"): schemas.of(unread) # The schema it wrote for a nested shape before it failed is also # dropped, and so are the uses it counted. A later root of that shape, # used once, is written inline. held = _closed("Held", {"x": int}) with pytest.raises(TypeError, match="is not a shape JSON takes"): schemas.of(_closed("Partly", {"held": held, "value": bytes})) assert schemas.document({}) == {"$schema": DIALECT} assert schemas.document(schemas.of(held)) == { "$schema": DIALECT, **Schemas().object_of(held), } def test_json_schema_refuses_metadata_check_does_not_hold_a_value_to() -> None: with pytest.raises(TypeError, match="is not a constraint the checker reads"): json_schema(_holding(Annotated[tuple[int, ...], MinLen(1)])) _PROPERTY = settings(max_examples=300, deadline=None, suppress_health_check=[HealthCheck.too_slow]) _MARKERS: dict[str, Callable[[int], object]] = {"ge": Ge, "gt": Gt, "le": Le, "lt": Lt} _LOW = st.none() | st.tuples(st.sampled_from(("ge", "gt")), st.integers(-3, 3)) _HIGH = st.none() | st.tuples(st.sampled_from(("le", "lt")), st.integers(-3, 3)) _LAYERS = st.lists( st.tuples(st.sampled_from(("annotated", "newtype", "alias")), _LOW, _HIGH), min_size=1, max_size=3, ) _NUMBERS = st.integers(-5, 5) | st.floats(-5, 5, allow_nan=False) | st.sampled_from((True, "1")) def _integral_as_int(value: object) -> object: """`value` as JSON Schema reads a number: `1.0` is the integer 1.""" return int(value) if isinstance(value, float) and value.is_integer() else value @_PROPERTY @given(base=st.sampled_from((int, float)), layers=_LAYERS, values=st.lists(_NUMBERS, max_size=8)) def test_bounds_in_layers_are_written_as_check_holds_a_value_to_them( base: type, layers: list[tuple[str, object, object]], values: list[object] ) -> None: # Bounds on a number, on a `NewType` of it, or on an alias of it, in # several layers. The schema accepts what `check` accepts, where a number # with no fraction is read as the integer it equals. annotation: object = base for index, (how, low, high) in enumerate(layers): sides = [cast("tuple[str, int]", side) for side in (low, high) if side is not None] bounds = [_MARKERS[name](bound) for name, bound in sides] bounded = _annotated[(annotation, *bounds)] if len(bounds) != 0 else annotation if how == "newtype": annotation = _new_type(f"Bounded{index}", bounded) elif how == "alias": annotation = _alias_type(f"Bounded{index}", bounded) else: annotation = bounded holder = _holding(annotation) try: schema = json_schema(holder) except TypeError: # A second bound on the same side, which neither `json_schema` nor `check` supports. event("refused") with pytest.raises(TypeError): check({}, holder) return validator = Draft202012Validator(schema) for value in values: accepted = check({"value": _integral_as_int(value)}, holder)[1] == () event("accepted" if accepted else "refused a value") assert validator.is_valid(cast("Any", {"value": value})) == accepted, (value, schema) # --- values changed in one place -------------------------------------------- _GONE = object() _NAMES = ("r16", "r16\n", "r*", "gzip", "bytes", "crc32c", "int8", "regular", "default", "acme.x") _KEYS = ("name", "configuration", "must_understand", "level", "x") _SCALARS = ( st.none() | st.booleans() | st.integers(-3, 300) | st.floats(-3, 3, allow_nan=False) | st.sampled_from(_NAMES) | st.text(max_size=3) ) _VALUES = st.recursive( _SCALARS, lambda inner: ( st.lists(inner, max_size=2) | st.dictionaries(st.sampled_from(_KEYS) | st.text(max_size=2), inner, max_size=2) ), max_leaves=4, ) def _places(value: object, path: tuple[str | int, ...] = ()) -> list[tuple[str | int, ...]]: """Every place in `value`: itself, and each member or entry inside it, depth first.""" found = [path] if isinstance(value, dict): for key, entry in cast("dict[str, object]", value).items(): found += _places(entry, (*path, key)) elif isinstance(value, list): for index, entry in enumerate(cast("list[object]", value)): found += _places(entry, (*path, index)) return found def _at(value: object, path: tuple[str | int, ...]) -> object: for step in path: value = cast("Any", value)[step] return value def _put(value: object, path: tuple[str | int, ...], new: object) -> object: """`value` with the item at `path` replaced by `new`, or removed when `new` is `_GONE`.""" if len(path) == 0: return new step, rest = path[0], path[1:] copy: Any = ( dict(cast("dict[str, object]", value)) if isinstance(value, dict) else list(cast("list[object]", value)) ) if len(rest) == 0 and new is _GONE: del copy[step] else: copy[step] = _put(copy[step], rest, new) return copy @st.composite def _changed(draw: st.DrawFn, value: object) -> object: """`value` changed in one or two places: an item replaced, dropped or added, a string extended by one character, or a field renamed.""" for _ in range(draw(st.integers(1, 2))): path = draw(st.sampled_from(_places(value))) here = _at(value, path) how = draw(st.sampled_from(("replace", "drop", "add", "extend", "rename"))) if how == "drop" and len(path) != 0: value = _put(value, path, _GONE) elif how == "add" and isinstance(here, dict): members = cast("dict[str, object]", here) value = _put(value, path, {**members, draw(st.sampled_from(_KEYS)): draw(_VALUES)}) elif how == "add" and isinstance(here, list): value = _put(value, path, [*cast("list[object]", here), draw(_VALUES)]) elif how == "extend" and isinstance(here, str): value = _put(value, path, here + draw(st.sampled_from(("\n", " ", "0", "x")))) elif how == "rename" and isinstance(here, dict) and "name" in here: value = _put(value, (*path, "name"), draw(st.sampled_from(_NAMES))) elif how == "rename" and isinstance(here, str): value = _put(value, path, draw(st.sampled_from(_NAMES))) else: value = _put(value, path, draw(_VALUES)) return value _CONTEXTS = (CORE, CORE_AND_EXTENSIONS, ZarrV3Context()) _FIELD_BASES: tuple[tuple[str, object], ...] = ( *CASES, # Names that nothing claims, so nothing checks their configuration. ("codecs:acme.x", {"name": "acme.x", "configuration": {"x": [1]}}), ("data_type:acme.t", {"name": "acme.t", "configuration": {"bits": 8}}), ("chunk_grid:acme.g", {"name": "acme.g", "configuration": {"chunk_shape": [0]}}), ) _FIELD_SCHEMAS: dict[tuple[type[ZarrV3Kind], int], Draft202012Validator] = {} def _field_validator(kind: type[ZarrV3Kind], context: int) -> Draft202012Validator: if (kind, context) not in _FIELD_SCHEMAS: schema = field_json_schema_v3(kind, context=_CONTEXTS[context]) _FIELD_SCHEMAS[(kind, context)] = Draft202012Validator(schema) return _FIELD_SCHEMAS[(kind, context)] # --- a field in a scope ----------------------------------------------------- _INDEXED = {"chunk_shape": [2], "codecs": ["bytes"]} # Each field, the kind and scope it is read in, whether the schema accepts # it, and whether the scope reads it without a problem. Where the two # differ, a rule found a problem that the schema cannot express. FIELDS: list[tuple[str, object, type[ZarrV3Kind], ZarrV3Context, bool, bool]] = [ ("object", {"name": "gzip", "configuration": {"level": 5}}, CodecDefinition, CORE, True, True), ( "out-of-bounds", {"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE, False, False, ), ( "unknown-key", {"name": "gzip", "configuration": {"level": 5, "window": 15}}, CodecDefinition, CORE, False, False, ), ("bare-name-needing-a-configuration", "gzip", CodecDefinition, CORE, False, False), ("bare-name", "bytes", CodecDefinition, CORE, True, True), ( "understood", {"name": "gzip", "configuration": {"level": 5}, "must_understand": True}, CodecDefinition, CORE, True, True, ), ( "not-understood", {"name": "gzip", "configuration": {"level": 5}, "must_understand": False}, CodecDefinition, CORE, False, False, ), ( "stray-member", {"name": "gzip", "configuration": {"level": 5}, "version": 2}, CodecDefinition, CORE, False, False, ), ( "unclaimed", {"name": "zfpy", "configuration": {"mode": 4}}, CodecDefinition, CORE, True, True, ), ("unclaimed-bare", "zfpy", CodecDefinition, CORE, True, True), ( "unclaimed-not-understood", {"name": "zfpy", "must_understand": False}, CodecDefinition, CORE, False, False, ), # Nothing in an empty scope claims gzip, so nothing checks it. ( "out-of-scope", {"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, ZarrV3Context(), True, True, ), ( "a-rule", { "name": "blosc", "configuration": {"cname": "lz4", "clevel": 1, "shuffle": "shuffle", "blocksize": 0}, }, CodecDefinition, CORE, True, False, ), ( "static-index-codec", { "name": "sharding_indexed", "configuration": {**_INDEXED, "index_codecs": ["bytes", "crc32c"]}, }, CodecDefinition, CORE, True, True, ), ( "dynamic-index-codec", { "name": "sharding_indexed", "configuration": { **_INDEXED, "index_codecs": [{"name": "gzip", "configuration": {"level": 1}}], }, }, CodecDefinition, CORE, False, False, ), ( "unclaimed-index-codec", { "name": "sharding_indexed", "configuration": {**_INDEXED, "index_codecs": ["bytes", "acme.sum"]}, }, CodecDefinition, CORE, True, True, ), ( "inner-codec-out-of-bounds", { "name": "sharding_indexed", "configuration": { **_INDEXED, "codecs": ["bytes", {"name": "gzip", "configuration": {"level": 12}}], "index_codecs": ["bytes"], }, }, CodecDefinition, CORE, False, False, ), ("raw-bits", "r16", DataTypeDefinition, CORE, True, True), ("raw-bits-object", {"name": "r16", "configuration": {}}, DataTypeDefinition, CORE, True, True), # The values a raw-bits name carries are not also written in a configuration. ( "raw-bits-configured", {"name": "r16", "configuration": {"bits": 16}}, DataTypeDefinition, CORE, False, False, ), ("raw-bits-of-a-size-the-spec-refuses", "r12", DataTypeDefinition, CORE, True, False), # `r*` is the notation of the spec's table, not a name: `*` is not # allowed in a name. ("raw-bits-notation", "r*", DataTypeDefinition, CORE, False, False), # The pattern is matched to the end of the name, as the package matches # it, so a validator that matches as Python does must not accept a final # newline. It is also not a name, because a newline is not allowed in one. ( "raw-bits-and-a-newline", {"name": "r16\n", "configuration": {"x": 1}}, DataTypeDefinition, CORE, False, False, ), ( "struct-field-out-of-bounds", { "name": "struct", "configuration": { "fields": [ { "name": "acme.t", "data_type": { "name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 0}, }, } ] }, }, DataTypeDefinition, CORE_AND_EXTENSIONS, False, False, ), ( "grid-out-of-bounds", {"name": "regular", "configuration": {"chunk_shape": [0]}}, ChunkGridDefinition, CORE, False, False, ), ( "encoding", {"name": "v2", "configuration": {"separator": "/"}}, ChunkKeyEncodingDefinition, CORE, True, True, ), ("transformer", {"name": "acme.cache"}, StorageTransformerDefinition, CORE, True, True), ] @pytest.mark.parametrize( ("field", "kind", "context", "accepted", "clean"), [case[1:] for case in FIELDS], ids=[case[0] for case in FIELDS], ) def test_field_json_schema_says_what_the_scope_reads( field: object, kind: type[ZarrV3Kind], context: ZarrV3Context, accepted: bool, clean: bool ) -> None: schema = field_json_schema_v3(kind, context=context) Draft202012Validator.check_schema(schema) assert Draft202012Validator(schema).is_valid(cast("Any", field)) == accepted assert (resolve(field, kind, context)[1] == ()) == clean @pytest.mark.parametrize( ("key", "field"), CASES, ids=[f"{key}:{index}" for index, (key, _) in enumerate(CASES)] ) def test_every_example_field_is_one_its_schema_accepts(key: str, field: object) -> None: schema = field_json_schema_v3(KINDS[key.split(":")[0]], context=CORE_AND_EXTENSIONS) assert Draft202012Validator(schema).is_valid(cast("Any", field)) @_PROPERTY @given(st.data()) def test_a_field_read_without_a_problem_is_one_its_schema_accepts(data: st.DataObject) -> None: # Each example is changed in one place and read in one of three scopes. # If the scope still reads it without a problem, the schema accepts it. key, example = data.draw(st.sampled_from(_FIELD_BASES), label="example") kind = KINDS[key.split(":")[0]] context = data.draw(st.sampled_from(range(len(_CONTEXTS))), label="context") field = data.draw(_changed(json.loads(json.dumps(example))), label="field") read = resolve(field, kind, _CONTEXTS[context])[1] == () event("read without a problem" if read else "a problem") if read: assert _field_validator(kind, context).is_valid(cast("Any", field)) def test_error_field_json_schema_v3_refuses_a_scope_of_another_format() -> None: """A v2 scope cannot read a v3 field, so `field_json_schema_v3` raises `TypeError` for one, as every v3 reader does.""" with pytest.raises(TypeError, match="v3 document is read in a scope of that format"): field_json_schema_v3(CodecDefinition, context=CORE_V2) # pyright: ignore[reportArgumentType] def test_field_json_schema_refuses_what_is_not_a_kind() -> None: with pytest.raises(TypeError, match="is not a kind of metadata"): field_json_schema_v3(Definition, context=CORE) # pyright: ignore[reportArgumentType] # --- a zarr.json ------------------------------------------------------------ ARRAY: dict[str, Any] = { "zarr_format": 3, "node_type": "array", "shape": [4, 4], "data_type": "int8", "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [2, 2]}}, "chunk_key_encoding": {"name": "default"}, "fill_value": 0, "codecs": [{"name": "bytes"}, {"name": "gzip", "configuration": {"level": 5}}], } FLOATS: dict[str, Any] = { **ARRAY, "data_type": "float32", "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], } GROUP: dict[str, Any] = {"zarr_format": 3, "node_type": "group"} def _consolidated(**metadata: object) -> dict[str, Any]: return { **GROUP, "consolidated_metadata": {"kind": "inline", "must_understand": False, "metadata": metadata}, } # Each document, whether the schema accepts it, and whether # `read_node_metadata_v3` finds no problem with it. DOCUMENTS: list[tuple[str, dict[str, Any], bool, bool]] = [ ("array", ARRAY, True, True), ("group", {**GROUP, "attributes": {"a": [1, None]}}, True, True), ("fill-value-out-of-range", {**ARRAY, "fill_value": 300}, False, False), ("fill-value-of-the-wrong-type", {**ARRAY, "fill_value": "0"}, False, False), ("float-fill-value", {**FLOATS, "fill_value": "NaN"}, True, True), ("hex-fill-value", {**FLOATS, "fill_value": "0x7fc00000"}, True, True), # A string that is not a float value is found by a rule, not the schema. ("fill-value-a-rule-refuses", {**FLOATS, "fill_value": "nan"}, True, False), ("unclaimed-data-type", {**ARRAY, "data_type": "acme.int7", "fill_value": "any"}, True, True), ( "a-name-raw-bits-and-a-newline", {**ARRAY, "data_type": "r16\n", "fill_value": "any"}, False, False, ), ("raw-bits", {**ARRAY, "data_type": "r16", "fill_value": [0, 255]}, True, True), ( "raw-bits-byte-out-of-range", {**ARRAY, "data_type": "r16", "fill_value": [0, 256]}, False, False, ), ( "codec-not-understood", {**ARRAY, "codecs": [{"name": "bytes", "must_understand": False}]}, False, False, ), ( "codec-out-of-bounds", {**ARRAY, "codecs": [{"name": "bytes"}, {"name": "gzip", "configuration": {"level": 12}}]}, False, False, ), ("negative-shape", {**ARRAY, "shape": [-1, 4]}, False, False), ("wrong-format", {**ARRAY, "zarr_format": 2}, False, False), ( "no-node-type", {key: value for key, value in ARRAY.items() if key != "node_type"}, False, False, ), ("an-extension-field", {**ARRAY, "acme": {"must_understand": False}}, True, True), # Constraints across several members are checked by the rules, not the schema. ("names-for-another-rank", {**ARRAY, "dimension_names": ["x"]}, True, False), ( "codecs-out-of-order", {**ARRAY, "codecs": [{"name": "gzip", "configuration": {"level": 5}}, "bytes"]}, True, False, ), ("consolidated", _consolidated(a=ARRAY, b=GROUP), True, True), ("consolidated-null", {**GROUP, "consolidated_metadata": None}, False, False), # A group's validator checks the envelope of its consolidated metadata, # and does not read the documents it lists. Nor does the schema. ("consolidated-entries-not-read", _consolidated(a={**ARRAY, "fill_value": 300}), True, True), ( "consolidated-kind", { **GROUP, "consolidated_metadata": {"kind": "sidecar", "must_understand": False, "metadata": {}}, }, False, False, ), ] NODE_SCHEMA = node_metadata_json_schema_v3() @pytest.mark.parametrize( ("document", "accepted", "clean"), [case[1:] for case in DOCUMENTS], ids=[case[0] for case in DOCUMENTS], ) def test_node_metadata_json_schema_says_what_a_zarr_json_holds( document: dict[str, Any], accepted: bool, clean: bool ) -> None: assert Draft202012Validator(NODE_SCHEMA).is_valid(document) == accepted assert (read_node_metadata_v3(document).problems == ()) == clean _BASES: tuple[dict[str, Any], ...] = ( ARRAY, FLOATS, {**ARRAY, "data_type": "r16", "fill_value": [0, 0]}, { **ARRAY, "codecs": [ { "name": "sharding_indexed", "configuration": { "chunk_shape": [1, 1], "codecs": ["bytes", {"name": "gzip", "configuration": {"level": 1}}], "index_codecs": ["bytes", "crc32c"], }, } ], }, _consolidated(a=ARRAY, b=GROUP), ) _NODE_SCHEMAS: dict[int, Draft202012Validator] = {} def _node_validator(context: int) -> Draft202012Validator: if context not in _NODE_SCHEMAS: _NODE_SCHEMAS[context] = Draft202012Validator( node_metadata_json_schema_v3(context=_CONTEXTS[context]) ) return _NODE_SCHEMAS[context] @_PROPERTY @given(st.data()) def test_a_document_read_without_a_problem_is_one_its_schema_accepts(data: st.DataObject) -> None: # Each document is changed in one place and read in one of three scopes. # If `read_node_metadata_v3` finds no problem, the schema accepts it. base = data.draw(st.sampled_from(_BASES), label="base") context = data.draw(st.sampled_from(range(len(_CONTEXTS))), label="context") document = data.draw(_changed(json.loads(json.dumps(base))), label="document") read = read_node_metadata_v3(document, context=_CONTEXTS[context]).problems == () event("read without a problem" if read else "a problem") if read: assert _node_validator(context).is_valid(cast("Any", document)) @pytest.mark.parametrize( "context", [CORE, CORE_AND_EXTENSIONS, ZarrV3Context()], ids=["core", "core-and-extensions", "empty"], ) def test_every_schema_is_a_json_schema_written_the_same_each_time(context: ZarrV3Context) -> None: schema = node_metadata_json_schema_v3(context=context) Draft202012Validator.check_schema(schema) assert json.loads(json.dumps(schema)) == schema assert schema == node_metadata_json_schema_v3(context=context) defs = cast("dict[str, JSONValue]", schema["$defs"]) assert {"ZarrV3ArrayMetadataJSON", "ZarrV3GroupMetadataJSON"} <= defs.keys() for kind in ( CodecDefinition, DataTypeDefinition, ChunkGridDefinition, ChunkKeyEncodingDefinition, StorageTransformerDefinition, ): Draft202012Validator.check_schema(field_json_schema_v3(kind, context=context)) @pytest.mark.parametrize( ("inner", "outer", "expected"), [ (Gt(1), Gt(3), {"type": "integer", "exclusiveMinimum": 3}), (Gt(3), Gt(1), {"type": "integer", "exclusiveMinimum": 3}), (Lt(9), Lt(5), {"type": "integer", "exclusiveMaximum": 5}), (Lt(5), Lt(9), {"type": "integer", "exclusiveMaximum": 5}), (Le(5), Le(9), {"type": "integer", "maximum": 5}), (Le(9), Le(5), {"type": "integer", "maximum": 5}), (Ge(5), Ge(9), {"type": "integer", "minimum": 9}), (Ge(9), Ge(5), {"type": "integer", "minimum": 9}), ], ids=[ "gt-looser-inside", "gt-stricter-inside", "lt", "lt-stricter-inside", "le", "le-stricter-inside", "ge", "ge-stricter-inside", ], ) def test_the_stricter_of_two_bounds_of_one_keyword_is_kept( inner: object, outer: object, expected: dict[str, object] ) -> None: # A bounded NewType with another bound on top: for each keyword, the # stricter bound is kept, whichever layer it is on. # pyright does not allow `Annotated` in `NewType`; the runtime does. Bounded = cast("Callable[[str, object], type]", NewType)("Bounded", _annotated[int, inner]) assert json_schema(_holding(_annotated[Bounded, outer])) == _held(expected) def test_a_stray_member_beside_an_unclaimed_name_is_refused_by_the_schema() -> None: # The schema rejects it, as the reader does: the envelope of an # unclaimed name holds only the name and its configuration. field = {"name": "zfpy", "version": 2} assert [(p.loc, p.kind) for p in resolve(field, CodecDefinition, CORE)[1]] == [ (("version",), "unknown_key") ] assert not Draft202012Validator(field_json_schema_v3(CodecDefinition, context=CORE)).is_valid( field ) @pytest.mark.parametrize( "name", [ "zstd", "numcodecs.adler32", "vlen-utf8", "acme_x", "r16", "https://example.com/c", "urn:acme:c", "urn:acme:%C3%BC", "", " ", "Int8", "9x", "int8 ", "int8\n", "foo/bar", "-int8", "a", "r*", "x:", "a: b", "urn:acme:c\n", "urn:acme:ü", "urn:a\x1c", "urn:a", ], ) def test_the_schema_names_an_extension_as_the_reader_does(name: str) -> None: # A name matches the spec's regex or is a URI. The schema's `pattern` # accepts exactly the names that `well_named` accepts. field = {"name": name} _, problems = resolve(field, CodecDefinition, CORE) accepted = len(problems) == 0 assert ( Draft202012Validator(field_json_schema_v3(CodecDefinition, context=CORE)).is_valid(field) is accepted ) assert accepted is ( name in ( "zstd", "numcodecs.adler32", "vlen-utf8", "acme_x", "https://example.com/c", "urn:acme:c", "urn:acme:%C3%BC", ) or name == "r16" ) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/test_problem_data.py000066400000000000000000000356041527014137600312130ustar00rootroot00000000000000"""The data a problem carries besides its message: what was found, and what was expected. Like pydantic's errors and zod's issues, a problem carries the data behind its message. `input` is the part of the input value at the problem's `loc`. `ctx` is what was expected there, when that is more than a type: for example, a type's bounds or the values of a closed set. Every reader fills in `input`, so a rule only needs to say where a problem is. """ from __future__ import annotations import copy import dataclasses import math import pickle from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Any, Literal, cast import pytest from annotated_types import Le from typing_extensions import TypedDict from zarr_metadata._json import ( arrays_to_tuples, is_canonical_json, validate_json, value_at, with_input, within, ) from zarr_metadata.definition import ( fill_value_problems, resolve, ) from zarr_metadata.model import ( UNSET, ZarrV2ConsolidatedMetadata, ZarrV3ArrayMetadata, read_array_metadata_v2, read_array_metadata_v3, read_group_metadata_v2, read_group_metadata_v3, read_node_metadata_v3, validate_node_name_v3, validate_node_path_v3, ) from zarr_metadata.problems import MetadataValidationError, ValidationProblem from zarr_metadata.typed_json import ( check, ) from zarr_metadata.v3.codec.gzip import GZIP_CODEC from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS, CodecDefinition, DataTypeDefinition, ) if TYPE_CHECKING: from collections.abc import Callable, Sequence from zarr_metadata.typed_json import JSONValue @pytest.mark.parametrize( ("found", "ctx"), [ (UNSET, {}), (12, {"ge": 0, "le": 9}), (None, {"expected": ["C", "F"]}), ], ids=["nothing-found", "a-bound", "a-null-found"], ) def test_a_problem_carries_what_was_found_and_what_was_expected( found: JSONValue | UNSET, ctx: dict[str, JSONValue] ) -> None: written = dict(ctx) problem = ValidationProblem(("level",), "bad level", "invalid_value", input=found, ctx=written) assert problem.input is found # `ctx` is stored as a read-only view of a copy, with arrays as tuples. assert dict(problem.ctx) == arrays_to_tuples(ctx) # The message and location define the problem; the data it carries is detail. bare = ValidationProblem(("level",), "bad level", "invalid_value") assert problem == bare assert hash(problem) == hash(bare) assert repr(problem) == repr(bare) assert str(problem) == "level: bad level" # A problem is a finished report: changes to the dict passed as `ctx` do not reach it. written["more"] = 1 assert dict(problem.ctx) == arrays_to_tuples(ctx) with pytest.raises(TypeError): problem.ctx["ge"] = 1 # pyright: ignore[reportIndexIssue] @pytest.mark.parametrize( "ctx", [["ge", 0], {1: 0}, {"ge": {0}}, {"ge": math.nan}], ids=["an-array", "a-key-that-is-not-a-string", "a-set", "nan"], ) def test_error_a_problem_refuses_a_ctx_that_is_not_an_object_of_json_values(ctx: object) -> None: with pytest.raises(TypeError, match="ctx is an object of JSON values"): ValidationProblem(("level",), "bad level", "invalid_value", ctx=ctx) # pyright: ignore[reportArgumentType] def test_an_error_about_what_is_not_json_pickles() -> None: # A problem holds no input that is not JSON, because such input might # not pickle. So the error that holds it can be sent to another process. value = {"shape": [lambda: 4]} problems = validate_json(value) assert [(problem.loc, problem.input) for problem in problems] == [(("shape", 0), UNSET)] again = pickle.loads(pickle.dumps(MetadataValidationError(problems))) assert again.problems == problems def test_a_problem_pickles_and_copies_with_what_it_carries() -> None: problem = ValidationProblem(("shape",), "bad", "invalid_value", input=[12], ctx={"ge": 0}) error = MetadataValidationError([problem]) for again in ( pickle.loads(pickle.dumps(problem)), copy.copy(problem), copy.deepcopy(problem), pickle.loads(pickle.dumps(error)).problems[0], ): assert again == problem assert again.input == [12] assert dict(again.ctx) == {"ge": 0} with pytest.raises(TypeError): again.ctx["ge"] = 1 # pyright: ignore[reportIndexIssue] def _deep(levels: int) -> list[object]: """An array nested `levels` deep, deeper than the interpreter can recurse.""" value: list[object] = [] for _ in range(levels): value = [value] return value @pytest.mark.parametrize( ("value", "at", "loc", "held", "found"), [ ({"a": [1, {"b": 2}]}, (), ("a", 1, "b"), UNSET, 2), ({"a": None}, (), ("a",), UNSET, None), ({"a": 1}, (), ("b",), UNSET, UNSET), ({"a": [1]}, (), ("a", 1), UNSET, UNSET), ({"a": "text"}, (), ("a", 0), UNSET, UNSET), ({"a": [1]}, (), ("a", "b"), UNSET, UNSET), ([1, 2], ("codecs",), ("codecs", 1), UNSET, 2), ([1, 2], ("codecs",), ("storage_transformers", 1), UNSET, UNSET), ({"a": {1, 2}}, (), ("a",), UNSET, UNSET), ({"a": _deep(100_000)}, (), ("a",), UNSET, UNSET), ({"a": 1}, (), ("a",), "what a reader inside found", 1), ({"a": {1, 2}}, (), ("a",), [1, 2], [1, 2]), ], ids=[ "nested", "a-null", "a-missing-key", "past-the-end", "a-string-is-no-array", "an-array-has-no-keys", "under-where-the-value-sits", "not-under-it", "what-is-not-json", "too-deep-to-walk", "the-value-handed-in-wins", "what-is-not-json-keeps-what-it-holds", ], ) def test_a_problem_holds_what_its_value_holds_at_its_loc( value: object, at: tuple[str | int, ...], loc: tuple[str | int, ...], held: JSONValue | UNSET, found: JSONValue | UNSET, ) -> None: (problem,) = with_input([ValidationProblem(loc, "bad", "invalid_value", input=held)], value, at) assert problem.input == found assert (problem.input is UNSET) is (found is UNSET) class GzipLevelOnly(TypedDict, closed=True): level: int def _raised(read: Callable[[], object]) -> Sequence[ValidationProblem]: with pytest.raises(MetadataValidationError) as caught: read() return caught.value.problems BAD_ARRAY: dict[str, object] = { "zarr_format": 3, "node_type": "array", "shape": [4, 4], "data_type": "uint8", "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [2]}}, "chunk_key_encoding": {"name": "default"}, "fill_value": 300, "codecs": [ {"name": "transpose", "configuration": {"order": [0, 1, 2]}}, {"name": "bytes"}, {"name": "gzip", "configuration": {"level": 12, "extra": [1]}}, ], "dimension_names": ["x"], "attributes": {}, } """An array document as `json.loads` returns it, with arrays as lists, and with a problem in each part that a reader reads.""" BAD_ARRAY_LOCS = { ("chunk_grid", "configuration", "chunk_shape"), ("fill_value",), ("codecs", 0, "configuration", "order"), ("codecs", 2, "configuration", "level"), ("codecs", 2, "configuration", "extra"), ("dimension_names",), } READERS: list[tuple[Callable[[object], Sequence[ValidationProblem]], object]] = [ (lambda value: check(value, GzipLevelOnly)[1], {"level": 12, "extra": [1]}), (lambda value: GZIP_CODEC.read_configuration(value)[1], {"level": 12, "extra": [1]}), ( lambda value: resolve(value, CodecDefinition, CORE_AND_EXTENSIONS)[1], {"name": "gzip", "configuration": {"level": 12}, "must_understand": "yes"}, ), ( lambda value: fill_value_problems( resolve("uint8", DataTypeDefinition, CORE_AND_EXTENSIONS)[0], value ), 300, ), (validate_json, {"a": [math.inf]}), ( lambda value: resolve(value, CodecDefinition, CORE_AND_EXTENSIONS)[1], {"name": 1, "configuration": {"a": [math.nan]}, "extra": [1]}, ), (validate_node_name_v3, "__/"), (validate_node_path_v3, "a//"), (lambda value: read_array_metadata_v3(value).problems, BAD_ARRAY), (lambda value: _raised(lambda: ZarrV3ArrayMetadata.from_json(value)), BAD_ARRAY), (lambda value: read_node_metadata_v3(value).problems, BAD_ARRAY), ( lambda value: read_group_metadata_v3(value).problems, { "zarr_format": 3, "node_type": "group", "consolidated_metadata": { "kind": "inline", "must_understand": [False], "metadata": {"a": BAD_ARRAY, "__b": BAD_ARRAY, "a/c": BAD_ARRAY, "d/e": BAD_ARRAY}, }, }, ), ( lambda value: read_array_metadata_v2(value).problems, {"zarr_format": 2, "shape": [4], "chunks": [2, 2], "order": "Q", "filters": [[1]]}, ), (lambda value: read_group_metadata_v2(value).problems, {"zarr_format": 3, "extra": [1]}), ( lambda value: _raised(lambda: ZarrV2ConsolidatedMetadata.from_json(value)), {"zarr_consolidated_format": [1], "metadata": {"a/.zarray": [math.inf]}}, ), ( lambda value: _raised(lambda: ZarrV3ArrayMetadata.from_key_value(value)), # pyright: ignore[reportArgumentType] {"zarr.json": b"{"}, ), ] @pytest.mark.parametrize( ("read", "value"), READERS, ids=[ "check", "read_configuration", "resolve", "fill-value-problems", "validate-json", "resolve-a-field-that-is-not-json", "validate-node-name", "validate-node-path", "read-array", "array-from-json", "read-node", "read-group-and-what-it-holds", "read-array-v2", "read-group-v2", "v2-consolidated-from-json", "from-key-value", ], ) def test_every_problem_a_reader_finds_holds_what_the_value_handed_in_holds_there( read: Callable[[object], Sequence[ValidationProblem]], value: object ) -> None: # Each problem holds part of the input value itself, not of a copy the # reader made along the way. This holds for problems in a field, a # pipeline, a chunk grid checked against the shape, and a document held # by a group. A missing key holds no input, and neither does a value # that is not JSON, such as a store's bytes. problems = read(value) assert len(problems) != 0 for problem in problems: there = value_at(value, problem.loc) assert problem.input is (there if is_canonical_json(there, finite=False) else UNSET), ( problem ) if value is BAD_ARRAY: assert {problem.loc for problem in problems} >= BAD_ARRAY_LOCS class WideBits(TypedDict, closed=True): bits: Annotated[int, Le(16)] def test_a_problem_with_what_a_name_carries_is_the_field_s() -> None: # `r24` carries `{"bits": 24}`, which this scope's raw-bits definition # refuses. The problem is located at the field, because the name is what # the document wrote there. The bound on `bits` is not a bound on the # name, so the problem's `ctx` is empty. context = CORE_AND_EXTENSIONS.update( data_types={ **CORE_AND_EXTENSIONS.data_types, "r*": DataTypeDefinition(name="r*", configuration=WideBits), } ) _, problems = resolve("r24", DataTypeDefinition, context, ("data_type",)) assert [(p.loc, p.input, dict(p.ctx)) for p in problems] == [(("data_type",), "r24", {})] assert problems[0].message == "expected an integer <= 16, got 24" class Ordered(TypedDict, closed=True): order: Literal["F", "C"] @pytest.mark.parametrize( ("problems", "loc", "expected"), [ (check({"order": "Q"}, Ordered)[1], ("order",), ("C", "F")), ( read_array_metadata_v3({**BAD_ARRAY, "zarr_format": 2}).problems, ("zarr_format",), (3,), ), ( read_node_metadata_v3({"node_type": "dataset"}).problems, ("node_type",), ("array", "group"), ), (read_array_metadata_v2({"order": "Q"}).problems, ("order",), ("C", "F")), ( read_group_metadata_v3( { "zarr_format": 3, "node_type": "group", "consolidated_metadata": { "kind": "inline", "must_understand": True, "metadata": {}, }, } ).problems, ("consolidated_metadata", "must_understand"), (False,), ), ], ids=["a-literal", "zarr-format", "node-type", "v2-order", "consolidated-must-understand"], ) def test_a_value_outside_a_closed_set_is_told_the_set( problems: Sequence[ValidationProblem], loc: tuple[str | int, ...], expected: tuple[object, ...] ) -> None: # The values are listed in the same order as in the message, written as JSON. (problem,) = [problem for problem in problems if problem.loc == loc] assert dict(problem.ctx) == {"expected": expected} def test_error_a_ctx_of_another_mapping_type_is_checked() -> None: # Only a `ctx` taken from another problem, which was checked when that # problem was made, is trusted. Any other mapping is checked, whatever # its type. with pytest.raises(TypeError, match="ctx is an object of JSON values"): ValidationProblem( (), "m", "invalid_value", ctx=cast("Any", MappingProxyType({"x": object()})) ) first = ValidationProblem((), "m", "invalid_value", ctx={"ge": 0}) again = dataclasses.replace(first, loc=("a",)) assert again.ctx == {"ge": 0} assert again.ctx is first.ctx def test_a_problem_at_the_first_element_holds_it() -> None: # Compared with the literal value, not with `value_at`, which the reader uses. (problem,) = with_input([ValidationProblem(("a", 0), "bad", "invalid_value")], {"a": [7]}) assert problem.input == 7 assert value_at([5, 6], (0,)) == 5 read = resolve("bytes", DataTypeDefinition, CORE_AND_EXTENSIONS)[0] (found,) = fill_value_problems(read, [-1]) assert (found.loc, found.input) == ((0,), -1) def test_a_problem_s_ctx_is_its_own_at_every_level() -> None: # `ctx` is copied at every level when the problem is made, so later # changes the caller makes to the value it passed in do not reach it. inner: dict[str, int] = {"a": 1} problem = ValidationProblem((), "m", "invalid_value", ctx={"nested": inner}) inner["a"] = 2 assert problem.ctx["nested"] == {"a": 1} def test_error_a_problem_not_below_where_a_reader_counts_from_is_refused() -> None: # `within` moves a problem's location from the document passed in to # the document that was read. A problem located from a different root # would end up in the wrong place, so it is rejected. problem = ValidationProblem(("a", "b"), "m", "invalid_value") assert within((problem,), ("a",))[0].loc == ("b",) with pytest.raises(TypeError, match="does not sit below"): within((problem,), ("c",)) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/test_public_api.py000066400000000000000000000555521527014137600306750ustar00rootroot00000000000000"""Test that the curated front-door names are accessible from the top-level zarr_metadata package.""" import importlib import pkgutil import re from typing import Literal, get_args, get_origin import pytest import zarr_metadata as zm from zarr_metadata.definition import Definition from zarr_metadata.v2.definition import ZarrV2Context from zarr_metadata.v3.definition import ZarrV3Context def _group_rank(s: str) -> int: """RUF022 groups `__all__` as: SCREAMING_SNAKE (0), then TitleCase (1), then dunders (2). The exact intra-group ordering is ruff's own natural sort and is enforced by ruff itself (pre-commit + CI); this test only asserts the grouping, not the fragile tie-breaking, so it can't drift out of sync with ruff's implementation. """ if s.startswith("__") and s.endswith("__"): return 2 stripped = re.sub(r"[\d_]", "", s) return 0 if stripped.isupper() else 1 EXPECTED = [ # Category A — metadata-document types "ZarrV2ArrayMetadataJSON", "ZarrV2ZArrayJSON", "ZarrV2GroupMetadataJSON", "ZarrV2ZGroupJSON", "ZarrV2ConsolidatedMetadataJSON", "ZarrV2ZAttrsJSON", "ZarrV2CodecMetadata", "ZarrV3ArrayMetadataJSON", "ZarrV3ExtensionField", "ZarrV3GroupMetadataJSON", "ZarrV3ConsolidatedMetadataJSON", "ZarrV3ConsolidatedMetadataEnvelopeJSON", "ZarrV3NamedConfigJSON", "ZarrV3MetadataFieldJSON", "JSONValue", # Category A' — metadata models (in-memory dataclasses over the documents) "ZarrV2ArrayMetadata", "ZarrV2ArrayMetadataUpdate", "ZarrV3ArrayMetadata", "ZarrV3ArrayMetadataUpdate", "ZarrV2GroupMetadata", "ZarrV2GroupMetadataUpdate", "ZarrV3GroupMetadata", "ZarrV3GroupMetadataUpdate", "ZarrV3ConsolidatedMetadataInput", "ZarrV3NodeMetadataInput", "ZarrV2ConsolidatedMetadata", "ZarrV3ConsolidatedMetadata", "ValidationProblem", "MetadataValidationError", "ProblemKind", "UNSET", # Store keys — the names the documents are persisted under. Defined in the # v2/v3 spec modules, re-exported through `zarr_metadata.model`. "ZARR_V2_ARRAY_METADATA_STORE_KEY", "ZarrV2ArrayMetadataStoreKey", "ZARR_V2_GROUP_METADATA_STORE_KEY", "ZarrV2GroupMetadataStoreKey", "ZARR_V2_ATTRIBUTES_STORE_KEY", "ZarrV2AttributesStoreKey", "ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY", "ZarrV2ConsolidatedMetadataStoreKey", "ZARR_V3_ARRAY_METADATA_STORE_KEY", "ZarrV3ArrayMetadataStoreKey", "ZARR_V3_GROUP_METADATA_STORE_KEY", "ZarrV3GroupMetadataStoreKey", # Not a store key: v3 consolidated metadata is embedded in the group's own # `zarr.json`, so it has no paired Literal alias. "ZARR_V3_CONSOLIDATED_METADATA_KEY", # v2 data-type encoding union "ZarrV2DataTypeMetadata", # Category B — codec canonical unions "BloscCodecMetadata", "BytesCodecMetadata", "CastValueCodecMetadata", "Crc32cCodecMetadata", "GzipCodecMetadata", "ScaleOffsetCodecMetadata", "ShardingIndexedCodecMetadata", "TransposeCodecMetadata", "ZstdCodecMetadata", # Category C — grid/key canonical unions "RegularChunkGridMetadata", "RectilinearChunkGridMetadata", "DefaultChunkKeyEncodingMetadata", "V2ChunkKeyEncodingMetadata", # Category D — dtype trios # bool "BoolDataTypeName", "BOOL_DATA_TYPE_NAME", "BoolFillValue", # int8/16/32/64 "Int8DataTypeName", "INT8_DATA_TYPE_NAME", "Int8FillValue", "Int16DataTypeName", "INT16_DATA_TYPE_NAME", "Int16FillValue", "Int32DataTypeName", "INT32_DATA_TYPE_NAME", "Int32FillValue", "Int64DataTypeName", "INT64_DATA_TYPE_NAME", "Int64FillValue", # uint8/16/32/64 (actual casing is Uint, not UInt) "Uint8DataTypeName", "UINT8_DATA_TYPE_NAME", "Uint8FillValue", "Uint16DataTypeName", "UINT16_DATA_TYPE_NAME", "Uint16FillValue", "Uint32DataTypeName", "UINT32_DATA_TYPE_NAME", "Uint32FillValue", "Uint64DataTypeName", "UINT64_DATA_TYPE_NAME", "Uint64FillValue", # float16/32/64 "Float16DataTypeName", "FLOAT16_DATA_TYPE_NAME", "Float16FillValue", "Float32DataTypeName", "FLOAT32_DATA_TYPE_NAME", "Float32FillValue", "Float64DataTypeName", "FLOAT64_DATA_TYPE_NAME", "Float64FillValue", # complex64/128 "Complex64DataTypeName", "COMPLEX64_DATA_TYPE_NAME", "Complex64FillValue", "Complex128DataTypeName", "COMPLEX128_DATA_TYPE_NAME", "Complex128FillValue", # bytes "BytesDataTypeName", "BYTES_DATA_TYPE_NAME", "BytesFillValue", # string "StringDataTypeName", "STRING_DATA_TYPE_NAME", "StringFillValue", # numpy_datetime64 "NumpyDatetime64DataTypeName", "NUMPY_DATETIME64_DATA_TYPE_NAME", "NumpyDatetime64FillValue", # numpy_timedelta64 "NumpyTimedelta64DataTypeName", "NUMPY_TIMEDELTA64_DATA_TYPE_NAME", "NumpyTimedelta64FillValue", # struct "StructDataTypeName", "STRUCT_DATA_TYPE_NAME", "StructFillValue", # raw (no _DATA_TYPE_NAME constant) "RawBytesDataTypeName", "RawBytesFillValue", # Category E — constant+Literal pairs "ZARR_V2_ARRAY_ORDER", "ZarrV2ArrayOrder", "ZARR_V2_ARRAY_DIMENSION_SEPARATOR", "ZarrV2ArrayDimensionSeparator", "ENDIANNESS", "Endianness", "BYTES_CODEC_NAME", "BytesCodecName", "BLOSC_CODEC_NAME", "BloscCodecName", "BLOSC_CNAME", "BloscCName", "BLOSC_SHUFFLE", "BloscShuffle", "CAST_ROUNDING_MODE", "CastRoundingMode", "CAST_OUT_OF_RANGE_MODE", "CastOutOfRangeMode", "CAST_VALUE_CODEC_NAME", "CastValueCodecName", "CRC32C_CODEC_NAME", "Crc32cCodecName", "GZIP_CODEC_NAME", "GzipCodecName", "SCALE_OFFSET_CODEC_NAME", "ScaleOffsetCodecName", "SHARDING_INDEX_LOCATION", "ShardingIndexLocation", "SHARDING_INDEXED_CODEC_NAME", "ShardingIndexedCodecName", "TRANSPOSE_CODEC_NAME", "TransposeCodecName", "ZSTD_CODEC_NAME", "ZstdCodecName", "REGULAR_CHUNK_GRID_NAME", "RegularChunkGridName", "RECTILINEAR_CHUNK_GRID_NAME", "RectilinearChunkGridName", "DEFAULT_CHUNK_KEY_ENCODING_NAME", "DefaultChunkKeyEncodingName", "DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR", "DefaultChunkKeyEncodingSeparator", "V2_CHUNK_KEY_ENCODING_NAME", "V2ChunkKeyEncodingName", "V2_CHUNK_KEY_ENCODING_SEPARATOR", "V2ChunkKeyEncodingSeparator", "NUMPY_TIME_UNIT", "NumpyTimeUnit", ] def test_front_door_names_public() -> None: missing = [n for n in EXPECTED if n not in zm.__all__ or not hasattr(zm, n)] assert not missing, f"missing from top-level API: {missing}" def test_front_door_is_exactly_expected() -> None: """`__all__` must contain exactly the curated names (plus `__version__`). Guards against a name being promoted to the front door without a corresponding, deliberate entry in `EXPECTED` — i.e. an accidental addition to the public API surface. """ assert set(zm.__all__) - {"__version__"} == set(EXPECTED) def test_all_is_grouped_and_unique() -> None: ranks = [_group_rank(n) for n in zm.__all__] assert ranks == sorted(ranks), "`__all__` groups out of order (SCREAMING, TitleCase, dunder)" assert len(zm.__all__) == len(set(zm.__all__)) # --- naming grammar --------------------------------------------------------- # Core document/model names: the format version comes first (`ZarrV2` / # `ZarrV3`), then the CamelCase entity, then an optional role suffix # (`JSON`, `Update`, `Reading`, `StoreKey`) — validated loosely here # because `JSON` decomposes into single-letter words under any strict # word-splitting regex. _CORE_NAME = re.compile(r"^ZarrV[23](?:[A-Z][a-z0-9]*)+$") # Zarr v3 extension-entity names: the registered entity comes first (`Blosc`, # `Uint8`, ... — `V2` here is the *entity name* of the v2-compatibility chunk # key encoding, not a format-version marker, which is always spelled # `ZarrV2`/`ZarrV3`), followed by exactly one role suffix. _EXTENSION_ROLES = ( "CodecConfiguration", "CodecMetadata", "CodecName", "CodecObject", "ChunkGridConfiguration", "ChunkGridMetadata", "ChunkGridName", "ChunkGridObject", "ChunkKeyEncodingConfiguration", "ChunkKeyEncodingMetadata", "ChunkKeyEncodingName", "ChunkKeyEncodingObject", "ChunkKeyEncodingSeparator", "DataTypeName", "FillValue", "Configuration", "Component", # The definition of one kind of extension metadata as JSON: for example, # `CodecDefinition`, which holds a codec's name, configuration TypedDict # and rules. "Definition", # A configuration member that holds a metadata field of that kind: for # example, `ZarrV3CodecField`, read in the same scope as the field that # contains it. "Field", ) _EXTENSION_NAME = re.compile(r"^(?:[A-Z][a-z0-9]*)+?(?:" + "|".join(_EXTENSION_ROLES) + r")$") # Standalone vocabulary: scalar Literal aliases, structural helper shapes, and # the validation diagnostics. Closed by hand — a new name belongs here only if # it is genuinely role-less; anything document- or entity-shaped must fit the # grammars above instead. _STANDALONE_VOCAB = frozenset( { "Base64Bytes", "BloscCName", "BloscShuffle", "Chunk", # The algebra of scopes: the key of a claim, and a conflict between # two definitions of one name. "ClaimKey", "Conflict", "ScopeConflictError", # A reader's problems, as the signatures of every reader name them. "Problems", "CodecKind", "CodecSize", "Context", "Definition", "Lengths", "Loc", "Nested", "NodeName", "NodePath", # The result of reading a field in a scope: `AcceptedField` (read by the # definition that claims its name), `UnclaimedField`, or `RefusedField`. # `ResolvedField` is the union of the three. "AcceptedField", "RefusedField", "ResolvedField", "Stage", "StorageClass", "UnclaimedField", "CastOutOfRangeMode", "CastRoundingMode", "Endianness", "HexFloat16", "HexFloat32", "HexFloat64", "JSONSchema", "JSONValue", "UserJSONValue", # A pydantic field's scope, fixed: `Annotated[Model, InScope(CONTEXT)]`. "InScope", "MetadataValidationError", "NumpyDatetime64", "NumpyTimeUnit", "NumpyTimedelta64", "ProblemKind", "RectilinearDimSpec", # What a repair of a writer's bug changed, in the same way that # `ValidationProblem` and `ProblemKind` describe what a read found. "Repair", "RepairKind", "ScalarMap", "ScalarMapEntry", "ShardingIndexLocation", "Struct", "StructField", "ValidationProblem", } ) def _iter_module_names() -> set[str]: """Every public module in the package, including the top-level namespace.""" module_names = {"zarr_metadata"} for info in pkgutil.walk_packages(zm.__path__, prefix="zarr_metadata."): if not any(part.startswith("_") for part in info.name.split(".")[1:]): module_names.add(info.name) return module_names def _public_type_names() -> set[tuple[str, str]]: """Every (module, CamelCase name) pair exported via a public `__all__`.""" out: set[tuple[str, str]] = set() for module_name in _iter_module_names(): module = importlib.import_module(module_name) for name in getattr(module, "__all__", ()): if name.startswith("_") or name.isupper() or name.islower(): continue out.add((module_name, name)) return out def test_public_type_names_comply_with_naming_grammar() -> None: """Every public type name parses against the package naming grammar: version-first core names, entity-plus-role extension names, or the closed standalone vocabulary.""" exported = _public_type_names() violations = [ f"{module}.{name}" for module, name in sorted(exported) if name not in _STANDALONE_VOCAB and not _CORE_NAME.match(name) and not _EXTENSION_NAME.match(name) ] assert not violations, f"names outside the naming grammar: {violations}" def test_standalone_vocab_is_not_stale() -> None: """Every allowlisted vocabulary name is still actually exported.""" exported_names = {name for _, name in _public_type_names()} assert exported_names >= _STANDALONE_VOCAB def test_promoted_pairs_drift() -> None: """Each promoted runtime constant holds exactly the values of the `Literal` type it manifests, so the two cannot drift apart.""" pairs = [ (zm.ENDIANNESS, zm.Endianness), (zm.BLOSC_CNAME, zm.BloscCName), (zm.BLOSC_SHUFFLE, zm.BloscShuffle), (zm.SHARDING_INDEX_LOCATION, zm.ShardingIndexLocation), (zm.NUMPY_TIME_UNIT, zm.NumpyTimeUnit), (zm.CAST_ROUNDING_MODE, zm.CastRoundingMode), (zm.CAST_OUT_OF_RANGE_MODE, zm.CastOutOfRangeMode), (zm.ZARR_V2_ARRAY_ORDER, zm.ZarrV2ArrayOrder), (zm.ZARR_V2_ARRAY_DIMENSION_SEPARATOR, zm.ZarrV2ArrayDimensionSeparator), (zm.DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR, zm.DefaultChunkKeyEncodingSeparator), (zm.V2_CHUNK_KEY_ENCODING_SEPARATOR, zm.V2ChunkKeyEncodingSeparator), ] for const, lit in pairs: assert set(const) == set(get_args(lit)) def constant_name_for(type_name: str) -> str: """Derive a constant's name from the name of the type it manifests. The transformation is purely syntactic: split at each lowercase-to-uppercase boundary and before an uppercase run that starts a new word, then uppercase. Digit runs stay glued to the token they follow (`Uint8` -> `UINT8`, `Crc32c` -> `CRC32C`), because a digit boundary in CamelCase does not mark a word boundary in the spec vocabulary these names model. Consecutive capitals do not split, so acronym-adjacent names derive badly: `ZarrV2ZArrayJSON` -> `ZARR_V2ZARRAY_JSON`. Every such name in the package today is a `TypedDict` or `TypeAliasType` that backs no constant, so none reaches this function — but a future `Literal` spelled that way would silently be held to a bad name. Splitting acronyms correctly needs a vocabulary, not a regex, so the rule stays syntactic and this stays a known limit. """ return re.sub(r"(?<=[a-z0-9])(?=[A-Z][a-z])|(?<=[a-z])(?=[A-Z])", "_", type_name).upper() def _literal_backed_constants() -> list[tuple[str, str, str]]: """Every (module, constant, type) triple where a module-level SCREAMING_SNAKE constant holds exactly the values of a `Literal` type in the same module. Pairing is by value, not by proximity: a constant manifests the type whose members it enumerates. Constants with no such type (extension-field keys, key sets, canonical bit patterns) are exempt from the naming rule and are simply absent from the result. """ out: list[tuple[str, str, str]] = [] for module_name in _iter_module_names(): module = importlib.import_module(module_name) literals = { name: frozenset(get_args(obj)) for name, obj in vars(module).items() if not name.startswith("_") and not name.isupper() and get_origin(obj) is Literal and get_args(obj) } if not literals: continue for const_name, value in vars(module).items(): if const_name.startswith("_") or not const_name.isupper(): continue if isinstance(value, tuple): members = frozenset(value) elif isinstance(value, str): members = frozenset({value}) else: # A value that is not a name -- a definition, a scope -- # backs no Literal type and carries no signal here. continue if not all(isinstance(m, str) for m in members): continue matches = [t for t, args in literals.items() if args == members] # A single unambiguous type means this constant manifests it. Ties # (two Literals with identical members) carry no signal about which # name the constant should take, so they are skipped. if len(matches) == 1: out.append((module_name, const_name, matches[0])) return out def _value_tied_constants() -> set[str]: """Constants whose manifested type is ambiguous because two or more `Literal` types in the same module share its exact members. These are invisible to the derivation check, so they are surfaced here and counted, rather than silently dropped inside the pairing helper.""" tied: set[str] = set() for module_name in _iter_module_names(): module = importlib.import_module(module_name) literals = [ frozenset(get_args(obj)) for name, obj in vars(module).items() if not name.startswith("_") and not name.isupper() and get_origin(obj) is Literal and get_args(obj) ] for const_name, value in vars(module).items(): if const_name.startswith("_") or not const_name.isupper(): continue if isinstance(value, tuple): members = frozenset(value) elif isinstance(value, str): members = frozenset({value}) else: # A value that is not a name -- a definition, a scope -- # backs no Literal type and carries no signal here. continue if not all(isinstance(m, str) for m in members): continue if sum(1 for args in literals if args == members) > 1: tied.add(f"{module_name}.{const_name}") return tied # Constants whose `Literal` type cannot be identified by value because another # `Literal` in the same module has identical members. Pairing is by value, so a # tie carries no signal about which name the constant should take. These are # checked by eye; the count below fails if the tied set grows silently. KNOWN_VALUE_TIES = 7 def test_constant_names_derive_from_their_type_names() -> None: """Every `Literal`-backed constant whose type can be identified by value has a name that is the mechanical transform of that type's name. Constants tied to more than one identically-valued `Literal` are exempt (see `KNOWN_VALUE_TIES`), as are constants in private modules and those backing no `Literal` at all — so this pins the rule for most of the package, not all of it.""" pairs = _literal_backed_constants() assert pairs, "found no Literal-backed constants to check" violations = [ f"{module}: {const} should be {constant_name_for(type_name)} (manifests {type_name})" for module, const, type_name in pairs if const != constant_name_for(type_name) ] assert not violations, "constants whose names do not derive from their type:\n" + "\n".join( violations ) def test_value_tied_constants_are_a_known_set() -> None: """The derivation check cannot see constants whose type is ambiguous by value. Pin how many there are, so the exempt set cannot grow unnoticed and quietly shrink the rule's coverage.""" tied = _value_tied_constants() assert len(tied) == KNOWN_VALUE_TIES, ( f"value-tied constants changed (expected {KNOWN_VALUE_TIES}, got {len(tied)}); " f"these are unchecked by the derivation rule and must be named by hand:\n" + "\n".join(sorted(tied)) ) _FRONT_DOORS = ("zarr_metadata.v2", "zarr_metadata.v3") """The module of each format that also exports every definition and scope of that format, and the format's scope class.""" _SCOPE_CLASSES = (ZarrV2Context, ZarrV3Context) """The class of the scope of each format.""" def _builds_scopes(value: object) -> bool: """Whether a front door may alias `value`: a definition, a scope, or a scope class.""" return isinstance(value, (Definition, *_SCOPE_CLASSES)) or any( value is scope for scope in _SCOPE_CLASSES ) def _homes() -> dict[str, list[str]]: """Each public name, and every public submodule that exports it, with a format's front door left out for a definition or scope it aliases.""" homes: dict[str, list[str]] = {} for module_name in sorted(_iter_module_names() - {"zarr_metadata"}): for name in getattr(importlib.import_module(module_name), "__all__", ()): homes.setdefault(name, []).append(module_name) for name, modules in homes.items(): others = [module for module in modules if module not in _FRONT_DOORS] if len(others) == 1 and len(modules) == 2: homes[name] = others return homes def test_every_name_has_one_public_home() -> None: """Each public name is exported by exactly one public submodule, its home. The top-level `zarr_metadata` namespace may also export it, as a convenience alias of the same object, and so may the front door of its format (`zarr_metadata.v2` or `zarr_metadata.v3`) when the name is a definition, a scope or the scope class of that format.""" homes = _homes() duplicated = {name: modules for name, modules in homes.items() if len(modules) > 1} assert duplicated == {} homeless = [name for name in zm.__all__ if name not in homes and name != "__version__"] assert homeless == [], "a top-level name must also be exported by its home submodule" aliases = [ name for name in zm.__all__ if name in homes and getattr(zm, name) is not getattr(importlib.import_module(homes[name][0]), name) ] assert aliases == [], "a top-level name must be the same object as the name in its home module" for door in _FRONT_DOORS: module = importlib.import_module(door) for name in module.__all__: (home,) = homes[name] if home == door: continue value = getattr(importlib.import_module(home), name) assert home.startswith(f"{door}."), f"{door} aliases {name} of another format" assert _builds_scopes(value), f"{door} aliases {name}" assert getattr(module, name) is value, f"{door}.{name} is another object" @pytest.mark.parametrize("door", _FRONT_DOORS) def test_a_front_door_exports_every_definition_and_scope_of_its_format(door: str) -> None: """`zarr_metadata.v2` and `zarr_metadata.v3` each export every definition, scope and scope class that a submodule of theirs exports, so one import builds or extends a scope of that format.""" exported = set(importlib.import_module(door).__all__) missing = sorted( name for module_name in _iter_module_names() if module_name.startswith(f"{door}.") for name in getattr(importlib.import_module(module_name), "__all__", ()) if _builds_scopes(getattr(importlib.import_module(module_name), name)) and name not in exported ) assert missing == [] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/test_typed_json.py000066400000000000000000001646721527014137600307500ustar00rootroot00000000000000"""The checker: parsers compiled from annotations for the shapes that JSON values take. These tests use the checker on its own, without any metadata field. A leaf stands in for whatever a caller adds (the definition layer adds the shape of a member that holds another metadata field), and `no_leaf` adds nothing. """ from __future__ import annotations import dataclasses import importlib import itertools import json import math import pkgutil import sys import textwrap import types from collections.abc import Mapping from typing import ( TYPE_CHECKING, Annotated, Final, Generic, Literal, NewType, NotRequired, Required, TypeVar, cast, ) import pydantic import pytest from annotated_types import ( BaseMetadata, Ge, Gt, Interval, Le, Len, Lt, MinLen, MultipleOf, Predicate, Timezone, ) from typing_extensions import Doc, ReadOnly, TypeAliasType, TypedDict, is_typeddict import zarr_metadata.definition import zarr_metadata.v2 import zarr_metadata.v3 from zarr_metadata._typed_json import ( Parsed, Parser, Unread, UnreadConstraintError, describe, no_leaf, parser, parser_for, shape_of, typeddict_keys, unread_in, ) from zarr_metadata.model import ( ZarrV2ArrayMetadata, ZarrV3ArrayMetadata, ) from zarr_metadata.typed_json import ( JSONValue, Loc, check, ) from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON from zarr_metadata.v2.data_type import ZarrV2DataTypeMetadata from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.data_type.float32 import Float32FillValue if TYPE_CHECKING: from _pytest.mark import ParameterSet Level = NewType("Level", int) Width = TypeAliasType("Width", int) class Options(TypedDict, closed=True): level: int note: NotRequired[str] tag: ReadOnly[NotRequired[str]] class WithExtras(TypedDict, extra_items=int): level: int class Open(TypedDict, closed=False): level: int class Small(TypedDict, closed=True): x: int class Large(TypedDict, closed=True): x: int y: int class Tree(TypedDict, closed=True): label: str children: tuple[Tree, ...] class GzipLevel(TypedDict, closed=True): level: int class Gzip(TypedDict, closed=True): name: Literal["gzip"] configuration: GzipLevel class Blosc(TypedDict, closed=True): name: Literal["blosc"] configuration: Options class Pair(TypedDict, closed=True): a: int b: int class Triple(TypedDict, closed=True): a: str b: str c: str class Option1(TypedDict, closed=False): one: NotRequired[int] class Option2(TypedDict, closed=False): two: NotRequired[tuple[int, ...]] def _read(annotation: object, value: object) -> Parsed: return parser(annotation, no_leaf)(value, ()) def _found(parsed: Parsed) -> list[tuple[Loc, str]]: return [(found.loc, found.kind) for found in parsed[1]] @pytest.mark.parametrize( ("annotation", "value", "parsed"), [ (int, 3, 3), (float, 3, 3), (float, 2.5, 2.5), (bool, True, True), (str, "x", "x"), (None, None, None), (Literal["a", "b"], "a", "a"), (Literal[1, 2], 2, 2), (tuple[int, ...], [1, 2], (1, 2)), (tuple[int, str], (1, "a"), (1, "a")), (tuple[str | None, ...], ["a", None], ("a", None)), (int | str, "a", "a"), (Mapping[str, int], {"k": 1}, {"k": 1}), (JSONValue, {"any": [1, None]}, {"any": [1, None]}), (Level, 5, 5), (Width, 5, 5), ], ids=[ "int", "number-from-int", "number", "bool", "str", "null", "literal-str", "literal-int", "sequence", "fixed-tuple", "sequence-of-optional", "union", "mapping", "json", "newtype", "alias", ], ) def test_a_value_of_the_shape_reads_as_itself( annotation: object, value: object, parsed: object ) -> None: # The shapes JSON values take, and no others, each compiled once from its # annotation. An array is returned as a tuple; other values unchanged. assert _read(annotation, value) == (parsed, ()) @pytest.mark.parametrize( ("annotation", "value", "loc", "kind"), [ (int, True, (), "invalid_type"), (int, 2.0, (), "invalid_type"), (float, "2", (), "invalid_type"), (str, 1, (), "invalid_type"), (None, 0, (), "invalid_type"), (Literal["a"], "b", (), "invalid_value"), (Literal[1], True, (), "invalid_type"), (Literal["0.5"], 0.5, (), "invalid_type"), (tuple[int, ...], (1, "x"), (1,), "invalid_type"), (tuple[int, ...], 5, (), "invalid_type"), (int | str, 2.5, (), "invalid_type"), (Mapping[str, int], {"k": "v"}, ("k",), "invalid_type"), (JSONValue, {"k": object()}, (), "invalid_type"), ( Blosc | Gzip, {"name": "gzip", "configuration": {"level": "x"}}, ("configuration", "level"), "invalid_type", ), (Blosc | Gzip, {"name": "zstd", "configuration": {}}, ("name",), "invalid_value"), (Blosc | Gzip, {"name": 5, "configuration": {}}, ("name",), "invalid_type"), (Blosc | Gzip, {"configuration": {}}, ("name",), "missing_key"), (Triple | Pair, {"a": 1, "b": "x"}, ("b",), "invalid_type"), ], ids=[ "bool-is-not-int", "float-is-not-int", "str-is-not-number", "int-is-not-str", "zero-is-not-null", "literal-other", "literal-bool-is-not-int", "literal-of-another-type", "element", "not-a-sequence", "no-branch", "mapping-value", "not-json", "a-tag-picks-the-branch-that-reports", "a-tag-no-branch-has", "a-tag-of-another-type", "a-tag-missing", "untagged-the-closest-branch-reports", ], ) def test_error_a_value_of_another_shape_is_located( annotation: object, value: object, loc: Loc, kind: str ) -> None: parsed, problems = _read(annotation, value) assert parsed == value assert [(problem.loc, problem.kind) for problem in problems] == [(loc, kind)] @pytest.mark.parametrize( ("annotation", "value", "parsed", "found"), [ (Options, {"level": 1}, {"level": 1}, []), (Options, {"note": "n", "level": 1}, {"note": "n", "level": 1}, []), (Options, {"level": 1, "extra": 2}, {"level": 1}, [(("extra",), "unknown_key")]), (WithExtras, {"level": 1, "more": 2}, {"level": 1, "more": 2}, []), ( WithExtras, {"level": 1, "more": "two"}, {"level": 1, "more": "two"}, [(("more",), "invalid_type")], ), (Open, {"level": 1, "more": [2]}, {"level": 1, "more": [2]}, []), (Small | Large, {"x": 1, "y": 2}, {"x": 1, "y": 2}, []), (Small | Large, {"x": 1, "z": 2}, {"x": 1}, [(("z",), "unknown_key")]), (Option1 | Option2, {"two": [1]}, {"two": (1,)}, []), ( Tree, {"label": "a", "children": [{"label": "b", "children": []}]}, {"label": "a", "children": ({"label": "b", "children": ()},)}, [], ), ( Tree, {"label": "a", "children": [{"label": 1, "children": []}]}, {"label": "a", "children": ({"label": 1, "children": ()},)}, [(("children", 0, "label"), "invalid_type")], ), ], ids=[ "closed", "closed-keeps-the-order-keys-came-in", "closed-leaves-an-unknown-key-out", "extra-items", "extra-items-of-the-wrong-type", "open", "union-takes-the-branch-that-accepts", "union-falls-back-to-the-first-that-tried", "union-takes-the-branch-declaring-the-most-keys", "recursive", "recursive-located", ], ) def test_an_object_reads_as_what_its_typeddict_admits( annotation: object, value: object, parsed: dict[str, object], found: list[tuple[Loc, str]] ) -> None: # The result is a new dict of the keys the type allows, in input order. # A closed TypedDict reports a key it does not declare and leaves it # out. `extra_items` gives the type of every other key. An open # TypedDict keeps other keys. result = _read(annotation, value) assert result[0] == parsed assert list(cast_dict(result[0])) == list(parsed) assert _found(result) == found def cast_dict(value: object) -> dict[str, object]: assert isinstance(value, dict) return value def test_error_a_typed_dict_misses_a_required_key() -> None: value, problems = parser(Options, no_leaf)({"note": "n"}, ()) assert value == {"note": "n"} assert [(problem.loc, problem.kind) for problem in problems] == [(("level",), "missing_key")] # --- a TypedDict, read as the typing spec defines it ----------------------- _PRELUDE = """\ from typing import Annotated, Generic, NotRequired, Required, TypeVar from typing_extensions import Never, ReadOnly from {typeddicts} import TypedDict V = TypeVar("V") """ _modules = itertools.count() _TYPING_TAKES_PEP_728 = sys.version_info >= (3, 15) """Whether `typing.TypedDict` takes `closed=` and `extra_items=`, as `typing_extensions` does.""" def _declared( source: str, *, postponed: bool, monkeypatch: pytest.MonkeyPatch, typeddicts: str = "typing_extensions", ) -> types.ModuleType: """The module defined by `source`, written with or without `from __future__ import annotations`. The module is registered in `sys.modules`, because a string annotation is resolved in the module where its class was defined. `typeddicts` is the module that provides the `TypedDict` used in the declaration. """ name = f"tests._typeddicts_{next(_modules)}" module = types.ModuleType(name) monkeypatch.setitem(sys.modules, name, module) header = "from __future__ import annotations\n" if postponed else "" text = header + _PRELUDE.format(typeddicts=typeddicts) + textwrap.dedent(source) exec(text, module.__dict__) # noqa: S102 - the declaration under test return module _DECLARATIONS = [ pytest.param( """ class T(TypedDict, closed=True): a: int b: NotRequired[int] c: ReadOnly[NotRequired[str]] d: Annotated[NotRequired[int], 'meta'] """, {"a", "b", "c", "d"}, {"a"}, "Never", True, id="qualifiers-in-any-order", ), pytest.param( """ class T(TypedDict, total=False, closed=True): a: int b: Required[int] c: Annotated[Required[int], 'meta'] """, {"a", "b", "c"}, {"b", "c"}, "Never", True, id="total-false-and-required", ), pytest.param( """ class Base(TypedDict, total=False): a: int b: Required[int] class T(Base, closed=True): c: int d: NotRequired[int] """, {"a", "b", "c", "d"}, {"b", "c"}, "Never", True, id="totality-is-per-class", ), pytest.param( """ class Base(TypedDict): a: ReadOnly[NotRequired[int]] class T(Base): a: ReadOnly[int] """, {"a"}, {"a"}, "object", False, id="read-only-key-required-by-a-subclass", ), pytest.param( """ class Base(TypedDict): a: ReadOnly[int | str] b: ReadOnly[NotRequired[int]] class T(Base): a: ReadOnly[int] b: ReadOnly[Required[int]] """, {"a", "b"}, {"a", "b"}, "object", False, id="read-only-keys-narrowed", ), pytest.param( """ class Base(TypedDict, total=False): a: int class T(Base): a: int """, {"a"}, {"a"}, "object", False, id="redeclared-by-a-total-subclass", ), pytest.param( """ class T(TypedDict, total=False): a: Annotated['Annotated[Required[str], "x"]', "y"] b: str """, {"a", "b"}, {"a"}, "object", False, id="a-qualifier-quoted-inside-annotated", ), pytest.param( """ T = TypedDict("T", {"class": int, "that's": NotRequired[str], "": ReadOnly[bool]}) """, {"class", "that's", ""}, {"class", ""}, "object", False, id="functional-syntax-with-keys-that-are-not-names", ), pytest.param( "class T(TypedDict):\n a: int\n", {"a"}, {"a"}, "object", False, id="open-by-default", ), pytest.param( "class T(TypedDict, closed=False):\n a: int\n", {"a"}, {"a"}, "object", True, id="open-when-said", ), pytest.param( "class T(TypedDict, extra_items=int):\n a: int\n", {"a"}, {"a"}, "int", True, id="extra-items", ), pytest.param( "class T(TypedDict, extra_items='int'):\n a: int\n", {"a"}, {"a"}, "int", True, id="extra-items-written-as-a-string", ), pytest.param( "class T(TypedDict, extra_items='tuple[T, ...]'):\n a: int\n", {"a"}, {"a"}, "tuple[T, ...]", True, id="extra-items-naming-its-own-class", ), pytest.param( "class T(TypedDict, extra_items=ReadOnly[str]):\n a: int\n", {"a"}, {"a"}, "str", True, id="extra-items-read-only", ), pytest.param( """ class Base(TypedDict, closed=True): a: int class T(Base): pass """, {"a"}, {"a"}, "Never", True, id="closed-is-inherited", ), pytest.param( """ class Base(TypedDict, extra_items=ReadOnly[int]): a: int class T(Base): b: NotRequired[int] """, {"a", "b"}, {"a"}, "int", True, id="extra-items-are-inherited", ), pytest.param( """ class Base(TypedDict, extra_items=int): a: int class Left(Base): pass class Right(Base): pass class T(Left, Right): pass """, {"a"}, {"a"}, "int", True, id="a-diamond-over-extra-items", ), pytest.param( """ class Base(TypedDict): a: int class T(Base, closed=True): pass """, {"a"}, {"a"}, "Never", True, id="a-subclass-closes-an-open-base", ), pytest.param( """ class Base(TypedDict, closed=False): a: int class T(Base): pass """, {"a"}, {"a"}, "object", True, id="open-said-by-a-base", ), pytest.param( """ class G(TypedDict, Generic[V], closed=True): a: int class T(G[int]): pass """, {"a"}, {"a"}, "Never", True, id="through-a-generic-base", ), pytest.param( """ class A(TypedDict, closed=True): a: int class B(TypedDict, closed=True): b: NotRequired[int] class T(A, B): pass """, {"a", "b"}, {"a"}, "Never", True, id="bases-that-agree", ), pytest.param( """ class A(TypedDict, closed=False): a: int class B(TypedDict, closed=True): b: int class T(A, B): pass """, {"a", "b"}, {"a", "b"}, "Never", True, id="a-closed-base-beside-an-open-one", ), ] def _under_both_typeddicts(declarations: list[ParameterSet]) -> list[ParameterSet]: """Each declaration using `typing_extensions.TypedDict`, and also using `typing.TypedDict` where it can express the declaration.""" both: list[ParameterSet] = [] for declaration in declarations: source = cast("str", declaration.values[0]) both.append(pytest.param(*declaration.values, "typing_extensions", id=f"{declaration.id}")) if _TYPING_TAKES_PEP_728 or ("closed=" not in source and "extra_items=" not in source): both.append(pytest.param(*declaration.values, "typing", id=f"{declaration.id}-typing")) return both @pytest.mark.parametrize( ("source", "members", "required", "extra_items", "declared", "typeddicts"), _under_both_typeddicts(_DECLARATIONS), ) @pytest.mark.parametrize("postponed", [False, True], ids=["evaluated", "postponed"]) def test_typeddict_keys_follow_the_typing_spec( source: str, members: set[str], required: set[str], extra_items: str, declared: bool, typeddicts: str, postponed: bool, monkeypatch: pytest.MonkeyPatch, ) -> None: # When annotations are strings, as `from __future__ import annotations` # writes every annotation, `__required_keys__` does not see a qualifier. # Neither does `typing.TypedDict` on 3.11 under `ReadOnly` or # `Annotated`. The evaluated annotations do show it, so every spelling # of a declaration is read the same way. Openness is inherited, but the # runtime does not record that. module = _declared(source, postponed=postponed, monkeypatch=monkeypatch, typeddicts=typeddicts) keys = typeddict_keys(module.T) assert set(keys.members) == members assert keys.required == required assert keys.extra_items == eval(extra_items, vars(module)) assert keys.declared is declared _WRITTEN_WHERE = """ class _Hidden(TypedDict): x: int class Base(TypedDict, total=False): nested: tuple['_Hidden', ...] top: '_Hidden' kept: ReadOnly[int | str] """ @pytest.mark.parametrize( "child_postponed", [False, True], ids=["child-evaluated", "child-postponed"] ) @pytest.mark.parametrize("base_postponed", [False, True], ids=["base-evaluated", "base-postponed"]) def test_a_key_reads_where_its_class_was_written( base_postponed: bool, child_postponed: bool, monkeypatch: pytest.MonkeyPatch ) -> None: # The subclass's module never imports `_Hidden`, and the runtime ties a # string to its module only at the top level of an annotation. Each key # is still resolved in the module of the class that declared it. base = _declared(_WRITTEN_WHERE, postponed=base_postponed, monkeypatch=monkeypatch) child = _declared( f"from {base.__name__} import Base\nclass T(Base):\n own: int\n kept: ReadOnly[int]\n", postponed=child_postponed, monkeypatch=monkeypatch, ) assert typeddict_keys(child.T).required == {"kept", "own"} value = {"nested": [{"x": 1}], "top": {"x": 2}, "kept": 3, "own": 4} assert parser(child.T, no_leaf)(value, ()) == ({**value, "nested": ({"x": 1},)}, ()) @pytest.mark.parametrize( "postponed", [ pytest.param( False, id="evaluated", marks=pytest.mark.skipif( sys.version_info < (3, 14), reason="an unquoted later name needs PEP 649" ), ), pytest.param(True, id="postponed"), ], ) def test_a_typeddict_may_name_a_class_written_after_it( postponed: bool, monkeypatch: pytest.MonkeyPatch ) -> None: module = _declared( """ class T(TypedDict): later: NotRequired[Later] class Later(TypedDict): x: int """, postponed=postponed, monkeypatch=monkeypatch, ) assert parser(module.T, no_leaf)({"later": {"x": 1}}, ()) == ({"later": {"x": 1}}, ()) @pytest.mark.parametrize( ("base_source", "postponed"), [ ("Foo = int\nclass Base(TypedDict):\n x: Foo\n items: tuple[Foo, ...]\n", True), ("Foo = int\nclass Base(TypedDict):\n x: 'Foo'\n items: tuple['Foo', ...]\n", False), ], ids=["postponed", "quoted-by-hand"], ) def test_a_name_reads_as_the_module_that_wrote_it_means_it( base_source: str, postponed: bool, monkeypatch: pytest.MonkeyPatch ) -> None: # In the subclass's module, `Foo` means something else. The keys are # still read as the base class declared them, at the top level of an # annotation or nested inside it. base = _declared(base_source, postponed=postponed, monkeypatch=monkeypatch) child = _declared( f"from {base.__name__} import Base\nFoo = str\nclass T(Base):\n pass\n", postponed=True, monkeypatch=monkeypatch, ) read = parser(child.T, no_leaf) assert read({"x": 1, "items": [2]}, ())[1] == () assert [(found.loc, found.kind) for found in read({"x": "s", "items": ["t"]}, ())[1]] == [ (("x",), "invalid_type"), (("items", 0), "invalid_type"), ] def test_error_a_key_both_required_and_not_required(monkeypatch: pytest.MonkeyPatch) -> None: # The spec allows only one; the runtime and other checkers disagree on which wins. module = _declared( "class T(TypedDict):\n a: Required[NotRequired[int]]\n", postponed=True, monkeypatch=monkeypatch, ) with pytest.raises(TypeError, match="T.a: Required and NotRequired both"): typeddict_keys(module.T) def test_error_a_typeddict_whose_runtime_keys_are_not_its_annotations( monkeypatch: pytest.MonkeyPatch, ) -> None: # If the two disagreed, a key could stop being required without notice. module = _declared( "class T(TypedDict):\n a: int\n", postponed=False, monkeypatch=monkeypatch ) monkeypatch.setattr(module.T, "__required_keys__", frozenset({"a", "b"})) with pytest.raises(TypeError, match="T: its annotations declare"): typeddict_keys(module.T) def test_error_a_typeddict_whose_bases_disagree_on_other_keys( monkeypatch: pytest.MonkeyPatch, ) -> None: module = _declared( """ class A(TypedDict, closed=True): a: int class B(TypedDict, extra_items=int): b: int class T(A, B): pass """, postponed=False, monkeypatch=monkeypatch, ) with pytest.raises(TypeError, match="T: its bases disagree"): typeddict_keys(module.T) @pytest.mark.parametrize("name", ["T", "Outer"]) def test_error_a_typeddict_whose_annotations_do_not_resolve( name: str, monkeypatch: pytest.MonkeyPatch ) -> None: # The error names the TypedDict that holds the annotation, at any depth, # not an outer TypedDict whose parser would contain it. module = _declared( """ class T(TypedDict): a: Missing class Outer(TypedDict): inner: T """, postponed=True, monkeypatch=monkeypatch, ) with pytest.raises(TypeError, match="T: name 'Missing' is not defined"): parser_for(getattr(module, name), no_leaf) def test_error_a_key_that_does_not_resolve_where_it_was_written_names_its_class( monkeypatch: pytest.MonkeyPatch, ) -> None: # When asked through a subclass in another module, the error names the # class that declared the key. base = _declared( "class Base(TypedDict):\n items: tuple['Missing', ...]\n", postponed=False, monkeypatch=monkeypatch, ) child = _declared( f"from {base.__name__} import Base\nclass T(Base):\n own: int\n", postponed=False, monkeypatch=monkeypatch, ) with pytest.raises(TypeError, match="Base: name 'Missing' is not defined"): typeddict_keys(child.T) def test_error_a_typeddict_written_in_a_function_resolves_only_in_its_module() -> None: # This module postpones annotations, and `Local` is not a module-level name here. class Local(TypedDict, closed=True): x: int class Holder(TypedDict, closed=True): local: Local with pytest.raises(TypeError, match="Holder: name 'Local' is not defined"): typeddict_keys(Holder) T = TypeVar("T") class Generic_(TypedDict, Generic[T], closed=True): x: T @pytest.mark.parametrize( "annotation", [set[int], list[int], Literal[b"x"], Generic_, Generic_[int]], ids=["set", "list", "literal-bytes", "generic-typeddict", "generic-alias"], ) def test_error_an_annotation_outside_the_shapes_has_no_parser(annotation: object) -> None: assert parser_for(annotation, no_leaf) is None with pytest.raises(TypeError, match="is not a shape JSON takes"): parser(annotation, no_leaf) @pytest.mark.skipif(sys.version_info < (3, 12), reason="the `type` statement is 3.12 syntax") def test_error_a_type_statement_alias_that_names_what_is_not_defined( monkeypatch: pytest.MonkeyPatch, ) -> None: """A `type` statement evaluates its value when it is read. A name in it that is not defined is a `TypeError` naming the alias, as for an alias made with `TypeAliasType`, not a `NameError`.""" module = _declared("type Flag = Missing | bool\n", postponed=False, monkeypatch=monkeypatch) with pytest.raises(TypeError, match="Flag: name 'Missing' is not defined"): parser(module.Flag, no_leaf) @pytest.mark.skipif(sys.version_info < (3, 12), reason="the `type` statement is 3.12 syntax") def test_a_type_statement_alias_may_hold_itself(monkeypatch: pytest.MonkeyPatch) -> None: module = _declared( "type Nested = int | tuple[Nested, ...]\n", postponed=False, monkeypatch=monkeypatch ) assert parser(module.Nested, no_leaf)([1, [2, [3]]], ()) == ((1, (2, (3,))), ()) def test_a_leaf_is_asked_first_at_every_depth() -> None: # The only hook: a caller's own shapes, checked before the built-in ones, # inside an array or an object as well as at the top level. class Marker: pass def leaf(annotation: object) -> Parser | None: if annotation is not Marker: return None def parse(value: object, loc: Loc) -> Parsed: if isinstance(value, str) and value.startswith("m:"): return value[2:], () return value, ((loc, "expected a marker"),) # pyright: ignore[reportReturnType] return parse read = parser(tuple[Marker, ...], leaf) assert read(["m:a", "m:b"], ()) == (("a", "b"), ()) @pytest.mark.parametrize( ("annotation", "shape"), [ (Literal[True], "bool"), (Literal[1, 2], "int"), (Literal["a", "b"], "str"), (Literal[0, "auto"], None), (None, "null"), (tuple[int, ...], "tuple"), (Options, "mapping"), (Width, "int"), (JSONValue, None), ], ) def test_a_shape_is_what_a_union_dispatches_on(annotation: object, shape: str | None) -> None: assert shape_of(annotation) == shape @pytest.mark.parametrize( ("annotation", "value", "message"), [ (Blosc | Gzip, 3, "expected an object, got 3"), (Literal["0.5"], 0.5, 'expected "0.5", got 0.5'), (Literal["C", "F"], "Q", 'expected one of ["C", "F"], got "Q"'), (tuple[str, ...], "nuclei", 'expected an array, got "nuclei"'), (str, None, "expected a string, got null"), ], ids=["a-union-of-objects", "one-value", "values", "array", "null"], ) def test_a_message_names_what_was_expected_and_shows_the_json_it_got( annotation: object, value: object, message: str ) -> None: assert [problem.message for problem in _read(annotation, value)[1]] == [message] def test_a_shape_is_described_as_a_message_would_name_it() -> None: assert describe(Literal[0, "auto"]) == 'one of ["auto", 0]' assert describe(Literal["0.5"]) == '"0.5"' assert describe(int | None) == "an integer or null" assert describe(Blosc | Gzip) == "an object" assert describe(float | int | None) == "a number or null" # An array's elements are named in the plural. assert describe(tuple[int, ...]) == "an array of integers" assert describe(tuple[int | None, ...]) == "an array of integers or nulls" assert describe(tuple[tuple[str, ...], ...]) == "an array of arrays of strings" assert describe(tuple[Literal["C", "F"], ...]) == 'an array of values in ["C", "F"]' assert describe(int | tuple[int, ...]) == "an integer or an array of integers" # A broader shape includes a narrower one: "a string" covers the hex # string spelling of a float32 and the strings of its non-finite values. assert describe(Float32FillValue) == "a number or a string" assert describe(Literal["C", "F"] | None) == 'one of ["C", "F"] or null' assert describe(Width) == "an integer" assert describe(ZarrV2DataTypeMetadata) == "a ZarrV2DataTypeMetadata" # --- constraints ------------------------------------------------------------ Digit = Annotated[int, Interval(ge=0, le=9)] class Bounded(TypedDict, extra_items=Annotated[int, Ge(0)]): level: Digit shape: ReadOnly[NotRequired[tuple[Annotated[int, Ge(1)], ...]]] name: Annotated[NotRequired[Annotated[str, "the inner note"]], Doc("the outer note")] class Noted(TypedDict, extra_items=Annotated[object, "any JSON"]): level: int @pytest.mark.parametrize( ("annotation", "value", "found"), [ (Digit, 9, []), (Digit, 10, [((), "expected an integer in [0, 9], got 10", {"ge": 0, "le": 9})]), (Digit, -1, [((), "expected an integer in [0, 9], got -1", {"ge": 0, "le": 9})]), (Annotated[int, Gt(0)], 0, [((), "expected an integer > 0, got 0", {"gt": 0})]), (Annotated[float, Lt(1)], 1, [((), "expected a number < 1, got 1", {"lt": 1})]), (Annotated[float, Le(0.5)], 0.5, []), ( Annotated[float, Interval(gt=0, le=0.5)], 0.0, [((), "expected a number in (0, 0.5], got 0.0", {"gt": 0, "le": 0.5})], ), ( Annotated[float, Interval(ge=0.0, le=9.0)], 10, [((), "expected a number in [0, 9], got 10", {"ge": 0, "le": 9})], ), (Annotated[int, Ge(True)], 0, [((), "expected an integer >= 1, got 0", {"ge": 1})]), ( tuple[Digit, ...], [0, 10, 9], [((1,), "expected an integer in [0, 9], got 10", {"ge": 0, "le": 9})], ), (Annotated[Width, Le(9)], 10, [((), "expected an integer <= 9, got 10", {"le": 9})]), (Digit | str, "ten", []), (Digit | str, 10, [((), "expected an integer in [0, 9], got 10", {"ge": 0, "le": 9})]), (Digit, "nine", [((), 'expected an integer, got "nine"', {})]), (Annotated[int, "a note", Doc("another")], -1, []), ], ids=[ "within", "above", "below", "exclusive-lower", "exclusive-upper", "inclusive-upper", "half-open", "an-integral-bound-is-an-integer", "a-bool-bound-is-the-integer-it-equals", "each-element-at-its-index", "on-an-alias", "a-branch-it-does-not-bound", "the-branch-it-bounds", "a-value-of-another-type-is-that-problem-alone", "notes", ], ) def test_a_value_is_held_to_the_bounds_its_type_carries( annotation: object, value: object, found: list[tuple[Loc, str, dict[str, JSONValue]]] ) -> None: # annotated-types bounds, read as pydantic reads them. A value outside # any bound gives one problem that says what the type allows, with the # bounds in the problem's `ctx`. Bounds are checked only for a value of # the type. A bound is compared as the number it equals, whichever of # the equal values Python's `Annotated` cache returns. _, problems = _read(annotation, value) assert [(problem.loc, problem.message, dict(problem.ctx)) for problem in problems] == found assert all(problem.kind == "invalid_value" for problem in problems if problem.ctx) def test_a_key_s_type_keeps_the_metadata_its_annotation_carries() -> None: # Qualifiers are removed wherever they sit among the `Annotated` layers. # The metadata is kept, inner layer first, as `Annotated` flattens # nested layers. A note on `object` leaves the type open. keys = typeddict_keys(Bounded) assert keys.members["level"] == (Digit, True) assert keys.members["shape"] == (tuple[Annotated[int, Ge(1)], ...], False) assert keys.members["name"] == ( Annotated[str, "the inner note", Doc("the outer note")], False, ) assert keys.extra_items == Annotated[int, Ge(0)] typed, problems = check({"level": 3, "shape": [0], "other": -1}, Bounded) assert typed is None assert [(problem.loc, problem.input, dict(problem.ctx)) for problem in problems] == [ (("other",), -1, {"ge": 0}), (("shape", 0), 0, {"ge": 1}), ] assert typeddict_keys(Noted).open assert check({"level": 1, "more": [None]}, Noted) == ({"level": 1, "more": (None,)}, ()) def test_error_a_bound_on_what_is_not_a_number() -> None: with pytest.raises(TypeError, match="ge: a bound is on a number, and a string is not one"): parser(Annotated[str, Ge(0)], no_leaf) def test_error_a_bound_on_the_values_an_open_typeddict_takes() -> None: class Loose(TypedDict, extra_items=Annotated[object, Ge(0)]): level: int with pytest.raises(TypeError, match="ge: a bound is on a number, and a value is not one"): parser(Loose, no_leaf) @dataclasses.dataclass(frozen=True) class Later(BaseMetadata): """A constraint that a later release of annotated-types might add.""" @pytest.mark.parametrize( "constraint", [ MinLen(1), Len(1, 3), MultipleOf(2), Predicate(str.isdigit), Timezone(None), Later(), pydantic.Field(ge=0), pydantic.StringConstraints(pattern="^[a-z]+$"), ], ids=[ "min-len", "len", "multiple-of", "predicate", "timezone", "one-a-later-release-adds", "pydantic-field", "pydantic-string-constraints", ], ) def test_error_a_constraint_the_checker_does_not_read(constraint: object) -> None: # If the checker ignored a constraint, the type would promise something # that is not checked, and pydantic and the checker would disagree. with pytest.raises(UnreadConstraintError, match="is not a constraint the checker reads"): parser(Annotated[str, constraint], no_leaf) class _ConstrainedMember(TypedDict, closed=True): a: Annotated[int, MultipleOf(2)] class _HoldsAConstrainedMember(TypedDict, closed=True): inner: _ConstrainedMember class _NotAJSONShape(TypedDict, closed=True): a: set[int] class _HoldsAMemberNotAJSONShape(TypedDict, closed=True): inner: _NotAJSONShape class _Readable(TypedDict, closed=True): a: int # `_Missing` is not defined on purpose: the TypedDict cannot be read at all. class _Unresolvable(TypedDict, closed=True): a: "_Missing" # noqa: F821 # pyright: ignore[reportUndefinedVariable] class _HoldsAnUnresolvableTypedDict(TypedDict, closed=True): inner: _Unresolvable class _ConstraintOnAReadableMember(TypedDict, closed=True): inner: Annotated[_Readable, MultipleOf(2)] class _ConstraintOnAConstrainedMember(TypedDict, closed=True): inner: Annotated[_ConstrainedMember, MultipleOf(2)] class _SelfReferentialWithAConstraint(TypedDict, closed=True): parent: NotRequired[_SelfReferentialWithAConstraint] a: Annotated[int, MultipleOf(2)] class _SelfReferentialNotAJSONShape(TypedDict, closed=True): parent: NotRequired[_SelfReferentialNotAJSONShape] a: set[int] class _SelfReferentialConstraintOnItself(TypedDict, closed=True): parent: NotRequired[Annotated[_SelfReferentialConstraintOnItself, MultipleOf(2)]] a: int class _CycleOut(TypedDict, closed=True): b: _CycleBack class _CycleBack(TypedDict, closed=True): a: NotRequired[_CycleOut] x: set[int] class _OuterWithAConstraintAfterACycle(TypedDict, closed=True): inner: _InnerLeadingBackToAConstraint a: Annotated[int, MultipleOf(2)] class _InnerLeadingBackToAConstraint(TypedDict, closed=True): back: NotRequired[_OuterWithAConstraintAfterACycle] class _OuterNotAJSONShapeAfterACycle(TypedDict, closed=True): inner: _InnerLeadingBackToAShape a: set[int] class _InnerLeadingBackToAShape(TypedDict, closed=True): back: NotRequired[_OuterNotAJSONShapeAfterACycle] class _OuterWithAConstraintOnTheCycle(TypedDict, closed=True): inner: Annotated[_InnerLeadingBackToTheConstraint, MultipleOf(2)] a: int class _InnerLeadingBackToTheConstraint(TypedDict, closed=True): back: NotRequired[_OuterWithAConstraintOnTheCycle] class _SelfReferentialTwiceWithAConstraintOnTheSecond(TypedDict, closed=True): plain: NotRequired[_SelfReferentialTwiceWithAConstraintOnTheSecond] constrained: NotRequired[ Annotated[_SelfReferentialTwiceWithAConstraintOnTheSecond, MultipleOf(2)] ] class _DocumentedThenConstrainedBackReferences(TypedDict, closed=True): documented: NotRequired[Annotated[_DocumentedThenConstrainedBackReferences, Doc("parent")]] constrained: NotRequired[Annotated[_DocumentedThenConstrainedBackReferences, MultipleOf(2)]] class _OuterWithAConstraintAfterAnOptionalCycle(TypedDict, closed=True): inner: _InnerLeadingBackOptionally a: Annotated[int, MultipleOf(2)] class _InnerLeadingBackOptionally(TypedDict, closed=True): back: _OuterWithAConstraintAfterAnOptionalCycle | None class _HoldsAnOptionalNotAJSONShape(TypedDict, closed=True): inner: _NotAJSONShape | None class _HoldsATupleOfAConstrainedMember(TypedDict, closed=True): inner: tuple[_ConstrainedMember, ...] class _ConstraintOnANotAJSONShapeMember(TypedDict, closed=True): inner: Annotated[_NotAJSONShape, MultipleOf(2)] _Children = TypeAliasType("_Children", "tuple[_SelfReferentialThroughAnAlias, ...]") class _SelfReferentialThroughAnAlias(TypedDict, closed=True): children: NotRequired[_Children] a: set[int] class _BoundOnATupleThatLeadsBack(TypedDict, closed=True): children: NotRequired[Annotated[tuple[_BoundOnATupleThatLeadsBack, ...], Ge(0)]] a: int class _BoundOnATupleThatLeadsBackBeforeANotAJSONShape(TypedDict, closed=True): children: NotRequired[ Annotated[tuple[_BoundOnATupleThatLeadsBackBeforeANotAJSONShape, ...], Ge(0)] ] a: set[int] class _SelfReferentialThroughANewType(TypedDict, closed=True): parent: NotRequired[_ParentOfANode] a: set[int] # A type checker refuses a NewType of a TypedDict, but the checker reads # one at run time, so `unread_in` must follow it too. _ParentOfANode = NewType("_ParentOfANode", _SelfReferentialThroughANewType) # pyright: ignore[reportGeneralTypeIssues] # `_MissingType` is not defined on purpose: the alias cannot be resolved. _UndefinedAlias = TypeAliasType("_UndefinedAlias", "_MissingType") # noqa: F821 # pyright: ignore[reportUndefinedVariable] class _HoldsAnUndefinedAlias(TypedDict, closed=True): a: _UndefinedAlias class _SelfReferentialReadable(TypedDict, closed=True): parent: NotRequired[_SelfReferentialReadable] a: int @pytest.mark.parametrize( ("typeddict", "unread"), [ ( _ConstrainedMember, Unread( "_ConstrainedMember.a: MultipleOf(multiple_of=2) is not a constraint the checker " "reads, which are Gt, Ge, Lt and Le", constraint=True, ), ), ( _HoldsAConstrainedMember, Unread( "_HoldsAConstrainedMember.inner: _ConstrainedMember.a: MultipleOf(multiple_of=2) is " "not a constraint the checker reads, which are Gt, Ge, Lt and Le", constraint=True, ), ), ( _NotAJSONShape, Unread( "_NotAJSONShape: a is not a shape JSON takes; write it as one -- a number, string, " "boolean, null, array, object, or an alias of one", constraint=False, ), ), ( _HoldsAMemberNotAJSONShape, Unread( "_HoldsAMemberNotAJSONShape.inner: _NotAJSONShape: a is not a shape JSON takes; " "write it as one -- a number, string, boolean, null, array, object, or an alias of " "one", constraint=False, ), ), ( _HoldsAnUnresolvableTypedDict, Unread( "_HoldsAnUnresolvableTypedDict.inner: _Unresolvable: name '_Missing' is not " "defined; its annotations must resolve in its module", constraint=False, ), ), ( _ConstraintOnAReadableMember, Unread( "_ConstraintOnAReadableMember.inner: MultipleOf(multiple_of=2) is not a " "constraint the checker reads, which are Gt, Ge, Lt and Le", constraint=True, ), ), ( _ConstraintOnAConstrainedMember, Unread( "_ConstraintOnAConstrainedMember.inner: _ConstrainedMember.a: " "MultipleOf(multiple_of=2) is not a constraint the checker reads, which are Gt, " "Ge, Lt and Le", constraint=True, ), ), ( _SelfReferentialWithAConstraint, Unread( "_SelfReferentialWithAConstraint.a: MultipleOf(multiple_of=2) is not a constraint " "the checker reads, which are Gt, Ge, Lt and Le", constraint=True, ), ), ( _SelfReferentialNotAJSONShape, Unread( "_SelfReferentialNotAJSONShape: a is not a shape JSON takes; write it as one -- a " "number, string, boolean, null, array, object, or an alias of one", constraint=False, ), ), ( _SelfReferentialConstraintOnItself, Unread( "_SelfReferentialConstraintOnItself.parent: MultipleOf(multiple_of=2) is not a " "constraint the checker reads, which are Gt, Ge, Lt and Le", constraint=True, ), ), ( _CycleOut, Unread( "_CycleOut.b: _CycleBack: x is not a shape JSON takes; write it as one -- a " "number, string, boolean, null, array, object, or an alias of one", constraint=False, ), ), ( _OuterWithAConstraintAfterACycle, Unread( "_OuterWithAConstraintAfterACycle.a: MultipleOf(multiple_of=2) is not a " "constraint the checker reads, which are Gt, Ge, Lt and Le", constraint=True, ), ), ( _OuterNotAJSONShapeAfterACycle, Unread( "_OuterNotAJSONShapeAfterACycle: a is not a shape JSON takes; write it as one -- a " "number, string, boolean, null, array, object, or an alias of one", constraint=False, ), ), ( _OuterWithAConstraintOnTheCycle, Unread( "_OuterWithAConstraintOnTheCycle.inner: MultipleOf(multiple_of=2) is not a " "constraint the checker reads, which are Gt, Ge, Lt and Le", constraint=True, ), ), ( _SelfReferentialTwiceWithAConstraintOnTheSecond, Unread( "_SelfReferentialTwiceWithAConstraintOnTheSecond.constrained: " "MultipleOf(multiple_of=2) is not a constraint the checker reads, which are Gt, " "Ge, Lt and Le", constraint=True, ), ), ( _DocumentedThenConstrainedBackReferences, Unread( "_DocumentedThenConstrainedBackReferences.constrained: MultipleOf(multiple_of=2) " "is not a constraint the checker reads, which are Gt, Ge, Lt and Le", constraint=True, ), ), ( _OuterWithAConstraintAfterAnOptionalCycle, Unread( "_OuterWithAConstraintAfterAnOptionalCycle.a: MultipleOf(multiple_of=2) is not a " "constraint the checker reads, which are Gt, Ge, Lt and Le", constraint=True, ), ), ( _HoldsAnOptionalNotAJSONShape, Unread( "_HoldsAnOptionalNotAJSONShape.inner: _NotAJSONShape: a is not a shape JSON takes; " "write it as one -- a number, string, boolean, null, array, object, or an alias of " "one", constraint=False, ), ), ( _HoldsATupleOfAConstrainedMember, Unread( "_HoldsATupleOfAConstrainedMember.inner: _ConstrainedMember.a: " "MultipleOf(multiple_of=2) is not a constraint the checker reads, which are Gt, " "Ge, Lt and Le", constraint=True, ), ), ( _ConstraintOnANotAJSONShapeMember, Unread( "_ConstraintOnANotAJSONShapeMember.inner: _NotAJSONShape: a is not a shape JSON " "takes; write it as one -- a number, string, boolean, null, array, object, or an " "alias of one", constraint=False, ), ), ( _SelfReferentialThroughAnAlias, Unread( "_SelfReferentialThroughAnAlias: a is not a shape JSON takes; write it as one -- a " "number, string, boolean, null, array, object, or an alias of one", constraint=False, ), ), ( _BoundOnATupleThatLeadsBack, Unread( "_BoundOnATupleThatLeadsBack.children: ge: a bound is on a number, and an array of " "objects is not one", constraint=False, ), ), ( _BoundOnATupleThatLeadsBackBeforeANotAJSONShape, Unread( "_BoundOnATupleThatLeadsBackBeforeANotAJSONShape.children: ge: a bound is on a " "number, and an array of objects is not one", constraint=False, ), ), ( _SelfReferentialThroughANewType, Unread( "_SelfReferentialThroughANewType: a is not a shape JSON takes; write it as one -- a " "number, string, boolean, null, array, object, or an alias of one", constraint=False, ), ), ( _HoldsAnUndefinedAlias, Unread( "_HoldsAnUndefinedAlias.a: _UndefinedAlias: name '_MissingType' is not defined; " "its value must resolve in its module", constraint=False, ), ), (_SelfReferentialReadable, None), (_Readable, None), ], ids=[ "a-constraint", "a-constraint-in-a-nested-typeddict", "not-a-json-shape", "not-a-json-shape-in-a-nested-typeddict", "a-nested-typeddict-that-cannot-be-read", "a-constraint-on-a-member-that-is-a-typeddict", "a-constraint-on-the-member-and-inside-its-typeddict", "self-referential-with-a-constraint", "self-referential-not-a-json-shape", "self-referential-with-a-constraint-on-the-reference", "a-cycle-of-two-typeddicts", "a-constraint-after-a-member-that-leads-back", "not-a-json-shape-after-a-member-that-leads-back", "a-constraint-on-a-member-that-leads-back", "a-constraint-on-a-second-member-that-leads-back", "a-documented-member-that-leads-back-before-a-constrained-one", "a-constraint-after-a-member-that-leads-back-optionally", "a-typeddict-in-a-union", "a-typeddict-in-a-tuple", "a-constraint-on-a-member-whose-typeddict-is-not-a-json-shape", "self-referential-through-an-alias", "a-bound-on-a-tuple-that-leads-back", "a-bound-on-a-tuple-that-leads-back-before-a-member-not-a-json-shape", "self-referential-through-a-newtype", "an-alias-that-does-not-resolve", "self-referential-and-readable", "readable", ], ) def test_unread_in_names_the_part_without_a_parser_and_whether_it_is_a_constraint( typeddict: type, unread: Unread | None ) -> None: """`unread_in` names the first part of a TypedDict that the checker has no parser for, down to the TypedDict that contains it, also through a union or a container, and says whether it is a constraint the checker does not read, so a definition can advise checking it in its `rules`. A member that is not a JSON shape is not such a constraint. A nested TypedDict that cannot be read at all is named with the reason. When both a member and the TypedDict it holds have such a part, the one inside is named first, as the checker meets it first. A TypedDict that holds itself, directly or through another, is named where its own members are unread, and a member that leads back, also through a union, is named only when nothing else is, unless metadata of its own, such as a constraint but not a `Doc`, is unread. A TypedDict the checker reads whole has no unread part.""" assert unread_in(typeddict, no_leaf) == unread @pytest.mark.parametrize( "annotation", [ Annotated[Digit, Ge(1)], Annotated[int, Ge(0), Gt(0)], Annotated[int, Interval(le=9), Lt(5)], ], ids=["narrowing-an-alias", "gt-and-ge", "interval-and-lt"], ) def test_error_a_second_bound_from_one_side(annotation: object) -> None: # Pydantic uses the last bound given; write only one. with pytest.raises(TypeError, match="is a second bound from (below|above)"): parser(annotation, no_leaf) @pytest.mark.parametrize( "bound", ["0", math.nan, math.inf, None], ids=["string", "nan", "infinity", "null"] ) def test_error_a_bound_that_is_not_a_finite_number(bound: object) -> None: with pytest.raises(TypeError, match="a bound is a finite number"): parser(Annotated[int, Ge(bound)], no_leaf) # pyright: ignore[reportArgumentType] # --- check: the public door ------------------------------------------------ _V3_DOCUMENT = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) _V2_DOCUMENT = { **ZarrV2ArrayMetadata.create_default(shape=(4,)).to_json(), "dtype": [["a", " None: # For any TypedDict, from the package or not, the result is a value of # it, with arrays as tuples, holding only what the TypedDict allows. typed, problems = check(value, shape) assert [(problem.loc, problem.kind) for problem in problems] == found assert typed is not None assert parser(shape, no_leaf)(typed, ())[1] == () assert set(typed) == {key for key in value if (key,) not in {loc for loc, _ in found}} def test_error_check_is_asked_of_a_typeddict() -> None: with pytest.raises(TypeError, match="is not a TypedDict"): check({}, dict) def test_error_check_names_what_no_parser_reads() -> None: # The error names the path down to the TypedDict that holds it, at any depth. with pytest.raises(TypeError, match=r"Holds.unreadable: Unreadable: members is not a shape"): check({}, Holds) @pytest.mark.parametrize( ("value", "loc", "kind"), [ ({"level": math.nan}, ("level",), "invalid_value"), ({"level": 1, "note": math.inf}, ("note",), "invalid_value"), ({"level": 1, 2: 3}, (), "invalid_type"), ({"level": 1, "note": {1}}, ("note",), "invalid_type"), ({"level": 1, "note": [{3: 1}]}, ("note", 0), "invalid_type"), ], ids=[ "nan", "infinity", "a-key-that-is-not-a-string", "a-set", "nested-key-that-is-not-a-string", ], ) def test_error_check_locates_a_value_that_is_not_json( value: dict[object, object], loc: Loc, kind: str ) -> None: # Rejected before any TypedDict is checked: `check` compares only JSON values. typed, problems = check(value, Options) assert typed is None assert [(problem.loc, problem.kind) for problem in problems] == [(loc, kind)] def test_error_check_refuses_null() -> None: # JSON null is valid JSON but not an object. Refining it gives `None` # with no problem, which would not be a usable result. typed, problems = check(None, Options) assert typed is None assert [(problem.loc, problem.kind) for problem in problems] == [((), "invalid_type")] def test_error_check_refuses_a_value_of_another_type() -> None: typed, problems = check({"level": "high"}, Options) assert typed is None assert [(problem.loc, problem.kind) for problem in problems] == [(("level",), "invalid_type")] _NOT_DOCUMENTS: Final = frozenset({"ZarrV3ContextSlots", "ZarrV2ContextSlots"}) """TypedDicts that describe no document: the keyword arguments of a scope class, which hold definitions, not JSON.""" def _declared_typeddicts() -> list[type]: """Every TypedDict that describes a document, exported by a public module of the package's JSON types or of its definitions.""" found: dict[type, None] = {} for package in (zarr_metadata.definition, zarr_metadata.v2, zarr_metadata.v3): modules = [package.__name__] + [ info.name for info in pkgutil.walk_packages(package.__path__, prefix=f"{package.__name__}.") if not any(part.startswith("_") for part in info.name.split(".")) ] for name in modules: module = importlib.import_module(name) for exported in getattr(module, "__all__", ()): value = getattr(module, exported) if ( isinstance(value, type) and is_typeddict(value) and exported not in _NOT_DOCUMENTS ): found[value] = None return list(found) @pytest.mark.parametrize( "typeddict", _declared_typeddicts(), ids=lambda typeddict: typeddict.__name__ ) def test_every_typeddict_the_package_declares_compiles(typeddict: type) -> None: # A declaration that no parser can read would be a document type that # `check` cannot check. assert parser_for(typeddict, no_leaf) is not None def test_a_union_of_a_string_and_another_shape_names_both() -> None: # "a string" absorbs only the string Literals, not the other branches. assert describe(str | int) == "a string or an integer" assert describe(str | None) == "a string or null" (problem,) = parser(str | int, no_leaf)(None, ())[1] assert problem.message == "expected a string or an integer, got null" _PQ = TypeAliasType("_PQ", Literal["p", "q"]) @pytest.mark.parametrize( ("annotation", "value", "message", "kind", "expected"), [ ( float | Literal["NaN"] | None, "abc", 'expected a number or "NaN" or null, got "abc"', "invalid_value", ("NaN",), ), ( Literal[-1, 0, 1, 2] | bool, 3, "expected one of [-1, 0, 1, 2] or a boolean, got 3", "invalid_value", (-1, 0, 1, 2), ), ( Literal[1, "a"] | None, True, 'expected one of ["a", 1] or null, got true', "invalid_type", ("a", 1), ), ( _PQ | None, "r", 'expected one of ["p", "q"] or null, got "r"', "invalid_value", ("p", "q"), ), ( _PQ | Literal[3], 4, 'expected one of ["p", "q"] or 3, got 4', "invalid_value", ("p", "q", 3), ), ( Literal[3] | _PQ, 4, 'expected 3 or one of ["p", "q"], got 4', "invalid_value", (3, "p", "q"), ), (_PQ | Literal[3], None, 'expected one of ["p", "q"] or 3, got null', "invalid_type", None), ( Literal[0, 1, 2] | Literal[False, True], # noqa: PYI030 - two branches on purpose 5, "expected one of [0, 1, 2] or one of [false, true], got 5", "invalid_value", (0, 1, 2, False, True), ), ], ids=[ "string-outside-the-literal", "number-outside-the-literal", "mixed-literal", "literal-behind-an-alias", "literals-of-two-types", "literals-in-the-order-the-message-names-them", "a-shape-no-literal-has", "numbers-and-booleans", ], ) def test_error_a_value_outside_every_literal_of_a_union_names_the_whole_union( annotation: object, value: object, message: str, kind: str, expected: tuple[object, ...] | None, ) -> None: """A value whose shape only `Literal` branches of a union have, and that none of them holds, is reported against the whole union, as a value of no branch's shape is, also when a `Literal` is behind a type alias. The kind is the one the `Literal` reports: a wrong value when its JSON type matches an entry, and a wrong type otherwise. When only `Literal` branches tried the value, the `ctx` holds every value the `Literal` branches allow under `expected`, in the order the message names them, as one `Literal`'s problem does. A value of a shape no branch has is not refused by a closed set of values, so its problem has no `expected`.""" (problem,) = parser(annotation, no_leaf)(value, ())[1] assert (problem.message, problem.kind) == (message, kind) # Compare as JSON, since `0 == False` in Python. assert json.dumps(dict(problem.ctx)) == json.dumps( {} if expected is None else {"expected": expected} ) class _Three(TypedDict, closed=True): x: Annotated[Required[Annotated[ReadOnly[Annotated[int, "inner"]], "middle"]], "outer"] def test_metadata_of_three_annotated_layers_is_kept_in_order() -> None: assert typeddict_keys(_Three).members["x"] == (Annotated[int, "inner", "middle", "outer"], True) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/test_typed_json_properties.py000066400000000000000000001012711527014137600332060ustar00rootroot00000000000000"""Compare the checker with a reference, over TypedDicts and JSON values drawn at random. Each test first draws a spec: a small description of a type in the checker's vocabulary. A spec can use scalars, `Literal`s, arrays, unions, `Mapping[str, V]`, `NewType`s and aliases, and TypedDicts with up to two bases. Each class has its own `total`, its keys' qualifiers in any order, and its own setting for other keys. Some classes contain themselves. The spec is written out as source text and built from that source twice: once with each class's annotations evaluated, and once with them left as the strings that `from __future__ import annotations` leaves. The reference is `conforms`, a predicate over JSON derived from the spec alone by the typing spec's rules, not by the checker's rules. Values are JSON with a capped depth. Some are drawn from the spec, so they conform. Some are drawn from all JSON, so most do not conform. Some are drawn from the spec and changed in one place, so they almost conform. For every value, the checker must agree with the reference, and the two builds must agree with each other. The JSON Schema generated for the type must agree too, except that JSON Schema reads a number with no fraction, such as `1.0`, as the integer it equals. """ from __future__ import annotations import copy import itertools import json import sys import types import typing from collections.abc import Mapping from dataclasses import dataclass, field from typing import Annotated, Any, Literal, NewType, NotRequired, Required, TypeAlias, Union, cast import pytest from hypothesis import HealthCheck, event, find, given, note, settings from hypothesis import strategies as st from jsonschema import Draft202012Validator from typing_extensions import ReadOnly, TypeAliasType, TypedDict from zarr_metadata._typed_json import Parsed, Schemas, no_leaf, parser, typeddict_keys from zarr_metadata.typed_json import JSONValue, Loc # --- specs ----------------------------------------------------------------- Scalar: TypeAlias = Literal["int", "float", "bool", "str", "null", "json"] @dataclass(frozen=True) class Leaf: name: Scalar @dataclass(frozen=True) class OneOf: values: tuple[str | int | bool | None, ...] @dataclass(frozen=True) class ArrayOf: element: Spec @dataclass(frozen=True) class Fixed: elements: tuple[Spec, ...] @dataclass(frozen=True) class AnyOf: branches: tuple[Spec, ...] @dataclass(frozen=True) class MappingOf: value: Spec @dataclass(frozen=True) class Named: how: Literal["newtype", "alias"] inner: Spec @dataclass(frozen=True) class Itself: """The TypedDict that contains the member: a class that contains itself.""" @dataclass(frozen=True) class ExtraItems: spec: Spec Says: TypeAlias = Literal["unsaid", "closed", "open"] | ExtraItems """How a class treats the keys it does not declare: not stated, `closed=True`, `closed=False`, or `extra_items=`.""" @dataclass(frozen=True) class Member: key: str spec: Spec wrappers: tuple[str, ...] """`Required`, `NotRequired`, `ReadOnly` and `Annotated`, as drawn, innermost first.""" @dataclass(frozen=True) class Layer: """One class in a TypedDict's ancestry: its own keys and keywords, and how its module writes its annotations.""" total: bool members: tuple[Member, ...] says: Says postponed: bool = False """In a mixed build, whether the class's annotations are left as strings.""" quoted: bool = False """When annotations are evaluated, whether the class writes the name of another class as a string inside an annotation.""" @dataclass(frozen=True) class Object: layers: tuple[Layer, ...] """Base first; the last is the TypedDict itself. Each is written in a module of its own.""" stdlib: bool = False """Whether the classes are `typing.TypedDict`s rather than `typing_extensions` ones.""" Spec: TypeAlias = Leaf | OneOf | ArrayOf | Fixed | AnyOf | MappingOf | Named | Itself | Object def members_of(spec: Object) -> dict[str, tuple[Spec, bool]]: """Each key the TypedDict declares, its spec, and whether it is required, by the typing spec. A `Required` or `NotRequired` qualifier decides; otherwise the `total` of the class that declared the key decides. """ found: dict[str, tuple[Spec, bool]] = {} for layer in spec.layers: for member in layer.members: found[member.key] = (member.spec, _required(member, layer.total)) return found def _required(member: Member, total: bool) -> bool: if "Required" in member.wrappers: return True if "NotRequired" in member.wrappers: return False return total def says_of(spec: Object) -> Says: """How the TypedDict treats other keys: the setting of the nearest class that states one, as the typing spec defines it.""" said: Says = "unsaid" for layer in spec.layers: if layer.says != "unsaid": said = layer.says return said def conforms(value: object, spec: Spec, itself: Object | None = None) -> bool: """Whether JSON `value` is a value of `spec`. This is the reference, derived from the spec alone.""" if isinstance(spec, Leaf): return { "int": type(value) is int, "float": type(value) in (int, float), "bool": type(value) is bool, "str": type(value) is str, "null": value is None, "json": True, }[spec.name] if isinstance(spec, OneOf): return any(value == entry and type(value) is type(entry) for entry in spec.values) if isinstance(spec, ArrayOf): return isinstance(value, (list, tuple)) and all( conforms(entry, spec.element, itself) for entry in value ) if isinstance(spec, Fixed): return ( isinstance(value, (list, tuple)) and len(value) == len(spec.elements) and all( conforms(entry, element, itself) for entry, element in zip(value, spec.elements, strict=True) ) ) if isinstance(spec, AnyOf): return any(conforms(value, branch, itself) for branch in spec.branches) if isinstance(spec, MappingOf): return isinstance(value, dict) and all( conforms(entry, spec.value, itself) for entry in value.values() ) if isinstance(spec, Named): return conforms(value, spec.inner, itself) if isinstance(spec, Itself): assert itself is not None return conforms(value, itself, itself) if not isinstance(value, dict): return False members = members_of(spec) for key, (member, required) in members.items(): if key in value: if not conforms(value[key], member, spec): return False elif required: return False others = [key for key in value if key not in members] said = says_of(spec) if said == "closed": return len(others) == 0 if isinstance(said, ExtraItems): return all(conforms(value[key], said.spec, spec) for key in others) return True # --- building a spec, evaluated or postponed -------------------------------- _PRELUDE: dict[str, object] = { "Annotated": Annotated, "JSONValue": JSONValue, "Literal": Literal, "Mapping": Mapping, "NewType": NewType, "NotRequired": NotRequired, "ReadOnly": ReadOnly, "Required": Required, "TypeAliasType": TypeAliasType, "Union": Union, } _serial = itertools.count() _LEAF_TEXT: dict[str, str] = { "int": "int", "float": "float", "bool": "bool", "str": "str", "null": "None", "json": "JSONValue", } Mode: TypeAlias = Literal["evaluated", "postponed", "mixed"] """How a build writes its classes' annotations: all evaluated, all strings, or as each class chooses.""" _ONE_MODULE_FOR_STDLIB = sys.version_info < (3, 12) """`typing.TypedDict` records a class's bases from 3.12 on. Before 3.12, a subclass reads all its keys in one module.""" def _module() -> types.ModuleType: """A new module that holds only the prelude, registered in `sys.modules` so string annotations resolve in it.""" module = types.ModuleType(f"tests._typed_json_properties_{next(_serial)}") module.__dict__.update(_PRELUDE) sys.modules[module.__name__] = module return module def _evaluated(text: str, home: types.ModuleType) -> object: return eval(text, home.__dict__) @dataclass(frozen=True) class Build: """A spec built from source text, with each class in its own module and the names it uses imported there.""" mode: Mode source: list[str] = field(default_factory=list) """The source text of what was built, for the notes of a failing example.""" def annotation(self, spec: Spec) -> object: home = _module() return _evaluated(self.text(spec, None, home, quoted=False), home) def _named(self, name: str, made: object, home: types.ModuleType, *, quoted: bool) -> str: home.__dict__[name] = made return repr(name) if quoted else name def text(self, spec: Spec, itself: str | None, home: types.ModuleType, *, quoted: bool) -> str: """The source text of `spec` in module `home`. A name inside another annotation is quoted when `quoted` is true.""" if isinstance(spec, Leaf): return _LEAF_TEXT[spec.name] if isinstance(spec, OneOf): return f"Literal[{', '.join(repr(value) for value in spec.values)}]" if isinstance(spec, ArrayOf): return f"tuple[{self.text(spec.element, itself, home, quoted=quoted)}, ...]" if isinstance(spec, Fixed): elements = ", ".join( self.text(element, itself, home, quoted=quoted) for element in spec.elements ) return f"tuple[{elements}]" if isinstance(spec, AnyOf): branches = ", ".join( self.text(branch, itself, home, quoted=quoted) for branch in spec.branches ) return f"Union[{branches}]" if isinstance(spec, MappingOf): return f"Mapping[str, {self.text(spec.value, itself, home, quoted=quoted)}]" if isinstance(spec, Named): name = f"Named{next(_serial)}" maker = "NewType" if spec.how == "newtype" else "TypeAliasType" inner = self.text(spec.inner, itself, home, quoted=quoted and spec.how == "alias") written = f"{maker}({name!r}, {inner})" self.source.append(f"# {home.__name__}\n{name} = {written}") return self._named(name, _evaluated(written, home), home, quoted=quoted) if isinstance(spec, Itself): assert itself is not None return repr(itself) # a forward reference: the class is not built yet name, built = self.typeddict(spec) return self._named(name, built, home, quoted=quoted) def member(self, member: Member, itself: str, home: types.ModuleType, *, quoted: bool) -> str: text = self.text(member.spec, itself, home, quoted=quoted) for wrapper in member.wrappers: text = f"Annotated[{text}, 'meta']" if wrapper == "Annotated" else f"{wrapper}[{text}]" return text def typeddict(self, spec: Object) -> tuple[str, object]: """Build the TypedDict described by `spec`, one class per layer, each in its own module. Return its name and the class.""" names = [f"T{next(_serial)}" for _ in spec.layers] shared = _module() if spec.stdlib and _ONE_MODULE_FOR_STDLIB else None base: object = typing.TypedDict if spec.stdlib else TypedDict base_name = "typing.TypedDict" if spec.stdlib else "TypedDict" for layer, name in zip(spec.layers, names, strict=True): home = shared or _module() postponed = self.mode == "postponed" or (self.mode == "mixed" and layer.postponed) quoted = not postponed and layer.quoted texts = { member.key: self.member(member, names[-1], home, quoted=quoted) for member in layer.members } annotations = ( texts if postponed else {key: _evaluated(text, home) for key, text in texts.items()} ) keywords: dict[str, object] = {"total": layer.total} if layer.says == "closed": keywords["closed"] = True elif layer.says == "open": keywords["closed"] = False elif isinstance(layer.says, ExtraItems): extra = self.text(layer.says.spec, None, home, quoted=False) keywords["extra_items"] = extra if postponed else _evaluated(extra, home) namespace = {"__annotations__": annotations, "__module__": home.__name__} built = types.new_class(name, (base,), keywords, lambda body: body.update(namespace)) # noqa: B023 - run at once home.__dict__[name] = built written = "".join( f"\n {key}: {text!r}" if postponed else f"\n {key}: {text}" for key, text in texts.items() ) header = "from __future__ import annotations\n" if postponed else "" self.source.append( f"# {home.__name__}\n{header}class {name}({base_name}, {keywords!r}):{written or ' ...'}" ) base, base_name = built, name return names[-1], base def text_of(self) -> str: return "\n\n".join(self.source) # --- strategies ------------------------------------------------------------ _KEYS = ("a", "b", "c", "d", "x-y") _LITERALS: tuple[str | int | bool | None, ...] = (None, True, False, 0, 1, -1, "", "a", "b") _SCALARS = st.one_of( st.none(), st.booleans(), st.integers(-2, 2), st.floats(allow_nan=False, allow_infinity=False, width=32), st.text(max_size=2), ) _leaves = st.one_of( st.builds(Leaf, st.sampled_from(("int", "float", "bool", "str", "null", "json"))), st.lists( st.sampled_from(_LITERALS), min_size=1, max_size=3, unique_by=lambda v: (type(v), v) ).map(lambda values: OneOf(tuple(values))), ) @st.composite def _wrappers(draw: st.DrawFn) -> tuple[str, ...]: chosen = [draw(st.sampled_from(("", "Required", "NotRequired")))] chosen += ["ReadOnly"] if draw(st.booleans()) else [] chosen += ["Annotated"] if draw(st.booleans()) else [] return tuple(draw(st.permutations([wrapper for wrapper in chosen if wrapper != ""]))) _TYPING_TAKES_PEP_728 = sys.version_info >= (3, 15) """Whether `typing.TypedDict` takes `closed=` and `extra_items=`, as `typing_extensions` does.""" @st.composite def _objects(draw: st.DrawFn, inner: st.SearchStrategy[Spec]) -> Object: keys = draw(st.lists(st.sampled_from(_KEYS), unique=True, max_size=4)) count = draw(st.integers(1, 3)) owners = [draw(st.integers(0, count - 1)) for _ in keys] layers: list[Layer] = [] said: Says = "unsaid" declared: dict[str, tuple[Member, bool]] = {} for index in range(count): # A subclass of a closed TypedDict, or of one with extra items, is # limited to what the typing spec allows: it may repeat the same # setting or state none, and it may not add a key. restricted = said == "closed" or isinstance(said, ExtraItems) choices: list[Says] = ["unsaid", said] if restricted else ["unsaid", "closed", "open"] if not restricted and draw(st.booleans()): choices.append(ExtraItems(draw(inner))) says = draw(st.sampled_from(choices)) total = draw(st.booleans()) members = [ Member(key, draw(inner), draw(_wrappers())) for key, owner in zip(keys, owners, strict=True) if owner == index and not restricted ] if not restricted: members += draw(_redeclared(declared, total)) if index == count - 1 and not restricted and draw(st.booleans()): members.append(Member("self", ArrayOf(Itself()), draw(_wrappers()))) postponed, quoted = draw(st.booleans()), draw(st.booleans()) layers.append(Layer(total, tuple(members), says, postponed, quoted)) declared.update({member.key: (member, _required(member, total)) for member in members}) said = says if says != "unsaid" else said speaks = any(layer.says != "unsaid" for layer in layers) # From 3.13, `typing.TypedDict` rejects a read-only key that redeclares a # key it thinks is mutable, and it cannot see `ReadOnly` in a postponed # base. So it would not build some classes the spec allows, and no key is # declared twice when it is used. redeclares = sum(len(layer.members) for layer in layers) > len( {member.key for layer in layers for member in layer.members} ) stdlib = (_TYPING_TAKES_PEP_728 or not speaks) and not redeclares and draw(st.booleans()) return Object(tuple(layers), stdlib) @st.composite def _redeclared( draw: st.DrawFn, declared: dict[str, tuple[Member, bool]], total: bool ) -> list[Member]: """No key, or one read-only key declared by a base, narrowed as the typing spec allows a subclass to. Its type may narrow (a union to some of its branches), and a key that was not required may become required. The reverse is not allowed. """ candidates = sorted( key for key, (member, _) in declared.items() if "ReadOnly" in member.wrappers ) if len(candidates) == 0 or not draw(st.booleans()): return [] key = draw(st.sampled_from(candidates)) earlier, was_required = declared[key] spec = earlier.spec if isinstance(spec, AnyOf): kept = sorted(draw(st.sets(st.integers(0, len(spec.branches) - 1), min_size=1))) branches = tuple(spec.branches[index] for index in kept) spec = branches[0] if len(branches) == 1 else AnyOf(branches) wrappers = draw(_wrappers()) if was_required and not _required(Member(key, spec, wrappers), total): wrappers = ("Required", *(wrapper for wrapper in wrappers if wrapper != "NotRequired")) return [Member(key, spec, wrappers)] def specs(depth: int) -> st.SearchStrategy[Spec]: if depth == 0: return _leaves inner = specs(depth - 1) return st.one_of( _leaves, st.builds(ArrayOf, inner), st.lists(inner, min_size=1, max_size=3).map(lambda elements: Fixed(tuple(elements))), st.lists(inner, min_size=2, max_size=3).map(lambda branches: AnyOf(tuple(branches))), st.builds(MappingOf, inner), st.builds(Named, st.sampled_from(("newtype", "alias")), inner), _objects(inner), ) def json_values(depth: int) -> st.SearchStrategy[object]: """Any JSON, nested no deeper than `depth`.""" if depth <= 0: return _SCALARS inner = json_values(depth - 1) keys = st.sampled_from((*_KEYS, "self", "z")) | st.text(max_size=2) return st.one_of( _SCALARS, st.lists(inner, max_size=2), st.dictionaries(keys, inner, max_size=2) ) def conforming(spec: Spec, depth: int, itself: Object | None = None) -> st.SearchStrategy[object]: """JSON values of `spec`, with arrays and objects of any key nested no deeper than `depth`.""" if isinstance(spec, Leaf): return { "int": st.integers(), "float": st.integers() | st.floats(allow_nan=False, allow_infinity=False), "bool": st.booleans(), "str": st.text(max_size=3), "null": st.none(), "json": json_values(depth), }[spec.name] if isinstance(spec, OneOf): return st.sampled_from(spec.values) if isinstance(spec, ArrayOf): if depth <= 0: return st.just([]) return st.lists(conforming(spec.element, depth - 1, itself), max_size=2) if isinstance(spec, Fixed): elements = [conforming(element, depth - 1, itself) for element in spec.elements] return st.tuples(*elements).map(list) if isinstance(spec, AnyOf): return st.one_of(*(conforming(branch, depth, itself) for branch in spec.branches)) if isinstance(spec, MappingOf): if depth <= 0: return st.just({}) return st.dictionaries( st.text(max_size=2), conforming(spec.value, depth - 1, itself), max_size=2 ) if isinstance(spec, Named): return conforming(spec.inner, depth, itself) if isinstance(spec, Itself): assert itself is not None return conforming(itself, depth, itself) return _object_values(spec, depth) @st.composite def _object_values(draw: st.DrawFn, spec: Object, depth: int) -> dict[str, object]: members = members_of(spec) value: dict[str, object] = {} for key, (member, required) in members.items(): if required or draw(st.booleans()): value[key] = draw(conforming(member, depth - 1, spec)) said = says_of(spec) others = st.text(max_size=3).filter(lambda key: key not in members) if said in ("unsaid", "open"): value.update(draw(st.dictionaries(others, json_values(depth - 1), max_size=2))) elif isinstance(said, ExtraItems): value.update( draw(st.dictionaries(others, conforming(said.spec, depth - 1, spec), max_size=2)) ) return value def _positions(value: object, path: Loc = ()) -> list[Loc]: """Every place in `value`: itself, and each position inside an array or object, depth first.""" found: list[Loc] = [path] if isinstance(value, dict): for key, entry in value.items(): found += _positions(entry, (*path, key)) elif isinstance(value, (list, tuple)): for index, entry in enumerate(value): found += _positions(entry, (*path, index)) return found def _changed(value: object, path: Loc, change: object) -> object: """`value` with the item at `path` replaced by `change`, or removed if `change` is `_GONE`.""" if len(path) == 0: return change step, rest = path[0], path[1:] if isinstance(value, dict): entries = dict(value) if len(rest) == 0 and change is _GONE: del entries[step] else: entries[step] = _changed(entries[step], rest, change) return entries assert isinstance(value, (list, tuple)) assert isinstance(step, int) entries_ = list(value) if len(rest) == 0 and change is _GONE: del entries_[step] else: entries_[step] = _changed(entries_[step], rest, change) return entries_ _GONE = object() @st.composite def nearly(draw: st.DrawFn, spec: Spec, depth: int) -> object: """A value of `spec`, changed in one place: a value replaced, a key or entry dropped, or one added.""" value = draw(conforming(spec, depth)) path = draw(st.sampled_from(_positions(value))) how = draw(st.sampled_from(("replace", "drop", "add"))) if how == "drop" and len(path) != 0: return _changed(value, path, _GONE) target = _at(value, path) if how == "add" and isinstance(target, dict): key = draw(st.sampled_from((*_KEYS, "self", "z"))) return _changed(value, path, {**target, key: draw(json_values(1))}) if how == "add" and isinstance(target, list): return _changed(value, path, [*target, draw(json_values(1))]) return _changed(value, path, draw(json_values(1))) def _at(value: object, loc: Loc) -> object: for step in loc: if isinstance(value, dict): value = cast("dict[str, object]", value)[cast("str", step)] else: value = cast("list[object]", value)[cast("int", step)] return value def _is_place(value: object, loc: Loc) -> bool: """Whether `loc` names a place in `value`.""" for step in loc: in_object = isinstance(value, dict) and isinstance(step, str) and step in value in_array = ( isinstance(value, (list, tuple)) and isinstance(step, int) and 0 <= step < len(value) ) if not (in_object or in_array): return False value = _at(value, (step,)) return True def _same_json(left: object, right: object) -> bool: """Whether two values are equal as JSON, where a list and a tuple are both arrays.""" if isinstance(left, (list, tuple)) and isinstance(right, (list, tuple)): return len(left) == len(right) and all(map(_same_json, left, right)) if isinstance(left, dict) and isinstance(right, dict): return left.keys() == right.keys() and all( _same_json(left[key], right[key]) for key in left ) return type(left) is type(right) and left == right # --- the properties --------------------------------------------------------- _MODES: tuple[Mode, ...] = ("evaluated", "postponed", "mixed") _DEPTH = 3 """The depth limit for both a spec and a value drawn from it.""" _EXAMPLES = settings(max_examples=300, deadline=None, suppress_health_check=[HealthCheck.too_slow]) def _values(spec: Spec) -> st.SearchStrategy[object]: return st.one_of(conforming(spec, _DEPTH), nearly(spec, _DEPTH), json_values(_DEPTH)) def _read(annotation: object, value: object) -> Parsed: return parser(annotation, no_leaf)(value, ()) @_EXAMPLES @given(st.data()) def test_the_checker_agrees_with_the_reference(data: st.DataObject) -> None: # The checker reports no problem exactly when the value is of the type. # The returned value equals the input as JSON. Every problem is located # at a place in the value, or, for a missing key, at the object that # lacks it. When every problem is a key that a closed TypedDict does not # declare, the returned value is of the type. spec = data.draw(specs(_DEPTH), label="spec") build = Build(data.draw(st.sampled_from(_MODES), label="mode")) annotation = build.annotation(spec) note(build.text_of()) value = data.draw(_values(spec), label="value") event(f"spec {type(spec).__name__}") before = copy.deepcopy(value) typed, problems = _read(annotation, value) expected = conforms(value, spec) event("conforms" if expected else "does not conform") assert value == before assert (len(problems) == 0) == expected, problems if expected: assert _same_json(typed, value) for found in problems: place = found.loc[:-1] if found.kind == "missing_key" else found.loc assert _is_place(value, place), found if len(problems) != 0 and all(found.kind == "unknown_key" for found in problems): event("only unknown keys") assert conforms(typed, spec) def _as_json_schema_reads(value: object) -> object: """`value` as JSON Schema reads it: a number with no fraction is an integer, so `1.0` is `1`.""" if isinstance(value, float) and value.is_integer(): return int(value) if isinstance(value, list): return [_as_json_schema_reads(entry) for entry in cast("list[object]", value)] if isinstance(value, dict): entries = cast("dict[str, object]", value) return {key: _as_json_schema_reads(entry) for key, entry in entries.items()} return value @_EXAMPLES @given(st.data()) def test_the_json_schema_agrees_with_the_reference(data: st.DataObject) -> None: # The schema of a type accepts exactly the values of the type, as the # checker does. The one difference is that JSON Schema reads `1.0` as # the integer it equals, and the checker does not. spec = data.draw(specs(_DEPTH), label="spec") build = Build(data.draw(st.sampled_from(_MODES), label="mode")) annotation = build.annotation(spec) note(build.text_of()) schemas = Schemas() schema = schemas.document(schemas.of(annotation)) note(json.dumps(schema, indent=1)) Draft202012Validator.check_schema(schema) value = data.draw(_values(spec), label="value") expected = conforms(_as_json_schema_reads(value), spec) event("conforms" if expected else "does not conform") assert Draft202012Validator(schema).is_valid(cast("Any", value)) == expected @_EXAMPLES @given(st.data()) def test_a_postponed_typeddict_reads_as_an_evaluated_one(data: st.DataObject) -> None: # The same classes, with annotations evaluated or left as the strings # that `from __future__ import annotations` leaves, give the same # result, including each problem's message. spec = data.draw(specs(_DEPTH), label="spec") evaluated = Build("evaluated") written = Build(data.draw(st.sampled_from(("postponed", "mixed")), label="mode")) read_evaluated, read_written = evaluated.annotation(spec), written.annotation(spec) note(evaluated.text_of()) note(written.text_of()) value = data.draw(_values(spec), label="value") assert _read(read_evaluated, value) == _read(read_written, value) @_EXAMPLES @given(st.data()) def test_typeddict_keys_follow_the_typing_spec(data: st.DataObject) -> None: # Which keys are required, from each class's `total` and each key's own # qualifiers, and what other keys may hold, inherited when a class does # not state it. The result follows the typing spec, whether the # annotations are evaluated or not. spec = data.draw(_objects(specs(_DEPTH - 1)), label="spec") members = members_of(spec) said = says_of(spec) event(f"{len(spec.layers)} classes") event(f"says {said if isinstance(said, str) else 'extra_items'}") for mode in _MODES: build = Build(mode) keys = typeddict_keys(cast("type", build.annotation(spec))) note(build.text_of()) assert set(keys.members) == set(members) assert keys.required == {key for key, (_, required) in members.items() if required} assert keys.closed == (said == "closed") assert keys.open == (said in ("unsaid", "open")) assert keys.declared == (said != "unsaid") @_EXAMPLES @given(st.data()) def test_a_key_a_closed_typeddict_does_not_declare_is_reported_and_left_out( data: st.DataObject, ) -> None: # The value is still read, and equals the value before the key was # added. The key is the only problem, located at the key. spec = data.draw( _objects(specs(_DEPTH - 1)).filter(lambda drawn: says_of(drawn) == "closed"), label="spec" ) declared = members_of(spec) value = data.draw(_object_values(spec, _DEPTH), label="value") key = data.draw(st.text(max_size=3).filter(lambda drawn: drawn not in declared), label="key") build = Build(data.draw(st.sampled_from(_MODES), label="mode")) annotation = build.annotation(spec) note(build.text_of()) typed, problems = _read(annotation, {**value, key: data.draw(json_values(1), label="held")}) assert [(found.loc, found.kind) for found in problems] == [((key,), "unknown_key")] assert _same_json(typed, value) # --- reach: what the strategy has to keep drawing ----------------------------- def _nests_a_name(spec: Spec, *, nested: bool = False) -> bool: """Whether `spec` names another class or alias inside an annotation, where a quoted name loses track of its module.""" if isinstance(spec, (Named, Object)): return nested if isinstance(spec, ArrayOf): return _nests_a_name(spec.element, nested=True) if isinstance(spec, MappingOf): return _nests_a_name(spec.value, nested=True) if isinstance(spec, Fixed): return any(_nests_a_name(element, nested=True) for element in spec.elements) if isinstance(spec, AnyOf): return any(_nests_a_name(branch, nested=True) for branch in spec.branches) return False def _hides_a_qualifier(spec: Object) -> bool: """Whether a postponed class writes a qualifier that contradicts its `total`, which the runtime then does not see.""" return any( layer.postponed and any( ("NotRequired" in member.wrappers and layer.total) or ("Required" in member.wrappers and not layer.total) for member in layer.members ) for layer in spec.layers ) _REACHES = { "a-key-redeclared": lambda spec: any( member.key in {earlier.key for layer in spec.layers[:index] for earlier in layer.members} for index, later in enumerate(spec.layers) for member in later.members ), "openness-inherited": lambda spec: ( spec.layers[-1].says == "unsaid" and any(layer.says != "unsaid" for layer in spec.layers) ), "typing-typeddict": lambda spec: spec.stdlib and len(spec.layers) > 1, "classes-evaluated-and-postponed": lambda spec: ( len({layer.postponed for layer in spec.layers}) == 2 ), "a-qualifier-the-runtime-cannot-see": _hides_a_qualifier, "a-quoted-name-a-subclass-inherits": lambda spec: ( len(spec.layers) > 1 and any( layer.quoted and any(_nests_a_name(member.spec) for member in layer.members) for layer in spec.layers[:-1] ) ), "a-class-that-holds-itself": lambda spec: any( member.key == "self" for layer in spec.layers for member in layer.members ), } @pytest.mark.parametrize("reach", sorted(_REACHES)) def test_the_strategy_reaches(reach: str) -> None: # Each case the properties are meant to cover must be drawn at least once # in a small search, so an edit to the strategy cannot silently lose one. find( _objects(specs(_DEPTH - 1)), _REACHES[reach], settings=settings(max_examples=2000, database=None, deadline=None), ) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v2/000077500000000000000000000000001527014137600254705ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v2/__init__.py000066400000000000000000000000001527014137600275670ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v2/array/000077500000000000000000000000001527014137600266065ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v2/array/__init__.py000066400000000000000000000000001527014137600307050ustar00rootroot00000000000000blosc_compressor_with_filters.json000066400000000000000000000004741527014137600355700ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v2/array{ "zarr_format": 2, "shape": [200], "chunks": [50], "dtype": " None: ADAPTER.validate_python(json.loads(fixture.read_text())) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v2/consolidated/000077500000000000000000000000001527014137600301405ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v2/consolidated/__init__.py000066400000000000000000000000001527014137600322370ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v2/consolidated/minimal.json000066400000000000000000000000661527014137600324630ustar00rootroot00000000000000{ "zarr_consolidated_format": 1, "metadata": {} } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v2/consolidated/test_fixtures.py000066400000000000000000000010711527014137600334210ustar00rootroot00000000000000"""Decode v2 consolidated metadata fixtures via pydantic.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) ADAPTER = TypeAdapter(ZarrV2ConsolidatedMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) def test_validate(fixture: Path) -> None: ADAPTER.validate_python(json.loads(fixture.read_text())) with_array_and_group.json000066400000000000000000000006121527014137600351620ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v2/consolidated{ "zarr_consolidated_format": 1, "metadata": { ".zgroup": {"zarr_format": 2}, ".zattrs": {"description": "root group attrs"}, "data/.zarray": { "zarr_format": 2, "shape": [100], "chunks": [10], "dtype": " None: ADAPTER.validate_python(json.loads(fixture.read_text())) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v2/test_codecs.py000066400000000000000000000371041527014137600303460ustar00rootroot00000000000000"""Reading each v2 codec from its numcodecs configuration.""" from __future__ import annotations import json from typing import Any import pytest from zarr_metadata.definition import ( AcceptedField, RefusedField, UnclaimedField, canonical_of, resolve, ) from zarr_metadata.v2.codec import CODECS_V2, ZarrV2CodecMetadata from zarr_metadata.v2.definition import ZarrV2CodecDefinition, ZarrV2Context CONTEXT = ZarrV2Context(codecs=CODECS_V2) Loc = tuple[str | int, ...] # The output of numcodecs 0.16.5 `get_config()` for a default instance, where one exists, and # other configurations that numcodecs 0.15 and 0.16 read and round-trip. EXAMPLES: dict[str, tuple[dict[str, Any], ...]] = { # -1 is zlib's default-level constant, which numcodecs writes as given. "zlib": ({"id": "zlib", "level": 1}, {"id": "zlib"}, {"id": "zlib", "level": -1}), "gzip": ({"id": "gzip", "level": 9}, {"id": "gzip", "level": -1}), "bz2": ({"id": "bz2", "level": 1},), "lzma": ({"id": "lzma", "format": 1, "check": -1, "preset": None, "filters": None},), "blosc": ( {"id": "blosc", "cname": "lz4", "clevel": 5, "shuffle": 1, "blocksize": 0}, {"id": "blosc", "clevel": 5}, {"id": "blosc", "cname": "zstd", "clevel": 3, "shuffle": 2, "blocksize": 0, "typesize": 4}, # numcodecs writes a `shuffle` given as a bool as it was given. {"id": "blosc", "cname": "lz4", "clevel": 5, "shuffle": True, "blocksize": 0}, {"id": "blosc", "cname": "lz4", "clevel": 5, "shuffle": False, "blocksize": 0}, {"id": "blosc", "cname": "lz4", "clevel": 5, "shuffle": 1, "blocksize": -1}, # numcodecs passes `typesize` to C as an `int`, and Blosc refuses one past it. {"id": "blosc", "cname": "lz4", "clevel": 5, "shuffle": 0, "typesize": 2**31 - 1}, ), # zstd clamps a level outside its own range. "zstd": ( {"id": "zstd", "level": 0, "checksum": False}, {"id": "zstd", "level": 23, "checksum": False}, {"id": "zstd", "level": -200000, "checksum": False}, {"id": "zstd", "level": 2**31 - 1, "checksum": False}, ), # numcodecs passes `acceleration` to C as an `int`, and lz4 reads one below 1 as 1. "lz4": ( {"id": "lz4", "acceleration": 1}, {"id": "lz4", "acceleration": 2**31 - 1}, {"id": "lz4", "acceleration": -(2**31)}, ), # An element size of 1 or less leaves the bytes as they are. numcodecs passes a larger # one to C as a `Py_ssize_t`. "shuffle": ( {"id": "shuffle", "elementsize": 4}, {"id": "shuffle", "elementsize": 0}, {"id": "shuffle", "elementsize": -(2**63) - 1}, {"id": "shuffle", "elementsize": 2**63 - 1}, ), "delta": ( {"id": "delta", "dtype": "M8"}, ), "packbits": ({"id": "packbits"},), "vlen-utf8": ({"id": "vlen-utf8"},), "vlen-bytes": ({"id": "vlen-bytes"},), "vlen-array": ({"id": "vlen-array", "dtype": " None: """Every definition in `CODECS_V2` has an example below.""" assert {definition.name for definition in CODECS_V2} == set(EXAMPLES) @pytest.mark.parametrize( ("name", "field"), CASES, ids=[f"{n}:{i}" for i, (n, _) in enumerate(CASES)] ) def test_every_example_reads_and_is_written_back_as_it_was( name: str, field: ZarrV2CodecMetadata ) -> None: """A numcodecs configuration is read by the definition its `id` names, with no problem, and is written back unchanged: the `id` next to the parameters. Its canonical form reads as the same codec.""" resolved, problems = resolve(field, ZarrV2CodecDefinition, CONTEXT) assert isinstance(resolved, AcceptedField) assert resolved.definition.name == name assert problems == () assert resolved.to_json() == field assert resolve(canonical_of(resolved, problems), ZarrV2CodecDefinition, CONTEXT)[0] == resolved BLOSC_DEFAULTS: dict[str, Any] = { "id": "blosc", "cname": "lz4", "clevel": 5, "shuffle": 1, "blocksize": 0, } CHECKSUM_DEFAULTS: dict[str, Any] = {"location": "start"} @pytest.mark.parametrize( ("field", "canonical"), [ ({"id": "zlib"}, {"id": "zlib", "level": 1}), ({"id": "gzip"}, {"id": "gzip", "level": 1}), ({"id": "gzip", "level": 9}, {"id": "gzip", "level": 9}), ({"id": "bz2"}, {"id": "bz2", "level": 1}), ( {"id": "lzma"}, {"id": "lzma", "format": 1, "check": -1, "preset": None, "filters": None}, ), ({"id": "blosc"}, BLOSC_DEFAULTS), ({"id": "blosc", "shuffle": True}, BLOSC_DEFAULTS), ({"id": "blosc", "shuffle": False}, {**BLOSC_DEFAULTS, "shuffle": 0}), ({"id": "blosc", "typesize": None}, BLOSC_DEFAULTS), ({"id": "blosc", "typesize": 4}, {**BLOSC_DEFAULTS, "typesize": 4}), ( {"id": "blosc", "shuffle": 0, "typesize": 4}, {**BLOSC_DEFAULTS, "shuffle": 0, "typesize": 4}, ), ( {"id": "blosc", "shuffle": False, "typesize": 4}, {**BLOSC_DEFAULTS, "shuffle": 0, "typesize": 4}, ), ( {"id": "blosc", "shuffle": 2, "typesize": 4}, {**BLOSC_DEFAULTS, "shuffle": 2, "typesize": 4}, ), ({"id": "zstd"}, {"id": "zstd", "level": 0, "checksum": False}), ({"id": "zstd", "level": 0}, {"id": "zstd", "level": 0, "checksum": False}), ({"id": "lz4"}, {"id": "lz4", "acceleration": 1}), ({"id": "shuffle"}, {"id": "shuffle", "elementsize": 4}), ({"id": "delta", "dtype": " None: """The canonical form of a codec fills in each parameter that numcodecs gives a default, where the configuration omits it or writes null, as `get_config()` writes it. A blosc `shuffle` of `true` or `false` is 1 or 0, which numcodecs reads it as, and an `astype` left out is the `dtype`. It writes a dtype parameter as NumPy spells it (` None: """Two configurations that numcodecs encodes the same data with differently are two codecs: a parameter away from its default; a blosc `typesize` under `shuffle` 0, which still sets Blosc's block split and header; a fixedscaleoffset `offset` or `scale` written as an integer or as a float, which NumPy computes with differently for integer and half-precision dtypes; and a crc32c with no `location` and one at the start, since `numcodecs.CRC32C` places its checksum at the end by default, unlike crc32 and adler32.""" one, _ = resolve(first, ZarrV2CodecDefinition, CONTEXT) other, _ = resolve(second, ZarrV2CodecDefinition, CONTEXT) assert one != other @pytest.mark.parametrize( "field", [{"id": "categorize", "labels": ["a"]}, {"id": "pickle"}, {"id": "n5_wrapper", "inner": 1}], ) def test_an_id_the_package_does_not_model_is_unclaimed(field: dict[str, Any]) -> None: """A codec id that no definition in scope claims is read as `UnclaimedField`. Its parameters are kept and not checked, as for an unclaimed v3 extension.""" resolved, problems = resolve(field, ZarrV2CodecDefinition, CONTEXT) assert isinstance(resolved, UnclaimedField) assert problems == () assert json.dumps(resolved.to_json()) == json.dumps(field) @pytest.mark.parametrize( ("field", "at", "kind"), [ ({"id": "zlib", "level": 10}, ("c", "level"), "invalid_value"), ({"id": "gzip", "level": -2}, ("c", "level"), "invalid_value"), ({"id": ""}, ("c", "id"), "invalid_value"), ({"id": "bz2", "level": 0}, ("c", "level"), "invalid_value"), ({"id": "blosc", "cname": "brotli"}, ("c", "cname"), "invalid_value"), ({"id": "blosc", "clevel": -1}, ("c", "clevel"), "invalid_value"), ({"id": "blosc", "shuffle": 3}, ("c", "shuffle"), "invalid_value"), ({"id": "blosc", "blocksize": 2**31}, ("c", "blocksize"), "invalid_value"), ({"id": "blosc", "typesize": 2**31}, ("c", "typesize"), "invalid_value"), ({"id": "blosc", "shuffle": None}, ("c", "shuffle"), "invalid_type"), ({"id": "zstd", "level": 2**31}, ("c", "level"), "invalid_value"), ({"id": "zstd", "level": -(2**31) - 1}, ("c", "level"), "invalid_value"), ({"id": "lz4", "acceleration": 2**31}, ("c", "acceleration"), "invalid_value"), ({"id": "lz4", "acceleration": -(2**31) - 1}, ("c", "acceleration"), "invalid_value"), ({"id": "shuffle", "elementsize": 2**63}, ("c", "elementsize"), "invalid_value"), ({"id": "shuffle", "elementsize": 1.5}, ("c", "elementsize"), "invalid_type"), ({"id": "delta"}, ("c", "dtype"), "missing_key"), ({"id": "delta", "dtype": "float32"}, ("c", "dtype"), "invalid_value"), ({"id": "delta", "dtype": " None: """Each of these is reported at the parameter's location inside the field: a parameter out of the range numcodecs reads, a parameter of the wrong type, a parameter that is missing when numcodecs has no default for it, a dtype parameter that an array's dtype could not be, and a key that no codec declares.""" resolved, problems = resolve(field, ZarrV2CodecDefinition, CONTEXT, ("c",)) assert [(p.loc, p.kind) for p in problems] == [(at, kind)] assert isinstance(resolved, AcceptedField if kind == "unknown_key" else RefusedField) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v2/test_data_types.py000066400000000000000000000355521527014137600312500ustar00rootroot00000000000000"""Reading each v2 data type by its family: which typestrs and which fill values each family accepts.""" from __future__ import annotations from typing import TYPE_CHECKING import pytest from zarr_metadata.definition import ( AcceptedField, RefusedField, UnclaimedField, canonical_fill_value, canonical_of, fill_value_problems, resolve, ) from zarr_metadata.definition._definition import fields_of from zarr_metadata.v2.data_type import DATA_TYPES_V2 from zarr_metadata.v2.definition import ZarrV2Context, ZarrV2DataTypeDefinition if TYPE_CHECKING: from zarr_metadata import JSONValue CONTEXT = ZarrV2Context(data_types=DATA_TYPES_V2) Loc = tuple[str | int, ...] def _read(value: object) -> tuple[type, list[tuple[Loc, str]]]: resolved, problems = resolve(value, ZarrV2DataTypeDefinition, CONTEXT, ("dtype",)) return type(resolved), [(p.loc, p.kind) for p in problems] @pytest.mark.parametrize( ("value", "family", "canonical"), [ ("|b1", "bool", "|b1"), ("i4", "int", ">i4"), ("u8", "uint", ">u8"), ("f8", "float", ">f8"), ("c16", "complex", ">c16"), ("|S0", "bytes", "|S0"), ("|S12", "bytes", "|S12"), ("U0", "str", ">U0"), ("|V8", "void", "|V8"), ("|O", "object", "|O"), ("M8[10s]", "datetime64", ">M8[10s]"), (" None: """Each typestr of a family is read by the family's definition. Its canonical spelling is the typestr NumPy writes: `|` for a type of one byte or with no byte order, and `us` for a microsecond unit. A records array is read as `struct`, and the type of each record is read too. A records array may be empty.""" resolved, problems = resolve(value, ZarrV2DataTypeDefinition, CONTEXT) assert isinstance(resolved, AcceptedField) assert resolved.definition.name == family assert problems == () assert canonical_of(resolved, problems) == canonical @pytest.mark.parametrize("value", [" None: """A typestr whose type code is not listed in the v2 spec is filed under its own name, which no definition in scope claims. It is read as `UnclaimedField` with no problem, and written back unchanged.""" resolved, problems = resolve(value, ZarrV2DataTypeDefinition, CONTEXT) assert isinstance(resolved, UnclaimedField) assert problems == () assert resolved.to_json() == value @pytest.mark.parametrize( ("value", "at"), [ ("float32", ("dtype",)), (" None: """A typestr with a size, byte order or unit that its family does not accept is refused, and the problem is located at the field. A struct's problems are located under `fields`, at the record and the position inside it. If a record's type is refused, the problem is reported there and the struct is still read.""" kind, problems = _read(value) assert kind is RefusedField or "fields" in at assert next(loc for loc, _ in problems) == at def test_error_a_str_typestr_without_a_byte_order_states_its_size_in_bytes() -> None: """`|U1` is refused, because a `U` type always has a byte order. The size in a `U` typestr counts code points, so the message gives the type's size in bytes, four per code point.""" _, problems = resolve("|U1", ZarrV2DataTypeDefinition, CONTEXT) assert [p.message for p in problems] == [ "expected a byte order '<' or '>' for a type of 4 bytes, got '|'" ] @pytest.mark.parametrize(("value", "example"), [(" None: """` None: """Each family accepts the fill values the v2 spec allows for it, and always accepts `null`: booleans; integers in the type's range; numbers or the named non-finite strings for floats and for each complex component; base64 for bytes, void and struct; a string for str; any JSON for object; and ticks or `NaT` for times. The canonical spelling turns an integer written for a float into a float, and turns the ticks value of `NaT` into `"NaT"`.""" resolved, _ = resolve(dtype, ZarrV2DataTypeDefinition, CONTEXT) assert fill_value_problems(resolved, value) == () assert canonical_fill_value(resolved, value) == canonical @pytest.mark.parametrize( ("dtype", "value", "at"), [ ("|b1", 1, ()), (" None: """A fill value outside the shape or range its family accepts is reported, at the value or at the wrong component. A v2 float does not accept a hex string, and a struct's fill value is the base64 of the record, not an object of fields.""" resolved, _ = resolve(dtype, ZarrV2DataTypeDefinition, CONTEXT) problems = fill_value_problems(resolved, value, ("fill_value",)) assert len(problems) != 0 assert problems[0].loc == ("fill_value", *at) def test_error_a_float_fill_value_string_that_is_not_a_named_value_names_every_choice() -> None: """A string fill value for a float dtype that is not a named non-finite value is refused with a message that names every fill value a float takes, numbers and null included, as for a value of another JSON type.""" resolved, _ = resolve(" None: """Every definition in `DATA_TYPES_V2` has an example above, so no definition goes untested.""" names = {definition.name for definition in DATA_TYPES_V2} assert names == { "bool", "int", "uint", "float", "complex", "bytes", "str", "void", "datetime64", "timedelta64", "object", "struct", } def test_a_struct_record_type_sits_where_the_document_writes_it() -> None: """`fields_of` places a record's type under the struct's own configuration location, `("dtype", "fields", 0, 1)`, not under a `configuration` key a v2 document does not have.""" resolved, _ = resolve( [["a", " None: """A float fill value that is too large for a float of the dtype's width (`U2"]], "AAAAMQAAAAA=", True), ([["a", [["s", " None: """A void or struct fill value must decode to exactly the item size, or be null. A record's size is the sum of its fields' sizes, each multiplied by the product of its subarray shape, including nested structs. A `U` field's size counts code points, of four bytes each, and a struct with no fields takes no bytes, also as the field of another struct. A byte string's fill value must decode to at most the item size, because NumPy pads a shorter one. A record with a field of no fixed size (`|O`) is not checked. zarr-python 3 fails to open an array whose fill value has a different size.""" from zarr_metadata.model import ZarrV2ArrayMetadata, read_array_metadata_v2 document = { **ZarrV2ArrayMetadata.create_default(shape=(4,), chunks=(2,)).to_json(), "dtype": dtype, "fill_value": fill_value, } found = read_array_metadata_v2(document).problems assert (len(found) == 0) is accepted, [p.message for p in found] if not accepted: assert [(p.loc, p.kind) for p in found] == [(("fill_value",), "invalid_value")] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v2/test_definition.py000066400000000000000000000046401527014137600312350ustar00rootroot00000000000000"""The public entry point for v2 definitions: the scope that zarr-python 2.x reads in, and reading one field in it.""" from __future__ import annotations import pickle import pytest from zarr_metadata.definition import ( AcceptedField, UnclaimedField, resolve, ) from zarr_metadata.v2.codec import CODECS_V2 from zarr_metadata.v2.data_type import DATA_TYPES_V2 from zarr_metadata.v2.definition import ( CORE_V2, ZarrV2CodecDefinition, ZarrV2Context, ZarrV2DataTypeDefinition, ) from zarr_metadata.v3.codec.gzip import GZIP_CODEC from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS, CodecDefinition, ) def test_core_v2_files_every_v2_definition_apart_from_v3() -> None: """`CORE_V2` files the 12 data types and 21 codecs by their v2 kinds, and is a scope of format 2: a v2 `gzip` and a v3 `gzip` are two definitions of two different kinds. No scope can file both, because `union` rejects scopes of two formats.""" assert set(CORE_V2.data_types.values()) == set(DATA_TYPES_V2) assert set(CORE_V2.codecs.values()) == set(CODECS_V2) assert CORE_V2.zarr_format == 2 assert CORE_V2.claimant(ZarrV2CodecDefinition, "gzip") is not GZIP_CODEC assert CORE_V2.claimant(ZarrV2CodecDefinition, "gzip") is not None assert CORE_AND_EXTENSIONS.claimant(CodecDefinition, "gzip") is GZIP_CODEC assert CORE_V2.claimant(ZarrV2DataTypeDefinition, " None: """`resolve` reads a v2 dtype or codec with its kind in a v2 scope, prefixing every problem with `loc`, as it reads a v3 field.""" dtype, problems = resolve("i8", "i", {"byteorder": ">", "itemsize": 8}), ("|b1", "b", {"byteorder": "|", "itemsize": 1}), ("|S12", "S", {"byteorder": "|", "itemsize": 12}), (" None: """A NumPy typestr splits into its type code and the values the name carries: the byte order, the item size if one is written, and a time unit with its multiplier if one is given in brackets.""" assert parse_typestr(name) == (code, carried) @pytest.mark.parametrize( "name", [ "float32", "f4", " None: """A string without a byte order, a type code and a size in that order is not a typestr.""" assert parse_typestr(name) is None @pytest.mark.parametrize( ("name", "filed", "carried"), [ (" None: """A typestr is filed under its family and carries its byte order, size and unit. `struct` is filed under itself. A typestr whose type code the spec does not list is filed under itself, so a scope leaves it unclaimed. A family name is not filed under anything, because no document writes it.""" assert ZarrV2DataTypeDefinition._filed_under(name) == (filed, carried) @pytest.mark.parametrize( ("value", "name", "configuration"), [ (" None: """A dtype field is either a string, which is its name, or an array of field records, which is a `struct` whose configuration is the records. Any other value has no name.""" assert ZarrV2DataTypeDefinition._named_configuration(value) == (name, configuration, ()) @pytest.mark.parametrize( ("value", "problems"), [ (" None: """The envelope of a dtype field is valid when it is a typestr or an array of records. A string that is neither is reported as an invalid value, and the message names the typestr form. Any other value is reported as an invalid type.""" found = ZarrV2DataTypeDefinition._envelope_problems(value) assert [(p.loc, p.kind) for p in found] == problems if value == "float32": assert "typestr" in found[0].message def test_a_v2_dtype_writes_back_as_a_string_or_records() -> None: """`envelope_json` writes a dtype as its string, and a struct as its records. Both the name and the configuration are located at the field.""" assert ZarrV2DataTypeDefinition._envelope_json(" None: """A codec field is an object. Its `id` is its name and its other members are its parameters. A value that is not an object, or has no string `id`, has no name, and the envelope problems say why.""" assert ZarrV2CodecDefinition._named_configuration(value) == (name, configuration, ()) assert [(p.loc, p.kind) for p in ZarrV2CodecDefinition._envelope_problems(value)] == problems def test_a_v2_codec_writes_back_with_its_id_beside_its_parameters() -> None: """`envelope_json` writes a codec as `{"id": name, **parameters}`. The parameters are located at the field and the name is located under `id`.""" assert ZarrV2CodecDefinition._envelope_json("zlib", {"level": 1}) == {"id": "zlib", "level": 1} assert ZarrV2CodecDefinition._configuration_loc(("compressor",)) == ("compressor",) assert ZarrV2CodecDefinition._name_loc(("compressor",)) == ("compressor", "id") assert ZarrV2CodecDefinition._filed_under("zlib") == ("zlib", None) def test_error_a_v2_definition_is_named_as_its_format_names_one() -> None: """Building a v2 codec definition with an empty id raises an error. So does building a v2 data type definition named by a typestr that a document would write, instead of by a family name.""" with pytest.raises(TypeError, match="so no document names it"): ZarrV2CodecDefinition(name="", configuration=EmptyConfiguration) with pytest.raises(TypeError, match="is how a document writes"): ZarrV2DataTypeDefinition(name=" None: ADAPTER.validate_python(json.loads(fixture.read_text())) transpose_and_crc32c_codecs.json000066400000000000000000000006701527014137600347450ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/array{ "zarr_format": 3, "node_type": "array", "shape": [10, 20, 30], "data_type": "float32", "chunk_grid": { "name": "regular", "configuration": {"chunk_shape": [5, 10, 15]} }, "chunk_key_encoding": { "name": "default" }, "fill_value": "NaN", "codecs": [ {"name": "transpose", "configuration": {"order": [2, 1, 0]}}, {"name": "bytes", "configuration": {"endian": "little"}}, {"name": "crc32c"} ] } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/array/with_extra_field.json000066400000000000000000000007131527014137600330240ustar00rootroot00000000000000{ "zarr_format": 3, "node_type": "array", "shape": [10], "data_type": "int32", "chunk_grid": { "name": "regular", "configuration": {"chunk_shape": [10]} }, "chunk_key_encoding": { "name": "default" }, "fill_value": 0, "codecs": [ {"name": "bytes", "configuration": {"endian": "little"}} ], "my_custom_extension": { "must_understand": false, "purpose": "exercise the extra_items=ZarrV3ExtensionField path" } } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/array/with_optionals.json000066400000000000000000000010741527014137600325470ustar00rootroot00000000000000{ "zarr_format": 3, "node_type": "array", "shape": [10, 20, 30], "data_type": "float64", "chunk_grid": { "name": "regular", "configuration": {"chunk_shape": [5, 10, 15]} }, "chunk_key_encoding": { "name": "default", "configuration": {"separator": "/"} }, "fill_value": "NaN", "codecs": [ {"name": "bytes", "configuration": {"endian": "little"}} ], "attributes": { "description": "fixture exercising optional fields", "tags": ["test", "metadata"] }, "dimension_names": ["t", "y", "x"], "storage_transformers": [] } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/array/zstd_codec.json000066400000000000000000000005521527014137600316250ustar00rootroot00000000000000{ "zarr_format": 3, "node_type": "array", "shape": [128], "data_type": "int8", "chunk_grid": { "name": "regular", "configuration": {"chunk_shape": [64]} }, "chunk_key_encoding": { "name": "default" }, "fill_value": 0, "codecs": [ {"name": "bytes"}, {"name": "zstd", "configuration": {"level": 3, "checksum": false}} ] } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_grid/000077500000000000000000000000001527014137600276065ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_grid/__init__.py000066400000000000000000000000001527014137600317050ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_grid/rectilinear/000077500000000000000000000000001527014137600321075ustar00rootroot00000000000000__init__.py000066400000000000000000000000001527014137600341270ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_grid/rectilinearcases.json000066400000000000000000000010421527014137600340160ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_grid/rectilinear{ "explicit_per_dim": { "name": "rectilinear", "configuration": { "kind": "inline", "chunk_shapes": [ [10, 20, 30, 40], [50] ] } }, "uniform_dim_shorthand": { "name": "rectilinear", "configuration": { "kind": "inline", "chunk_shapes": [ [10, 20, 30, 40], 50 ] } }, "with_rle_pair": { "name": "rectilinear", "configuration": { "kind": "inline", "chunk_shapes": [ [[10, 5], 5, 5], [15, 15] ] } } } test_fixtures.py000066400000000000000000000010121527014137600353040ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_grid/rectilinear"""Validate rectilinear chunk grid fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.chunk_grid.rectilinear import RectilinearChunkGridMetadata CASES: dict[str, object] = json.loads((Path(__file__).parent / "cases.json").read_text()) @pytest.mark.parametrize("case", CASES.values(), ids=list(CASES)) def test_chunk_grid(case: object) -> None: TypeAdapter(RectilinearChunkGridMetadata).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_grid/regular/000077500000000000000000000000001527014137600312475ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_grid/regular/__init__.py000066400000000000000000000000001527014137600333460ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_grid/regular/cases.json000066400000000000000000000004011527014137600332330ustar00rootroot00000000000000{ "1d": { "name": "regular", "configuration": {"chunk_shape": [10]} }, "2d": { "name": "regular", "configuration": {"chunk_shape": [10, 20]} }, "3d": { "name": "regular", "configuration": {"chunk_shape": [5, 10, 15]} } } test_fixtures.py000066400000000000000000000007721527014137600344600ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_grid/regular"""Validate regular chunk grid fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.chunk_grid.regular import RegularChunkGridMetadata CASES: dict[str, object] = json.loads((Path(__file__).parent / "cases.json").read_text()) @pytest.mark.parametrize("case", CASES.values(), ids=list(CASES)) def test_chunk_grid(case: object) -> None: TypeAdapter(RegularChunkGridMetadata).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_key_encoding/000077500000000000000000000000001527014137600313175ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_key_encoding/__init__.py000066400000000000000000000000001527014137600334160ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_key_encoding/default/000077500000000000000000000000001527014137600327435ustar00rootroot00000000000000__init__.py000066400000000000000000000000001527014137600347630ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_key_encoding/defaultcases.json000066400000000000000000000004121527014137600346520ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_key_encoding/default{ "no_configuration": { "name": "default" }, "slash_separator": { "name": "default", "configuration": {"separator": "/"} }, "dot_separator": { "name": "default", "configuration": {"separator": "."} }, "short_hand_name": "default" } test_fixtures.py000066400000000000000000000010401527014137600361410ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_key_encoding/default"""Validate default chunk-key encoding fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.chunk_key_encoding.default import DefaultChunkKeyEncodingMetadata CASES: dict[str, object] = json.loads((Path(__file__).parent / "cases.json").read_text()) @pytest.mark.parametrize("case", CASES.values(), ids=list(CASES)) def test_chunk_key_encoding(case: object) -> None: TypeAdapter(DefaultChunkKeyEncodingMetadata).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_key_encoding/v2/000077500000000000000000000000001527014137600316465ustar00rootroot00000000000000__init__.py000066400000000000000000000000001527014137600336660ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_key_encoding/v2zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_key_encoding/v2/cases.json000066400000000000000000000003661527014137600336440ustar00rootroot00000000000000{ "no_configuration": { "name": "v2" }, "dot_separator": { "name": "v2", "configuration": {"separator": "."} }, "slash_separator": { "name": "v2", "configuration": {"separator": "/"} }, "short_hand_name": "v2" } test_fixtures.py000066400000000000000000000010321527014137600350450ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/chunk_key_encoding/v2"""Validate v2-compatibility chunk-key encoding fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.chunk_key_encoding.v2 import V2ChunkKeyEncodingMetadata CASES: dict[str, object] = json.loads((Path(__file__).parent / "cases.json").read_text()) @pytest.mark.parametrize("case", CASES.values(), ids=list(CASES)) def test_chunk_key_encoding(case: object) -> None: TypeAdapter(V2ChunkKeyEncodingMetadata).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/000077500000000000000000000000001527014137600265465ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/__init__.py000066400000000000000000000000001527014137600306450ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/blosc/000077500000000000000000000000001527014137600276505ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/blosc/__init__.py000066400000000000000000000000001527014137600317470ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/blosc/cases.json000066400000000000000000000005451527014137600316450ustar00rootroot00000000000000{ "with_typesize": { "name": "blosc", "configuration": { "cname": "zstd", "clevel": 5, "shuffle": "shuffle", "blocksize": 0, "typesize": 4 } }, "no_typesize": { "name": "blosc", "configuration": { "cname": "lz4", "clevel": 1, "shuffle": "noshuffle", "blocksize": 0 } } } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/blosc/test_fixtures.py000066400000000000000000000007331527014137600331350ustar00rootroot00000000000000"""Validate blosc codec fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.codec.blosc import BloscCodecMetadata CASES: dict[str, object] = json.loads((Path(__file__).parent / "cases.json").read_text()) @pytest.mark.parametrize("case", CASES.values(), ids=list(CASES)) def test_codec(case: object) -> None: TypeAdapter(BloscCodecMetadata).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/bytes/000077500000000000000000000000001527014137600276745ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/bytes/__init__.py000066400000000000000000000000001527014137600317730ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/bytes/cases.json000066400000000000000000000005011527014137600316610ustar00rootroot00000000000000{ "little_endian": { "name": "bytes", "configuration": {"endian": "little"} }, "big_endian": { "name": "bytes", "configuration": {"endian": "big"} }, "no_endian": { "name": "bytes", "configuration": {} }, "no_configuration": { "name": "bytes" }, "short_hand_name": "bytes" } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/bytes/test_fixtures.py000066400000000000000000000007331527014137600331610ustar00rootroot00000000000000"""Validate bytes codec fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.codec.bytes import BytesCodecMetadata CASES: dict[str, object] = json.loads((Path(__file__).parent / "cases.json").read_text()) @pytest.mark.parametrize("case", CASES.values(), ids=list(CASES)) def test_codec(case: object) -> None: TypeAdapter(BytesCodecMetadata).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/cast_value/000077500000000000000000000000001527014137600306745ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/cast_value/__init__.py000066400000000000000000000000001527014137600327730ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/cast_value/cases.json000066400000000000000000000023361527014137600326710ustar00rootroot00000000000000{ "minimal": { "name": "cast_value", "configuration": {"data_type": "uint8"} }, "named_config_dtype": { "name": "cast_value", "configuration": { "data_type": { "name": "numpy.datetime64", "configuration": {"unit": "ns", "scale_factor": 1} } } }, "with_rounding": { "name": "cast_value", "configuration": { "data_type": "int16", "rounding": "towards-zero" } }, "with_out_of_range_clamp": { "name": "cast_value", "configuration": { "data_type": "int8", "out_of_range": "clamp" } }, "with_out_of_range_wrap": { "name": "cast_value", "configuration": { "data_type": "uint8", "out_of_range": "wrap" } }, "with_scalar_map_encode_only": { "name": "cast_value", "configuration": { "data_type": "uint8", "scalar_map": { "encode": [["NaN", 0]] } } }, "numpy_compat_full_example": { "name": "cast_value", "configuration": { "data_type": "uint8", "rounding": "towards-zero", "out_of_range": "wrap", "scalar_map": { "encode": [ ["NaN", 0], ["+Infinity", 0], ["-Infinity", 0] ] } } } } test_fixtures.py000066400000000000000000000007551527014137600341060ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/cast_value"""Validate cast_value codec fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.codec.cast_value import CastValueCodecMetadata CASES: dict[str, object] = json.loads((Path(__file__).parent / "cases.json").read_text()) @pytest.mark.parametrize("case", CASES.values(), ids=list(CASES)) def test_codec(case: object) -> None: TypeAdapter(CastValueCodecMetadata).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/crc32c/000077500000000000000000000000001527014137600276255ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/crc32c/__init__.py000066400000000000000000000000001527014137600317240ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/crc32c/cases.json000066400000000000000000000002421527014137600316140ustar00rootroot00000000000000{ "no_configuration": { "name": "crc32c" }, "empty_configuration": { "name": "crc32c", "configuration": {} }, "short_hand_name": "crc32c" } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/crc32c/test_fixtures.py000066400000000000000000000007371527014137600331160ustar00rootroot00000000000000"""Validate crc32c codec fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.codec.crc32c import Crc32cCodecMetadata CASES: dict[str, object] = json.loads((Path(__file__).parent / "cases.json").read_text()) @pytest.mark.parametrize("case", CASES.values(), ids=list(CASES)) def test_codec(case: object) -> None: TypeAdapter(Crc32cCodecMetadata).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/gzip/000077500000000000000000000000001527014137600275175ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/gzip/__init__.py000066400000000000000000000000001527014137600316160ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/gzip/cases.json000066400000000000000000000001201527014137600315010ustar00rootroot00000000000000{ "with_level": { "name": "gzip", "configuration": {"level": 5} } } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/gzip/test_fixtures.py000066400000000000000000000007271527014137600330070ustar00rootroot00000000000000"""Validate gzip codec fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.codec.gzip import GzipCodecMetadata CASES: dict[str, object] = json.loads((Path(__file__).parent / "cases.json").read_text()) @pytest.mark.parametrize("case", CASES.values(), ids=list(CASES)) def test_codec(case: object) -> None: TypeAdapter(GzipCodecMetadata).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/scale_offset/000077500000000000000000000000001527014137600312035ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/scale_offset/__init__.py000066400000000000000000000000001527014137600333020ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/scale_offset/cases.json000066400000000000000000000010611527014137600331720ustar00rootroot00000000000000{ "no_configuration": { "name": "scale_offset" }, "empty_configuration": { "name": "scale_offset", "configuration": {} }, "offset_only": { "name": "scale_offset", "configuration": {"offset": 5} }, "scale_only": { "name": "scale_offset", "configuration": {"scale": 0.1} }, "scale_and_offset": { "name": "scale_offset", "configuration": {"offset": 5, "scale": 0.1} }, "string_encoded_scalar": { "name": "scale_offset", "configuration": {"offset": "NaN"} }, "short_hand_name": "scale_offset" } test_fixtures.py000066400000000000000000000007651527014137600344160ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/scale_offset"""Validate scale_offset codec fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.codec.scale_offset import ScaleOffsetCodecMetadata CASES: dict[str, object] = json.loads((Path(__file__).parent / "cases.json").read_text()) @pytest.mark.parametrize("case", CASES.values(), ids=list(CASES)) def test_codec(case: object) -> None: TypeAdapter(ScaleOffsetCodecMetadata).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/sharding_indexed/000077500000000000000000000000001527014137600320455ustar00rootroot00000000000000__init__.py000066400000000000000000000000001527014137600340650ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/sharding_indexedcases.json000066400000000000000000000014001527014137600337520ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/sharding_indexed{ "with_index_location": { "name": "sharding_indexed", "configuration": { "chunk_shape": [64, 64], "codecs": [ {"name": "bytes", "configuration": {"endian": "little"}}, {"name": "gzip", "configuration": {"level": 1}} ], "index_codecs": [ {"name": "bytes", "configuration": {"endian": "little"}}, {"name": "crc32c"} ], "index_location": "end" } }, "no_index_location": { "name": "sharding_indexed", "configuration": { "chunk_shape": [128], "codecs": [ {"name": "bytes", "configuration": {"endian": "little"}} ], "index_codecs": [ {"name": "bytes", "configuration": {"endian": "little"}}, {"name": "crc32c"} ] } } } test_fixtures.py000066400000000000000000000010051527014137600352440ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/sharding_indexed"""Validate sharding_indexed codec fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.codec.sharding_indexed import ShardingIndexedCodecMetadata CASES: dict[str, object] = json.loads((Path(__file__).parent / "cases.json").read_text()) @pytest.mark.parametrize("case", CASES.values(), ids=list(CASES)) def test_codec(case: object) -> None: TypeAdapter(ShardingIndexedCodecMetadata).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/transpose/000077500000000000000000000000001527014137600305645ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/transpose/__init__.py000066400000000000000000000000001527014137600326630ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/transpose/cases.json000066400000000000000000000001361527014137600325550ustar00rootroot00000000000000{ "reversed_3d": { "name": "transpose", "configuration": {"order": [2, 1, 0]} } } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/transpose/test_fixtures.py000066400000000000000000000007531527014137600340530ustar00rootroot00000000000000"""Validate transpose codec fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.codec.transpose import TransposeCodecMetadata CASES: dict[str, object] = json.loads((Path(__file__).parent / "cases.json").read_text()) @pytest.mark.parametrize("case", CASES.values(), ids=list(CASES)) def test_codec(case: object) -> None: TypeAdapter(TransposeCodecMetadata).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/zstd/000077500000000000000000000000001527014137600275325ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/zstd/__init__.py000066400000000000000000000000001527014137600316310ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/zstd/cases.json000066400000000000000000000001401527014137600315160ustar00rootroot00000000000000{ "default": { "name": "zstd", "configuration": {"level": 3, "checksum": false} } } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/codec/zstd/test_fixtures.py000066400000000000000000000007271527014137600330220ustar00rootroot00000000000000"""Validate zstd codec fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.codec.zstd import ZstdCodecMetadata CASES: dict[str, object] = json.loads((Path(__file__).parent / "cases.json").read_text()) @pytest.mark.parametrize("case", CASES.values(), ids=list(CASES)) def test_codec(case: object) -> None: TypeAdapter(ZstdCodecMetadata).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/consolidated/000077500000000000000000000000001527014137600301415ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/consolidated/__init__.py000066400000000000000000000000001527014137600322400ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/consolidated/minimal.json000066400000000000000000000001051527014137600324560ustar00rootroot00000000000000{ "kind": "inline", "must_understand": false, "metadata": {} } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/consolidated/test_fixtures.py000066400000000000000000000010711527014137600334220ustar00rootroot00000000000000"""Decode v3 consolidated metadata fixtures via pydantic.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) ADAPTER = TypeAdapter(ZarrV3ConsolidatedMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) def test_validate(fixture: Path) -> None: ADAPTER.validate_python(json.loads(fixture.read_text())) with_array_and_group.json000066400000000000000000000010631527014137600351640ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/consolidated{ "kind": "inline", "must_understand": false, "metadata": { "child_group": { "zarr_format": 3, "node_type": "group" }, "child_array": { "zarr_format": 3, "node_type": "array", "shape": [10], "data_type": "int32", "chunk_grid": { "name": "regular", "configuration": {"chunk_shape": [10]} }, "chunk_key_encoding": { "name": "default" }, "fill_value": 0, "codecs": [ {"name": "bytes", "configuration": {"endian": "little"}} ] } } } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/000077500000000000000000000000001527014137600274435ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/__init__.py000066400000000000000000000000001527014137600315420ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/bool/000077500000000000000000000000001527014137600303765ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/bool/__init__.py000066400000000000000000000000001527014137600324750ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/bool/fill_values.json000066400000000000000000000000451527014137600335750ustar00rootroot00000000000000{ "true": true, "false": false } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/bool/test_fixtures.py000066400000000000000000000007731527014137600336670ustar00rootroot00000000000000"""Validate bool fill-value fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.bool import BoolFillValue FILL_VALUES: dict[str, object] = json.loads( (Path(__file__).parent / "fill_values.json").read_text() ) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(BoolFillValue).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/bytes/000077500000000000000000000000001527014137600305715ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/bytes/__init__.py000066400000000000000000000000001527014137600326700ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/bytes/fill_values.json000066400000000000000000000000551527014137600337710ustar00rootroot00000000000000{ "tuple": [1, 2, 3], "base64": "AQID" } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/bytes/test_fixtures.py000066400000000000000000000010171527014137600340520ustar00rootroot00000000000000"""Validate variable-length bytes fill-value fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.bytes import BytesFillValue FILL_VALUES: dict[str, object] = json.loads( (Path(__file__).parent / "fill_values.json").read_text() ) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(BytesFillValue).validate_python(case) test_validators.py000066400000000000000000000022321527014137600342720ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/bytes"""Cover the `base64_bytes` brand validator. The pydantic-driven fixture tests don't enforce the base64 alphabet or length-multiple-of-4 constraint because `Base64Bytes` is a `NewType`, which pydantic treats as plain `str`. Direct coverage of the validator function lives here. """ from __future__ import annotations import pytest from zarr_metadata.v3.data_type.bytes import base64_bytes VALID = [ "", # empty is valid base64 (length 0, multiple of 4) "AQID", # [1, 2, 3] "AAAA", # [0, 0, 0] "////", # [255, 255, 255] "abcd", "AB==", # padding "ABC=", # padding ] INVALID = [ "AB", # length 2, not multiple of 4 "ABC", # length 3, not multiple of 4 "ABCDE", # length 5 "AB-D", # url-safe alphabet, not standard "AB_D", # url-safe alphabet, not standard "AB!D", # not base64 char "AB CD", # whitespace ] @pytest.mark.parametrize("value", VALID) def test_valid(value: str) -> None: assert base64_bytes(value) == value @pytest.mark.parametrize("value", INVALID) def test_invalid(value: str) -> None: with pytest.raises(ValueError, match="standard-alphabet base64"): base64_bytes(value) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/complex128/000077500000000000000000000000001527014137600313455ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/complex128/__init__.py000066400000000000000000000000001527014137600334440ustar00rootroot00000000000000fill_values.json000066400000000000000000000003401527014137600344630ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/complex128{ "numeric": [1.5, 2.5], "zero": [0.0, 0.0], "with_sentinel_components": ["-Infinity", "NaN"], "mixed_numeric_and_sentinel": [1.0, "Infinity"], "with_hex_components": ["0x7ff8000000000000", "0x0000000000000000"] } test_fixtures.py000066400000000000000000000014361527014137600345540ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/complex128"""Validate complex128 fill-value fixtures. A v3 complex fill_value is a two-element JSON array `[real, imag]` where each component is shaped per the corresponding float's fill value: a number, one of the named sentinels (`"NaN"`, `"Infinity"`, `"-Infinity"`), or a hex string of the underlying float's bits. """ from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.complex128 import Complex128FillValue FILL_VALUES: dict[str, object] = json.loads( (Path(__file__).parent / "fill_values.json").read_text() ) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(Complex128FillValue).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/complex64/000077500000000000000000000000001527014137600312645ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/complex64/__init__.py000066400000000000000000000000001527014137600333630ustar00rootroot00000000000000fill_values.json000066400000000000000000000003201527014137600344000ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/complex64{ "numeric": [1.5, 2.5], "zero": [0.0, 0.0], "with_sentinel_components": ["-Infinity", "NaN"], "mixed_numeric_and_sentinel": [1.0, "Infinity"], "with_hex_components": ["0x7fc00000", "0x00000000"] } test_fixtures.py000066400000000000000000000014321527014137600344670ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/complex64"""Validate complex64 fill-value fixtures. A v3 complex fill_value is a two-element JSON array `[real, imag]` where each component is shaped per the corresponding float's fill value: a number, one of the named sentinels (`"NaN"`, `"Infinity"`, `"-Infinity"`), or a hex string of the underlying float's bits. """ from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.complex64 import Complex64FillValue FILL_VALUES: dict[str, object] = json.loads( (Path(__file__).parent / "fill_values.json").read_text() ) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(Complex64FillValue).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/float16/000077500000000000000000000000001527014137600307175ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/float16/__init__.py000066400000000000000000000000001527014137600330160ustar00rootroot00000000000000fill_values.json000066400000000000000000000002241527014137600340360ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/float16{ "zero": 0.0, "nan": "NaN", "infinity": "Infinity", "neg_infinity": "-Infinity", "hex_zero": "0x0000", "hex_signaling_nan": "0x7d00" } test_fixtures.py000066400000000000000000000013401527014137600341200ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/float16"""Validate float16 fill-value fixtures. A v3 float fill_value is a JSON number, one of the named non-finite sentinels (`"NaN"`, `"Infinity"`, `"-Infinity"`), or a hex string (`"0xYYYY"`) encoding the unsigned-integer representation of the IEEE 754 value. """ from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.float16 import Float16FillValue FILL_VALUES: dict[str, object] = json.loads( (Path(__file__).parent / "fill_values.json").read_text() ) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(Float16FillValue).validate_python(case) test_validators.py000066400000000000000000000016011527014137600344170ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/float16"""Cover the `hex_float16` brand validator. The pydantic-driven fixture tests don't enforce hex format because `HexFloat16` is a `NewType`, which pydantic treats as plain `str`. Direct coverage of the validator function lives here. """ from __future__ import annotations import pytest from zarr_metadata.v3.data_type.float16 import hex_float16 VALID = ["0x0000", "0x7c00", "0x7d00", "0xffff", "0xFFFF", "0xAbCd"] INVALID = [ "", "0000", # missing 0x "0x000", # too short "0x00000", # too long "0x000g", # non-hex char "0X0000", # uppercase X " 0x0000 ", # whitespace ] @pytest.mark.parametrize("value", VALID) def test_valid(value: str) -> None: assert hex_float16(value) == value @pytest.mark.parametrize("value", INVALID) def test_invalid(value: str) -> None: with pytest.raises(ValueError, match="Expected '0x'"): hex_float16(value) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/float32/000077500000000000000000000000001527014137600307155ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/float32/__init__.py000066400000000000000000000000001527014137600330140ustar00rootroot00000000000000fill_values.json000066400000000000000000000003011527014137600340300ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/float32{ "zero": 0.0, "nan": "NaN", "infinity": "Infinity", "neg_infinity": "-Infinity", "hex_zero": "0x00000000", "hex_canonical_nan": "0x7fc00000", "hex_signaling_nan": "0x7fa00000" } test_fixtures.py000066400000000000000000000010071527014137600341160ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/float32"""Validate float32 fill-value fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.float32 import Float32FillValue FILL_VALUES: dict[str, object] = json.loads( (Path(__file__).parent / "fill_values.json").read_text() ) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(Float32FillValue).validate_python(case) test_validators.py000066400000000000000000000017561527014137600344300ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/float32"""Cover the `hex_float32` brand validator. The pydantic-driven fixture tests don't enforce hex format because `HexFloat32` is a `NewType`, which pydantic treats as plain `str`. Direct coverage of the validator function lives here. """ from __future__ import annotations import pytest from zarr_metadata.v3.data_type.float32 import hex_float32 VALID = [ "0x00000000", "0x7fc00000", # canonical NaN "0x7fa00000", # signaling NaN "0xffffffff", "0xFFFFFFFF", "0xDeadBeef", ] INVALID = [ "", "00000000", # missing 0x "0x0000000", # too short "0x000000000", # too long "0x0000000g", # non-hex char "0X00000000", # uppercase X " 0x00000000 ", # whitespace ] @pytest.mark.parametrize("value", VALID) def test_valid(value: str) -> None: assert hex_float32(value) == value @pytest.mark.parametrize("value", INVALID) def test_invalid(value: str) -> None: with pytest.raises(ValueError, match="Expected '0x'"): hex_float32(value) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/float64/000077500000000000000000000000001527014137600307225ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/float64/__init__.py000066400000000000000000000000001527014137600330210ustar00rootroot00000000000000fill_values.json000066400000000000000000000003311527014137600340400ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/float64{ "zero": 0.0, "nan": "NaN", "infinity": "Infinity", "neg_infinity": "-Infinity", "hex_zero": "0x0000000000000000", "hex_canonical_nan": "0x7ff8000000000000", "hex_signaling_nan": "0x7ff4000000000000" } test_fixtures.py000066400000000000000000000010071527014137600341230ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/float64"""Validate float64 fill-value fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.float64 import Float64FillValue FILL_VALUES: dict[str, object] = json.loads( (Path(__file__).parent / "fill_values.json").read_text() ) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(Float64FillValue).validate_python(case) test_validators.py000066400000000000000000000021161527014137600344240ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/float64"""Cover the `hex_float64` brand validator. The pydantic-driven fixture tests don't enforce hex format because `HexFloat64` is a `NewType`, which pydantic treats as plain `str`. Direct coverage of the validator function lives here. """ from __future__ import annotations import pytest from zarr_metadata.v3.data_type.float64 import hex_float64 VALID = [ "0x0000000000000000", "0x7ff8000000000000", # canonical NaN "0x7ff4000000000000", # signaling NaN "0xffffffffffffffff", "0xFFFFFFFFFFFFFFFF", "0xDeadBeefCafeBabe", ] INVALID = [ "", "0000000000000000", # missing 0x "0x000000000000000", # too short "0x00000000000000000", # too long "0x000000000000000g", # non-hex char "0X0000000000000000", # uppercase X " 0x0000000000000000 ", # whitespace ] @pytest.mark.parametrize("value", VALID) def test_valid(value: str) -> None: assert hex_float64(value) == value @pytest.mark.parametrize("value", INVALID) def test_invalid(value: str) -> None: with pytest.raises(ValueError, match="Expected '0x'"): hex_float64(value) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/int16/000077500000000000000000000000001527014137600304045ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/int16/__init__.py000066400000000000000000000000001527014137600325030ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/int16/fill_values.json000066400000000000000000000001031527014137600335760ustar00rootroot00000000000000{ "zero": 0, "min": -32768, "max": 32767, "negative": -1 } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/int16/test_fixtures.py000066400000000000000000000007771527014137600337010ustar00rootroot00000000000000"""Validate int16 fill-value fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.int16 import Int16FillValue FILL_VALUES: dict[str, object] = json.loads( (Path(__file__).parent / "fill_values.json").read_text() ) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(Int16FillValue).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/int32/000077500000000000000000000000001527014137600304025ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/int32/__init__.py000066400000000000000000000000001527014137600325010ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/int32/fill_values.json000066400000000000000000000001151527014137600335770ustar00rootroot00000000000000{ "zero": 0, "min": -2147483648, "max": 2147483647, "negative": -1 } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/int32/test_fixtures.py000066400000000000000000000007771527014137600336770ustar00rootroot00000000000000"""Validate int32 fill-value fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.int32 import Int32FillValue FILL_VALUES: dict[str, object] = json.loads( (Path(__file__).parent / "fill_values.json").read_text() ) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(Int32FillValue).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/int64/000077500000000000000000000000001527014137600304075ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/int64/__init__.py000066400000000000000000000000001527014137600325060ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/int64/fill_values.json000066400000000000000000000001371527014137600336100ustar00rootroot00000000000000{ "zero": 0, "min": -9223372036854775808, "max": 9223372036854775807, "negative": -1 } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/int64/test_fixtures.py000066400000000000000000000007771527014137600337040ustar00rootroot00000000000000"""Validate int64 fill-value fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.int64 import Int64FillValue FILL_VALUES: dict[str, object] = json.loads( (Path(__file__).parent / "fill_values.json").read_text() ) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(Int64FillValue).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/int8/000077500000000000000000000000001527014137600303255ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/int8/__init__.py000066400000000000000000000000001527014137600324240ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/int8/fill_values.json000066400000000000000000000000771527014137600335310ustar00rootroot00000000000000{ "zero": 0, "min": -128, "max": 127, "negative": -1 } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/int8/test_fixtures.py000066400000000000000000000007731527014137600336160ustar00rootroot00000000000000"""Validate int8 fill-value fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.int8 import Int8FillValue FILL_VALUES: dict[str, object] = json.loads( (Path(__file__).parent / "fill_values.json").read_text() ) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(Int8FillValue).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/numpy_datetime64/000077500000000000000000000000001527014137600326415ustar00rootroot00000000000000__init__.py000066400000000000000000000000001527014137600346610ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/numpy_datetime64data_type.json000066400000000000000000000001431527014137600354250ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/numpy_datetime64{ "name": "numpy.datetime64", "configuration": { "unit": "ns", "scale_factor": 1 } } fill_values.json000066400000000000000000000000431527014137600357570ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/numpy_datetime64{ "int": 12345, "nat": "NaT" } test_fixtures.py000066400000000000000000000013361527014137600360470ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/numpy_datetime64"""Validate numpy.datetime64 dtype value and fill-value fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.numpy_datetime64 import ( NumpyDatetime64, NumpyDatetime64FillValue, ) DIR = Path(__file__).parent FILL_VALUES: dict[str, object] = json.loads((DIR / "fill_values.json").read_text()) def test_data_type() -> None: TypeAdapter(NumpyDatetime64).validate_python(json.loads((DIR / "data_type.json").read_text())) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(NumpyDatetime64FillValue).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/numpy_timedelta64/000077500000000000000000000000001527014137600330155ustar00rootroot00000000000000__init__.py000066400000000000000000000000001527014137600350350ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/numpy_timedelta64data_type.json000066400000000000000000000001431527014137600356010ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/numpy_timedelta64{ "name": "numpy.timedelta64", "configuration": { "unit": "s", "scale_factor": 1 } } fill_values.json000066400000000000000000000000401527014137600361300ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/numpy_timedelta64{ "int": 42, "nat": "NaT" } test_fixtures.py000066400000000000000000000017031527014137600362210ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/numpy_timedelta64"""Validate numpy.timedelta64 dtype value and fill-value fixtures.""" from __future__ import annotations import json from pathlib import Path from typing import get_args import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_TIME_UNIT, NumpyTimeUnit from zarr_metadata.v3.data_type.numpy_timedelta64 import NumpyTimedelta64, NumpyTimedelta64FillValue DIR = Path(__file__).parent FILL_VALUES: dict[str, object] = json.loads((DIR / "fill_values.json").read_text()) def test_data_type() -> None: TypeAdapter(NumpyTimedelta64).validate_python(json.loads((DIR / "data_type.json").read_text())) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(NumpyTimedelta64FillValue).validate_python(case) def test_time_unit_constant_matches_literal() -> None: assert set(NUMPY_TIME_UNIT) == set(get_args(NumpyTimeUnit)) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/raw/000077500000000000000000000000001527014137600302345ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/raw/__init__.py000066400000000000000000000000001527014137600323330ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/raw/fill_values.json000066400000000000000000000000471527014137600334350ustar00rootroot00000000000000{ "all_zero_4_bytes": [0, 0, 0, 0] } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/raw/test_fixtures.py000066400000000000000000000010201527014137600335070ustar00rootroot00000000000000"""Validate raw-bytes (`r`) fill-value fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.raw import RawBytesFillValue FILL_VALUES: dict[str, object] = json.loads( (Path(__file__).parent / "fill_values.json").read_text() ) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(RawBytesFillValue).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/raw/test_validators.py000066400000000000000000000020661527014137600340210ustar00rootroot00000000000000"""Cover the `raw_bytes_dtype_name` brand validator. The pydantic-driven fixture tests don't enforce the `r` shape because `RawBytesDataTypeName` is a `NewType`, which pydantic treats as plain `str`. Direct coverage of the validator function lives here. """ from __future__ import annotations import pytest from zarr_metadata.v3.data_type.raw import raw_bytes_dtype_name VALID = ["r8", "r16", "r24", "r256", "r1024"] INVALID_FORMAT = ["", "8", "R8", "r", "r-8", "r8 ", " r8", "r8r8"] INVALID_BITS = ["r0", "r1", "r7", "r9", "r15", "r17"] @pytest.mark.parametrize("value", VALID) def test_valid(value: str) -> None: assert raw_bytes_dtype_name(value) == value @pytest.mark.parametrize("value", INVALID_FORMAT) def test_invalid_format(value: str) -> None: with pytest.raises(ValueError, match="Expected 'r' followed by"): raw_bytes_dtype_name(value) @pytest.mark.parametrize("value", INVALID_BITS) def test_invalid_bit_count(value: str) -> None: with pytest.raises(ValueError, match="positive multiple of 8"): raw_bytes_dtype_name(value) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/string/000077500000000000000000000000001527014137600307515ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/string/__init__.py000066400000000000000000000000001527014137600330500ustar00rootroot00000000000000fill_values.json000066400000000000000000000001621527014137600340710ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/string{ "empty": "", "ascii": "hello", "unicode": "héllo 世界", "with_escapes": "line1\nline2\t\"quoted\"" } test_fixtures.py000066400000000000000000000010031527014137600341460ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/string"""Validate string fill-value fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.string import StringFillValue FILL_VALUES: dict[str, object] = json.loads( (Path(__file__).parent / "fill_values.json").read_text() ) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(StringFillValue).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/struct/000077500000000000000000000000001527014137600307675ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/struct/__init__.py000066400000000000000000000000001527014137600330660ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/struct/data_type.json000066400000000000000000000005151527014137600336350ustar00rootroot00000000000000{ "name": "struct", "configuration": { "fields": [ {"name": "x", "data_type": "float32"}, {"name": "y", "data_type": "float32"}, { "name": "when", "data_type": { "name": "numpy.datetime64", "configuration": {"unit": "ns", "scale_factor": 1} } } ] } } fill_values.json000066400000000000000000000001041527014137600341030ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/struct{ "all_fields": { "x": 0.0, "y": 1.0, "when": 0 } } test_fixtures.py000066400000000000000000000012311527014137600341670ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/struct"""Validate struct dtype value and fill-value fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.struct import Struct, StructFillValue DIR = Path(__file__).parent FILL_VALUES: dict[str, object] = json.loads((DIR / "fill_values.json").read_text()) def test_data_type() -> None: TypeAdapter(Struct).validate_python(json.loads((DIR / "data_type.json").read_text())) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(StructFillValue).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/test_dtype_names.py000066400000000000000000000062001527014137600333620ustar00rootroot00000000000000"""Validate every primitive Zarr v3 data-type name string. Primitive dtypes are encoded as bare strings in the `data_type` field of a v3 array metadata document (e.g. `"int32"`, `"float64"`). Each must validate as its declared per-dtype `*Name` literal type. """ from __future__ import annotations import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.bool import BOOL_DATA_TYPE_NAME, BoolDataTypeName from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE_NAME, BytesDataTypeName from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE_NAME, Complex64DataTypeName from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE_NAME, Complex128DataTypeName from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE_NAME, Float16DataTypeName from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE_NAME, Float32DataTypeName from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE_NAME, Float64DataTypeName from zarr_metadata.v3.data_type.int8 import INT8_DATA_TYPE_NAME, Int8DataTypeName from zarr_metadata.v3.data_type.int16 import INT16_DATA_TYPE_NAME, Int16DataTypeName from zarr_metadata.v3.data_type.int32 import INT32_DATA_TYPE_NAME, Int32DataTypeName from zarr_metadata.v3.data_type.int64 import INT64_DATA_TYPE_NAME, Int64DataTypeName from zarr_metadata.v3.data_type.raw import raw_bytes_dtype_name from zarr_metadata.v3.data_type.string import STRING_DATA_TYPE_NAME, StringDataTypeName from zarr_metadata.v3.data_type.uint8 import UINT8_DATA_TYPE_NAME, Uint8DataTypeName from zarr_metadata.v3.data_type.uint16 import UINT16_DATA_TYPE_NAME, Uint16DataTypeName from zarr_metadata.v3.data_type.uint32 import UINT32_DATA_TYPE_NAME, Uint32DataTypeName from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE_NAME, Uint64DataTypeName # (name_string, per-dtype literal type) PRIMITIVE_DTYPES = [ (BOOL_DATA_TYPE_NAME, BoolDataTypeName), (INT8_DATA_TYPE_NAME, Int8DataTypeName), (INT16_DATA_TYPE_NAME, Int16DataTypeName), (INT32_DATA_TYPE_NAME, Int32DataTypeName), (INT64_DATA_TYPE_NAME, Int64DataTypeName), (UINT8_DATA_TYPE_NAME, Uint8DataTypeName), (UINT16_DATA_TYPE_NAME, Uint16DataTypeName), (UINT32_DATA_TYPE_NAME, Uint32DataTypeName), (UINT64_DATA_TYPE_NAME, Uint64DataTypeName), (FLOAT16_DATA_TYPE_NAME, Float16DataTypeName), (FLOAT32_DATA_TYPE_NAME, Float32DataTypeName), (FLOAT64_DATA_TYPE_NAME, Float64DataTypeName), (COMPLEX64_DATA_TYPE_NAME, Complex64DataTypeName), (COMPLEX128_DATA_TYPE_NAME, Complex128DataTypeName), (STRING_DATA_TYPE_NAME, StringDataTypeName), (BYTES_DATA_TYPE_NAME, BytesDataTypeName), ] @pytest.mark.parametrize(("name", "literal_type"), PRIMITIVE_DTYPES, ids=lambda x: str(x)) def test_primitive_against_literal(name: str, literal_type: object) -> None: """The dtype name validates against its declared Literal type.""" TypeAdapter(literal_type).validate_python(name) @pytest.mark.parametrize("raw_name", ["r8", "r16", "r24", "r256", "r1024"], ids=str) def test_raw_bytes_name(raw_name: str) -> None: """`r` names pass the raw_bytes_dtype_name validator.""" raw_bytes_dtype_name(raw_name) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/uint16/000077500000000000000000000000001527014137600305715ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/uint16/__init__.py000066400000000000000000000000001527014137600326700ustar00rootroot00000000000000fill_values.json000066400000000000000000000000401527014137600337040ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/uint16{ "zero": 0, "max": 65535 } test_fixtures.py000066400000000000000000000010031527014137600337660ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/uint16"""Validate uint16 fill-value fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.uint16 import Uint16FillValue FILL_VALUES: dict[str, object] = json.loads( (Path(__file__).parent / "fill_values.json").read_text() ) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(Uint16FillValue).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/uint32/000077500000000000000000000000001527014137600305675ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/uint32/__init__.py000066400000000000000000000000001527014137600326660ustar00rootroot00000000000000fill_values.json000066400000000000000000000000451527014137600337070ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/uint32{ "zero": 0, "max": 4294967295 } test_fixtures.py000066400000000000000000000010031527014137600337640ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/uint32"""Validate uint32 fill-value fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.uint32 import Uint32FillValue FILL_VALUES: dict[str, object] = json.loads( (Path(__file__).parent / "fill_values.json").read_text() ) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(Uint32FillValue).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/uint64/000077500000000000000000000000001527014137600305745ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/uint64/__init__.py000066400000000000000000000000001527014137600326730ustar00rootroot00000000000000fill_values.json000066400000000000000000000000571527014137600337170ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/uint64{ "zero": 0, "max": 18446744073709551615 } test_fixtures.py000066400000000000000000000010031527014137600337710ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/uint64"""Validate uint64 fill-value fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.uint64 import Uint64FillValue FILL_VALUES: dict[str, object] = json.loads( (Path(__file__).parent / "fill_values.json").read_text() ) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(Uint64FillValue).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/uint8/000077500000000000000000000000001527014137600305125ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/uint8/__init__.py000066400000000000000000000000001527014137600326110ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/uint8/fill_values.json000066400000000000000000000000361527014137600337110ustar00rootroot00000000000000{ "zero": 0, "max": 255 } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/data_type/uint8/test_fixtures.py000066400000000000000000000007771527014137600340070ustar00rootroot00000000000000"""Validate uint8 fill-value fixtures.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.data_type.uint8 import Uint8FillValue FILL_VALUES: dict[str, object] = json.loads( (Path(__file__).parent / "fill_values.json").read_text() ) @pytest.mark.parametrize("case", FILL_VALUES.values(), ids=list(FILL_VALUES)) def test_fill_value(case: object) -> None: TypeAdapter(Uint8FillValue).validate_python(case) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/group/000077500000000000000000000000001527014137600266255ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/group/__init__.py000066400000000000000000000000001527014137600307240ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/group/minimal.json000066400000000000000000000000571527014137600311500ustar00rootroot00000000000000{ "zarr_format": 3, "node_type": "group" } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/group/test_fixtures.py000066400000000000000000000010351527014137600321060ustar00rootroot00000000000000"""Decode v3 group metadata fixtures via pydantic.""" from __future__ import annotations import json from pathlib import Path import pytest from pydantic import TypeAdapter from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) ADAPTER = TypeAdapter(ZarrV3GroupMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) def test_validate(fixture: Path) -> None: ADAPTER.validate_python(json.loads(fixture.read_text())) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/group/with_attributes.json000066400000000000000000000002031527014137600327340ustar00rootroot00000000000000{ "zarr_format": 3, "node_type": "group", "attributes": { "label": "root", "spatial_units": ["meter", "meter"] } } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/group/with_extra_field.json000066400000000000000000000002301527014137600330340ustar00rootroot00000000000000{ "zarr_format": 3, "node_type": "group", "consolidated_metadata": { "must_understand": false, "kind": "inline", "metadata": {} } } zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/test_definitions.py000066400000000000000000002053311527014137600314210ustar00rootroot00000000000000"""Reading one metadata field against its definition: type-checked, checked by rules, and read in a scope.""" from __future__ import annotations import copy import dataclasses import math import pickle import re from collections.abc import ( Mapping, ) from dataclasses import dataclass, replace from typing import TYPE_CHECKING, Annotated, Any, Final, NotRequired, cast import pytest from annotated_types import Ge, Predicate from typing_extensions import TypedDict from zarr_metadata import ( JSONValue, ValidationProblem, ) from zarr_metadata._json import refine_json from zarr_metadata.definition import ( AcceptedField, Context, Definition, EmptyConfiguration, Nested, RefusedField, UnclaimedField, canonical_of, configuration_of, resolve, ) from zarr_metadata.definition._definition import resolve_refined from zarr_metadata.model import ( UNSET, ZarrV3ArrayMetadata, read_array_metadata_v3, ) from zarr_metadata.typed_json import ( check, ) from zarr_metadata.v2.definition import CORE_V2, ZarrV2CodecDefinition, ZarrV2DataTypeField from zarr_metadata.v3 import ZarrV3MetadataFieldJSON from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID from zarr_metadata.v3.codec.gzip import GZIP_CODEC, GzipCodecConfiguration from zarr_metadata.v3.data_type.int8 import INT8_DATA_TYPE from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE from zarr_metadata.v3.definition import ( CORE, CORE_AND_EXTENSIONS, ChunkGridDefinition, ChunkKeyEncodingDefinition, CodecDefinition, DataTypeDefinition, StorageTransformerDefinition, ZarrV3CodecField, ZarrV3Context, ) if TYPE_CHECKING: from collections.abc import Callable, Iterable, Iterator class AcmeStackConfiguration(TypedDict, closed=True): """A third-party container: a codec that holds a pipeline of codecs.""" codecs: tuple[ZarrV3CodecField, ...] def acme_stack_rules( configuration: AcmeStackConfiguration, nested: Nested ) -> Iterator[ValidationProblem]: # A rule may read a nested field's name, because rules run only on a # configuration whose nested envelopes are valid. for index, codec in enumerate(configuration["codecs"]): if (codec if isinstance(codec, str) else codec["name"]) == "acme.stack": yield ValidationProblem( ("codecs", index), "a stack does not hold itself", "invalid_value" ) # The rule also uses what the scope read: a codec of another kind is # refused, and a codec that nothing in scope claims is not checked. inner = nested.get(("codecs", index)) definition = inner.definition if isinstance(inner, (AcceptedField, RefusedField)) else None if isinstance(definition, CodecDefinition) and definition.kind != "bytes_bytes": yield ValidationProblem( ("codecs", index), "a stack holds bytes -> bytes codecs", "invalid_value" ) ACME_STACK = CodecDefinition( name="acme.stack", configuration=AcmeStackConfiguration, kind="bytes_bytes", size="dynamic", rules=acme_stack_rules, ) class AcmeLevelConfiguration(TypedDict, closed=True): """With postponed annotations, as in this module, `NotRequired` is written as a string.""" level: NotRequired[int] class AcmeTotalConfiguration(TypedDict, total=False, closed=True): level: int class AcmeTreeConfiguration(TypedDict, closed=True): """A recursive configuration, nested to any depth, as JSON can be.""" label: str children: tuple[AcmeTreeConfiguration, ...] class ByCodec(TypedDict, closed=True): codec: ZarrV3CodecField weight: int class Raw(TypedDict, closed=True): codec: Mapping[str, JSONValue] note: str class AcmeFallbackConfiguration(TypedDict, closed=True): """A union whose branches disagree on whether `codec` is a metadata field.""" fallback: ByCodec | Raw class AcmeRoutesConfiguration(TypedDict, extra_items=ZarrV3CodecField): """Every key is a route, and holds the codec for that route.""" ACME_BOUNDS: Final = {"level": (0, 9), "window": (9, 15)} class AcmeBoundedConfiguration(TypedDict, closed=True): level: int window: NotRequired[int] def acme_bounded_rules( configuration: AcmeBoundedConfiguration, nested: Nested ) -> Iterator[ValidationProblem]: # Every key the rule sees is declared by its TypedDict. for key, value in cast("Mapping[str, int]", configuration).items(): low, high = ACME_BOUNDS[key] if not low <= value <= high: yield ValidationProblem((key,), f"expected {low} to {high}", "invalid_value") class AcmePairedConfiguration(TypedDict, closed=True): first: NotRequired[int] second: NotRequired[int] def acme_paired_rules( configuration: AcmePairedConfiguration, nested: Nested ) -> Iterator[ValidationProblem]: if ("first" in configuration) != ("second" in configuration): yield ValidationProblem((), "expected first and second, or neither", "invalid_value") ACME_LEVEL = CodecDefinition( name="acme.level", configuration=AcmeLevelConfiguration, kind="bytes_bytes", size="dynamic" ) ACME_TOTAL = CodecDefinition( name="acme.total", configuration=AcmeTotalConfiguration, kind="bytes_bytes", size="dynamic" ) ACME_TREE = CodecDefinition( name="acme.tree", configuration=AcmeTreeConfiguration, kind="bytes_bytes", size="dynamic" ) ACME_FALLBACK = CodecDefinition( name="acme.fallback", configuration=AcmeFallbackConfiguration, kind="bytes_bytes", size="dynamic", ) ACME_ROUTES = CodecDefinition( name="acme.routes", configuration=AcmeRoutesConfiguration, kind="bytes_bytes", size="dynamic" ) ACME_BOUNDED = CodecDefinition( name="acme.bounded", configuration=AcmeBoundedConfiguration, kind="bytes_bytes", size="dynamic", rules=acme_bounded_rules, ) ACME_PAIRED = CodecDefinition( name="acme.paired", configuration=AcmePairedConfiguration, kind="bytes_bytes", size="dynamic", rules=acme_paired_rules, ) CONTEXT = CORE_AND_EXTENSIONS.update( codecs=[ *CORE_AND_EXTENSIONS.codecs.values(), ACME_STACK, ACME_LEVEL, ACME_TOTAL, ACME_TREE, ACME_FALLBACK, ACME_ROUTES, ACME_BOUNDED, ACME_PAIRED, ] ) INT8: Final = AcceptedField(json="int8", name="int8", definition=INT8_DATA_TYPE, configuration={}) """An `int8` field as a scope reads it: read by its definition, with no nested fields.""" def _locs(problems: tuple[ValidationProblem, ...]) -> list[tuple[tuple[str | int, ...], str]]: return [(found.loc, found.kind) for found in problems] def test_core_is_a_subset_of_core_and_extensions() -> None: """Each slot of `CORE` files a subset of what the same slot of `CORE_AND_EXTENSIONS` files.""" for kind in ZarrV3Context.kinds.values(): assert CORE.definitions_of(kind).items() <= CORE_AND_EXTENSIONS.definitions_of(kind).items() def test_a_read_field_keeps_the_fields_it_read_inside() -> None: # Keyed by location in the configuration, as the scope read them. shard = { "name": "sharding_indexed", "configuration": { "chunk_shape": [2], "codecs": ["bytes"], "index_codecs": ["bytes", "crc32c"], }, } resolved, _ = resolve(shard, CodecDefinition, CORE_AND_EXTENSIONS) assert isinstance(resolved, AcceptedField) assert {loc: inner.json for loc, inner in resolved.nested.items()} == { ("codecs", 0): "bytes", ("index_codecs", 0): "bytes", ("index_codecs", 1): "crc32c", } cast_value = {"name": "cast_value", "configuration": {"data_type": "int8"}} resolved, _ = resolve(cast_value, CodecDefinition, CORE_AND_EXTENSIONS) assert isinstance(resolved, AcceptedField) assert resolved.nested[("data_type",)] == INT8 # A field with no nested fields has nothing inside. A field that nothing # in scope claims holds no fields at all. A field refused by its check # or rules keeps the nested fields it read. assert resolve("int8", DataTypeDefinition, CORE)[0] == INT8 unclaimed = {"name": "acme.cast", "configuration": {"data_type": "int8"}} assert isinstance(resolve(unclaimed, CodecDefinition, CORE_AND_EXTENSIONS)[0], UnclaimedField) refused = {"name": "cast_value", "configuration": {"data_type": "int8", "rounding": 1}} resolved, _ = resolve(refused, CodecDefinition, CORE_AND_EXTENSIONS) assert isinstance(resolved, RefusedField) assert resolved.nested[("data_type",)] == INT8 @pytest.mark.parametrize( ("field", "kind", "name"), [ ("int8", DataTypeDefinition, "int8"), ({"name": "gzip", "configuration": {"level": 1}}, CodecDefinition, "gzip"), # Raw bits: the name as written, not the name its definition is filed under. ("r16", DataTypeDefinition, "r16"), ({"name": "acme.codec"}, CodecDefinition, "acme.codec"), ({"configuration": {}}, CodecDefinition, None), (5, CodecDefinition, None), ], ids=["bare-name", "object", "raw-bits", "out-of-scope", "no-name", "not-a-field"], ) def test_a_reading_says_the_name_the_field_was_written_with( field: JSONValue, kind: type[Definition[Any]], name: str | None ) -> None: assert resolve(field, kind, CORE_AND_EXTENSIONS)[0].name == name KINDLESS: Final = Definition(name="acme.kindless", configuration=EmptyConfiguration) """A definition with no kind, which no scope can file.""" @pytest.mark.parametrize( ("built", "read_as"), [ (INT8, DataTypeDefinition), # A name read by a definition filed under a different name, as raw bits are. ( AcceptedField( json="r16", name="r16", definition=RAW_BYTES_DATA_TYPE, configuration={"bits": 16} ), DataTypeDefinition, ), # Type arguments are dropped, as `resolve` drops them. ( UnclaimedField(json="acme.t", name="acme.t", read_as=DataTypeDefinition[Any]), DataTypeDefinition, ), ( RefusedField( json="int8", name="int8", read_as=DataTypeDefinition[Any], definition=INT8_DATA_TYPE, ), DataTypeDefinition, ), # Not a field, so it has no name and nothing claims it. (RefusedField(json=5, name=None, read_as=CodecDefinition), CodecDefinition), ], ids=["read", "raw-bits", "unclaimed", "refused", "refused-nameless"], ) def test_a_field_built_by_hand_is_of_the_kind_it_says( built: AcceptedField[Any] | UnclaimedField | RefusedField[Any], read_as: type[Definition[Any]] ) -> None: assert built.read_as is read_as class _HoldsAV2DataType(TypedDict, closed=True): """A v3 configuration with a member that holds a v2 data type field.""" dt: ZarrV2DataTypeField class _HoldsAV3Codec(TypedDict, closed=True): """A v2 configuration with a member that holds a v3 codec field.""" inner: ZarrV3CodecField class _HoldsV2DataTypes(TypedDict, closed=True): """A v3 configuration with a member that holds an array of v2 data type fields.""" dts: tuple[ZarrV2DataTypeField, ...] class _HoldsANestedV2DataType(TypedDict, closed=True): """A v3 configuration with a member that holds an object with a v2 data type field.""" nested: _HoldsAV2DataType class _HoldsV3Codecs(TypedDict, closed=True): """A v2 configuration with a member that holds an array of v3 codec fields.""" inner: tuple[ZarrV3CodecField, ...] @pytest.mark.parametrize( ("build", "named"), [ ( lambda: CodecDefinition( name="acme.mix", configuration=_HoldsAV2DataType, kind="array_array", size="static" ), "ZarrV2DataTypeField", ), ( lambda: ZarrV2CodecDefinition(name="acme_mix", configuration=_HoldsAV3Codec), "ZarrV3CodecField", ), ( lambda: CodecDefinition( name="acme.mix", configuration=_HoldsV2DataTypes, kind="array_array", size="static" ), "ZarrV2DataTypeField", ), ( lambda: CodecDefinition( name="acme.mix", configuration=_HoldsANestedV2DataType, kind="array_array", size="static", ), "ZarrV2DataTypeField", ), ( lambda: ZarrV2CodecDefinition(name="acme_mix", configuration=_HoldsV3Codecs), "ZarrV3CodecField", ), ], ids=[ "v3-holding-v2", "v2-holding-v3", "v3-holding-an-array-of-v2", "v3-holding-v2-nested", "v2-holding-an-array-of-v3", ], ) def test_error_a_configuration_holding_a_field_of_the_other_format( build: Callable[[], object], named: str ) -> None: """A definition reads the fields of its own format only: a document of one format never holds a field of the other. A configuration with a member that holds the other format's field alias, directly, as the items of an array, or inside a nested TypedDict, is refused when the definition is built, with a `TypeError` that names the alias, so no read of it fails part way.""" with pytest.raises(TypeError, match=named): build() @pytest.mark.parametrize("definition", [None, KINDLESS], ids=["none", "kindless"]) def test_error_a_field_read_by_hand_by_what_is_not_a_definition_of_a_kind( definition: Definition[Any] | None, ) -> None: with pytest.raises(TypeError, match="a field read is read by a definition of a kind"): AcceptedField( json="acme.kindless", name="acme.kindless", definition=definition, # pyright: ignore[reportArgumentType] configuration={}, ) @pytest.mark.parametrize("name", ["int16", "r16"]) def test_error_a_field_read_by_hand_by_a_definition_filed_under_another_name(name: str) -> None: with pytest.raises( TypeError, match=f"a field named '{name}' is read by the definition filed under it" ): AcceptedField(json=name, name=name, definition=INT8_DATA_TYPE, configuration={}) def test_error_a_field_refused_by_hand_by_a_definition_of_another_kind() -> None: with pytest.raises( TypeError, match="read as a DataTypeDefinition is read by one, got CodecDefinition" ): RefusedField(json="gzip", name="gzip", read_as=DataTypeDefinition, definition=GZIP_CODEC) @pytest.mark.parametrize("name", ["int16", None]) def test_error_a_field_refused_by_hand_by_a_definition_filed_under_another_name( name: str | None, ) -> None: with pytest.raises( TypeError, match=f"a field named {name!r} is read by the definition filed under it" ): RefusedField(json=name, name=name, read_as=DataTypeDefinition, definition=INT8_DATA_TYPE) @pytest.mark.parametrize( "build", [ lambda: UnclaimedField(json="acme.t", name="acme.t", read_as=Definition), lambda: RefusedField(json="acme.t", name="acme.t", read_as=Definition), ], ids=["unclaimed", "refused"], ) def test_error_a_field_built_by_hand_of_no_kind(build: Callable[[], object]) -> None: with pytest.raises(TypeError, match="is not a kind of metadata"): build() def test_error_a_field_nothing_claims_built_by_hand_without_a_name() -> None: with pytest.raises(TypeError, match="a field nothing in scope claims is named"): UnclaimedField(json=5, name=None, read_as=CodecDefinition) # pyright: ignore[reportArgumentType] @pytest.mark.parametrize( ("field", "kind", "written"), [ # A data type with nothing to configure is written as its bare name, # however it was written; every other field is written as an object. ("int8", DataTypeDefinition, "int8"), ( {"name": "int8", "configuration": {}, "must_understand": True}, DataTypeDefinition, "int8", ), ("crc32c", CodecDefinition, {"name": "crc32c"}), ("default", ChunkKeyEncodingDefinition, {"name": "default"}), ( {"name": "regular", "configuration": {"chunk_shape": [2, 3]}}, ChunkGridDefinition, {"name": "regular", "configuration": {"chunk_shape": (2, 3)}}, ), # Each field held in a configuration is written the same way. ( { "name": "sharding_indexed", "configuration": { "chunk_shape": [2], "codecs": ["bytes"], "index_codecs": ["bytes", {"name": "crc32c", "configuration": {}}], }, }, CodecDefinition, { "name": "sharding_indexed", "configuration": { "chunk_shape": (2,), "codecs": ({"name": "bytes"},), "index_codecs": ({"name": "bytes"}, {"name": "crc32c"}), }, }, ), ( {"name": "cast_value", "configuration": {"data_type": {"name": "int8"}}}, CodecDefinition, {"name": "cast_value", "configuration": {"data_type": "int8"}}, ), # Inside it, an unclaimed field is written the same way, and a # refused field is written as it was. ( { "name": "acme.stack", "configuration": { "codecs": [ "zfpy", {"name": "gzip", "configuration": {"level": 12}, "must_understand": True}, ] }, }, CodecDefinition, { "name": "acme.stack", "configuration": { "codecs": ( {"name": "zfpy"}, {"name": "gzip", "configuration": {"level": 12}, "must_understand": True}, ) }, }, ), # A name that carries its configuration is written on its own. ("r16", DataTypeDefinition, "r16"), ({"name": "r16", "configuration": {}}, DataTypeDefinition, "r16"), # Nothing in scope claims it, so its configuration is kept as written. ("zfpy", CodecDefinition, {"name": "zfpy"}), ( {"name": "zfpy", "configuration": {"x": [1]}}, CodecDefinition, {"name": "zfpy", "configuration": {"x": (1,)}}, ), ({"name": "acme.t", "configuration": {}}, DataTypeDefinition, "acme.t"), ], ids=[ "data-type", "data-type-spelled-out", "codec", "chunk-key-encoding", "configured", "nested", "nested-data-type", "nested-unclaimed-and-refused", "raw-bits", "raw-bits-spelled-out", "unclaimed-codec", "unclaimed-configured", "unclaimed-data-type", ], ) def test_a_field_is_written_as_every_reader_takes_it( field: JSONValue, kind: type[Definition[Any]], written: JSONValue ) -> None: resolved, _ = resolve(field, kind, CONTEXT) assert isinstance(resolved, (AcceptedField, UnclaimedField)) assert resolved.to_json() == written @pytest.mark.parametrize( ("field", "kind", "variant", "configuration", "problems"), [ ( {"name": "gzip", "configuration": {"level": 5}}, CodecDefinition, AcceptedField, {"level": 5}, [], ), ("crc32c", CodecDefinition, AcceptedField, {}, []), ({"name": "crc32c"}, CodecDefinition, AcceptedField, {}, []), ({"name": "crc32c", "configuration": {}}, CodecDefinition, AcceptedField, {}, []), ({"name": "crc32c", "must_understand": True}, CodecDefinition, AcceptedField, {}, []), ("bytes", CodecDefinition, AcceptedField, {}, []), ( {"name": "bytes", "configuration": {"endian": "big"}}, CodecDefinition, AcceptedField, {"endian": "big"}, [], ), ( {"name": "regular", "configuration": {"chunk_shape": [2, 3]}}, ChunkGridDefinition, AcceptedField, {"chunk_shape": (2, 3)}, [], ), # An unknown key is not fatal: it is reported and left out of the # configuration, and the field is still read. ( {"name": "gzip", "configuration": {"level": 5, "extra": 1}}, CodecDefinition, AcceptedField, {"level": 5}, [(("configuration", "extra"), "unknown_key")], ), # The rules see only keys that the TypedDict declares. ( {"name": "acme.bounded", "configuration": {"level": 5, "windw": 10}}, CodecDefinition, AcceptedField, {"level": 5}, [(("configuration", "windw"), "unknown_key")], ), # A key that is not required, written as a string under postponed # annotations, and every key of a `total=False` TypedDict. ("acme.level", CodecDefinition, AcceptedField, {}, []), ("acme.total", CodecDefinition, AcceptedField, {}, []), # A kind with type arguments is the same kind. ( {"name": "gzip", "configuration": {"level": 5}}, CodecDefinition[Any], AcceptedField, {"level": 5}, [], ), ( { "name": "acme.tree", "configuration": {"label": "a", "children": [{"label": "b", "children": []}]}, }, CodecDefinition, AcceptedField, {"label": "a", "children": ({"label": "b", "children": ()},)}, [], ), # Only the union branch that matched has its nested fields read. Here # `codec` is plain JSON, not a gzip with a missing configuration. ( { "name": "acme.fallback", "configuration": {"fallback": {"codec": {"name": "gzip"}, "note": "x"}}, }, CodecDefinition, AcceptedField, {"fallback": {"codec": {"name": "gzip"}, "note": "x"}}, [], ), # `extra_items` is a field alias: every other key holds a codec, # stored as a document writes it. ( {"name": "acme.routes", "configuration": {"fast": "crc32c"}}, CodecDefinition, AcceptedField, {"fast": {"name": "crc32c"}}, [], ), # Nothing in scope claims it, so it is not checked and not refused. ({"name": "zfpy", "configuration": {"x": 1}}, CodecDefinition, UnclaimedField, None, []), # A nested field is read in the same scope and stored as a document # writes it. A nested field that is out of scope is not checked. ( {"name": "acme.stack", "configuration": {"codecs": ["crc32c", "zfpy"]}}, CodecDefinition, AcceptedField, {"codecs": ({"name": "crc32c"}, {"name": "zfpy"})}, [], ), ], ids=[ "gzip", "bare-name", "object-without-configuration", "empty-configuration", "must-understand", "bytes-bare", "bytes-endian", "regular-grid", "unknown-key", "unknown-key-before-the-rules", "not-required-postponed", "total-false", "kind-with-type-arguments", "recursive", "union-branch-that-read", "extra-items-of-fields", "out-of-scope", "nested", ], ) def test_a_field_is_read_in_scope( field: object, kind: type[Definition[Any]], variant: type, configuration: dict[str, object] | None, problems: list[tuple[tuple[str | int, ...], str]], ) -> None: resolved, found = resolve(field, kind, CONTEXT) assert type(resolved) is variant assert ( resolved.configuration if isinstance(resolved, AcceptedField) else None ) == configuration assert _locs(found) == problems def test_error_a_rule_refuses_a_value() -> None: resolved, found = resolve( {"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CONTEXT ) assert isinstance(resolved, RefusedField) assert resolved.definition is GZIP_CODEC assert _locs(found) == [(("configuration", "level"), "invalid_value")] def test_error_a_member_of_the_wrong_type_is_not_asked_of_the_rules() -> None: resolved, found = resolve( {"name": "gzip", "configuration": {"level": "5"}}, CodecDefinition, CONTEXT ) assert isinstance(resolved, RefusedField) assert _locs(found) == [(("configuration", "level"), "invalid_type")] def test_error_a_required_configuration_is_missing() -> None: resolved, found = resolve("gzip", CodecDefinition, CONTEXT) assert isinstance(resolved, RefusedField) assert _locs(found) == [(("configuration",), "missing_key")] def test_error_the_configuration_is_not_an_object() -> None: # The result is a RefusedField, still claimed by the definition its name refers to. resolved, found = resolve({"name": "gzip", "configuration": 5}, CodecDefinition, CONTEXT) assert resolved == RefusedField( json={"name": "gzip", "configuration": 5}, name="gzip", read_as=CodecDefinition, definition=GZIP_CODEC, ) assert [found.loc for found in found] == [("configuration",)] def test_error_must_understand_false_is_refused() -> None: # The envelope's problem is reported with the field. The configuration # was read, so a later layer can still check the codec. resolved, found = resolve( {"name": "crc32c", "must_understand": False}, CodecDefinition, CONTEXT ) assert isinstance(resolved, AcceptedField) assert _locs(found) == [(("must_understand",), "invalid_value")] @pytest.mark.parametrize( ("member", "kind"), [ ("data_type", DataTypeDefinition), ("chunk_grid", ChunkGridDefinition), ("chunk_key_encoding", ChunkKeyEncodingDefinition), ("codecs", CodecDefinition), ("storage_transformers", StorageTransformerDefinition), ], ) def test_error_must_understand_false_is_refused_wherever_the_model_refuses_it( member: str, kind: type[Definition[Any]] ) -> None: # One interpretation of the spec, applied by two readers: reading a field # here, and the model's validator. If only one of them changes, this fails. field: dict[str, JSONValue] = {"name": "acme.example", "must_understand": False} listed = member in ("codecs", "storage_transformers") document: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) document[member] = [field] if listed else field at = (member, 0) if listed else (member,) by_model = [ (problem.loc[len(at) :], problem.kind) for problem in read_array_metadata_v3(document).problems if problem.loc[: len(at)] == at ] assert _locs(resolve(field, kind, CORE_AND_EXTENSIONS)[1]) == by_model assert by_model == [(("must_understand",), "invalid_value")] def _ruled_by( rules: Callable[[GzipCodecConfiguration, Nested], Iterable[ValidationProblem]], ) -> Context: lying = replace(GZIP_CODEC, name="acme.gzip", rules=rules) return CORE.update(codecs=[*CORE.codecs.values(), lying]) def test_error_a_rule_that_yields_something_else() -> None: def rules(configuration: GzipCodecConfiguration, nested: Nested) -> Iterator[ValidationProblem]: yield "level is too high" # pyright: ignore[reportReturnType] with pytest.raises(TypeError, match="'acme.gzip': its rules yield ValidationProblem values"): resolve( {"name": "acme.gzip", "configuration": {"level": 1}}, CodecDefinition, _ruled_by(rules), ) def test_error_a_rule_that_raises_says_whose_it_is() -> None: def rules(configuration: GzipCodecConfiguration, nested: Nested) -> Iterator[ValidationProblem]: yield from ({}[configuration["level"]],) with pytest.raises(KeyError) as raised: resolve( {"name": "acme.gzip", "configuration": {"level": 1}}, CodecDefinition, _ruled_by(rules), ) assert raised.value.__notes__ == [ "raised by the rules of 'acme.gzip', reading ('configuration',)" ] def test_error_a_rule_that_returns_none_says_whose_it_is() -> None: # A plain function that forgot to yield returns nothing to iterate. def rules(configuration: GzipCodecConfiguration, nested: Nested) -> None: return None with pytest.raises(TypeError, match="not iterable") as raised: resolve( {"name": "acme.gzip", "configuration": {"level": 1}}, CodecDefinition, _ruled_by(rules), # pyright: ignore[reportArgumentType] ) assert raised.value.__notes__ == [ "raised by the rules of 'acme.gzip', reading ('configuration',)" ] def test_error_null_is_not_a_field() -> None: # JSON null is valid JSON. Refining it gives `None` with no problem, # which is not a usable result. Read as a field or checked as a # configuration, it is a value of the wrong type. resolved, found = resolve(None, CodecDefinition, CONTEXT) assert (resolved, _locs(found)) == ( RefusedField(json=None, name=None, read_as=CodecDefinition), [((), "invalid_type")], ) configuration, found = GZIP_CODEC.read_configuration(None) assert (configuration, _locs(found)) == (None, [((), "invalid_type")]) def test_error_a_value_that_is_not_json() -> None: # The JSON is stored as `UNSET`, because no document can hold a value # that is not JSON. Its name still identifies the definition that claims it. resolved, found = resolve( {"name": "gzip", "configuration": {"level": math.nan}}, CodecDefinition, CONTEXT ) assert resolved == RefusedField( json=UNSET, name="gzip", read_as=CodecDefinition, definition=GZIP_CODEC ) assert _locs(found) == [(("configuration", "level"), "invalid_value")] def test_error_a_value_that_is_not_a_field() -> None: resolved, found = resolve(5, CodecDefinition, CONTEXT) assert resolved == RefusedField(json=5, name=None, read_as=CodecDefinition) assert len(found) == 1 def test_error_a_regular_grid_extent_is_negative() -> None: resolved, found = resolve( {"name": "regular", "configuration": {"chunk_shape": [2, -1]}}, ChunkGridDefinition, CONTEXT ) assert isinstance(resolved, RefusedField) assert _locs(found) == [(("configuration", "chunk_shape", 1), "invalid_value")] def test_error_a_nested_field_is_judged_where_it_sits() -> None: # The problem belongs to the nested field. The field that holds it is # still read, as a document holding it would be. field = { "name": "acme.stack", "configuration": {"codecs": ["crc32c", {"name": "gzip", "configuration": {"level": 12}}]}, } resolved, found = resolve(field, CodecDefinition, CONTEXT) assert isinstance(resolved, AcceptedField) assert isinstance(resolved.nested[("codecs", 1)], RefusedField) assert _locs(found) == [ (("configuration", "codecs", 1, "configuration", "level"), "invalid_value") ] def test_error_a_nested_field_in_extra_items_is_judged_where_it_sits() -> None: field = { "name": "acme.routes", "configuration": {"slow": {"name": "gzip", "configuration": {"level": 12}}}, } resolved, found = resolve(field, CodecDefinition, CONTEXT) assert isinstance(resolved, AcceptedField) assert isinstance(resolved.nested[("slow",)], RefusedField) assert _locs(found) == [(("configuration", "slow", "configuration", "level"), "invalid_value")] def test_error_a_nested_envelope_s_problem_is_its_own_and_the_rules_are_asked() -> None: # A nested `must_understand` of false is reported at its location, as # in a document, and the stack is still read. unread = {"name": "crc32c", "must_understand": False} resolved, found = resolve( {"name": "acme.stack", "configuration": {"codecs": [unread]}}, CodecDefinition, CONTEXT ) assert isinstance(resolved, AcceptedField) assert _locs(found) == [(("configuration", "codecs", 0, "must_understand"), "invalid_value")] # The stack's rules still run. One rule refuses the array -> bytes # codec it holds, which is a problem of the stack itself. resolved, found = resolve( {"name": "acme.stack", "configuration": {"codecs": [unread, "bytes"]}}, CodecDefinition, CONTEXT, ) assert isinstance(resolved, RefusedField) assert _locs(found) == [ (("configuration", "codecs", 1), "invalid_value"), (("configuration", "codecs", 0, "must_understand"), "invalid_value"), ] def test_error_a_container_rule_is_not_asked_of_a_malformed_nested_field() -> None: # The stack's rule reads each nested field's name. A nested field with no # name is reported at its location, and the rule does not run, as it # would not in `read_configuration`. field = {"name": "acme.stack", "configuration": {"codecs": [{"configuration": {}}]}} resolved, found = resolve(field, CodecDefinition, CONTEXT) assert isinstance(resolved, RefusedField) assert _locs(found) == [(("configuration", "codecs", 0, "name"), "missing_key")] assert ACME_STACK.read_configuration(field["configuration"])[0] is None def test_error_a_rule_about_the_whole_configuration_lands_on_it() -> None: # A rule reports locations relative to the configuration. An empty # location is the configuration, whether it is checked alone or read in # a field. _, judged = ACME_PAIRED.read_configuration({"first": 1}) _, read = resolve( {"name": "acme.paired", "configuration": {"first": 1}}, CodecDefinition, CONTEXT ) assert _locs(judged) == [((), "invalid_value")] assert _locs(read) == [(("configuration",), "invalid_value")] def test_error_a_rule_reads_the_fields_the_configuration_holds_as_the_scope_read_them() -> None: # `bytes` is an array -> bytes codec, which the stack's rule refuses # based on what the scope read. `read_configuration` reads in no scope, # so its rule sees no nested readings. field = {"name": "acme.stack", "configuration": {"codecs": ["crc32c", "bytes"]}} resolved, found = resolve(field, CodecDefinition, CONTEXT) assert isinstance(resolved, RefusedField) assert _locs(found) == [(("configuration", "codecs", 1), "invalid_value")] assert ACME_STACK.read_configuration(field["configuration"])[1] == () def test_error_a_nested_member_is_not_a_field() -> None: resolved, found = resolve( {"name": "acme.stack", "configuration": {"codecs": [5]}}, CodecDefinition, CONTEXT ) assert isinstance(resolved, RefusedField) assert _locs(found) == [(("configuration", "codecs", 0), "invalid_type")] def test_check_needs_nothing_but_the_value_and_a_typeddict() -> None: # The first step checks JSON against a TypedDict. A nested field is only # an envelope here, so no scope is needed and no name is checked. assert check({"level": 5}, GzipCodecConfiguration) == ({"level": 5}, ()) stack, problems = check({"codecs": ["gzip", {"name": "anything"}]}, AcmeStackConfiguration) assert (stack, problems) == ({"codecs": ("gzip", {"name": "anything"})}, ()) # The check and the reading agree on which union branch a value matches. fallback = {"fallback": {"codec": {"name": "gzip"}, "note": "x"}} assert check(fallback, AcmeFallbackConfiguration) == (fallback, ()) def test_check_reads_a_whole_array_document() -> None: # A null in `dimension_names`, a top-level key that the document type # does not declare (typed by its `extra_items`), and a dimension of # length 0. document = { **ZarrV3ArrayMetadata.create_default(shape=(0, 4)).to_json(), "dimension_names": [None, "x"], "acme": {"must_understand": False}, } typed, problems = check(document, ZarrV3ArrayMetadataJSON) assert problems == () assert typed is not None assert typed.get("dimension_names") == (None, "x") def test_error_check_holds_an_array_document_s_shape_to_its_bound() -> None: document = {**ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json(), "shape": [-1]} typed, problems = check(document, ZarrV3ArrayMetadataJSON) assert typed is None assert [(found.loc, found.kind, dict(found.ctx)) for found in problems] == [ (("shape", 0), "invalid_value", {"ge": 0}) ] def test_configuration_of_types_what_its_definition_read() -> None: """`configuration_of(field, definition)` returns, as a plain `dict`, the configuration of a field that `definition` read and accepted. It returns `None` for a field another definition read, for a field nothing claims, and for a field that `definition` claims but refused, whose configuration it did not read.""" resolved, _ = resolve({"name": "gzip", "configuration": {"level": 5}}, CodecDefinition, CONTEXT) assert configuration_of(resolved, GZIP_CODEC) == {"level": 5} assert type(configuration_of(resolved, GZIP_CODEC)) is dict assert configuration_of(resolved, ACME_LEVEL) is None unclaimed, _ = resolve( {"name": "acme.x", "configuration": {"level": 1}}, CodecDefinition, CONTEXT ) assert isinstance(unclaimed, UnclaimedField) assert configuration_of(unclaimed, GZIP_CODEC) is None refused, _ = resolve({"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CONTEXT) assert isinstance(refused, RefusedField) assert refused.definition is GZIP_CODEC assert configuration_of(refused, GZIP_CODEC) is None class _ClaimsWithNoFormat: """A value with a `claimant` but no `zarr_format`.""" def claimant(self, kind: object, name: str) -> None: return None class _BooleanFormatScope(_ClaimsWithNoFormat): """A value whose `zarr_format` is a boolean, which no format is.""" zarr_format = True class _ClassWithAFormat: """A class, not an instance, that declares a format.""" zarr_format = 3 class _RaisingClaimant: """A value whose `claimant` raises when it is looked up, and whose repr raises.""" zarr_format = 3 @property def claimant(self) -> object: raise RuntimeError def __repr__(self) -> str: raise RuntimeError class _HugeFormat(_ClaimsWithNoFormat): """A value whose `zarr_format` is an integer too long to write.""" zarr_format = 10**5000 @pytest.mark.parametrize( "context", [ None, "CORE", ZarrV3Context, _ClaimsWithNoFormat(), _BooleanFormatScope(), _ClassWithAFormat, _RaisingClaimant(), _HugeFormat(), ], ids=[ "none", "a-string", "the-scope-class", "no-format", "a-boolean-format", "a-class-with-a-format", "a-raising-claimant-and-repr", "a-huge-format", ], ) def test_error_resolve_refuses_what_is_not_a_scope(context: object) -> None: """`resolve` reads a field in a scope, which it takes with no default. `None`, which a model reader would take, or any other value that is not a scope raises `TypeError` naming it, rather than failing part way.""" with pytest.raises(TypeError, match="a field is read in a scope"): resolve({"name": "bytes"}, CodecDefinition, context) # pyright: ignore[reportArgumentType] def test_error_check_locates_what_is_not_the_typeddict() -> None: typed, problems = check({"level": "5", "extra": 1}, GzipCodecConfiguration) assert typed is None assert sorted(_locs(problems)) == [(("extra",), "unknown_key"), (("level",), "invalid_type")] def test_error_check_judges_a_nested_envelope() -> None: typed, problems = check( {"codecs": [{"name": "gzip", "configuration": 3}]}, AcmeStackConfiguration ) assert typed is None assert [found.loc for found in problems] == [("codecs", 0, "configuration")] def test_read_configuration_leaves_out_a_member_a_nested_field_s_envelope_does_not_declare() -> ( None ): # The member is reported as an unknown key and left out, as the checker # leaves out a key that a closed TypedDict does not declare. The # configuration is still returned. typed, problems = ACME_STACK.read_configuration({"codecs": [{"name": "crc32c", "x": 1}]}) assert typed == {"codecs": ({"name": "crc32c"},)} assert _locs(problems) == [(("codecs", 0, "x"), "unknown_key")] def test_error_read_configuration_refuses_a_nested_field_that_need_not_be_understood() -> None: # A `must_understand` of false is not an unknown key, so the # configuration is not returned. typed, problems = ACME_STACK.read_configuration( {"codecs": [{"name": "crc32c", "must_understand": False}]} ) assert typed is None assert _locs(problems) == [(("codecs", 0, "must_understand"), "invalid_value")] def test_read_configuration_is_the_check_and_then_the_rules() -> None: # For a caller with one configuration, the rules run only on a # configuration that passed the type check, so they never see a wrong type. assert GZIP_CODEC.read_configuration({"level": 5}) == ({"level": 5}, ()) refused, problems = GZIP_CODEC.read_configuration({"level": 12}) assert (refused, _locs(problems)) == (None, [(("level",), "invalid_value")]) mistyped, problems = GZIP_CODEC.read_configuration({"level": "x"}) assert (mistyped, _locs(problems)) == (None, [(("level",), "invalid_type")]) def test_a_scope_takes_a_name_over() -> None: strict = CodecDefinition( name="gzip", configuration=GzipCodecConfiguration, kind="bytes_bytes", size="dynamic", rules=lambda configuration, nested: ( [ValidationProblem(("level",), "level 0 stores uncompressed", "invalid_value")] if configuration["level"] == 0 else [] ), ) context = CORE_AND_EXTENSIONS.update(codecs={**CORE_AND_EXTENSIONS.codecs, "gzip": strict}) _, found = resolve({"name": "gzip", "configuration": {"level": 0}}, CodecDefinition, context) assert _locs(found) == [(("configuration", "level"), "invalid_value")] assert context.claimant(ChunkGridDefinition, "regular") is REGULAR_CHUNK_GRID def test_a_scope_is_shown_by_how_many_definitions_it_holds() -> None: # Kept short, because `help` shows it as a validator's default argument. count = sum(len(CORE.definitions_of(kind)) for kind in ZarrV3Context.kinds.values()) assert repr(CORE) == f"ZarrV3Context(<{count} definitions>)" def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: # A model holds the scope it was read in, and takes it along to another process. again = pickle.loads(pickle.dumps(CORE_AND_EXTENSIONS)) assert again == CORE_AND_EXTENSIONS assert again.codecs == CORE_AND_EXTENSIONS.codecs assert copy.copy(CORE) is CORE assert copy.deepcopy(CORE) is CORE LE: Final = {"name": "bytes", "configuration": {"endian": "little"}} SHARD: Final = {"chunk_shape": [2], "codecs": [LE], "index_location": "end"} NOSHUFFLE: Final = {"cname": "lz4", "clevel": 5, "shuffle": "noshuffle", "blocksize": 0} @pytest.mark.parametrize( ("one", "other", "kind", "equal"), [ ("uint8", {"name": "uint8"}, DataTypeDefinition, True), ({"name": "bytes"}, {"name": "bytes", "configuration": {}}, CodecDefinition, True), ( {"name": "gzip", "configuration": {"level": 1}}, {"name": "gzip", "configuration": {"level": 1}, "must_understand": True}, CodecDefinition, True, ), ("acme.codec", {"name": "acme.codec", "configuration": {}}, CodecDefinition, True), ( { "name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [LE, "crc32c"]}, }, { "name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [LE, {"name": "crc32c"}]}, }, CodecDefinition, True, ), ( {"name": "gzip", "configuration": {"level": 1}}, {"name": "gzip", "configuration": {"level": 2}}, CodecDefinition, False, ), ( {"name": "acme.codec", "configuration": {"a": 1}}, {"name": "acme.codec", "configuration": {"a": 2}}, CodecDefinition, False, ), ("int8", "uint8", DataTypeDefinition, False), # The spec's equivalences, which each definition's `canonical` # normalizes: a member at its default, a run-length encoding, and # leading zeros in a size. ( {"name": "blosc", "configuration": {**NOSHUFFLE}}, {"name": "blosc", "configuration": {**NOSHUFFLE, "typesize": 4}}, CodecDefinition, True, ), ( "default", {"name": "default", "configuration": {"separator": "/"}}, ChunkKeyEncodingDefinition, True, ), ( "default", {"name": "default", "configuration": {"separator": "."}}, ChunkKeyEncodingDefinition, False, ), ( {"name": "v2"}, {"name": "v2", "configuration": {"separator": "."}}, ChunkKeyEncodingDefinition, True, ), ( { "name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [LE, "crc32c"]}, }, { "name": "sharding_indexed", "configuration": { "chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, "crc32c"], }, }, CodecDefinition, True, ), ( {"name": "zstd", "configuration": {"level": 3}}, {"name": "zstd", "configuration": {"level": 3, "checksum": False}}, CodecDefinition, True, ), ( {"name": "zstd", "configuration": {"level": 3}}, {"name": "zstd", "configuration": {"level": 3, "checksum": True}}, CodecDefinition, False, ), ( {"name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[2, 2]]}}, { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[[2, 2]]]}, }, ChunkGridDefinition, True, ), ("r008", "r8", DataTypeDefinition, True), ], ids=[ "bare-or-object", "no-or-empty-configuration", "must-understand-true-or-absent", "unclaimed-bare-or-object", "a-field-it-holds-bare-or-object", "another-configuration", "unclaimed-another-configuration", "another-name", "blosc-typesize-noshuffle", "default-separator", "default-other-separator", "v2-separator", "sharding-index-location", "zstd-checksum-false", "zstd-checksum-true", "rectilinear-rle", "raw-bits-leading-zeros", ], ) def test_two_fields_are_equal_when_they_read_the_same( one: JSONValue, other: JSONValue, kind: type[Definition[Any]], equal: bool ) -> None: """However each field was spelled, equal fields have the same simplest spelling and the same hash, though each is written back as it was read.""" first, _ = resolve(one, kind, CORE_AND_EXTENSIONS) second, _ = resolve(other, kind, CORE_AND_EXTENSIONS) assert isinstance(first, (AcceptedField, UnclaimedField)) assert isinstance(second, (AcceptedField, UnclaimedField)) assert (first == second) is equal assert (canonical_of(first, ()) == canonical_of(second, ())) is equal if equal: assert hash(first) == hash(second) def test_a_field_copied_or_pickled_is_read_by_a_definition_equal_to_its_own() -> None: field, _ = resolve({"name": "gzip", "configuration": {"level": 1}}, CodecDefinition, CORE) for again in (pickle.loads(pickle.dumps(field)), copy.deepcopy(field)): assert again == field assert configuration_of(again, GZIP_CODEC) == {"level": 1} def test_a_definition_is_shown_by_its_kind_and_name() -> None: # Kept short, because a reading that holds the definition shows it this way. assert repr(GZIP_CODEC) == "CodecDefinition(name='gzip')" assert repr(INT8_DATA_TYPE) == "DataTypeDefinition(name='int8')" class Unreadable(TypedDict, closed=True): members: set[int] class Loose(TypedDict): level: int class AcmePlainConfiguration(TypedDict, closed=True): inner: ZarrV3MetadataFieldJSON class AcmeDecimal(TypedDict, closed=True): value: Decimal # noqa: F821 - a name the running module does not define # pyright: ignore[reportUndefinedVariable] class AcmeUnresolvedConfiguration(TypedDict, closed=True): inner: AcmeDecimal @dataclass(frozen=True, kw_only=True, slots=True) class AcmeCodecDefinition(CodecDefinition[Any]): """A reader's own codec subclass, under which no scope files anything.""" def test_error_a_definition_configuration_is_a_typeddict() -> None: with pytest.raises(TypeError, match="give the TypedDict"): CodecDefinition(name="acme.bad", configuration=dict, kind="bytes_bytes", size="dynamic") def test_error_a_definition_member_is_a_shape_json_takes() -> None: with pytest.raises(TypeError, match="Unreadable: members is not a shape JSON takes"): CodecDefinition( name="acme.bad", configuration=Unreadable, kind="bytes_bytes", size="dynamic" ) def test_error_a_configuration_says_what_its_other_keys_are() -> None: # An open TypedDict would accept a misspelled key without reporting it. with pytest.raises(TypeError, match="Loose says nothing of the keys it does not declare"): CodecDefinition(name="acme.loose", configuration=Loose, kind="bytes_bytes", size="dynamic") def test_error_a_member_typed_as_plain_field_json_is_refused() -> None: # It would be checked only as JSON, and its name would never be matched to a definition. with pytest.raises(TypeError, match="annotate it with the field alias of its kind"): CodecDefinition( name="acme.plain", configuration=AcmePlainConfiguration, kind="bytes_bytes", size="dynamic", ) def test_error_a_configuration_whose_annotations_do_not_resolve() -> None: # The error names the TypedDict that holds the annotation. with pytest.raises(TypeError, match="AcmeDecimal: name 'Decimal' is not defined"): CodecDefinition( name="acme.unresolved", configuration=AcmeUnresolvedConfiguration, kind="bytes_bytes", size="dynamic", ) class AcmeUncheckedConfiguration(TypedDict, closed=True): digits: Annotated[str, Predicate(str.isdigit)] def test_error_a_configuration_with_a_constraint_the_checker_does_not_read() -> None: # A constraint the checker did not enforce would promise something that is not checked. with pytest.raises( TypeError, match=( r"AcmeUncheckedConfiguration.digits: Predicate\(str.isdigit\) is not a constraint" r".*; check it in the definition's `rules`" ), ): CodecDefinition( name="acme.unchecked", configuration=AcmeUncheckedConfiguration, kind="bytes_bytes", size="dynamic", ) def test_error_a_fill_value_with_a_constraint_its_type_cannot_take() -> None: with pytest.raises(TypeError, match="fill_value: ge: a bound is on a number, and a string"): DataTypeDefinition( name="acme.bounded", configuration=EmptyConfiguration, fill_value=Annotated[str, Ge(0)] ) def test_error_a_definition_name_is_a_string() -> None: with pytest.raises(TypeError, match="a definition's name is a string"): CodecDefinition( name=5, # pyright: ignore[reportArgumentType] configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic", ) def test_error_a_codec_kind_is_one_of_three() -> None: with pytest.raises(TypeError, match="kind is one of"): CodecDefinition( name="acme.k", configuration=EmptyConfiguration, kind="bytes_to_array", # pyright: ignore[reportArgumentType] size="dynamic", ) def test_error_a_data_type_is_named_as_raw_bits_of_one_size_are_written() -> None: # `r16` is read as `r*`, so a definition filed under `r16` would never read anything. with pytest.raises(TypeError, match="to read raw bits your own way, define 'r\\*'"): DataTypeDefinition(name="r16", configuration=EmptyConfiguration) def test_error_a_data_type_fill_value_no_checker_reads() -> None: with pytest.raises(TypeError, match="'acme.set': fill_value: "): DataTypeDefinition(name="acme.set", configuration=EmptyConfiguration, fill_value=set[int]) def test_error_a_data_type_fill_value_holding_a_metadata_field() -> None: # A fill value is a value of the data type, which no scope reads as a field. with pytest.raises( TypeError, match="'acme.f': fill_value: ZarrV3CodecField holds a metadata field" ): DataTypeDefinition( name="acme.f", configuration=EmptyConfiguration, fill_value=tuple[ZarrV3CodecField, ...] ) @pytest.mark.parametrize( ("kind", "member"), [ (DataTypeDefinition, "rules"), (DataTypeDefinition, "canonical"), (DataTypeDefinition, "fill_value_rules"), (DataTypeDefinition, "storage"), (ChunkGridDefinition, "shape_rules"), (ChunkGridDefinition, "chunk_lengths"), (CodecDefinition, "chunk_rules"), (CodecDefinition, "transition"), (CodecDefinition, "pipelines"), ], ) def test_error_a_function_member_that_is_not_a_function( kind: type[Definition[Any]], member: str ) -> None: # Each member that the definition's annotations declare as a `Callable`. codec = {"kind": "array_array", "size": "static"} if kind is CodecDefinition else {} with pytest.raises(TypeError, match=f"'acme.t': {member} is a function, got 'none'"): kind(name="acme.t", configuration=EmptyConfiguration, **codec, **{member: "none"}) # pyright: ignore[reportArgumentType] @pytest.mark.parametrize("kind", [Definition, AcmeCodecDefinition]) def test_error_a_field_is_read_as_a_kind_of_metadata(kind: type[Definition[Any]]) -> None: # Nothing is filed under either kind, so the field would never be checked. with pytest.raises(TypeError, match="is not a kind of metadata"): resolve({"name": "gzip", "configuration": {"level": 99}}, kind, CONTEXT) def test_error_a_scope_refuses_a_definition_of_no_kind() -> None: """A definition that is no kind of v3 field has no slot, and a slot refuses it, at run time as a type checker does.""" kindless = Definition(name="acme.kindless", configuration=EmptyConfiguration) with pytest.raises(TypeError, match="a definition of no kind"): ZarrV3Context(codecs=[kindless]) # pyright: ignore[reportArgumentType] @pytest.mark.parametrize( ("name", "valid"), [ ("zstd", True), ("numcodecs.adler32", True), ("vlen-utf8", True), ("acme_x", True), ("r16", True), ("https://example.com/codec", True), ("urn:acme:codec", True), ("urn:acme:%C3%BC", True), ("x:", False), ("Int8:", False), ("a: b", False), ("urn:acme:codec\n", False), ("urn:acme:ü", False), # Whitespace in Python's `re` but not in ECMA-262, and the reverse. # Neither is a URI character, so both dialects reject them. ("urn:a\x1c", False), ("urn:a", False), ("", False), (" ", False), ("Int8", False), ("9x", False), ("int8 ", False), ("int8\n", False), ("foo/bar", False), ("-int8", False), ("a", False), ("r*", False), ], ) def test_an_extension_is_named_as_the_spec_names_one(name: str, valid: bool) -> None: """An extension name matches the spec's regex `^[a-z][a-z0-9-_.]+$`, or is a URI, which earlier versions of the spec required. Any other name is refused before any definition is consulted.""" for field, at in ((name, ()), ({"name": name}, ("name",))): resolved, problems = resolve(field, CodecDefinition, CORE) if valid: assert not isinstance(resolved, RefusedField) assert problems == () else: assert isinstance(resolved, RefusedField) assert [(p.loc, p.kind) for p in problems] == [(at, "invalid_value")] assert "expected an extension name" in problems[0].message def test_error_a_bad_name_is_a_problem_when_the_field_is_not_json_too() -> None: # On both paths the name is refused before any definition is consulted. # A field whose configuration is not JSON reports its name problem as # the JSON path does, and no definition is asked to claim it. field = {"name": "Acme", "configuration": {"x": float("nan")}} resolved, problems = resolve(field, CodecDefinition, CORE) assert isinstance(resolved, RefusedField) assert resolved.definition is None assert [(p.loc, p.kind) for p in problems] == [ (("name",), "invalid_value"), (("configuration", "x"), "invalid_value"), ] @pytest.mark.parametrize("name", ["Acme", "acme/x", "", "x"]) def test_error_a_definition_is_named_as_the_spec_names_an_extension(name: str) -> None: # No document could use this name, so the definition would never read anything. with pytest.raises(TypeError, match="so no document names it"): CodecDefinition( name=name, configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic" ) def test_error_a_field_built_by_hand_is_named_as_the_spec_names_one() -> None: with pytest.raises(TypeError, match="named as a document names a"): UnclaimedField(json="Int8", name="Int8", read_as=CodecDefinition) def test_error_a_field_without_a_name_is_missing_one() -> None: resolved, problems = resolve({"configuration": {}}, CodecDefinition, CORE) assert isinstance(resolved, RefusedField) assert [(p.loc, p.kind, p.message) for p in problems] == [ (("name",), "missing_key", "missing required key") ] @pytest.mark.parametrize("digits", [101, 4301]) def test_raw_bits_of_more_than_a_hundred_digits_are_no_size_but_a_name(digits: int) -> None: # `int` refuses to convert more than 4,300 digits, and no real size has a # hundred digits. Such a name is an extension name, which nothing in # scope claims. resolved, problems = resolve("r" + "1" * digits, DataTypeDefinition, CORE_AND_EXTENSIONS) assert type(resolved) is UnclaimedField assert problems == () def test_error_a_rule_that_writes_to_the_configuration_fails_there() -> None: # A definition's functions receive a read-only view of the field's own # configuration, so a function cannot change what a field holds. def writes( configuration: GzipCodecConfiguration, nested: Nested ) -> Iterator[ValidationProblem]: cast("dict[str, object]", configuration)["level"] = -1 yield from () context = CORE.update( codecs={**CORE.codecs, "gzip": dataclasses.replace(GZIP_CODEC, rules=writes)} ) with pytest.raises(TypeError, match="does not support item assignment") as raised: resolve({"name": "gzip", "configuration": {"level": 1}}, CodecDefinition, context) assert raised.value.__notes__ == ["raised by the rules of 'gzip', reading ('configuration',)"] def test_error_a_function_that_writes_to_the_configuration_fails_on_every_path() -> None: # `read_configuration` passes the rules the same read-only view that # `resolve` does, and `==` and `hash` pass one to `canonical`, as # `canonical_of` does. def writes( configuration: GzipCodecConfiguration, nested: Nested ) -> Iterator[ValidationProblem]: cast("dict[str, object]", configuration)["level"] = -1 yield from () with pytest.raises(TypeError, match="does not support item assignment"): dataclasses.replace(GZIP_CODEC, rules=writes).read_configuration({"level": 1}) def folds_in_place(configuration: GzipCodecConfiguration) -> GzipCodecConfiguration: cast("dict[str, object]", configuration)["level"] = 0 return configuration context = CORE.update( codecs={**CORE.codecs, "gzip": dataclasses.replace(GZIP_CODEC, canonical=folds_in_place)} ) read, _ = resolve({"name": "gzip", "configuration": {"level": 1}}, CodecDefinition, context) with pytest.raises(TypeError, match="does not support item assignment"): hash(read) def test_error_a_canonical_that_raises_says_which_definition_raised_it() -> None: def refuses(configuration: GzipCodecConfiguration) -> GzipCodecConfiguration: raise ValueError("no") context = CORE.update( codecs={**CORE.codecs, "gzip": dataclasses.replace(GZIP_CODEC, canonical=refuses)} ) read, _ = resolve({"name": "gzip", "configuration": {"level": 1}}, CodecDefinition, context) with pytest.raises(ValueError, match="no") as raised: canonical_of(read, ()) assert raised.value.__notes__ == ["raised by the canonical of 'gzip'"] def test_error_a_field_s_configuration_and_nested_fields_are_read_only() -> None: """A field's `json`, `configuration` and `nested` attributes are read-only at every level. So a field cannot be changed into a state where its key, `==` and `refines` disagree.""" whole = {**SHARD, "index_codecs": [LE, {"name": "crc32c"}]} shard, _ = resolve({"name": "sharding_indexed", "configuration": whole}, CodecDefinition, CORE) assert isinstance(shard, AcceptedField) with pytest.raises(TypeError): shard.configuration["index_location"] = "start" # pyright: ignore[reportIndexIssue] codecs = shard.configuration["codecs"] assert isinstance(codecs, tuple) inner = codecs[0] assert isinstance(inner, Mapping) with pytest.raises(TypeError): inner["name"] = "crc32c" # pyright: ignore[reportIndexIssue] with pytest.raises(TypeError): del shard.nested[("codecs", 0)] # pyright: ignore[reportIndexIssue] assert isinstance(shard.json, Mapping) with pytest.raises(TypeError): shard.json["name"] = "gzip" # pyright: ignore[reportIndexIssue] unclaimed, _ = resolve({"name": "acme.x", "configuration": {"a": [1]}}, CodecDefinition, CORE) assert isinstance(unclaimed, UnclaimedField) with pytest.raises(TypeError): unclaimed.configuration["a"] = 2 # pyright: ignore[reportIndexIssue] refused, _ = resolve( {"name": "sharding_indexed", "configuration": {**whole, "index_location": "x"}}, CodecDefinition, CORE, ) assert isinstance(refused, RefusedField) with pytest.raises(TypeError): del refused.nested[("codecs", 0)] # pyright: ignore[reportIndexIssue] def test_a_field_holding_fields_is_copied_and_pickled_whole() -> None: """A field pickles and deep-copies together with the fields it holds, and stays equal to itself, whether it is accepted, refused or unclaimed.""" whole = {**SHARD, "index_codecs": [LE, {"name": "crc32c"}]} fields = [ resolve({"name": "sharding_indexed", "configuration": whole}, CodecDefinition, CORE)[0], resolve({"name": "acme.x", "configuration": {"a": [1]}}, CodecDefinition, CORE)[0], resolve( {"name": "sharding_indexed", "configuration": {**whole, "index_location": "x"}}, CodecDefinition, CORE, )[0], ] for field in fields: for again in (pickle.loads(pickle.dumps(field)), copy.deepcopy(field)): assert again == field assert type(again) is type(field) assert again.nested == field.nested @pytest.mark.parametrize( "field", [ "crc32c", {"name": "gzip", "configuration": {"level": 1}}, {"name": "gzip", "configuration": {"level": 12}}, {"name": "gzip", "configuration": {"level": 1}, "must_understand": False}, {"name": "acme.x", "configuration": {"a": [1, {"b": None}]}}, {"name": "BAD NAME"}, {"name": "sharding_indexed", "configuration": {**SHARD, "index_location": "x"}}, {"configuration": {}}, ], ids=[ "bare", "read", "refused", "envelope", "unclaimed", "ill-named", "nested-refused", "nameless", ], ) def test_a_field_its_caller_refined_reads_as_resolve_reads_it(field: object) -> None: """`resolve_refined` is `resolve` for a caller that has already refined the field. It gives the same field and the same problems, at the same locations, with the same `input`.""" refined, problems = refine_json(field, ("codecs", 0)) assert problems == () expected = resolve(field, CodecDefinition, CORE, ("codecs", 0)) found = resolve_refined(field, refined, problems, CodecDefinition, CORE, ("codecs", 0)) assert found == expected assert [(p.loc, p.kind, p.input) for p in found[1]] == [ (p.loc, p.kind, p.input) for p in expected[1] ] def test_error_a_field_its_caller_found_not_to_be_json_is_refused_unread() -> None: """The problems `refine_json` found become the field's problems. The field is refused as not JSON, and its name still identifies the definition that claims it, as with `resolve`.""" field = {"name": "gzip", "configuration": {"level": {1, 2}}} refined, problems = refine_json(field, ("codecs", 0)) assert refined is None resolved, found = resolve_refined( field, refined, problems, CodecDefinition, CORE, ("codecs", 0) ) assert resolved == RefusedField( json=UNSET, name="gzip", read_as=CodecDefinition, definition=GZIP_CODEC ) assert [(p.loc, p.kind) for p in found] == [ (("codecs", 0, "configuration", "level"), "invalid_type") ] assert (resolved, found) == resolve(field, CodecDefinition, CORE, ("codecs", 0)) def test_error_a_refined_field_is_not_read_in_a_scope_of_the_other_format() -> None: with pytest.raises(TypeError, match="got a v2 scope"): resolve_refined("gzip", "gzip", (), CodecDefinition, CORE_V2) V2_IN_A_V3_SCOPE = ( "a v2 codec is a field of a Zarr v2 document, read in a scope of that format, got a v3 scope" ) V3_IN_A_V2_SCOPE = ( "a codec is a field of a Zarr v3 document, read in a scope of that format, got a v2 scope" ) @pytest.mark.parametrize( ("field", "kind", "context", "message"), [ ({"id": "gzip"}, ZarrV2CodecDefinition, CORE, V2_IN_A_V3_SCOPE), (5, ZarrV2CodecDefinition, CORE, V2_IN_A_V3_SCOPE), ({"name": "gzip"}, CodecDefinition, CORE_V2, V3_IN_A_V2_SCOPE), (5, CodecDefinition, CORE_V2, V3_IN_A_V2_SCOPE), ], ids=["v2-named", "v2-nameless", "v3-named", "v3-nameless"], ) def test_error_resolve_refuses_a_kind_of_the_other_format_whatever_the_field( field: object, kind: type[Definition[Any]], context: Context, message: str ) -> None: """`resolve` with a kind of one format and a scope of the other raises one `TypeError`, before it reads the field. So the message is the same for a field with a name that a scope could look up and for a value with no name.""" with pytest.raises(TypeError, match=f"^{re.escape(message)}$"): resolve(field, kind, context) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/test_every_definition.py000066400000000000000000001021021527014137600324400ustar00rootroot00000000000000"""Every extension the package defines, read against its definition. Each codec, data type, chunk grid and chunk key encoding is a definition. Its configuration is a TypedDict, and its rules are a function over that configuration. """ from __future__ import annotations import dataclasses import pickle from typing import Any import pytest from zarr_metadata import ValidationProblem from zarr_metadata._json import value_at from zarr_metadata.definition import ( AcceptedField, Definition, RefusedField, canonical_of, configuration_of, fill_value_problems, resolve, ) from zarr_metadata.definition._definition import canonicalize from zarr_metadata.v3.codec.gzip import GZIP_CODEC from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE, RawBytesConfiguration from zarr_metadata.v3.definition import ( CORE, CORE_AND_EXTENSIONS, ChunkGridDefinition, ChunkKeyEncodingDefinition, CodecDefinition, DataTypeDefinition, GroupExtraFieldDefinition, ZarrV3Context, ZarrV3Kind, ) KINDS: dict[str, type[ZarrV3Kind]] = { "codecs": CodecDefinition, "data_type": DataTypeDefinition, "chunk_grid": ChunkGridDefinition, "chunk_key_encoding": ChunkKeyEncodingDefinition, "group_extra_field": GroupExtraFieldDefinition, } # Fields that read without problems, per extension: at least one per definition in scope. EXAMPLES: dict[str, tuple[object, ...]] = { "codecs:blosc": ( { "name": "blosc", "configuration": { "cname": "zstd", "clevel": 5, "shuffle": "shuffle", "typesize": 4, "blocksize": 0, }, }, ), "codecs:bytes": ("bytes", {"name": "bytes", "configuration": {"endian": "little"}}), "codecs:cast_value": ( {"name": "cast_value", "configuration": {"data_type": "int8"}}, { "name": "cast_value", "configuration": {"data_type": "int8", "scalar_map": {"encode": [["NaN", 0]]}}, }, ), "codecs:crc32c": ("crc32c", {"name": "crc32c"}), "codecs:gzip": ({"name": "gzip", "configuration": {"level": 5}},), "codecs:scale_offset": ( "scale_offset", {"name": "scale_offset", "configuration": {"offset": 2, "scale": 0.5}}, ), "codecs:sharding_indexed": ( { "name": "sharding_indexed", "configuration": { "chunk_shape": [4], "codecs": ["bytes"], "index_codecs": [ {"name": "bytes", "configuration": {"endian": "little"}}, "crc32c", ], "index_location": "start", }, }, ), "codecs:transpose": ({"name": "transpose", "configuration": {"order": [2, 1, 0]}},), "codecs:zstd": ({"name": "zstd", "configuration": {"level": 3, "checksum": False}},), "chunk_grid:regular": ({"name": "regular", "configuration": {"chunk_shape": [4, 4]}},), "chunk_grid:rectilinear": ( { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[32, 32, 32]]}, }, {"name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[[32, 3]]]}}, ), "chunk_key_encoding:default": ( "default", {"name": "default", "configuration": {"separator": "."}}, ), "chunk_key_encoding:v2": ("v2", {"name": "v2", "configuration": {"separator": "/"}}), "data_type:numpy.datetime64": ( {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}}, ), "data_type:numpy.timedelta64": ( {"name": "numpy.timedelta64", "configuration": {"unit": "ms", "scale_factor": 10}}, ), "data_type:bool": ("bool",), "data_type:int8": ("int8",), "data_type:int16": ("int16",), "data_type:int32": ("int32",), "data_type:int64": ("int64",), "data_type:uint8": ("uint8",), "data_type:uint16": ("uint16",), "data_type:uint32": ("uint32",), "data_type:uint64": ("uint64",), "data_type:float16": ("float16",), "data_type:float32": ("float32",), "data_type:float64": ("float64",), "data_type:complex64": ("complex64",), "data_type:complex128": ("complex128",), "data_type:bytes": ("bytes",), "data_type:struct": ( { "name": "struct", "configuration": { "fields": [ {"name": "a", "data_type": "uint8"}, { "name": "b", "data_type": { "name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}, }, }, ] }, }, ), "data_type:string": ("string",), "data_type:r*": ("r16", "r008"), "group_extra_field:consolidated_metadata": ( {"consolidated_metadata": {"kind": "inline", "must_understand": False, "metadata": {}}}, { "consolidated_metadata": { "kind": "inline", "must_understand": False, "metadata": {"a": {"attributes": {"x": float("nan")}}}, } }, ), } CASES = [(key, field) for key, fields in EXAMPLES.items() for field in fields] def _read(key: str, field: object) -> tuple[type, list[tuple[tuple[str | int, ...], str]]]: """The type of the result of reading `field` in the scope (`AcceptedField`, `UnclaimedField` or `RefusedField`), and the location of each problem.""" resolved, problems = resolve(field, KINDS[key.split(":")[0]], CORE_AND_EXTENSIONS) return type(resolved), [(found.loc, found.kind) for found in problems] def _problems(key: str, field: object) -> list[tuple[tuple[str | int, ...], str]]: return _read(key, field)[1] def test_a_scope_s_definitions_pickle() -> None: # This lets a scope be sent to another process and rebuilt there. data_types = pickle.loads(pickle.dumps(tuple(CORE_AND_EXTENSIONS.data_types.values()))) context = ZarrV3Context(data_types=data_types) resolved, _ = resolve("complex64", DataTypeDefinition, context) assert [(p.loc, p.kind) for p in fill_value_problems(resolved, [1, "x"])] == [ ((1,), "invalid_value") ] def _every(context: ZarrV3Context) -> set[Definition[Any]]: """Every definition in `context`, from each of its slots.""" return { definition for kind in ZarrV3Context.kinds.values() for definition in context.definitions_of(kind).values() } def test_every_definition_in_scope_has_an_example() -> None: names = {definition.name for definition in _every(CORE_AND_EXTENSIONS)} assert names == {key.split(":")[1] for key in EXAMPLES} assert _every(CORE) < _every(CORE_AND_EXTENSIONS) @pytest.mark.parametrize( ("key", "field"), CASES, ids=[f"{k}:{i}" for i, (k, _) in enumerate(CASES)] ) def test_every_example_reads_and_its_simplest_spelling_is_stable(key: str, field: object) -> None: # The field is an AcceptedField with no problems. Its simplest spelling # reads the same way, and is its own simplest spelling. assert _read(key, field) == (AcceptedField, []) kind = KINDS[key.split(":")[0]] simplest, problems = canonicalize(field, kind, CORE_AND_EXTENSIONS) assert problems == () assert simplest is not None assert canonicalize(simplest, kind, CORE_AND_EXTENSIONS) == (simplest, ()) # A field that was already read gives the same spelling as its JSON, without reading it again. assert canonical_of(*resolve(field, kind, CORE_AND_EXTENSIONS)) == simplest @pytest.mark.parametrize( ("field", "simplest"), [ # `noshuffle` ignores `typesize`, so the simplest spelling drops it. ( { "name": "blosc", "configuration": { "cname": "lz4", "clevel": 1, "shuffle": "noshuffle", "typesize": 4, "blocksize": 0, }, }, { "name": "blosc", "configuration": { "cname": "lz4", "clevel": 1, "shuffle": "noshuffle", "blocksize": 0, }, }, ), # With nothing configured, the simplest spelling is an object with the # name alone, which every reader accepts, and no `must_understand: true`. ({"name": "crc32c", "configuration": {}, "must_understand": True}, {"name": "crc32c"}), # But a data type with nothing configured is written as its bare name, # as core data types are written. ({"name": "int8", "configuration": {}, "must_understand": True}, "int8"), # Each nested field is written in its own simplest spelling. ( { "name": "sharding_indexed", "configuration": { "chunk_shape": [2], "codecs": [{"name": "bytes"}], "index_codecs": [ {"name": "bytes", "configuration": {"endian": "little"}}, {"name": "crc32c"}, ], }, }, { "name": "sharding_indexed", "configuration": { "chunk_shape": (2,), "codecs": ({"name": "bytes"},), "index_codecs": ( {"name": "bytes", "configuration": {"endian": "little"}}, {"name": "crc32c"}, ), }, }, ), ( { "name": "cast_value", "configuration": {"data_type": {"name": "int8", "configuration": {}}}, }, {"name": "cast_value", "configuration": {"data_type": "int8"}}, ), # A dimension's runs of equal sizes are run-length encoded. ( { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[32, 32, 32, 16], 8]}, }, { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": (((32, 3), 16), 8)}, }, ), # A member set to its spec default is left out: a key encoding's # separator, a shard's index location, and zstd's checksum. ({"name": "default", "configuration": {"separator": "/"}}, {"name": "default"}), ({"name": "v2", "configuration": {"separator": "."}}, {"name": "v2"}), ( { "name": "sharding_indexed", "configuration": { "chunk_shape": [2], "codecs": ["bytes"], "index_codecs": [ {"name": "bytes", "configuration": {"endian": "little"}}, "crc32c", ], "index_location": "end", }, }, { "name": "sharding_indexed", "configuration": { "chunk_shape": (2,), "codecs": ({"name": "bytes"},), "index_codecs": ( {"name": "bytes", "configuration": {"endian": "little"}}, {"name": "crc32c"}, ), }, }, ), ( {"name": "zstd", "configuration": {"level": 3, "checksum": False}}, {"name": "zstd", "configuration": {"level": 3}}, ), ], ids=[ "blosc-noshuffle", "name-alone", "data-type-bare-name", "sharding-nested", "cast-value-target", "rectilinear-rle", "default-separator", "v2-separator", "sharding-index-location", "zstd-checksum", ], ) def test_the_simplest_spelling(field: dict[str, Any], simplest: object) -> None: kinds = { "rectilinear": ChunkGridDefinition, "int8": DataTypeDefinition, "default": ChunkKeyEncodingDefinition, "v2": ChunkKeyEncodingDefinition, } kind = kinds.get(field["name"], CodecDefinition) assert canonicalize(field, kind, CORE_AND_EXTENSIONS) == (simplest, ()) assert canonical_of(*resolve(field, kind, CORE_AND_EXTENSIONS)) == simplest @pytest.mark.parametrize( ("field", "bits", "simplest"), [ ("r16", 16, "r16"), ({"name": "r8"}, 8, "r8"), ({"name": "r24", "configuration": {}}, 24, "r24"), # The size is a number, and its simplest spelling is in decimal. ("r008", 8, "r8"), ], ids=["bare", "object", "empty-configuration", "leading-zeros"], ) def test_raw_bits_read_as_r_star_with_the_size_their_name_carries( field: object, bits: int, simplest: str ) -> None: resolved, problems = resolve(field, DataTypeDefinition, CORE_AND_EXTENSIONS) assert problems == () assert isinstance(resolved, AcceptedField) assert resolved.definition is RAW_BYTES_DATA_TYPE assert resolved.json == field assert configuration_of(resolved, RAW_BYTES_DATA_TYPE) == {"bits": bits} assert canonicalize(field, DataTypeDefinition, CORE_AND_EXTENSIONS) == (simplest, ()) def test_a_reader_reads_raw_bits_its_own_way_by_defining_r_star() -> None: # This definition accepts any size. Filed under `r*` in place of the # package's own, it reads every raw-bits name. Filed back, the package's # definition reads them again. mine = DataTypeDefinition(name="r*", configuration=RawBytesConfiguration) context = CORE_AND_EXTENSIONS.update(data_types={**CORE_AND_EXTENSIONS.data_types, "r*": mine}) resolved, problems = resolve("r12", DataTypeDefinition, context) assert problems == () assert isinstance(resolved, AcceptedField) assert resolved.definition is mine # The package's own definition does not accept 12 bits, and refuses them. again, _ = resolve( "r12", DataTypeDefinition, context.update(data_types={**context.data_types, "r*": RAW_BYTES_DATA_TYPE}), ) assert isinstance(again, RefusedField) assert again.definition is RAW_BYTES_DATA_TYPE @pytest.mark.parametrize( ("field", "at"), [("r*", ()), ({"name": "r*", "configuration": {"bits": 16}}, ("name",))], ids=["bare", "object"], ) def test_error_r_star_is_notation_and_no_name(field: object, at: tuple[str, ...]) -> None: # `r*` is how the specification's table writes raw bits, not a name a # document can use: `*` is not allowed in an extension name. resolved, problems = resolve(field, DataTypeDefinition, CORE_AND_EXTENSIONS) assert type(resolved) is RefusedField assert [(p.loc, p.kind) for p in problems] == [(at, "invalid_value")] @pytest.mark.parametrize( ("field", "kind", "simplest"), [ ({"name": "zfpy"}, CodecDefinition, {"name": "zfpy"}), # Only its own definition can decide its simplest form, so its # configuration is kept as written. Its envelope follows its kind. ( {"name": "zfpy", "configuration": {"level": [1]}}, CodecDefinition, {"name": "zfpy", "configuration": {"level": (1,)}}, ), ("zfpy", CodecDefinition, {"name": "zfpy"}), ( {"name": "zfpy", "configuration": {}, "must_understand": True}, CodecDefinition, {"name": "zfpy"}, ), ({"name": "acme.decimal"}, DataTypeDefinition, "acme.decimal"), ], ids=["object", "configured", "bare-codec", "spelled-out", "data-type"], ) def test_an_unclaimed_field_keeps_its_configuration_in_its_kind_s_envelope( field: object, kind: type[Definition[Any]], simplest: object ) -> None: assert canonicalize(field, kind, CORE) == (simplest, ()) def _shard(codecs: list[object], index_codecs: list[object]) -> dict[str, object]: configuration = {"chunk_shape": [2], "codecs": codecs, "index_codecs": index_codecs} return {"name": "sharding_indexed", "configuration": configuration} @pytest.mark.parametrize( ("field", "kind", "found"), [ ( {"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, [(("configuration", "level"), "invalid_value")], ), ( {"name": "bytes", "configuration": {"endain": "big"}}, CodecDefinition, [(("configuration", "endain"), "unknown_key")], ), ( { "name": "struct", "configuration": {"fields": [{"name": "a", "data_type": "int8", "x": 1}]}, }, DataTypeDefinition, [(("configuration", "fields", 0, "x"), "unknown_key")], ), ( {"name": "crc32c", "must_understand": False}, CodecDefinition, [(("must_understand",), "invalid_value")], ), ( {"name": "zfpy", "configuration": {}, "extra": 1}, CodecDefinition, [(("extra",), "unknown_key")], ), (None, CodecDefinition, [((), "invalid_type")]), ( _shard(["bytes", {"name": "gzip", "configuration": {"level": 12}}], ["bytes"]), CodecDefinition, [(("configuration", "codecs", 1, "configuration", "level"), "invalid_value")], ), ( _shard(["bytes"], ["bytes", {"name": "gzip", "configuration": {"level": 1}}]), CodecDefinition, [(("configuration", "index_codecs", 1), "invalid_value")], ), ], ids=[ "refused-value", "unknown-key", "unknown-key-nested", "must-understand-false", "stray", "null", "holding-a-refused-field", "a-codec-of-dynamic-size-where-one-of-static-size-goes", ], ) def test_error_a_field_with_a_problem_has_no_simplest_spelling( field: dict[str, Any] | None, kind: type[Definition[Any]], found: list[object] ) -> None: # What the author wrote is kept. A simpler spelling would drop the # unknown key, the stray member or the `must_understand`, or would # write a value that is not valid. A field that was already read, # together with its problems, has no simplest spelling either. simplest, problems = canonicalize(field, kind, CORE_AND_EXTENSIONS) assert simplest is None assert [(problem.loc, problem.kind) for problem in problems] == found assert canonical_of(*resolve(field, kind, CORE_AND_EXTENSIONS)) is None def test_error_a_canonical_that_does_not_hold_is_the_definitions_fault() -> None: # The output of `canonical` is checked again. A level the rules refuse # is a bug in the definition, not in the field it was asked about. lying = dataclasses.replace( GZIP_CODEC, name="acme.lying", canonical=lambda configuration: {"level": -7} ) with pytest.raises(ValueError, match="'acme.lying': its canonical gave"): canonicalize( {"name": "acme.lying", "configuration": {"level": 1}}, CodecDefinition, CORE_AND_EXTENSIONS.update(codecs=[*CORE_AND_EXTENSIONS.codecs.values(), lying]), ) def _one(key: str, configuration: object) -> list[tuple[tuple[str | int, ...], str]]: return _problems(key, {"name": key.split(":")[1], "configuration": configuration}) BLOSC = {"cname": "lz4", "clevel": 5, "shuffle": "shuffle", "typesize": 4, "blocksize": 0} def test_error_blosc_clevel_is_out_of_range() -> None: assert _one("codecs:blosc", {**BLOSC, "clevel": 10}) == [ (("configuration", "clevel"), "invalid_value") ] def test_error_blosc_blocksize_is_negative() -> None: assert _one("codecs:blosc", {**BLOSC, "blocksize": -1}) == [ (("configuration", "blocksize"), "invalid_value") ] def test_error_blosc_typesize_is_missing_while_shuffling() -> None: configuration = {key: value for key, value in BLOSC.items() if key != "typesize"} assert _one("codecs:blosc", configuration) == [(("configuration", "typesize"), "missing_key")] def test_a_rule_is_asked_only_of_a_configuration_within_its_bounds() -> None: # This works like pydantic's after-validators and zod's refinements. A # rule relies on the bounds its type declares, so a value outside them # is the only problem reported until it is fixed. shuffled = {key: value for key, value in BLOSC.items() if key != "typesize"} assert _one("codecs:blosc", {**shuffled, "clevel": 12}) == [ (("configuration", "clevel"), "invalid_value") ] assert _one("codecs:blosc", shuffled) == [(("configuration", "typesize"), "missing_key")] def test_error_blosc_typesize_is_not_positive() -> None: assert _one("codecs:blosc", {**BLOSC, "typesize": 0}) == [ (("configuration", "typesize"), "invalid_value") ] def test_error_zstd_level_is_out_of_range() -> None: assert _one("codecs:zstd", {"level": 23}) == [(("configuration", "level"), "invalid_value")] def test_error_transpose_order_is_not_a_permutation() -> None: assert _one("codecs:transpose", {"order": [0, 0]}) == [ (("configuration", "order"), "invalid_value") ] def test_error_scale_offset_scalar_is_null() -> None: assert _one("codecs:scale_offset", {"scale": None}) == [ (("configuration", "scale"), "invalid_value") ] def test_error_regular_chunk_length_is_zero() -> None: # This applies along a dimension of length 0 too: a grid's chunks must have a nonzero size. assert _one("chunk_grid:regular", {"chunk_shape": [4, 0]}) == [ (("configuration", "chunk_shape", 1), "invalid_value") ] def test_error_sharding_inner_chunk_extent_is_zero() -> None: configuration = {"chunk_shape": [0], "codecs": ["bytes"], "index_codecs": ["bytes"]} assert _one("codecs:sharding_indexed", configuration) == [ (("configuration", "chunk_shape", 0), "invalid_value") ] @pytest.mark.parametrize( ("kind", "field", "loc", "ctx"), [ ( CodecDefinition, {"name": "gzip", "configuration": {"level": 10}}, ("configuration", "level"), {"ge": 0, "le": 9}, ), ( CodecDefinition, {"name": "zstd", "configuration": {"level": -131073}}, ("configuration", "level"), {"ge": -131072, "le": 22}, ), ( CodecDefinition, {"name": "blosc", "configuration": {**BLOSC, "clevel": -1}}, ("configuration", "clevel"), {"ge": 0, "le": 9}, ), ( CodecDefinition, {"name": "blosc", "configuration": {**BLOSC, "blocksize": -1}}, ("configuration", "blocksize"), {"ge": 0}, ), ( CodecDefinition, { "name": "sharding_indexed", "configuration": { "chunk_shape": [2, 0], "codecs": ["bytes"], "index_codecs": ["bytes"], }, }, ("configuration", "chunk_shape", 1), {"ge": 1}, ), ( ChunkGridDefinition, {"name": "regular", "configuration": {"chunk_shape": [2, 0]}}, ("configuration", "chunk_shape", 1), {"ge": 1}, ), ( ChunkGridDefinition, {"name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [0]}}, ("configuration", "chunk_shapes", 0), {"ge": 1}, ), ( ChunkGridDefinition, { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[4, [2, 0]]]}, }, ("configuration", "chunk_shapes", 0, 1, 1), {"ge": 1}, ), ( DataTypeDefinition, {"name": "numpy.timedelta64", "configuration": {"unit": "s", "scale_factor": 2**31}}, ("configuration", "scale_factor"), {"ge": 1, "le": 2**31 - 1}, ), ], ids=[ "gzip-level", "zstd-level", "blosc-clevel", "blosc-blocksize", "sharding-inner-chunk-extent", "regular-chunk-extent", "rectilinear-extent", "rectilinear-run-count", "numpy-time-scale-factor", ], ) def test_a_bound_is_its_member_s_type_and_its_problem_holds_it( kind: type[Definition[Any]], field: dict[str, Any], loc: tuple[str | int, ...], ctx: dict[str, int], ) -> None: # The bound is declared on the TypedDict, as pydantic reads a bound, # not in a rule. The problem holds the bound and the value found. _, problems = resolve(field, kind, CORE_AND_EXTENSIONS) assert [(p.loc, p.kind, p.input, dict(p.ctx)) for p in problems] == [ (loc, "invalid_value", value_at(field, loc), ctx) ] STATIC_SIZE = ("bytes", "cast_value", "crc32c", "scale_offset", "transpose") DYNAMIC_SIZE = ("blosc", "gzip", "sharding_indexed", "zstd") def test_every_codec_says_whether_its_size_is_static() -> None: # This matches zarr-python's `is_fixed_size`. A compressor, and a shard # (whose empty inner chunks take no bytes), output a size that depends # on the values. sizes = {definition.name: definition.size for definition in CORE_AND_EXTENSIONS.codecs.values()} assert sizes == dict.fromkeys(STATIC_SIZE, "static") | dict.fromkeys(DYNAMIC_SIZE, "dynamic") @pytest.mark.parametrize( "index_codecs", [ ["bytes"], ["bytes", "crc32c"], [{"name": "bytes", "configuration": {"endian": "little"}}, {"name": "crc32c"}], # A name that nothing in scope claims is not checked, and its size is unknown. ["bytes", {"name": "acme.lz9"}], ], ids=["bytes", "bytes-crc32c", "spelled-out", "unclaimed"], ) def test_a_shard_s_index_codecs_are_of_static_size(index_codecs: list[object]) -> None: # The codecs for the inner chunks may be of either size. configuration = { "chunk_shape": [2], "codecs": ["bytes", {"name": "gzip", "configuration": {"level": 5}}], "index_codecs": index_codecs, } assert _one("codecs:sharding_indexed", configuration) == [] @pytest.mark.parametrize( "codec", [ {"name": "gzip", "configuration": {"level": 5}}, {"name": "zstd", "configuration": {"level": 3}}, { "name": "blosc", "configuration": { "cname": "lz4", "clevel": 5, "shuffle": "noshuffle", "blocksize": 0, }, }, ], ids=["gzip", "zstd", "blosc"], ) def test_error_a_shard_s_index_codec_is_of_dynamic_size(codec: dict[str, object]) -> None: # The spec: "Codecs that produce variable-sized encoded representation, # such as compression codecs, MUST NOT be used for index codecs." configuration = {"chunk_shape": [2], "codecs": ["bytes"], "index_codecs": ["bytes", codec]} assert _one("codecs:sharding_indexed", configuration) == [ (("configuration", "index_codecs", 1), "invalid_value") ] def test_error_a_codec_size_that_is_neither() -> None: with pytest.raises(TypeError, match="size is one of"): dataclasses.replace(GZIP_CODEC, size="fixed") def test_error_a_codec_inside_a_shard_is_judged_where_it_sits() -> None: configuration = { "chunk_shape": [2], "codecs": [{"name": "gzip", "configuration": {"level": 12}}], "index_codecs": ["bytes"], } assert _one("codecs:sharding_indexed", configuration) == [ (("configuration", "codecs", 0, "configuration", "level"), "invalid_value") ] @pytest.mark.parametrize( "configuration", [ {"data_type": "bool"}, {"data_type": "complex64"}, {"data_type": "string"}, {"data_type": "r16"}, # Only one problem: the wrap problem follows from the target data type. {"data_type": "bool", "out_of_range": "wrap"}, ], ) def test_error_a_cast_to_a_data_type_that_models_no_real_numbers(configuration: object) -> None: assert _one("codecs:cast_value", configuration) == [ (("configuration", "data_type"), "invalid_value") ] def test_error_a_cast_that_wraps_to_a_data_type_that_is_not_integral() -> None: assert _one("codecs:cast_value", {"data_type": "float32", "out_of_range": "wrap"}) == [ (("configuration", "out_of_range"), "invalid_value") ] @pytest.mark.parametrize( ("data_type", "scalar_map", "loc"), [ # The output of encoding and the input of decoding are values of the # target data type. ("uint8", {"encode": [[0, 300]]}, ("scalar_map", "encode", 0, 1)), ("uint8", {"decode": [[-1, 0]]}, ("scalar_map", "decode", 0, 0)), # A float's infinity is "Infinity", as the core encoding writes it. ("float32", {"decode": [["+Infinity", 0]]}, ("scalar_map", "decode", 0, 0)), ], ) def test_error_a_scalar_the_cast_maps_to_that_is_not_of_its_data_type( data_type: str, scalar_map: object, loc: tuple[str | int, ...] ) -> None: assert _one("codecs:cast_value", {"data_type": data_type, "scalar_map": scalar_map}) == [ (("configuration", *loc), "invalid_value") ] def test_error_a_cast_target_is_judged_where_it_sits() -> None: target = {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 0}} assert _one("codecs:cast_value", {"data_type": target}) == [ (("configuration", "data_type", "configuration", "scale_factor"), "invalid_value") ] def test_error_rectilinear_extent_is_not_positive() -> None: configuration = {"kind": "inline", "chunk_shapes": [0, [4, 0], [[4, 0]]]} assert _one("chunk_grid:rectilinear", configuration) == [ (("configuration", "chunk_shapes", 0), "invalid_value"), (("configuration", "chunk_shapes", 1, 1), "invalid_value"), (("configuration", "chunk_shapes", 2, 0, 1), "invalid_value"), ] def test_error_numpy_time_scale_factor_is_out_of_range() -> None: assert _one("data_type:numpy.timedelta64", {"unit": "s", "scale_factor": 2**31}) == [ (("configuration", "scale_factor"), "invalid_value") ] def test_error_struct_has_no_fields() -> None: assert _one("data_type:struct", {"fields": []}) == [ (("configuration", "fields"), "invalid_value") ] def test_error_struct_field_name_is_empty() -> None: fields = [{"name": "", "data_type": "uint8"}] assert _one("data_type:struct", {"fields": fields}) == [ (("configuration", "fields", 0, "name"), "invalid_value") ] def test_error_struct_field_name_is_repeated() -> None: fields = [{"name": "a", "data_type": "uint8"}, {"name": "a", "data_type": "int8"}] assert _one("data_type:struct", {"fields": fields}) == [ (("configuration", "fields", 1, "name"), "invalid_value") ] @pytest.mark.parametrize( ("field_type", "at"), [ ("string", ()), ("bytes", ()), # A struct holding a struct that holds a string is refused at the # location of the string. ( {"name": "struct", "configuration": {"fields": [{"name": "s", "data_type": "string"}]}}, ("configuration", "fields", 0, "data_type"), ), ], ) def test_error_a_struct_field_whose_values_vary_in_size( field_type: object, at: tuple[str | int, ...] ) -> None: fields = [{"name": "a", "data_type": "int8"}, {"name": "b", "data_type": field_type}] assert _one("data_type:struct", {"fields": fields}) == [ (("configuration", "fields", 1, "data_type", *at), "invalid_value") ] def test_error_struct_field_type_is_judged_where_it_sits() -> None: fields = [ { "name": "a", "data_type": { "name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 0}, }, } ] assert _one("data_type:struct", {"fields": fields}) == [ ( ("configuration", "fields", 0, "data_type", "configuration", "scale_factor"), "invalid_value", ) ] @pytest.mark.parametrize("name", ["r12", "r0"]) def test_error_a_raw_bytes_width_is_not_a_positive_multiple_of_8(name: str) -> None: # Raw bits of a size the spec does not allow are reported, not left # unchecked. The document wrote a name, so the problem is located at # the field. assert _problems("data_type:r*", name) == [((), "invalid_value")] def test_error_a_configuration_written_beside_a_raw_bits_name() -> None: # The name carries the configuration, so a configuration written next # to it may hold nothing: each member is a key that nothing declares. assert _read("data_type:r*", {"name": "r16", "configuration": {"bits": 16}}) == ( AcceptedField, [(("configuration", "bits"), "unknown_key")], ) def test_error_a_chunk_key_separator_is_not_one_the_encoding_takes() -> None: assert _one("chunk_key_encoding:v2", {"separator": "-"}) == [ (("configuration", "separator"), "invalid_value") ] def test_a_rule_is_a_function_over_the_typeddict() -> None: # The rules are plain functions over JSON. A caller with one # configuration can call them without a scope or a field. from zarr_metadata.v3.codec.blosc import BLOSC_CODEC _, problems = BLOSC_CODEC.read_configuration({**BLOSC, "clevel": 10}) assert problems == ( ValidationProblem(("clevel",), "expected an integer in [0, 9], got 10", "invalid_value"), ) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/test_fill_values.py000066400000000000000000000503161527014137600314140ustar00rootroot00000000000000"""Fill values, checked against their data type. Each data type's definition declares the JSON shape of its fill value and the rules for a value of that shape. `fill_value_problems` checks a fill value against a data type field that a scope read. The v3 array validators check a document's `fill_value` against its `data_type`. """ from __future__ import annotations import dataclasses import json import math from typing import TYPE_CHECKING, Any, cast, get_args import pytest from hypothesis import given from hypothesis import strategies as st from typing_extensions import TypedDict from zarr_metadata import ( JSONValue, ValidationProblem, ) from zarr_metadata._json import JSON_DEPTH, value_at from zarr_metadata.definition import ( AcceptedField, EmptyConfiguration, Nested, canonical_fill_value, fill_value_problems, resolve, ) from zarr_metadata.model import ( UNSET, ZarrV3ArrayMetadata, read_array_metadata_v3, read_consolidated_metadata_v3, ) from zarr_metadata.v3.data_type._float import FloatWidth, complex_fill_value_rules, float_bits from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS, DataTypeDefinition, ) if TYPE_CHECKING: from collections.abc import Iterator DATETIME: JSONValue = { "name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}, } TIMEDELTA: JSONValue = { "name": "numpy.timedelta64", "configuration": {"unit": "ms", "scale_factor": 1}, } STRUCT: JSONValue = { "name": "struct", "configuration": { "fields": [ {"name": "a", "data_type": "int8"}, {"name": "b", "data_type": "float32"}, ] }, } class AcmePointFillValue(TypedDict, closed=True): x: int def acme_point_fill_value_rules( configuration: EmptyConfiguration, nested: Nested, value: AcmePointFillValue ) -> Iterator[ValidationProblem]: if value["x"] > 10: yield ValidationProblem(("x",), "expected x <= 10", "invalid_value") ACME_POINT = DataTypeDefinition( name="acme.point", configuration=EmptyConfiguration, fill_value=AcmePointFillValue, fill_value_rules=acme_point_fill_value_rules, ) def _problems(data_type: JSONValue, value: object) -> list[tuple[tuple[str | int, ...], str]]: resolved, found = resolve(data_type, DataTypeDefinition, CORE_AND_EXTENSIONS) assert found == () return [(problem.loc, problem.kind) for problem in fill_value_problems(resolved, value)] @pytest.mark.parametrize( ("data_type", "fill_value"), [ ("bool", False), ("int8", -128), ("int8", 127), ("uint8", 255), ("int64", -(2**63)), ("uint64", 2**64 - 1), ("float16", 1.5), # A number may have any value: a reader rounds it to the type. ("float16", 1e10), ("float32", 0), ("float32", "NaN"), ("float32", "-Infinity"), ("float16", "0x7e00"), ("float32", "0x7fc00000"), ("float64", "0x7ff8000000000000"), ("complex64", [1.5, "NaN"]), ("complex128", ["0x7ff8000000000000", 0]), ("r8", [255]), ("r16", [0, 1]), ("r008", [7]), ("bytes", [0, 255]), ("bytes", "AQI="), ("bytes", ""), ("string", "a"), (DATETIME, "NaT"), (DATETIME, -(2**63)), (TIMEDELTA, 2**63 - 1), (STRUCT, {"a": 1, "b": "NaN"}), # If nothing in scope claims a field's type, its fill value is not checked. ( { "name": "struct", "configuration": {"fields": [{"name": "a", "data_type": "acme.decimal"}]}, }, {"a": "anything"}, ), # The same holds for a data type that nothing in scope claims, or one that is not read. ("acme.decimal", "anything"), ( {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 0}}, "anything", ), ], ) def test_every_data_type_takes_its_fill_values(data_type: JSONValue, fill_value: object) -> None: resolved, _ = resolve(data_type, DataTypeDefinition, CORE_AND_EXTENSIONS) assert fill_value_problems(resolved, fill_value) == () @pytest.mark.parametrize( ("data_type", "fill_value"), [ ("bool", 0), ("int8", True), ("int8", 1.0), ("int8", "5"), ("float32", None), ("string", 1), ("complex64", [1]), ("r16", "AQI="), (DATETIME, 1.5), (STRUCT, [1, 2]), ], ) def test_error_a_fill_value_of_the_wrong_json_type( data_type: JSONValue, fill_value: object ) -> None: assert _problems(data_type, fill_value) == [((), "invalid_type")] def test_error_a_fill_value_that_is_not_json() -> None: # A float's NaN is written as the string "NaN"; the number NaN is not JSON. assert _problems("float32", math.nan) == [((), "invalid_value")] @pytest.mark.parametrize( ("data_type", "fill_value"), [ ("int8", 128), ("int8", -129), ("uint8", -1), ("uint64", 2**64), ("int64", -(2**63) - 1), ], ) def test_error_an_integer_fill_value_out_of_its_type_s_range( data_type: str, fill_value: int ) -> None: assert _problems(data_type, fill_value) == [((), "invalid_value")] @pytest.mark.parametrize( ("data_type", "fill_value", "loc"), [ # Not one of the named values: the spec allows only its exact spelling. ("float32", "nan", ()), # A hex string of a different type's width. ("float32", "0x7e00", ()), ("float16", "0x7fc00000", ()), ("complex64", [1, "0x7ff8000000000000"], (1,)), ], ) def test_error_a_float_string_that_is_not_a_named_value_or_a_hex_string_of_its_width( data_type: str, fill_value: object, loc: tuple[int, ...] ) -> None: assert _problems(data_type, fill_value) == [(loc, "invalid_value")] @pytest.mark.parametrize(("data_type", "fill_value"), [("r16", [1]), ("r8", [1, 2])]) def test_error_a_raw_bits_fill_value_of_another_size(data_type: str, fill_value: object) -> None: # The fill value has one byte value for each 8 bits of the size. assert _problems(data_type, fill_value) == [((), "invalid_value")] @pytest.mark.parametrize( ("data_type", "fill_value", "loc"), [("r16", [1, 256], (1,)), ("bytes", [-1], (0,))] ) def test_error_a_byte_value_out_of_range( data_type: str, fill_value: object, loc: tuple[int, ...] ) -> None: assert _problems(data_type, fill_value) == [(loc, "invalid_value")] @pytest.mark.parametrize( ("data_type", "fill_value", "loc", "ctx"), [ ("int8", 128, (), {"ge": -128, "le": 127}), ("uint16", -1, (), {"ge": 0, "le": 2**16 - 1}), ("int64", 2**63, (), {"ge": -(2**63), "le": 2**63 - 1}), ("uint64", 2**64, (), {"ge": 0, "le": 2**64 - 1}), ("r16", [1, 256], (1,), {"ge": 0, "le": 255}), ("bytes", [-1], (0,), {"ge": 0, "le": 255}), (DATETIME, 2**63, (), {"ge": -(2**63), "le": 2**63 - 1}), ], ids=["int8", "uint16", "int64", "uint64", "raw-bits-byte", "bytes-byte", "numpy-time"], ) def test_a_fill_value_s_range_is_its_type_s( data_type: JSONValue, fill_value: object, loc: tuple[int, ...], ctx: dict[str, int] ) -> None: # `Int8FillValue` is `Annotated[int, Interval(ge=-128, le=127)]`. The # problem holds the range and the value that was found. resolved, _ = resolve(data_type, DataTypeDefinition, CORE_AND_EXTENSIONS) problems = fill_value_problems(resolved, fill_value) assert [(p.loc, p.input, dict(p.ctx)) for p in problems] == [ (loc, value_at(fill_value, loc), ctx) ] @pytest.mark.parametrize("fill_value", ["!!", "AQI"]) def test_error_a_bytes_fill_value_that_is_not_base64(fill_value: str) -> None: assert _problems("bytes", fill_value) == [((), "invalid_value")] @pytest.mark.parametrize( ("data_type", "fill_value"), [(DATETIME, 2**63), (TIMEDELTA, -(2**63) - 1)] ) def test_error_a_time_fill_value_outside_a_signed_64_bit_integer( data_type: JSONValue, fill_value: int ) -> None: assert _problems(data_type, fill_value) == [((), "invalid_value")] def test_error_a_struct_fill_value_missing_a_field() -> None: assert _problems(STRUCT, {"a": 1}) == [(("b",), "missing_key")] def test_error_a_struct_fill_value_for_no_field() -> None: assert _problems(STRUCT, {"a": 1, "b": 0, "c": 2}) == [(("c",), "unknown_key")] def test_error_a_struct_field_s_fill_value_its_own_type_refuses() -> None: # Each value is checked against the field's type, as the scope read it, # and the problem is located at the field's name. assert _problems(STRUCT, {"a": 300, "b": "x"}) == [ (("a",), "invalid_value"), (("b",), "invalid_value"), ] def test_error_an_array_document_s_fill_value_its_data_type_refuses() -> None: # Checked against the data type the document names. In consolidated # metadata, the problem is located where the array sits. document = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"fill_value": 300} assert [(p.loc, p.kind) for p in read_array_metadata_v3(document).problems] == [ (("fill_value",), "invalid_value") ] member = {"kind": "inline", "must_understand": False, "metadata": {"a": document}} assert [(p.loc, p.kind) for p in read_consolidated_metadata_v3(member).problems] == [ (("metadata", "a", "fill_value"), "invalid_value") ] def test_error_a_key_the_fill_value_shape_does_not_declare_hides_no_rule() -> None: # The extra key is reported and hidden from the rules, which still check the other keys. context = CORE_AND_EXTENSIONS.update( data_types=[*CORE_AND_EXTENSIONS.data_types.values(), ACME_POINT] ) resolved, _ = resolve("acme.point", DataTypeDefinition, context) found = fill_value_problems(resolved, {"x": 99, "extra": 1}) assert [(problem.loc, problem.kind) for problem in found] == [ (("extra",), "unknown_key"), (("x",), "invalid_value"), ] def test_a_fill_value_nested_as_deep_as_a_reader_walks_is_read() -> None: def deep(levels: int) -> dict[str, object]: value: dict[str, object] = {} for _ in range(levels): value = {"x": value} return value document = dict(ZarrV3ArrayMetadata.create_default().to_json()) | { "data_type": STRUCT, "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], # `c` is two levels down, and holds the deepest value the depth limit allows. "fill_value": {"a": 1, "b": 0.5, "c": deep(JSON_DEPTH - 3)}, } assert [(p.loc, p.kind) for p in read_array_metadata_v3(document).problems] == [ (("fill_value", "c"), "unknown_key") ] def test_a_struct_read_without_its_field_types_leaves_its_fields_unjudged() -> None: # A reading built by hand, with no reading of the field types. configuration = {"fields": ({"name": "a", "data_type": "int8"},)} struct = AcceptedField( json=STRUCT, name="struct", definition=STRUCT_DATA_TYPE, configuration=configuration ) assert fill_value_problems(struct, {"a": 300}) == () # --- canonical spellings --------------------------------------------------- def _alike(left: object, right: object) -> bool: """Whether two JSON values are written the same way. `==` is not enough, because it treats `-0.0` as equal to `0.0`.""" return json.dumps(left, sort_keys=True) == json.dumps(right, sort_keys=True) def _read(data_type: JSONValue) -> AcceptedField[DataTypeDefinition[Any]]: resolved, found = resolve(data_type, DataTypeDefinition, CORE_AND_EXTENSIONS) assert found == () assert isinstance(resolved, AcceptedField) return resolved @pytest.mark.parametrize( ("data_type", "spellings", "canonical"), [ ("float32", ["NaN", "0x7fc00000", "0x7FC00000"], "NaN"), # Any other NaN is written as its bits: the spec names only this one. ("float32", ["0xffc00000"], "0xffc00000"), ("float32", ["0x7fc00001", "0x7FC00001"], "0x7fc00001"), ("float32", ["Infinity", "0x7f800000", 3.5e38, 10**39], "Infinity"), ("float32", ["-Infinity", "0xff800000", -(10**39)], "-Infinity"), ("float32", [0, 0.0, "0x00000000", 1e-46], 0.0), # Negative zero is a distinct value. ("float32", [-0.0, "0x80000000"], -0.0), ("float32", [1, 1.0, "0x3f800000"], 1.0), # The number with the fewest digits that rounds to the value. ("float32", [0.1, 0.10000000149011612, "0x3dcccccd"], 0.1), # A number is read as a float64, as a JSON parser reads it, and then # rounded to the type, as zarrs, tensorstore and numpy round it. An # integer and a number with a fraction that read as the same float64 # are the same value. An integer whose float64 is halfway between two # float32 values rounds to the even one, as those libraries store it. ("float32", [2**53 + 2**29 + 1, 9007199791611905.0, 2**53], 9007199000000000.0), ("float32", [2**60 + 2**36 + 1, 2**60, 1.1529215e18], 1.1529215e18), # The shortest number that rounds to the value can be farther from it # than the nearest decimal: at a power of two, the range of numbers # that round to it extends further above it than below. ("float16", [0.015625, "0x2400"], 0.01563), ("float32", ["0x6b000000"], 1.5474251e26), ("float16", [65504, 65519, "0x7bff"], 65500.0), ("float16", [65520, "Infinity", "0x7c00"], "Infinity"), # Halfway rounds to the even value. ("float16", [2049, 2048], 2048.0), ("float16", [2051, 2052], 2052.0), ("float16", ["NaN", "0x7e00", "0x7E00"], "NaN"), ("float16", [-0.0, "0x8000"], -0.0), ("float64", [2**53 + 1, 2**53], float(2**53)), ("float64", [10**400, "Infinity", "0x7ff0000000000000"], "Infinity"), ("float64", [-(10**400), "-Infinity", "0xfff0000000000000"], "-Infinity"), ("float64", ["NaN", "0x7ff8000000000000"], "NaN"), ("complex64", [["NaN", -0.0], ["0x7fc00000", "0x80000000"]], ("NaN", -0.0)), ("complex128", [[0.1, 1], ["0x3fb999999999999a", 1.0]], (0.1, 1.0)), ("bytes", [[65], "QQ==", "QR=="], "QQ=="), ("bytes", [[], ""], ""), (DATETIME, ["NaT", -(2**63)], "NaT"), (TIMEDELTA, ["NaT", -(2**63)], "NaT"), (TIMEDELTA, [5], 5), (STRUCT, [{"a": 1, "b": "0x7fc00000"}, {"b": "NaN", "a": 1}], {"a": 1, "b": "NaN"}), # If nothing in scope claims a field's type, its fill value is kept as written. ( { "name": "struct", "configuration": {"fields": [{"name": "a", "data_type": "acme.decimal"}]}, }, [{"a": -0.0}], {"a": -0.0}, ), # A type with only one spelling for each value keeps the value as written. ("int8", [-128], -128), ("bool", [True], True), ("string", ["NaN"], "NaN"), ("r16", [[0, 1]], (0, 1)), ], ) def test_a_fill_value_s_canonical_spelling_is_its_value_s( data_type: JSONValue, spellings: list[object], canonical: JSONValue ) -> None: read = _read(data_type) for spelling in spellings: spelled = canonical_fill_value(read, spelling) assert _alike(spelled, canonical), (spelling, spelled) # The canonical spelling is a valid fill value and is its own canonical spelling. assert fill_value_problems(read, canonical) == () assert _alike(canonical_fill_value(read, canonical), canonical) def test_a_data_type_nothing_in_scope_claims_spells_a_fill_value_as_written() -> None: unclaimed, _ = resolve("acme.decimal", DataTypeDefinition, CORE_AND_EXTENSIONS) values: list[JSONValue] = [-0.0, True, 1, [1.0], None] for value in values: assert _alike(canonical_fill_value(unclaimed, value), value) WIDTHS: tuple[FloatWidth, ...] = get_args(FloatWidth) @given(st.data()) def test_a_float_s_canonical_spelling_spells_its_bits(data: st.DataObject) -> None: # For every value of each float type, NaNs included, the canonical # spelling is a valid fill value of the type, has the same bits, and is # its own canonical spelling. width = data.draw(st.sampled_from(WIDTHS)) # Sign, exponent and mantissa are drawn separately, so normal values, # infinities and NaNs all appear. Bits drawn as one integer are almost # all subnormals. fraction = {16: 10, 32: 23, 64: 52}[width] sign = data.draw(st.integers(0, 1)) exponent = data.draw(st.integers(0, 2 ** (width - 1 - fraction) - 1)) mantissa = data.draw(st.integers(0, 2**fraction - 1)) bits = (sign << (width - 1)) | (exponent << fraction) | mantissa read = _read(f"float{width}") spelled = canonical_fill_value(read, f"0x{bits:0{width // 4}x}") assert spelled is not UNSET assert fill_value_problems(read, spelled) == () assert float_bits(cast("float | str", spelled), width) == bits assert _alike(canonical_fill_value(read, spelled), spelled) def test_error_a_fill_value_with_a_problem_has_no_canonical_spelling() -> None: # The result is `UNSET`, because `None` is the JSON null, which is a valid # fill value for a data type the scope did not read. assert canonical_fill_value(_read("float32"), "0x7fc0") is UNSET assert canonical_fill_value(_read("int8"), 1.0) is UNSET assert canonical_fill_value(_read("int8"), math.nan) is UNSET unclaimed, _ = resolve("acme.decimal", DataTypeDefinition, CORE_AND_EXTENSIONS) assert canonical_fill_value(unclaimed, None) is None def test_error_a_canonical_spelling_that_raises_says_which_data_type_raised_it() -> None: def refuses(configuration: EmptyConfiguration, nested: Nested, value: object) -> JSONValue: raise ValueError("no") context = CORE_AND_EXTENSIONS.update( data_types=[ *CORE_AND_EXTENSIONS.data_types.values(), dataclasses.replace(ACME_POINT, fill_value_canonical=refuses), ] ) resolved, _ = resolve("acme.point", DataTypeDefinition, context) with pytest.raises(ValueError, match="no") as raised: canonical_fill_value(resolved, {"x": 1}) assert raised.value.__notes__ == ["raised by the fill_value_canonical of 'acme.point'"] def test_error_a_canonical_spelling_that_is_not_json_is_refused() -> None: def not_json(configuration: EmptyConfiguration, nested: Nested, value: object) -> JSONValue: return cast("JSONValue", {1, 2}) context = CORE_AND_EXTENSIONS.update( data_types=[ *CORE_AND_EXTENSIONS.data_types.values(), dataclasses.replace(ACME_POINT, fill_value_canonical=not_json), ] ) resolved, _ = resolve("acme.point", DataTypeDefinition, context) with pytest.raises(TypeError, match="'acme.point': its fill_value_canonical gives JSON"): canonical_fill_value(resolved, {"x": 1}) @pytest.mark.parametrize( ("name", "low", "high"), [ ("int8", -(2**7), 2**7 - 1), ("int16", -(2**15), 2**15 - 1), ("int32", -(2**31), 2**31 - 1), ("int64", -(2**63), 2**63 - 1), ("uint8", 0, 2**8 - 1), ("uint16", 0, 2**16 - 1), ("uint32", 0, 2**32 - 1), ("uint64", 0, 2**64 - 1), ], ) def test_an_integer_type_takes_exactly_its_range(name: str, low: int, high: int) -> None: read = _read(name) assert fill_value_problems(read, low) == () assert fill_value_problems(read, high) == () for outside in (low - 1, high + 1): (problem,) = fill_value_problems(read, outside) assert (problem.loc, problem.kind, dict(problem.ctx)) == ( (), "invalid_value", {"ge": low, "le": high}, ) def test_a_complex_fill_value_keeps_what_its_component_rules_found() -> None: """A complex fill value's problems are the problems found by its component rules. Each problem is moved under its component's index and keeps its `input` and `ctx`.""" def at_least_one( configuration: EmptyConfiguration, nested: Nested, part: object ) -> Iterator[ValidationProblem]: if part == 0: yield ValidationProblem( ("x",), "expected >= 1", "invalid_value", input=0, ctx={"ge": 1} ) rules = complex_fill_value_rules(at_least_one) (found,) = rules({}, {}, (1.0, 0)) assert (found.loc, found.message, found.kind) == ((1, "x"), "expected >= 1", "invalid_value") assert (found.input, dict(found.ctx)) == (0, {"ge": 1}) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/test_grid_shapes.py000066400000000000000000000156211527014137600313770ustar00rootroot00000000000000"""Chunk grids, checked against the shape of the array they chunk. Each grid's definition declares two things: what the spec disallows for a grid with its configuration over an array of a given shape, and, for a shape the grid fits, the lengths its chunks take along each axis. `chunk_grid_lengths` reports both for a grid field that a scope read. The v3 array validators check a document's `chunk_grid` against its `shape`. """ from __future__ import annotations from typing import TYPE_CHECKING import pytest from zarr_metadata.definition import EmptyConfiguration, resolve from zarr_metadata.model import ( ZarrV3ArrayMetadata, read_array_metadata_v3, ) from zarr_metadata.v3._definition import ( chunk_grid_lengths, ) from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS, ChunkGridDefinition, ) if TYPE_CHECKING: from zarr_metadata import JSONValue def _regular(*lengths: int) -> JSONValue: return {"name": "regular", "configuration": {"chunk_shape": list(lengths)}} def _rectilinear(*specs: JSONValue) -> JSONValue: return {"name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": list(specs)}} def _problems(grid: JSONValue, shape: tuple[int, ...]) -> list[tuple[tuple[str | int, ...], str]]: resolved, found = resolve(grid, ChunkGridDefinition, CORE_AND_EXTENSIONS) assert found == () lengths, problems = chunk_grid_lengths(resolved, shape) # A grid that does not fit the shape gives no lengths along any axis. assert lengths == (None,) * len(shape) return [(problem.loc, problem.kind) for problem in problems] @pytest.mark.parametrize( ("grid", "shape", "lengths"), [ (_regular(), (), ()), (_regular(4, 4), (10, 3), ({4}, {4})), # A chunk longer than its dimension, and a chunk along a dimension of # length 0. (_regular(8, 1), (3, 0), ({8}, {1})), # A bare integer repeats until it covers its dimension. (_rectilinear(4), (10,), ({4},)), (_rectilinear([4, 4, 2]), (10,), ({4, 2},)), # Chunks may extend past the end of the dimension, and the lengths # include the chunk past the edge. (_rectilinear([4, 4, 4]), (10,), ({4},)), (_rectilinear([4, 4, 2, 7]), (10,), ({4, 2, 7},)), (_rectilinear([[4, 2], 2]), (10,), ({4, 2},)), # A dimension of length 0 and no chunks. (_rectilinear([]), (0,), (set(),)), (_rectilinear(2, [3, 3]), (7, 6), ({2}, {3})), # A grid that nothing in scope claims, or one that is not read, is # not checked, and its lengths are unknown along each axis. ({"name": "acme.grid", "configuration": {"x": 1}}, (3, 2), (None, None)), (_regular(-1), (3,), (None,)), ], ) def test_every_chunk_grid_gives_the_lengths_of_its_chunks_over_a_shape_it_fits( grid: JSONValue, shape: tuple[int, ...], lengths: tuple[set[int] | None, ...] ) -> None: resolved, _ = resolve(grid, ChunkGridDefinition, CORE_AND_EXTENSIONS) expected = tuple(None if axis is None else frozenset(axis) for axis in lengths) assert chunk_grid_lengths(resolved, shape) == (expected, ()) @pytest.mark.parametrize( ("grid", "shape"), [(_regular(4), (10, 3)), (_regular(4, 4), ()), (_regular(), (1,))] ) def test_error_a_regular_grid_of_another_rank(grid: JSONValue, shape: tuple[int, ...]) -> None: assert _problems(grid, shape) == [(("configuration", "chunk_shape"), "invalid_value")] @pytest.mark.parametrize( ("grid", "shape"), [(_rectilinear(4), (10, 3)), (_rectilinear(4, 4), (1,))] ) def test_error_a_rectilinear_grid_of_another_rank(grid: JSONValue, shape: tuple[int, ...]) -> None: assert _problems(grid, shape) == [(("configuration", "chunk_shapes"), "invalid_value")] @pytest.mark.parametrize("spec", [[4, 4], [[4, 2]], []]) def test_error_rectilinear_chunks_that_do_not_cover_their_dimension(spec: JSONValue) -> None: assert _problems(_rectilinear(2, spec), (4, 10)) == [ (("configuration", "chunk_shapes", 1), "invalid_value") ] def test_a_grid_that_says_nothing_of_the_shape_fits_every_one_its_lengths_unknown() -> None: lenient = ChunkGridDefinition(name="acme.grid", configuration=EmptyConfiguration) context = CORE_AND_EXTENSIONS.update( chunk_grids=[*CORE_AND_EXTENSIONS.chunk_grids.values(), lenient] ) resolved, _ = resolve("acme.grid", ChunkGridDefinition, context) assert chunk_grid_lengths(resolved, (3, 4)) == ((None, None), ()) def test_error_a_grid_whose_chunk_lengths_have_another_number_of_axes() -> None: # This is a bug in the definition, not in the field, so it raises. flat = ChunkGridDefinition( name="acme.grid", configuration=EmptyConfiguration, chunk_lengths=lambda c, n, s: (frozenset({1}),), ) context = CORE_AND_EXTENSIONS.update( chunk_grids=[*CORE_AND_EXTENSIONS.chunk_grids.values(), flat] ) resolved, _ = resolve("acme.grid", ChunkGridDefinition, context) with pytest.raises(ValueError, match="'acme.grid': its chunk_lengths gave 1 axes"): chunk_grid_lengths(resolved, (3, 4)) def test_error_an_array_document_s_chunk_grid_that_does_not_fit_its_shape() -> None: # The problem is located in the document's grid. document = dict(ZarrV3ArrayMetadata.create_default(shape=(4, 4)).to_json()) document["chunk_grid"] = _regular(4) assert [(p.loc, p.kind) for p in read_array_metadata_v3(document).problems] == [ (("chunk_grid", "configuration", "chunk_shape"), "invalid_value") ] def test_error_an_array_document_s_shape_that_is_not_read_leaves_its_grid_unjudged() -> None: document = dict(ZarrV3ArrayMetadata.create_default(shape=(4, 4)).to_json()) document["chunk_grid"] = _regular(4) document["shape"] = (4, -1) assert [(p.loc, p.kind) for p in read_array_metadata_v3(document).problems] == [ (("shape",), "invalid_value") ] @pytest.mark.parametrize( ("chunk_lengths", "match"), [ (lambda configuration, nested, shape: (4, 4), "its chunk_lengths give a frozenset"), (lambda configuration, nested, shape: {}["axis"], "'axis'"), ], ) def test_error_a_grid_whose_chunk_lengths_give_something_else_or_raise( chunk_lengths: object, match: str ) -> None: # A bug in the definition raises, and a raised error names the definition. odd = ChunkGridDefinition( name="acme.grid", configuration=EmptyConfiguration, chunk_lengths=chunk_lengths, # pyright: ignore[reportArgumentType] ) resolved, _ = resolve( "acme.grid", ChunkGridDefinition, CORE_AND_EXTENSIONS.update(chunk_grids=[*CORE_AND_EXTENSIONS.chunk_grids.values(), odd]), ) with pytest.raises((TypeError, KeyError), match=match) as raised: chunk_grid_lengths(resolved, (3, 4), ("chunk_grid",)) if isinstance(raised.value, KeyError): assert raised.value.__notes__ == [ "raised by the chunk lengths of 'acme.grid', reading ('chunk_grid', 'configuration')" ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/test_hierarchy.py000066400000000000000000000241151527014137600310630ustar00rootroot00000000000000"""A Zarr v3 hierarchy: the names and paths of its nodes, and the tree they make, as the core specification constrains them. Names: https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L818-L837 Paths: https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L211-L229 The tree: https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L177-L181 """ from __future__ import annotations import time from typing import TYPE_CHECKING, Literal, cast import pytest from zarr_metadata.model import ( is_node_name_v3, is_node_path_v3, parse_node_name_v3, parse_node_path_v3, validate_node_name_v3, validate_node_path_v3, ) from zarr_metadata.problems import MetadataValidationError, ValidationProblem from zarr_metadata.v3._hierarchy import hierarchy_problems if TYPE_CHECKING: from collections.abc import Callable @pytest.mark.parametrize( ("value", "message"), [ # The root's name, and names of any code points except the reserved ones. ("", None), ("a", None), ("FOO", None), ("a.b", None), ("...a", None), ("_a", None), ("a__", None), ("zarr.json.bak", None), ("ñ", None), ("/", 'expected a node name, got "/", which holds "/"'), ("a/b", 'expected a node name, got "a/b", which holds "/"'), (".", 'expected a node name, got ".", which is periods alone'), ("..", 'expected a node name, got "..", which is periods alone'), ("__a", 'expected a node name, got "__a", which starts with the reserved "__"'), ("zarr.json", 'expected a node name, got "zarr.json", which is the reserved "zarr.json"'), # Every reason the string is not a name, reported in one problem. ( "__/", 'expected a node name, got "__/", which holds "/" and starts with the reserved "__"', ), ], ) def test_node_names(value: str, message: str | None) -> None: problems = validate_node_name_v3(value) assert [problem.message for problem in problems] == ([] if message is None else [message]) assert {(problem.loc, problem.kind, problem.input) for problem in problems} <= { ((), "invalid_value", value) } assert is_node_name_v3(value) is (message is None) if message is None: assert parse_node_name_v3(value) == value @pytest.mark.parametrize( ("value", "message"), [ # The root's path, and the paths below it. ("/", None), ("/a", None), ("/a/b", None), ("/a/.b/c..", None), ("", 'expected a node path, got "", which does not start with "/"'), ("a/b", 'expected a node path, got "a/b", which does not start with "/"'), ("/a/", 'expected a node path, got "/a/", which ends with "/"'), ( "//", 'expected a node path, got "//", which ends with "/" and holds an empty name between two "/"', ), ("/a//b", 'expected a node path, got "/a//b", which holds an empty name between two "/"'), ( "/a/../b", 'expected a node path, got "/a/../b", which holds "..", a name that is periods alone', ), ( "/__a", ( 'expected a node path, got "/__a", which holds "__a", a name that starts with the ' 'reserved "__"' ), ), ( "/a/zarr.json", ( 'expected a node path, got "/a/zarr.json", which holds "zarr.json", a name that is ' 'the reserved "zarr.json"' ), ), # Every reason the string is not a path, reported in one problem. The # message names the first name that is not a node name and counts the rest. ( "/./__a/", ( 'expected a node path, got "/./__a/", which ends with "/", holds ".", a name that is ' "periods alone and holds 1 more name that is not a node name" ), ), ], ) def test_node_paths(value: str, message: str | None) -> None: problems = validate_node_path_v3(value) assert [problem.message for problem in problems] == ([] if message is None else [message]) assert {(problem.loc, problem.kind, problem.input) for problem in problems} <= { ((), "invalid_value", value) } assert is_node_path_v3(value) is (message is None) if message is None: assert parse_node_path_v3(value) == value def test_error_parse_node_name_raises_every_reason_a_string_is_not_a_node_name() -> None: with pytest.raises(MetadataValidationError) as raised: parse_node_name_v3("__/") assert raised.value.problems == validate_node_name_v3("__/") assert len(raised.value.problems) == 1 assert 'holds "/"' in raised.value.problems[0].message assert 'starts with the reserved "__"' in raised.value.problems[0].message def test_error_parse_node_path_raises_every_reason_a_string_is_not_a_node_path() -> None: with pytest.raises(MetadataValidationError) as raised: parse_node_path_v3("/./__a/") assert raised.value.problems == validate_node_path_v3("/./__a/") assert len(raised.value.problems) == 1 assert 'ends with "/"' in raised.value.problems[0].message assert "1 more name" in raised.value.problems[0].message @pytest.mark.parametrize( ("validate", "parse", "what"), [ (validate_node_name_v3, parse_node_name_v3, "a node name"), (validate_node_path_v3, parse_node_path_v3, "a node path"), ], ids=["name", "path"], ) def test_error_a_node_name_or_path_is_a_string( validate: Callable[[object], tuple[ValidationProblem, ...]], parse: Callable[[object], str], what: str, ) -> None: problems = validate(3) assert problems == (ValidationProblem((), f"expected {what}, got 3", "invalid_type"),) assert problems[0].input == 3 with pytest.raises(MetadataValidationError) as raised: parse(b"a") assert [(problem.loc, problem.kind) for problem in raised.value.problems] == [ ((), "invalid_type") ] @pytest.mark.parametrize( "nodes", [ # No nodes, a root alone, and a tree of groups whose leaves are # arrays. The root may be an array, which is then the whole hierarchy. {}, {"/": "group"}, {"/": "array"}, {"/": "group", "/a": "array", "/g": "group", "/g/b": "array", "/g/h": "group"}, # A node of unknown type is treated as a group. {"/": "group", "/x": None, "/x/a": "array"}, ], ids=["empty", "root-group", "root-array", "tree", "unknown-type"], ) def test_a_hierarchy(nodes: dict[str, Literal["array", "group"] | None]) -> None: assert hierarchy_problems(nodes) == () def test_error_a_hierarchy_holds_its_nodes_at_node_paths() -> None: assert hierarchy_problems({"/": "group", "a": "array", "/b/": "array"}) == ( ValidationProblem( ("a",), 'expected a node path, got "a", which does not start with "/"', "invalid_value" ), ValidationProblem( ("/b/",), 'expected a node path, got "/b/", which ends with "/"', "invalid_value" ), ) @pytest.mark.parametrize( ("nodes", "path"), [ ({"/": "group", "/a": "array", "/a/b": "group"}, "/a/b"), # However many groups are missing between them, adding them would not help. ({"/": "group", "/a": "array", "/a/b/c": "array"}, "/a/b/c"), # The root is an array, so the hierarchy is only that array. ({"/": "array", "/a": "group"}, "/a"), ], ids=["child", "descendant", "below-the-root"], ) def test_error_no_node_is_below_an_array( nodes: dict[str, Literal["array", "group"] | None], path: str ) -> None: holder = next(ancestor for ancestor in ("/a", "/") if nodes.get(ancestor) == "array") message = f'expected a node below a group, got "{path}", below the array "{holder}"' assert hierarchy_problems(nodes) == (ValidationProblem((path,), message, "invalid_value"),) def test_error_a_hierarchy_holds_the_group_holding_each_node() -> None: # The nearest missing group above a node is reported once, with a count # of the missing groups above it, including the root. assert hierarchy_problems({"/a/b/c": "array", "/a/b/d": "array"}) == ( ValidationProblem( ("/a/b",), 'missing the group holding "/a/b/c", and 2 groups above it', "missing_key" ), ) assert hierarchy_problems({"/": "group", "/a/b": "array"}) == ( ValidationProblem(("/a",), 'missing the group holding "/a/b"', "missing_key"), ) def test_problems_stay_proportional_to_the_path() -> None: # One problem per value, however many names are wrong and however many # groups are missing, so a hostile path costs time in proportion to its size. path = "/" * 10_000 problems = validate_node_path_v3(path) assert len(problems) == 1 assert len(problems[0].message) < len(path) + 200 deep = "/a" * 5_000 found = hierarchy_problems({deep: "array"}) assert [(problem.loc, problem.kind) for problem in found] == [ ((deep.rpartition("/")[0],), "missing_key") ] assert found[0].message.endswith(", and 4999 groups above it") assert len(found[0].message) < len(deep) + 200 def test_a_hierarchy_is_walked_in_time_proportional_to_its_paths() -> None: # Each path is split once and walked name by name. A key of a million # characters, and thousands of keys under one long missing prefix, each # cost time in proportion to their size. Walking up by ancestor strings # would cost quadratic time. started = time.perf_counter() long = "/a" * 500_000 assert len(hierarchy_problems({long: "array"})) == 1 prefix = "/a" * 5_000 many = {f"{prefix}/{index}": "array" for index in range(2_000)} assert len(hierarchy_problems(cast("dict[str, Literal['array', 'group'] | None]", many))) == 1 # This work takes a few seconds on a laptop and tens of seconds on a slow # CI runner; walking up by ancestor strings would take minutes. assert time.perf_counter() - started < 60 zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/test_kinds.py000066400000000000000000000144141527014137600302160ustar00rootroot00000000000000"""Kinds of definition: the class a definition's kind is, and the scope of each format, which holds one slot per kind.""" from __future__ import annotations from dataclasses import dataclass from typing import Any import pytest from zarr_metadata.definition import ( AcceptedField, Definition, EmptyConfiguration, UnclaimedField, resolve, ) from zarr_metadata.definition._definition import as_kind, kind_of from zarr_metadata.definition._scope import kind_name from zarr_metadata.json_schema import field_json_schema_v3 from zarr_metadata.v2.definition import CORE_V2, ZarrV2Context, ZarrV2DataTypeDefinition from zarr_metadata.v3.definition import ( CORE, CodecDefinition, DataTypeDefinition, ZarrV3Context, ) @dataclass(frozen=True, kw_only=True, slots=True, repr=False) class MyCodec(CodecDefinition[Any]): """A codec definition with an extra member of its own. It is still a codec.""" note: str = "" @dataclass(frozen=True, kw_only=True, slots=True, repr=False) class NoKind(Definition[Any]): """A definition subclass that declares no kind.""" def test_a_subclass_of_a_kind_is_a_definition_of_that_kind() -> None: """A definition built as a subclass of `CodecDefinition` is filed, read and named as a codec: the kind is the nearest class in the MRO declared with `kind=True`, not the class of the definition itself.""" mine = MyCodec(name="mine", configuration=EmptyConfiguration, kind="bytes_bytes", size="static") assert kind_of(mine) is CodecDefinition context = ZarrV3Context(codecs=[mine]) assert context.claimant(CodecDefinition, "mine") is mine assert isinstance(resolve({"name": "mine"}, CodecDefinition, context)[0], AcceptedField) def test_a_kind_is_named_by_its_class_and_its_label() -> None: """`as_kind` accepts a kind with or without type arguments, and messages name a kind by its label.""" assert as_kind(CodecDefinition) is CodecDefinition assert as_kind(CodecDefinition[Any]) is CodecDefinition assert kind_name(CodecDefinition) == "codec" assert kind_name(ZarrV2DataTypeDefinition) == "v2 data type" def test_error_a_class_that_declares_no_kind_is_of_none() -> None: """A `Definition` subclass declared without `kind=True` has no kind: `kind_of` returns None, no slot of a scope takes a definition of that class, and `as_kind` rejects the class.""" none = NoKind(name="nokind", configuration=EmptyConfiguration) assert kind_of(none) is None with pytest.raises(TypeError, match="a definition of no kind"): ZarrV3Context(codecs=[none]) # pyright: ignore[reportArgumentType] with pytest.raises(TypeError, match="is not a kind of metadata"): as_kind(NoKind) def test_error_the_json_schema_writer_writes_v3_fields_only() -> None: """`field_json_schema_v3` writes only the v3 envelope. For a kind of another format, it raises a `TypeError` that names this limit instead of writing a wrong schema.""" with pytest.raises(TypeError, match="Zarr v3"): field_json_schema_v3(ZarrV2DataTypeDefinition, context=CORE_V2) # pyright: ignore[reportArgumentType] def test_a_v3_value_that_is_no_field_is_shown_in_the_message() -> None: """A nested value that is not a metadata field at all is reported with the value in the message, as it was before the kind's envelope checked it.""" _, problems = resolve( {"name": "sharding_indexed", "configuration": {"chunk_shape": [1], "codecs": [3]}}, CodecDefinition, CORE, ) assert any(p.message.endswith("got 3") for p in problems), [p.message for p in problems] def test_error_a_kind_declares_its_format() -> None: """A kind declares which Zarr format its fields belong to, `format=2` or `format=3`, in the class header next to `kind=True`. Declaring a kind without a format, or with any other format, raises `TypeError` at class creation.""" with pytest.raises(TypeError, match="format"): type("NoFormat", (Definition,), {}, kind=True) with pytest.raises(TypeError, match="format"): type("WrongFormat", (Definition,), {}, kind=True, format=4) with pytest.raises(TypeError, match="format"): type("FormatOnADefinition", (CodecDefinition,), {}, format=3) def test_a_scope_has_the_format_of_its_class() -> None: """A scope's `zarr_format` is the format of every kind its class holds: 3 for a `ZarrV3Context` and 2 for a `ZarrV2Context`, including one that files nothing.""" assert CORE.zarr_format == 3 assert CORE_V2.zarr_format == 2 assert ZarrV3Context().zarr_format == 3 assert ZarrV2Context().zarr_format == 2 def test_error_a_scope_files_one_format() -> None: """Definitions of two formats cannot share a scope: a slot refuses a definition of the other format's kind, in the constructor and in `update`, and `union` refuses scopes of two formats. Each raises `TypeError`.""" from zarr_metadata.v2.data_type.scalar import UINT_DATA_TYPE_V2 with pytest.raises(TypeError, match="v2 data type"): ZarrV3Context(data_types=[UINT_DATA_TYPE_V2]) # pyright: ignore[reportArgumentType] with pytest.raises(TypeError, match="v2 data type"): CORE.update(data_types=[*CORE.data_types.values(), UINT_DATA_TYPE_V2]) # pyright: ignore[reportArgumentType] with pytest.raises(TypeError, match="format"): CORE.union(CORE_V2) # pyright: ignore[reportArgumentType] def test_error_a_field_is_read_in_a_scope_of_its_format() -> None: """`resolve` raises `TypeError` for a scope whose format differs from the kind's. In an empty scope of its format, it reads a field as unclaimed.""" with pytest.raises(TypeError, match="format"): resolve(" None: """A kind must be a class. A list or a dict passed as a kind is rejected with the same error as any other non-kind, not with the `TypeError` that hashing it would raise.""" with pytest.raises(TypeError, match="is not a kind of metadata"): as_kind(["codec"]) with pytest.raises(TypeError, match="is not a kind of metadata"): as_kind({"kind": CodecDefinition}) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/test_pipelines.py000066400000000000000000000522531527014137600311010ustar00rootroot00000000000000"""Reading codec pipelines: each codec together with the chunk it receives. `read_pipeline` checks a pipeline's order, and checks each codec against the chunk it receives. The v3 array validators read a document's `codecs` as the pipeline that the chunks of its grid, with its data type, pass through. """ from __future__ import annotations from typing import TYPE_CHECKING, Any import pytest from typing_extensions import TypedDict from zarr_metadata.definition import ( EmptyConfiguration, Nested, ResolvedField, resolve, ) from zarr_metadata.model import ( ZarrV3ArrayMetadata, read_array_metadata_v3, ) from zarr_metadata.v3._pipeline import ( read_pipeline, ) from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS, Chunk, CodecDefinition, DataTypeDefinition, ZarrV3CodecField, ZarrV3DataTypeField, ) if TYPE_CHECKING: from zarr_metadata import JSONValue, ValidationProblem from zarr_metadata.v3.definition import Stage def _tiled(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) -> Chunk: """Each chunk twice as long along every axis, of the same data type.""" if chunk.lengths is None: return Chunk(None, chunk.data_type) doubled = tuple( None if axis is None else frozenset(2 * length for length in axis) for axis in chunk.lengths ) return Chunk(doubled, chunk.data_type) ACME_TILE = CodecDefinition( name="acme.tile", configuration=EmptyConfiguration, kind="array_array", size="static", transition=_tiled, ) """An array -> array codec that declares its output chunk.""" ACME_SHUFFLE = CodecDefinition( name="acme.shuffle", configuration=EmptyConfiguration, kind="array_array", size="static" ) """An array -> array codec that does not declare its output chunk.""" CONTEXT = CORE_AND_EXTENSIONS.update( codecs=[*CORE_AND_EXTENSIONS.codecs.values(), ACME_TILE, ACME_SHUFFLE] ) UINT8 = resolve("uint8", DataTypeDefinition, CONTEXT)[0] CHUNK = Chunk((frozenset({4}), frozenset({2})), UINT8) """Chunks of 4 by 2 `uint8` values.""" GZIP: JSONValue = {"name": "gzip", "configuration": {"level": 1}} LITTLE: JSONValue = {"name": "bytes", "configuration": {"endian": "little"}} def _transpose(*order: int) -> JSONValue: return {"name": "transpose", "configuration": {"order": list(order)}} def _cast(data_type: JSONValue, **members: JSONValue) -> JSONValue: return {"name": "cast_value", "configuration": {"data_type": data_type, **members}} def _scale(**members: JSONValue) -> JSONValue: return {"name": "scale_offset", "configuration": members} def _of(data_type: JSONValue) -> Chunk: """Chunks of 4 by 2 values of `data_type`.""" return Chunk(CHUNK.lengths, resolve(data_type, DataTypeDefinition, CONTEXT)[0]) def _struct(*field_types: JSONValue) -> JSONValue: fields: list[JSONValue] = [ {"name": f"f{index}", "data_type": dt} for index, dt in enumerate(field_types) ] return {"name": "struct", "configuration": {"fields": fields}} def _read( codecs: list[JSONValue], chunk: Chunk ) -> tuple[tuple[Stage, ...], tuple[ValidationProblem, ...]]: """The stages, and every problem: first each codec's own problems at its location, then the pipeline's.""" read = [ resolve(codec, CodecDefinition, CONTEXT, (index,)) for index, codec in enumerate(codecs) ] stages, found = read_pipeline([resolved for resolved, _ in read], chunk) return stages, (*(problem for _, own in read for problem in own), *found) def _problems( codecs: list[JSONValue], chunk: Chunk = CHUNK ) -> list[tuple[tuple[str | int, ...], str]]: return [(problem.loc, problem.kind) for problem in _read(codecs, chunk)[1]] def _lengths(*axes: set[int]) -> tuple[frozenset[int], ...]: return tuple(frozenset(axis) for axis in axes) @pytest.mark.parametrize( ("codecs", "chunk", "incoming"), [ (["bytes"], CHUNK, [CHUNK]), # Each array -> array codec outputs the chunk its transition gives. # transpose permutes the axes; two transposes restore them. ( [_transpose(1, 0), "bytes"], CHUNK, [CHUNK, Chunk(_lengths({2}, {4}), UINT8)], ), ( [_transpose(1, 0), _transpose(1, 0), "bytes"], CHUNK, [CHUNK, Chunk(_lengths({2}, {4}), UINT8), CHUNK], ), (["acme.tile", "bytes"], CHUNK, [CHUNK, Chunk(_lengths({8}, {4}), UINT8)]), # After the array -> bytes codec, each codec receives bytes. (["bytes", GZIP, "crc32c"], CHUNK, [CHUNK, None, None]), # A codec that does not declare its output gives the next codec a # chunk with unknown shape and data type. (["acme.shuffle", "bytes"], CHUNK, [CHUNK, Chunk()]), # So does a codec that nothing in scope claims, which might be of any # kind. A pipeline that holds one may have it as its array -> bytes codec. (["zfpy", "bytes"], CHUNK, [CHUNK, Chunk()]), (["zfpy"], CHUNK, [CHUNK]), (["bytes", "zfpy"], CHUNK, [CHUNK, None]), # A chunk with unknown properties stays unknown, and no codec rejects it. ([_transpose(1, 0, 2), "bytes"], Chunk(), [Chunk(), Chunk()]), # `bytes` requires an `endian` for values of several bytes, and accepts # any for single-byte values. The spec says nothing about wider raw bits. ([LITTLE], _of("float32"), [_of("float32")]), ([LITTLE], _of("uint8"), [_of("uint8")]), (["bytes"], _of(_struct("int8", "uint8")), [_of(_struct("int8", "uint8"))]), (["bytes"], _of("r16"), [_of("r16")]), # cast_value outputs its data type, and the next codec is checked # against it. scale_offset outputs the same chunk it receives. ([_cast("float32"), LITTLE], _of("int16"), [_of("int16"), _of("float32")]), ( [_cast("uint8", out_of_range="wrap"), "bytes"], _of("float32"), [_of("float32"), _of("uint8")], ), ( [_cast("float32"), _scale(offset=0.5, scale=2), _cast("int8"), "bytes"], _of("int8"), [_of("int8"), _of("float32"), _of("float32"), _of("int8")], ), ( [_cast("uint8", scalar_map={"encode": [["NaN", 0]], "decode": [[0, "NaN"]]}), "bytes"], _of("float32"), [_of("float32"), _of("uint8")], ), # A cast to a data type that nothing in scope claims outputs that # field, which says nothing about the values. ([_cast("acme.decimal"), "bytes"], _of("float32"), [_of("float32"), _of("acme.decimal")]), # After a codec that nothing in scope claims, a cast still declares the # data type it outputs. An arithmetic codec that receives an unknown # chunk is not rejected. ( ["zfpy", _cast("float32"), LITTLE], CHUNK, [CHUNK, Chunk(), Chunk(None, _of("float32").data_type)], ), ([_scale(offset=1), "vlen-utf8"], Chunk(), [Chunk(), Chunk()]), # Where the specs disagree or say nothing, a data type is not checked. # The numpy time types say they accept any codec of 64-bit integers, # and scale_offset does not say whether it accepts complex numbers. ( [_cast("int64"), LITTLE], _of({"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}}), [ _of( {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}} ), _of("int64"), ], ), ([_scale(offset=[1, 0]), LITTLE], _of("complex64"), [_of("complex64"), _of("complex64")]), ], ) def test_every_pipeline_hands_each_codec_the_chunk_the_one_before_hands_on( codecs: list[JSONValue], chunk: Chunk, incoming: list[Chunk | None] ) -> None: stages, problems = _read(codecs, chunk) assert problems == () assert [stage.incoming for stage in stages] == incoming @pytest.mark.parametrize( ("codecs", "index"), [ ([GZIP, "bytes"], 1), (["bytes", _transpose(0, 1)], 1), (["bytes", "crc32c", _transpose(0, 1)], 2), (["bytes", GZIP, "bytes"], 2), ], ) def test_error_a_codec_out_of_order(codecs: list[JSONValue], index: int) -> None: # The problem is located at the codec. assert _problems(codecs) == [((index,), "invalid_value")] def test_error_a_codec_out_of_order_whose_configuration_is_not_an_object() -> None: # Its name still identifies the codec. assert _problems([{"name": "gzip", "configuration": 5}, "bytes"]) == [ ((0, "configuration"), "invalid_type"), ((1,), "invalid_value"), ] def test_error_a_second_array_to_bytes_codec() -> None: assert _problems(["bytes", "bytes"]) == [((1,), "invalid_value")] @pytest.mark.parametrize("codecs", [[], [GZIP], [_transpose(0, 1)]]) def test_error_a_pipeline_without_an_array_to_bytes_codec(codecs: list[JSONValue]) -> None: assert _problems(codecs) == [((), "invalid_value")] def test_error_a_transpose_order_with_another_number_of_axes_than_its_chunk() -> None: # The problem is located in the codec's configuration. assert _problems([_transpose(1, 0, 2), "bytes"]) == [ ((0, "configuration", "order"), "invalid_value") ] # It is checked against the chunk it receives from the codec before it, # which here still has two axes. assert _problems([_transpose(1, 0), _transpose(0), "bytes"]) == [ ((1, "configuration", "order"), "invalid_value") ] def test_error_a_codec_with_a_problem_of_its_own_hands_on_a_chunk_nothing_is_known_of() -> None: # The reader does not guess what such a codec does, so the next codec # receives an unknown chunk and is not checked against it. stages, problems = _read([_transpose(0, 0), "bytes"], CHUNK) assert [(problem.loc, problem.kind) for problem in problems] == [ ((0, "configuration", "order"), "invalid_value") ] assert [stage.incoming for stage in stages] == [CHUNK, Chunk()] @pytest.mark.parametrize( "data_type", [ "float32", "int16", _struct("int8", "float32"), {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}}, ], ) def test_error_a_bytes_codec_without_endian_handed_values_of_several_bytes( data_type: JSONValue, ) -> None: assert _problems(["bytes"], _of(data_type)) == [((0, "configuration", "endian"), "missing_key")] def test_error_a_bytes_codec_without_endian_after_a_cast_past_a_codec_nothing_claims() -> None: # The cast declares the data type it outputs, whatever it receives. assert _problems(["zfpy", _cast("float32"), "bytes"]) == [ ((2, "configuration", "endian"), "missing_key") ] @pytest.mark.parametrize("data_type", ["string", "bytes"]) def test_error_a_bytes_codec_handed_values_that_vary_in_size(data_type: JSONValue) -> None: assert _problems([LITTLE], _of(data_type)) == [((0, "configuration"), "invalid_value")] @pytest.mark.parametrize( ("scalar_map", "loc"), [ # The input of encoding and the output of decoding are values of the # data type the cast receives. ({"encode": [[300, 0]]}, ("scalar_map", "encode", 0, 0)), ({"decode": [[0, -129]]}, ("scalar_map", "decode", 0, 1)), ], ) def test_error_a_scalar_the_cast_maps_from_that_is_not_of_the_data_type_it_is_handed( scalar_map: JSONValue, loc: tuple[str | int, ...] ) -> None: codecs = [_cast("uint8", scalar_map=scalar_map), "bytes"] assert _problems(codecs, _of("int8")) == [((0, "configuration", *loc), "invalid_value")] @pytest.mark.parametrize("codec", [_cast("float32"), _scale()]) @pytest.mark.parametrize("data_type", ["bool", "string", _struct("int8")]) def test_error_an_arithmetic_codec_handed_values_that_are_no_numbers( codec: JSONValue, data_type: JSONValue ) -> None: # The codec comes before an array -> bytes codec that nothing in scope # claims, which is not checked. assert _problems([codec, "vlen-utf8"], _of(data_type)) == [ ((0, "configuration"), "invalid_value") ] def test_error_a_codec_handed_on_what_another_could_not_take_is_judged_too() -> None: # scale_offset outputs the chunk it receives, which the bytes codec # after it cannot accept either. assert _problems([_scale(), LITTLE], _of("string")) == [ ((0, "configuration"), "invalid_value"), ((1, "configuration"), "invalid_value"), ] def test_error_a_cast_handed_complex_numbers() -> None: assert _problems([_cast("float32"), "vlen-utf8"], _of("complex64")) == [ ((0, "configuration"), "invalid_value") ] @pytest.mark.parametrize( ("members", "data_type", "found"), [ ({"offset": 0.5}, "int16", [((0, "configuration", "offset"), "invalid_type")]), ({"scale": 70000}, "int16", [((0, "configuration", "scale"), "invalid_value")]), ({"offset": "nan"}, "float32", [((0, "configuration", "offset"), "invalid_value")]), ], ) def test_error_a_scale_or_offset_that_is_not_a_value_of_the_data_type_it_is_handed( members: dict[str, JSONValue], data_type: str, found: list[tuple[tuple[str | int, ...], str]] ) -> None: assert _problems([_scale(**members), LITTLE], _of(data_type)) == found def test_error_a_transition_that_gives_something_else() -> None: lying = CodecDefinition( name="acme.lying", configuration=EmptyConfiguration, kind="array_array", size="static", transition=lambda configuration, nested, chunk: "a chunk", # pyright: ignore[reportArgumentType] ) codec = resolve( "acme.lying", CodecDefinition, CONTEXT.update(codecs=[*CONTEXT.codecs.values(), lying]) )[0] with pytest.raises(TypeError, match="'acme.lying': its transition gives a Chunk"): read_pipeline([codec], CHUNK) def test_error_a_transition_that_raises_says_whose_it_is() -> None: def transition(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) -> Chunk: raise KeyError("axis") raising = CodecDefinition( name="acme.raising", configuration=EmptyConfiguration, kind="array_array", size="static", transition=transition, ) codec = resolve( "acme.raising", CodecDefinition, CONTEXT.update(codecs=[*CONTEXT.codecs.values(), raising]) )[0] with pytest.raises(KeyError) as raised: read_pipeline([codec], CHUNK, ("codecs",)) assert raised.value.__notes__ == [ "raised by the transition of 'acme.raising', reading ('codecs', 0, 'configuration')" ] @pytest.mark.parametrize( ("lengths", "data_type", "match"), [ ((4, 2), None, "a chunk's lengths are"), (None, "float32", "a chunk's data type is"), # A field read as a codec, though nothing claims it. (None, resolve("acme.t", CodecDefinition, CONTEXT)[0], "a chunk's data type is"), ], ) def test_error_a_transition_that_builds_a_chunk_of_something_else( lengths: object, data_type: object, match: str ) -> None: # A chunk checks its own contents, so the bug is in the transition. def transition(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) -> Chunk: return Chunk(lengths, data_type) # pyright: ignore[reportArgumentType] odd = CodecDefinition( name="acme.odd", configuration=EmptyConfiguration, kind="array_array", size="static", transition=transition, ) codec = resolve( "acme.odd", CodecDefinition, CONTEXT.update(codecs=[*CONTEXT.codecs.values(), odd]) )[0] with pytest.raises(TypeError, match=match) as raised: read_pipeline([codec], CHUNK, ("codecs",)) assert raised.value.__notes__ == [ "raised by the transition of 'acme.odd', reading ('codecs', 0, 'configuration')" ] @pytest.mark.parametrize( ("kind", "member"), [ ("bytes_bytes", "chunk_rules"), ("bytes_bytes", "transition"), ("bytes_bytes", "pipelines"), ("array_bytes", "transition"), ], ) def test_error_a_codec_hook_nothing_would_ask(kind: str, member: str) -> None: # A bytes -> bytes codec receives bytes, and only an array -> array # codec outputs a chunk, so these hooks would never be called. hook = {member: lambda configuration, nested, chunk: ()} with pytest.raises(TypeError, match=f"'acme.x': {member}, "): CodecDefinition( name="acme.x", configuration=EmptyConfiguration, kind=kind, # pyright: ignore[reportArgumentType] size="static", **hook, # pyright: ignore[reportArgumentType] ) def test_an_array_document_s_codecs_are_each_judged_against_the_chunk_they_are_handed() -> None: # scale_offset's offset is checked against the float32 that the first # cast outputs, not against the array's int8. document = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) document["data_type"] = "int8" document["codecs"] = [_cast("float32"), _scale(offset=0.5), _cast("int8"), "bytes"] assert read_array_metadata_v3(document).problems == () def test_error_a_transpose_of_another_rank_than_the_array_under_a_grid_nothing_claims() -> None: # A chunk has an extent for each dimension of the array, whatever the # grid says about the lengths. document = dict(ZarrV3ArrayMetadata.create_default(shape=(4, 4)).to_json()) document["chunk_grid"] = {"name": "acme.grid", "configuration": {}} document["codecs"] = [_transpose(0), "bytes"] assert [(p.loc, p.kind) for p in read_array_metadata_v3(document).problems] == [ (("codecs", 0, "configuration", "order"), "invalid_value") ] def test_error_an_array_document_s_codecs_out_of_order_or_not_fitting_its_chunks() -> None: # The problems are located in the document's codecs. The chunks come # from the grid, with the document's shape and data type. document = dict(ZarrV3ArrayMetadata.create_default(shape=(4, 4)).to_json()) document["data_type"] = "int16" document["codecs"] = [_transpose(0), "bytes", "crc32c", _transpose(0, 1)] assert [(p.loc, p.kind) for p in read_array_metadata_v3(document).problems] == [ (("codecs", 3), "invalid_value"), (("codecs", 0, "configuration", "order"), "invalid_value"), (("codecs", 1, "configuration", "endian"), "missing_key"), ] class AcmeHolderConfiguration(TypedDict, closed=True): codecs: tuple[ZarrV3CodecField, ...] types: tuple[ZarrV3DataTypeField, ...] def _holder( pipelines: object, types: JSONValue = ("uint8",) ) -> ResolvedField[CodecDefinition[Any]]: """A codec that holds a pipeline of codecs and a list of data types, `types`, and whose `pipelines` hook is `pipelines`.""" holder = CodecDefinition( name="acme.holder", configuration=AcmeHolderConfiguration, kind="array_bytes", size="dynamic", pipelines=pipelines, # pyright: ignore[reportArgumentType] ) field = {"name": "acme.holder", "configuration": {"codecs": [LITTLE], "types": types}} return resolve( field, CodecDefinition, CONTEXT.update(codecs=[*CONTEXT.codecs.values(), holder]) )[0] @pytest.mark.parametrize("given", ["a chunk", {"codecs": "a chunk"}, {("codecs",): Chunk()}]) def test_error_pipelines_that_give_something_else(given: object) -> None: with pytest.raises(TypeError, match="'acme.holder': its pipelines give a mapping of members"): read_pipeline([_holder(lambda configuration, nested, chunk: given)], CHUNK) @pytest.mark.parametrize( ("member", "types"), [ ("nowhere", ("uint8",)), ("types", ("uint8",)), # Data types that nothing in scope claims are still read as data types. ("types", ("acme.t",)), ], ) def test_error_pipelines_that_name_a_member_holding_no_codecs( member: str, types: JSONValue ) -> None: holder = _holder(lambda configuration, nested, chunk: {member: Chunk()}, types) with pytest.raises(TypeError, match=f"its pipelines name {member!r}, which holds no list"): read_pipeline([holder], CHUNK) def test_error_pipelines_that_raise_say_whose_they_are() -> None: holder = _holder(lambda configuration, nested, chunk: {}["codecs"]) with pytest.raises(KeyError) as raised: read_pipeline([holder], CHUNK, ("codecs",)) assert raised.value.__notes__ == [ "raised by the pipelines of 'acme.holder', reading ('codecs', 0, 'configuration')" ] def test_error_a_stage_s_inner_pipelines_are_read_only() -> None: """The pipelines of a shard's inner codecs held by a stage cannot be changed in place. A reading that holds them stays equal to itself after pickling and deep-copying.""" import copy import pickle shard: JSONValue = { "name": "sharding_indexed", "configuration": { "chunk_shape": [2], "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], "index_codecs": [ {"name": "bytes", "configuration": {"endian": "little"}}, {"name": "crc32c"}, ], "index_location": "end", }, } stages, _ = _read([shard], CHUNK) stage = stages[0] assert "codecs" in stage.inner with pytest.raises(TypeError): stage.inner["codecs"] = () # pyright: ignore[reportIndexIssue] for again in (pickle.loads(pickle.dumps(stages)), copy.deepcopy(stages)): assert again == stages zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/test_scope.py000066400000000000000000000513531527014137600302220ustar00rootroot00000000000000"""The algebra of scopes: value semantics for `Context`, claims, refinement, disagreements and unions.""" from __future__ import annotations import pickle from typing import Any import pytest from hypothesis import given from hypothesis import strategies as st from zarr_metadata.definition import ( Conflict, Definition, EmptyConfiguration, RefusedField, ResolvedField, ScopeConflictError, resolve, ) from zarr_metadata.definition._context import disagreements from zarr_metadata.definition._definition import fields_of from zarr_metadata.definition._scope import Disagreements, claims_of, kind_name, refines from zarr_metadata.v3.codec.bytes import BYTES_CODEC from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC from zarr_metadata.v3.codec.gzip import GZIP_CODEC from zarr_metadata.v3.codec.sharding_indexed import SHARDING_INDEXED_CODEC from zarr_metadata.v3.codec.zstd import ZSTD_CODEC from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE from zarr_metadata.v3.definition import ( CORE, CORE_AND_EXTENSIONS, ArrayExtraFieldDefinition, ChunkGridDefinition, ChunkKeyEncodingDefinition, CodecDefinition, DataTypeDefinition, GroupExtraFieldDefinition, StorageTransformerDefinition, ZarrV3Context, ) SHARD = { "name": "sharding_indexed", "configuration": { "chunk_shape": [1], "codecs": ["bytes"], "index_codecs": [{"name": "bytes", "configuration": {"endian": "little"}}, "crc32c"], }, } MY_GZIP = CodecDefinition( name="gzip", configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic" ) """A private definition of the name `gzip`: a second definition under the same name, which takes no configuration. Definitions compare by their contents, so a definition rebuilt from the core TypedDict and the core rules would equal the core definition. This one reads `gzip` differently. """ def _read(data: object, kind: type[Definition[Any]], context: ZarrV3Context) -> ResolvedField[Any]: return resolve(data, kind, context)[0] @pytest.mark.parametrize( ("left", "right", "equal"), [ ( ZarrV3Context(codecs=[GZIP_CODEC, BYTES_CODEC]), ZarrV3Context(codecs=[BYTES_CODEC, GZIP_CODEC]), True, ), (CORE, pickle.loads(pickle.dumps(CORE)), True), (CORE, CORE_AND_EXTENSIONS, False), (ZarrV3Context(codecs=[GZIP_CODEC]), ZarrV3Context(codecs=[GZIP_CODEC, ZSTD_CODEC]), False), (ZarrV3Context(), ZarrV3Context(), True), ], ids=["order", "pickle", "core-vs-extensions", "subset", "empty"], ) def test_a_scope_is_equal_to_another_by_the_definitions_it_files( left: ZarrV3Context, right: ZarrV3Context, equal: bool ) -> None: """Two scopes are equal when they file the same definitions under the same names, however they were built. Equal scopes have equal hashes.""" assert (left == right) is equal if equal: assert hash(left) == hash(right) def test_a_scope_is_a_set_member() -> None: """A scope is hashable, so it can be a dict key or a set member. It was not hashable when its tables were mapping proxies.""" assert len({CORE, CORE_AND_EXTENSIONS, pickle.loads(pickle.dumps(CORE))}) == 2 def test_error_a_scope_conflict_says_each_disagreement() -> None: """A scope conflict lists each `(kind, name)` with the claimed definition, the found definition, and the location, so a caller can see every disagreement at once.""" error = ScopeConflictError( ( Conflict((CodecDefinition, "bytes"), BYTES_CODEC, None, ("codecs", 0)), Conflict((CodecDefinition, "gzip"), None, GZIP_CODEC), ) ) assert error.conflicts[0].key == (CodecDefinition, "bytes") assert str(error) == ( "codec 'bytes' at ('codecs', 0): claimed CodecDefinition(name='bytes') of " "BytesCodecConfiguration, found no definition; " "codec 'gzip': claimed no definition, found CodecDefinition(name='gzip') of " "GzipCodecConfiguration" ) @pytest.mark.parametrize( ("field", "claims"), [ (_read("gzip", CodecDefinition, CORE), {(CodecDefinition, "gzip"): GZIP_CODEC}), (_read("zstd", CodecDefinition, CORE), {(CodecDefinition, "zstd"): None}), ( _read("r16", DataTypeDefinition, CORE), {(DataTypeDefinition, "r*"): RAW_BYTES_DATA_TYPE}, ), ( _read(SHARD, CodecDefinition, CORE), { (CodecDefinition, "sharding_indexed"): SHARDING_INDEXED_CODEC, (CodecDefinition, "bytes"): BYTES_CODEC, (CodecDefinition, "crc32c"): CRC32C_CODEC, }, ), ( _read({"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE), {(CodecDefinition, "gzip"): GZIP_CODEC}, ), (RefusedField(json=3, name=None, read_as=CodecDefinition), {}), ], ids=["read", "unclaimed", "raw-bits", "nested", "refused-claimed", "refused-nameless"], ) def test_claims_of_says_what_a_reading_claimed_of_each_name( field: ResolvedField[Any], claims: dict[object, object] ) -> None: """A reading's claims map each name written by the field, and by the fields it holds, to the definition that read it. Each name is keyed as the scope files it (raw bits under `r*`), with None where nothing claimed the name. A field refused by a definition still claims that definition. A field with no name claims nothing.""" assert claims_of(fields_of(field)) == claims def test_error_claims_of_refuses_one_name_read_two_ways() -> None: """Fields read in two scopes that give one name two different definitions have no single set of claims. `claims_of` raises a `ScopeConflictError` that names the key.""" fields = [ *fields_of(_read("gzip", CodecDefinition, CORE), ("codecs", 0)), *fields_of(_read("gzip", CodecDefinition, ZarrV3Context(codecs=[MY_GZIP])), ("codecs", 1)), ] with pytest.raises(ScopeConflictError) as raised: claims_of(fields) (conflict,) = raised.value.conflicts assert (conflict.key, conflict.claimed, conflict.found, conflict.loc) == ( (CodecDefinition, "gzip"), GZIP_CODEC, MY_GZIP, ("codecs", 1), ) GZIP_FIELD = {"name": "gzip", "configuration": {"level": 5}} ZSTD_FIELD = {"name": "zstd", "configuration": {"level": 3}} BLOSC = {"cname": "zstd", "shuffle": "noshuffle", "blocksize": 0} BLOSC_LEVEL_ONE = {"name": "blosc", "configuration": {**BLOSC, "clevel": 1}} BLOSC_LEVEL_TRUE = {"name": "blosc", "configuration": {**BLOSC, "clevel": True}} """Two documents that Python's `==` treats as equal, but that `json_text` tells apart.""" SHARD_HOLDING_REFUSED = { "name": "sharding_indexed", "configuration": { "chunk_shape": [1], "codecs": ["bytes", {"name": "gzip", "configuration": {"level": 12}}], "index_codecs": [{"name": "bytes", "configuration": {"endian": "little"}}, "crc32c"], }, } """A shard that is read, holding a gzip that its definition refuses.""" NESTED_ZSTD = { "name": "sharding_indexed", "configuration": { "chunk_shape": [1], "codecs": ["bytes", ZSTD_FIELD], "index_codecs": [{"name": "bytes", "configuration": {"endian": "little"}}, "crc32c"], }, } @pytest.mark.parametrize( ("field", "other", "expected"), [ (_read(GZIP_FIELD, CodecDefinition, CORE), _read(GZIP_FIELD, CodecDefinition, CORE), True), ( _read({"name": "crc32c"}, CodecDefinition, CORE), _read("crc32c", CodecDefinition, CORE), True, ), ( _read(ZSTD_FIELD, CodecDefinition, CORE_AND_EXTENSIONS), _read(ZSTD_FIELD, CodecDefinition, CORE), True, ), ( _read(ZSTD_FIELD, CodecDefinition, CORE), _read(ZSTD_FIELD, CodecDefinition, CORE_AND_EXTENSIONS), False, ), ( _read(GZIP_FIELD, CodecDefinition, CORE), _read(GZIP_FIELD, CodecDefinition, ZarrV3Context(codecs=[MY_GZIP])), False, ), ( _read(NESTED_ZSTD, CodecDefinition, CORE_AND_EXTENSIONS), _read(NESTED_ZSTD, CodecDefinition, CORE), True, ), ( _read(NESTED_ZSTD, CodecDefinition, CORE), _read(NESTED_ZSTD, CodecDefinition, CORE_AND_EXTENSIONS), False, ), ( _read({"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE), _read("gzip", CodecDefinition, CORE), False, ), (_read("zstd", CodecDefinition, CORE), _read("zstd", CodecDefinition, CORE), True), ( _read({"name": "zstd", "configuration": {"level": 1}}, CodecDefinition, CORE), _read({"name": "zstd", "configuration": {"level": 2}}, CodecDefinition, CORE), False, ), ( _read("crc32c", CodecDefinition, CORE), _read({"name": "crc32c"}, CodecDefinition, ZarrV3Context()), True, ), ( _read(BLOSC_LEVEL_ONE, CodecDefinition, CORE), _read(BLOSC_LEVEL_TRUE, CodecDefinition, ZarrV3Context()), False, ), ( _read(SHARD_HOLDING_REFUSED, CodecDefinition, CORE), _read(SHARD_HOLDING_REFUSED, CodecDefinition, CORE), True, ), ], ids=[ "same", "same-spelled-otherwise", "gain", "loss", "conflict", "nested-gain", "nested-loss", "refused", "both-unclaimed", "unclaimed-differ", "gain-over-another-spelling", "no-gain-over-true-for-one", "holding-a-refused-field", ], ) def test_refines_orders_readings_by_information( field: ResolvedField[Any], other: ResolvedField[Any], expected: bool ) -> None: """`field` refines `other` when it reads the same where both read, and gains where `other` left a name unclaimed, including in the fields it holds. A loss, a conflict, a refused field, or two unclaimed fields written differently do not refine.""" assert refines(field, other) is expected DOCUMENTS = [ GZIP_FIELD, "crc32c", {"name": "crc32c"}, {"name": "crc32c", "configuration": {}}, ZSTD_FIELD, NESTED_ZSTD, BLOSC_LEVEL_ONE, SHARD_HOLDING_REFUSED, ] CONTEXTS = [ZarrV3Context(), CORE, CORE_AND_EXTENSIONS] READINGS = [ _read(document, CodecDefinition, context) for document in DOCUMENTS for context in CONTEXTS ] @given(st.sampled_from(READINGS), st.sampled_from(READINGS), st.sampled_from(READINGS)) def test_refines_is_a_partial_order_whose_bottom_is_equality( a: ResolvedField[Any], b: ResolvedField[Any], c: ResolvedField[Any] ) -> None: """Over readings of documents in several scopes and spellings, `refines` is reflexive and transitive. Two fields that refine each other are equal, and equal fields are refined by the same fields.""" assert refines(a, a) if refines(a, b) and refines(b, c): assert refines(a, c) assert (refines(a, b) and refines(b, a)) is (a == b) if a == b: assert refines(c, a) is refines(c, b) @pytest.mark.parametrize( ("context", "claims", "gains", "conflicts"), [ (CORE, {(CodecDefinition, "gzip"): GZIP_CODEC}, (), ()), ( CORE_AND_EXTENSIONS, {(CodecDefinition, "zstd"): None}, ((CodecDefinition, "zstd"),), (), ), ( CORE, {(CodecDefinition, "zstd"): ZSTD_CODEC}, (), (Conflict((CodecDefinition, "zstd"), ZSTD_CODEC, None),), ), ( ZarrV3Context(codecs=[MY_GZIP]), {(CodecDefinition, "gzip"): GZIP_CODEC}, (), (Conflict((CodecDefinition, "gzip"), GZIP_CODEC, MY_GZIP),), ), (CORE, {(CodecDefinition, "acme.x"): None}, (), ()), (CORE, {(DataTypeDefinition, "r*"): RAW_BYTES_DATA_TYPE}, (), ()), ], ids=["agrees", "gain", "loss", "conflict", "unclaimed-both", "raw-bits"], ) def test_disagreements_says_where_a_scope_reads_claims_otherwise( context: ZarrV3Context, claims: dict[Any, Any], gains: tuple[Any, ...], conflicts: tuple[Conflict, ...], ) -> None: """A scope agrees with claims that it reads the same way. It gains where it claims a name the claims left unclaimed. It conflicts where it reads a name with another definition or with none. Models compare a scope they are moved to with their claims this way.""" found = disagreements(context, claims) assert isinstance(found, Disagreements) assert (found.gains, found.conflicts) == (gains, conflicts) assert found.agrees is (len(gains) == 0 and len(conflicts) == 0) @pytest.mark.parametrize( ("contexts", "united"), [ ((CORE, CORE_AND_EXTENSIONS), CORE_AND_EXTENSIONS), ( (ZarrV3Context(codecs=[GZIP_CODEC]), ZarrV3Context(codecs=[ZSTD_CODEC])), ZarrV3Context(codecs=[GZIP_CODEC, ZSTD_CODEC]), ), ((CORE,), CORE), ((CORE, CORE), CORE), ( (ZarrV3Context(codecs=[BYTES_CODEC]), ZarrV3Context(data_types=[BYTES_DATA_TYPE])), ZarrV3Context(data_types=[BYTES_DATA_TYPE], codecs=[BYTES_CODEC]), ), ], ids=["subset", "disjoint", "alone", "same", "same-name-two-kinds"], ) def test_union_is_the_least_scope_above_each( contexts: tuple[ZarrV3Context, ...], united: ZarrV3Context ) -> None: """The union of scopes, their join in the order `refines` defines, files every definition that any of them files, once. The same name in two kinds counts as two names.""" first, *others = contexts assert first.union(*others) == united def test_error_union_refuses_one_name_filed_two_ways() -> None: """Scopes that file different definitions under one name have no union. `union` raises a `ScopeConflictError` that names the key and both definitions.""" with pytest.raises(ScopeConflictError) as raised: CORE.union(ZarrV3Context(codecs=[MY_GZIP])) (conflict,) = raised.value.conflicts assert (conflict.key, conflict.claimed, conflict.found) == ( (CodecDefinition, "gzip"), GZIP_CODEC, MY_GZIP, ) @pytest.mark.parametrize( ("kind", "said"), [ (CodecDefinition, "codec"), (DataTypeDefinition, "data type"), (ChunkGridDefinition, "chunk grid"), (ChunkKeyEncodingDefinition, "chunk key encoding"), (StorageTransformerDefinition, "storage transformer"), ], ids=["codec", "data-type", "chunk-grid", "chunk-key-encoding", "storage-transformer"], ) def test_a_kind_is_named_in_words(kind: type[Definition[Any]], said: str) -> None: """`kind_name` names each kind of definition in the words a message uses: `ChunkKeyEncodingDefinition` is "chunk key encoding".""" assert kind_name(kind) == said assert said in str(Conflict((kind, "x"), None, None)) def test_a_gain_is_judged_by_what_the_definition_reads_not_by_spelling() -> None: """An accepted field refines an unclaimed one when its definition, reading the JSON of the unclaimed field, produces the accepted field. Two spellings of one configuration count as the same gain, so `refines` is transitive through `==`. A spelling that the definition reads as something else is not a gain.""" from zarr_metadata.definition import ( resolve, ) from zarr_metadata.v3.definition import ( CORE, ChunkKeyEncodingDefinition, ) nothing = ZarrV3Context() spelled_out, _ = resolve( {"name": "default", "configuration": {"separator": "/"}}, ChunkKeyEncodingDefinition, CORE ) bare, _ = resolve({"name": "default"}, ChunkKeyEncodingDefinition, CORE) unclaimed, _ = resolve({"name": "default"}, ChunkKeyEncodingDefinition, nothing) assert spelled_out == bare assert refines(bare, unclaimed) assert refines(spelled_out, unclaimed) other, _ = resolve( {"name": "default", "configuration": {"separator": "."}}, ChunkKeyEncodingDefinition, CORE ) assert not refines(other, unclaimed) shard = { "name": "sharding_indexed", "configuration": { "chunk_shape": [2], "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], "index_codecs": [{"name": "bytes", "configuration": {"endian": "little"}}, "crc32c"], "index_location": "end", }, } read, _ = resolve(shard, CodecDefinition, CORE) unread, _ = resolve({**shard, "must_understand": True}, CodecDefinition, nothing) assert refines(read, unread) OTHER_GZIP = CodecDefinition( name="gzip", configuration=EmptyConfiguration, kind="bytes_bytes", size="static" ) """A second definition under the core gzip's name, which makes a union conflict.""" def test_a_scope_conflict_error_pickles_and_copies_with_its_conflicts() -> None: """`ScopeConflictError` keeps its conflicts and message through pickling and copying, as `MetadataValidationError` does. So a conflict reported in another process reads the same here.""" import copy with pytest.raises(ScopeConflictError) as raised: CORE.union(ZarrV3Context(codecs=[OTHER_GZIP])) error = raised.value error.add_note("seen in a union") for again in (pickle.loads(pickle.dumps(error)), copy.copy(error), copy.deepcopy(error)): assert again.conflicts == error.conflicts assert str(again) == str(error) assert str(again).startswith("codec 'gzip'") assert again.__notes__ == ["seen in a union"] def test_a_conflict_names_the_field_as_the_document_writes_it_and_tells_the_definitions_apart() -> ( None ): """A conflict found in a document reports the name the document writes, `r16`, not the name its definition is filed under, `r*`. It tells two definitions of one name apart by the configuration each declares, as the messages for consolidated entries do.""" import dataclasses from zarr_metadata.model import ZarrV3ArrayMetadata from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS document = dict( ZarrV3ArrayMetadata.create_default(shape=(2,), data_type="r16", fill_value=[0, 0]).to_json() ) model = ZarrV3ArrayMetadata(document, CORE_AND_EXTENSIONS) other = dataclasses.replace(RAW_BYTES_DATA_TYPE, configuration=EmptyConfiguration) with pytest.raises(ScopeConflictError) as raised: model.refined_in( CORE_AND_EXTENSIONS.update(data_types={**CORE_AND_EXTENSIONS.data_types, "r*": other}) ) message = str(raised.value) assert message.startswith("data type 'r16'") assert "RawBytesConfiguration" in message assert "EmptyConfiguration" in message with pytest.raises(ScopeConflictError) as united: ZarrV3Context(codecs=[GZIP_CODEC]).union(ZarrV3Context(codecs=[OTHER_GZIP])) assert str(united.value).count("gzip") >= 2 assert "EmptyConfiguration" in str(united.value) def test_each_slot_is_a_read_only_view_by_name() -> None: """A v3 scope has one slot per kind, as `ZarrV3Context.kinds` lists them: the five extension points, and the extra fields of arrays and of groups. Each slot maps the name each definition is filed under to the definition, and cannot be changed in place.""" assert dict(ZarrV3Context.kinds) == { "data_types": DataTypeDefinition, "chunk_grids": ChunkGridDefinition, "chunk_key_encodings": ChunkKeyEncodingDefinition, "codecs": CodecDefinition, "storage_transformers": StorageTransformerDefinition, "array_extra_fields": ArrayExtraFieldDefinition, "group_extra_fields": GroupExtraFieldDefinition, } assert CORE.codecs["gzip"] is GZIP_CODEC assert CORE.data_types["r*"] is RAW_BYTES_DATA_TYPE assert dict(CORE.storage_transformers) == {} with pytest.raises(TypeError): CORE.codecs["gzip"] = MY_GZIP # pyright: ignore[reportIndexIssue] def test_error_a_slot_refuses_a_definition_of_another_kind() -> None: """A definition of another kind than its slot holds raises `TypeError` naming the slot, the definition and its kind, at run time as a type checker does.""" with pytest.raises( TypeError, match="codecs holds codec definitions, got 'bytes', a data type definition" ): ZarrV3Context(codecs=[BYTES_DATA_TYPE]) # pyright: ignore[reportArgumentType] def test_error_a_slot_refuses_two_definitions_under_one_name() -> None: """Two different definitions under one name in a slot raise `ScopeConflictError` naming both, because the scope could not tell which one reads the name. `update` is the way to replace a definition.""" with pytest.raises(ScopeConflictError) as raised: ZarrV3Context(codecs=[GZIP_CODEC, OTHER_GZIP]) (conflict,) = raised.value.conflicts assert (conflict.key, conflict.claimed, conflict.found) == ( (CodecDefinition, "gzip"), GZIP_CODEC, OTHER_GZIP, ) zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/test_sharding.py000066400000000000000000000212101527014137600306750ustar00rootroot00000000000000"""Reading the sharding codec: the shard it receives, and the two pipelines it holds. `sharding_indexed` divides the chunk it receives into inner chunks, which its inner codecs receive. It indexes them in a shard index, which its index codecs receive. Each pipeline is read in the same way as the array's pipeline. """ from __future__ import annotations from typing import TYPE_CHECKING, Any import pytest from zarr_metadata import JSONValue from zarr_metadata.definition import ( ResolvedField, resolve, ) from zarr_metadata.model import ( ZarrV3ArrayMetadata, read_array_metadata_v3, ) from zarr_metadata.v3._pipeline import ( read_pipeline, ) from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS, Chunk, CodecDefinition, DataTypeDefinition, ) if TYPE_CHECKING: from zarr_metadata import ValidationProblem from zarr_metadata.v3.definition import Stage Where = tuple[str | int, ...] LITTLE: JSONValue = {"name": "bytes", "configuration": {"endian": "little"}} INDEX: list[JSONValue] = [LITTLE, "crc32c"] def _dt(name: JSONValue) -> ResolvedField[DataTypeDefinition[Any]]: return resolve(name, DataTypeDefinition, CORE_AND_EXTENSIONS)[0] FLOAT32 = _dt("float32") UINT64 = _dt("uint64") def _chunk( *axes: set[int] | None, data_type: ResolvedField[DataTypeDefinition[Any]] = FLOAT32 ) -> Chunk: return Chunk(tuple(None if axis is None else frozenset(axis) for axis in axes), data_type) def _index(*axes: set[int] | None) -> Chunk: return _chunk(*axes, data_type=UINT64) def _shard( chunk_shape: list[int], codecs: list[JSONValue] | None = None, index: list[JSONValue] | None = None, ) -> JSONValue: configuration: dict[str, JSONValue] = { "chunk_shape": list[JSONValue](chunk_shape), "codecs": [LITTLE] if codecs is None else codecs, "index_codecs": INDEX if index is None else index, } return {"name": "sharding_indexed", "configuration": configuration} def _transpose(*order: int) -> JSONValue: return {"name": "transpose", "configuration": {"order": list(order)}} def _read( codecs: list[JSONValue], chunk: Chunk ) -> tuple[tuple[Stage, ...], tuple[ValidationProblem, ...]]: """The stages, and every problem: first each codec's own problems at its location, then the pipeline's.""" read = [ resolve(codec, CodecDefinition, CORE_AND_EXTENSIONS, (index,)) for index, codec in enumerate(codecs) ] stages, found = read_pipeline([resolved for resolved, _ in read], chunk) return stages, (*(problem for _, own in read for problem in own), *found) def _problems(codecs: list[JSONValue], chunk: Chunk) -> list[tuple[Where, str]]: return [(problem.loc, problem.kind) for problem in _read(codecs, chunk)[1]] def _handed(stages: tuple[Stage, ...], at: Where = ()) -> dict[Where, Chunk | None]: """The chunk each codec receives, for every pipeline held by the codecs, keyed by the codec's location.""" found: dict[Where, Chunk | None] = {} for index, stage in enumerate(stages): for member, held in stage.inner.items(): found |= {(*at, index, member, place): s.incoming for place, s in enumerate(held)} found |= _handed(held, (*at, index, member)) return found @pytest.mark.parametrize( ("codecs", "chunk", "handed"), [ # Chunks of 8 by 8, shards of 2 by 2 inner chunks. ( [_shard([4, 4])], _chunk({8}, {8}), { (0, "codecs", 0): _chunk({4}, {4}), (0, "index_codecs", 0): _index({2}, {2}, {2}), (0, "index_codecs", 1): None, }, ), # The shard is the chunk the codec receives. After the transpose, 4 by # 6 divides into inner chunks of 4 by 3, though 6 by 4 would not. ( [_transpose(1, 0), _shard([4, 3])], _chunk({6}, {4}), { (1, "codecs", 0): _chunk({4}, {3}), (1, "index_codecs", 0): _index({1}, {2}, {2}), (1, "index_codecs", 1): None, }, ), # A rectilinear grid's shards differ in size, and so do their counts # of inner chunks. ( [_shard([4])], _chunk({8, 4}), { (0, "codecs", 0): _chunk({4}), (0, "index_codecs", 0): _index({2, 1}, {2}), (0, "index_codecs", 1): None, }, ), # When it receives a chunk with unknown properties, a shard still has # the inner chunks it declares, and an index of uint64. ( [_shard([4])], Chunk(), { (0, "codecs", 0): Chunk((frozenset({4}),)), (0, "index_codecs", 0): _index(None, {2}), (0, "index_codecs", 1): None, }, ), # A shard within a shard is read the same way. ( [_shard([4, 4], codecs=[_shard([2, 2])])], _chunk({8}, {8}), { (0, "codecs", 0): _chunk({4}, {4}), (0, "codecs", 0, "codecs", 0): _chunk({2}, {2}), (0, "codecs", 0, "index_codecs", 0): _index({2}, {2}, {2}), (0, "codecs", 0, "index_codecs", 1): None, (0, "index_codecs", 0): _index({2}, {2}, {2}), (0, "index_codecs", 1): None, }, ), ], ) def test_every_shard_hands_its_inner_codecs_its_inner_chunks_and_its_index_codecs_its_index( codecs: list[JSONValue], chunk: Chunk, handed: dict[Where, Chunk | None] ) -> None: stages, problems = _read(codecs, chunk) assert problems == () assert _handed(stages) == handed def test_error_a_shard_whose_inner_chunks_have_another_number_of_axes() -> None: # One problem. It is not known which of the two is wrong, so the chunks # the pipelines receive are also unknown, and the pipelines are not # checked against them. These transposes would fit if `chunk_shape` had # two lengths. shard = _shard([4], codecs=[_transpose(1, 0), LITTLE], index=[_transpose(2, 1, 0), LITTLE]) assert _problems([shard], _chunk({8}, {8})) == [ ((0, "configuration", "chunk_shape"), "invalid_value") ] @pytest.mark.parametrize( ("codecs", "chunk", "axis"), [ ([_shard([4, 4])], _chunk({8}, {6}), 1), ([_shard([4])], _chunk({8, 6}), 0), # The shard is divided along its axes in the order it receives them. # After the transpose, a shard of 6 by 2 does not divide into inner # chunks of 2 by 3, though a shard of 2 by 6 would. ([_transpose(1, 0), _shard([2, 3])], _chunk({2}, {6}), 1), ], ) def test_error_an_inner_chunk_length_that_does_not_divide_the_shard( codecs: list[JSONValue], chunk: Chunk, axis: int ) -> None: at = len(codecs) - 1 assert _problems(codecs, chunk) == [ ((at, "configuration", "chunk_shape", axis), "invalid_value") ] @pytest.mark.parametrize( ("shard", "found"), [ # Each pipeline is read as the array's pipeline is, at its own location. (_shard([4, 4], codecs=[]), [((0, "configuration", "codecs"), "invalid_value")]), ( _shard([4, 4], codecs=["crc32c", LITTLE]), [((0, "configuration", "codecs", 1), "invalid_value")], ), ( _shard([4, 4], codecs=["bytes"]), [((0, "configuration", "codecs", 0, "configuration", "endian"), "missing_key")], ), # The index has data type uint64, whose values take several bytes, and # has one more axis than the shard. ( _shard([4, 4], index=["bytes"]), [((0, "configuration", "index_codecs", 0, "configuration", "endian"), "missing_key")], ), ( _shard([4, 4], index=[_transpose(1, 0), LITTLE]), [((0, "configuration", "index_codecs", 0, "configuration", "order"), "invalid_value")], ), ], ) def test_error_a_pipeline_a_shard_holds_that_does_not_fit_what_it_is_handed( shard: JSONValue, found: list[tuple[Where, str]] ) -> None: assert _problems([shard], _chunk({8}, {8})) == found def test_error_an_array_document_s_shard_is_read_where_it_sits() -> None: document = dict(ZarrV3ArrayMetadata.create_default(shape=(16, 16)).to_json()) document["chunk_grid"] = {"name": "regular", "configuration": {"chunk_shape": [8, 6]}} document["codecs"] = [_shard([4, 4], index=["bytes"])] assert [(p.loc, p.kind) for p in read_array_metadata_v3(document).problems] == [ (("codecs", 0, "configuration", "chunk_shape", 1), "invalid_value"), ( ("codecs", 0, "configuration", "index_codecs", 0, "configuration", "endian"), "missing_key", ), ] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/tests/v3/test_storage.py000066400000000000000000000102331527014137600305450ustar00rootroot00000000000000"""How a data type's values are stored: in single bytes, in several bytes each, or each in as many bytes as it needs. Each data type's definition declares its storage. `storage_of` returns the storage of a data type field that a scope read. The `bytes` codec asks for the storage of the data type it receives, and a struct asks for the storage of each of its fields. """ from __future__ import annotations from typing import TYPE_CHECKING import pytest from zarr_metadata.definition import ( EmptyConfiguration, Nested, resolve, ) from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS, DataTypeDefinition, StorageClass, storage_of, ) if TYPE_CHECKING: from zarr_metadata import JSONValue def _struct(*field_types: JSONValue) -> JSONValue: fields: list[JSONValue] = [ {"name": f"f{index}", "data_type": dt} for index, dt in enumerate(field_types) ] return {"name": "struct", "configuration": {"fields": fields}} DATETIME: JSONValue = { "name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}, } @pytest.mark.parametrize( ("data_type", "storage"), [ ("bool", "single_byte"), ("int8", "single_byte"), ("uint8", "single_byte"), ("r8", "single_byte"), ("int16", "multi_byte"), ("uint64", "multi_byte"), ("float16", "multi_byte"), ("float64", "multi_byte"), ("complex64", "multi_byte"), (DATETIME, "multi_byte"), ("string", "variable_length"), ("bytes", "variable_length"), # For raw bits wider than a byte, the spec does not say whether a # byte order applies. ("r16", None), # A struct's values are its fields' values, packed together. (_struct("int8", "uint8"), "single_byte"), (_struct("int8", "float32"), "multi_byte"), (_struct("int8", _struct("uint8", "int16")), "multi_byte"), # A field with unknown storage makes the struct's storage unknown, # unless another field decides it. (_struct("int8", "r16"), None), (_struct("float32", "r16"), "multi_byte"), (_struct("int8", "acme.decimal"), None), # A data type that nothing in scope claims, or one that is not read. ("acme.decimal", None), ({"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 0}}, None), ], ) def test_every_data_type_says_how_its_values_are_stored( data_type: JSONValue, storage: StorageClass | None ) -> None: resolved, _ = resolve(data_type, DataTypeDefinition, CORE_AND_EXTENSIONS) assert storage_of(resolved) == storage def test_a_data_type_that_says_nothing_of_its_storage_leaves_it_unknown() -> None: quiet = DataTypeDefinition(name="acme.quiet", configuration=EmptyConfiguration) resolved, _ = resolve( "acme.quiet", DataTypeDefinition, CORE_AND_EXTENSIONS.update(data_types=[*CORE_AND_EXTENSIONS.data_types.values(), quiet]), ) assert storage_of(resolved) is None def test_error_a_storage_that_gives_something_else() -> None: lying = DataTypeDefinition( name="acme.lying", configuration=EmptyConfiguration, storage=lambda configuration, nested: "two bytes", # pyright: ignore[reportArgumentType] ) resolved, _ = resolve( "acme.lying", DataTypeDefinition, CORE_AND_EXTENSIONS.update(data_types=[*CORE_AND_EXTENSIONS.data_types.values(), lying]), ) with pytest.raises(TypeError, match="'acme.lying': its storage gives one of"): storage_of(resolved) def test_error_a_storage_that_raises_says_whose_it_is() -> None: def storage(configuration: EmptyConfiguration, nested: Nested) -> StorageClass: raise KeyError("bits") raising = DataTypeDefinition( name="acme.raising", configuration=EmptyConfiguration, storage=storage ) resolved, _ = resolve( "acme.raising", DataTypeDefinition, CORE_AND_EXTENSIONS.update(data_types=[*CORE_AND_EXTENSIONS.data_types.values(), raising]), ) with pytest.raises(KeyError) as raised: storage_of(resolved) assert raised.value.__notes__ == ["raised by the storage of 'acme.raising'"] zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/towncrier-template.md000066400000000000000000000140041527014137600301450ustar00rootroot00000000000000{#- This package's copy of towncrier's default Markdown template. The one change: each entry's continuation lines are indented by four spaces, not two, and entries are separated by a blank line, so an entry with a second paragraph or a nested list stays one list item in Python-Markdown, which renders the docs' release notes page, as well as in CommonMark. Without both, Python-Markdown ends the list at the second paragraph and runs the later entries together as one paragraph. -#} {#- ══════════════════════════════════════════════════════════════════════════════ TOWNCRIER MARKDOWN TEMPLATE ══════════════════════════════════════════════════════════════════════════════ ─── Macro: heading ───────────────────────────────────────────────────────── Purpose: Generates Markdown headings with the appropriate number of # characters. Based on header_prefix (default: "#") and the level argument. Arguments: level The relative heading level (1=#, 2=##, 3=###, etc.) -#} {%- macro heading(level) -%} {{- "#" * ( header_prefix | length + level -1 ) }} {%- endmacro -%} {#- ─── Variable: newline ────────────────────────────────────────────────────── Purpose: Consistent newline handling. -#} {%- set newline = "\n" -%} {#- ════════════════════════ TEMPLATE GENERATION ════════════════════════ -#} {#- ─── TITLE HEADING ─── #} {#- render_title is false when title_format is specified in the config #} {%- if render_title %} {%- if versiondata.name %} {{- heading(1) ~ " " ~ versiondata.name ~ " " ~ versiondata.version ~ " (" ~ versiondata.date ~ ")" ~ newline }} {%- else %} {{- heading(1) ~ " " ~ versiondata.version ~ " (" ~ versiondata.date ~ ")" ~ newline }} {%- endif %} {%- endif %} {#- If title_format is specified, we start with a new line #} {{- newline }} {%- for section, _ in sections.items() %} {#- ─── SECTION HEADING ─── #} {%- if section %} {{- newline }} {{- heading(2) ~ " " ~ section ~ newline }} {{- newline }} {%- endif %} {%- if sections[section] %} {%- for category, val in definitions.items() if category in sections[section] %} {%- set issue_pks = [] %} {#- ─── CATEGORY HEADING ─── #} {#- Increase heading level if section is not present #} {{- heading(3 if section else 2) ~" " ~ definitions[category]['name'] ~ newline }} {{- newline }} {#- ─── RENDER ENTRIES ─── #} {%- for text, values in sections[section][category].items() %} {#- A blank line between entries: Python-Markdown starts a new list item after an entry of several paragraphs only after one. #} {%- if not loop.first %} {{- newline }} {%- endif %} {#- Prepare the string of issue numbers (e.g., "#1, #9, #142") #} {%- set issue_pks = [] %} {%- for v_issue in values %} {%- set _= issue_pks.append(v_issue.split(": ", 1)[0]) %} {%- endfor %} {%- set issues_list = issue_pks | join(", ") %} {#- Check if text contains a sublist #} {%- set text_has_sublist = (("\n - " in text) or ("\n * " in text)) %} {#- CASE 1: No text, only issues #} {#- Output: - #1, #9, #142 #} {%- if not text and issues_list %} {{- "- " ~ issues_list ~ newline }} {#- Cases where both text and issues exist #} {%- elif text and issues_list %} {%- if text_has_sublist %} {#- CASE 3: Text with sublist #} {#- Output: - TEXT\n\n (#1, #9, #142) #} {{- "- " ~ (text | indent(2)) ~ newline ~ newline ~ " (" ~ issues_list ~ ")" ~ newline }} {%- else %} {#- CASE 2: Text, no sublist #} {#- Output: - TEXT (#1, #9, #142) #} {{- "- " ~ (text | indent(2)) ~ " (" ~ issues_list ~ ")" ~ newline }} {%- endif %} {%- elif text %} {#- Implicit Case: Text, but no issues #} {#- Output: - TEXT #} {{- "- " ~ (text | indent(2)) ~ newline }} {%- endif %} {%- endfor %} {#- New line between list and link references #} {{- newline }} {#- Link references #} {%- if issues_by_category[section][category] and "]: " in issues_by_category[section][category][0] %} {%- for issue in issues_by_category[section][category] %} {{- issue ~ newline }} {%- endfor %} {{- newline }} {%- endif %} {#- No changes in this category #} {%- if sections[section][category]|length == 0 %} {{- newline }} {{- "No significant changes." ~ newline * 2 }} {%- endif %} {%- endfor %} {%- else %} {#- No changes in this section #} {{- "No significant changes." ~ newline * 2 }} {%- endif %} {%- endfor %} {#- Newline at the end of the rendered newsfile content. In this way the there are 2 newlines between the latest release and the previous release content. -#} {{- newline -}} zarr-developers-zarr-python-a2f2415/packages/zarr-metadata/uv.lock000066400000000000000000007323371527014137600253220ustar00rootroot00000000000000version = 1 revision = 3 requires-python = ">=3.11" [[package]] name = "annotated-types" version = "0.8.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, ] [[package]] name = "attrs" version = "26.1.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] [[package]] name = "babel" version = "2.18.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d", size = 9959554, upload-time = "2026-02-01T12:30:56.078Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, ] [[package]] name = "backrefs" version = "8.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ec/56/4744bcd0c82184e80c52b0ac4076c261a8ffa1f1b343ff2f6e89ce0e1cef/backrefs-8.0.tar.gz", hash = "sha256:b556cd7d36c3a3a2f256b89590b176b8eddfb73bcfaee3a3ddd84ea66d21ce50", size = 7013081, upload-time = "2026-07-26T19:54:24.638Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e3/fd/9bf53b6a6f6f519ffaac765df2f2a25e5c2fc6d32cfd2b2747099e72c911/backrefs-8.0-py310-none-any.whl", hash = "sha256:4a627b817fd2dce43b79ab48da63613340509381cd8ce0897078a0bce79a2ab8", size = 380377, upload-time = "2026-07-26T19:54:17.457Z" }, { url = "https://files.pythonhosted.org/packages/e1/29/4bd7ae72a2634da00379c2b3bcc5439e7c94620235c6afea8af15229a973/backrefs-8.0-py311-none-any.whl", hash = "sha256:f0c35cf0102ba6b6070c12a492be3c1c1d3f5839529784b9a9565d6d04569a01", size = 392169, upload-time = "2026-07-26T19:54:18.782Z" }, { url = "https://files.pythonhosted.org/packages/29/13/232505664e8e2a0c7a2eb0c505cfade9d715538f89a5d62bc4c272968f62/backrefs-8.0-py312-none-any.whl", hash = "sha256:87f0fae8c5f207fe9f4b2887efc71d42f4900ac78faa1af08d675ef303692dc5", size = 398084, upload-time = "2026-07-26T19:54:19.954Z" }, { url = "https://files.pythonhosted.org/packages/8a/69/47a3dc20abc4fa5486655fde681bd55e63211b46c886d8c02223d6468431/backrefs-8.0-py313-none-any.whl", hash = "sha256:601ce68ca12385dbda06ce264406b4c4210cf5b79fd0fd627592365c92f29a88", size = 400040, upload-time = "2026-07-26T19:54:21.194Z" }, { url = "https://files.pythonhosted.org/packages/1c/cf/e5f9b68a5b0e939a2fb933a66c20180d0c9241bf8927f7a47fa48c1675e9/backrefs-8.0-py314-none-any.whl", hash = "sha256:9ec96efa080938be92323e8e730e57718c9c88eb15ad70bbef4e1766df591408", size = 411903, upload-time = "2026-07-26T19:54:23.221Z" }, ] [[package]] name = "black" version = "26.10.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "mypy-extensions" }, { name = "packaging" }, { name = "pathspec" }, { name = "platformdirs" }, { name = "pytokens" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f8/65/a9611a6ec0a8c88d86e59385da02d68d9533f7e86a05913d20c67be54029/black-26.10.1.tar.gz", hash = "sha256:5f9f83beae62437e060dafd53d7f1fc327e3d3494f74d72ee5c2b73eb90fc4e7", size = 741979, upload-time = "2026-10-10T04:13:40.776Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/6f/1e/c1d4f71299d45f4ff54c9c42ef9214d6b228223f196bc1bb2ee196aeaeee/black-26.10.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a0219b29cd70e49f920acb7081e6ce5025c719008447c521d0200dcad93206a", size = 2121821, upload-time = "2026-10-10T04:18:36.793Z" }, { url = "https://files.pythonhosted.org/packages/7f/e2/43944049c0945199bbd5c8b291b85306bfd0eeccbc908bf65538a69ff1b0/black-26.10.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7bdade400bfe24d78a7762896acc2f9a8e1a17fb0fd0536bf6b7c7097cf3eec7", size = 1961908, upload-time = "2026-10-10T04:18:38.942Z" }, { url = "https://files.pythonhosted.org/packages/d5/e9/9616d58f69f0e4bfa89ebe8686b6ef50f4b4f0f91237ae4c76f34dabf8aa/black-26.10.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff57f63029aa1353fa8b1b0c8971fd88a6c92dc766608d2eee33ad2deb23270e", size = 2019042, upload-time = "2026-10-10T04:18:40.612Z" }, { url = "https://files.pythonhosted.org/packages/c5/de/0415cce69e2402a5c186cdb5462a0edd3fa44d69d1a1630e68e58daaa6a4/black-26.10.1-cp311-cp311-win_amd64.whl", hash = "sha256:3414a0c52901964dceabd98c7c56beac0f964115a116ecedcce7247359b14017", size = 1608383, upload-time = "2026-10-10T04:18:42.25Z" }, { url = "https://files.pythonhosted.org/packages/a9/61/af18e716432e932f166ce9dfa419ce3f05fd5ef8c52c0510d27ae41aaa51/black-26.10.1-cp311-cp311-win_arm64.whl", hash = "sha256:1935b32f5326028019856e18cb42b4da63db23765dc84464cec723e0de478a9b", size = 1466048, upload-time = "2026-10-10T04:18:44.271Z" }, { url = "https://files.pythonhosted.org/packages/f7/ca/357ccdd12e8f8429539f8f52edc153de58acba84f6670ac36c254a276691/black-26.10.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fe85fc4019bee59bc495c0f2a8ee76c5cd02c7015508d94a967ba2376f39a52c", size = 2167369, upload-time = "2026-10-10T04:18:45.948Z" }, { url = "https://files.pythonhosted.org/packages/a5/a2/4709110a7ca326ba7b4f81a669fbebc20653c79a972b0f8de4ba5b5c5e68/black-26.10.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:182f6c32be38074b16d378498c498b32cb51928178ee611485344972c35ec9c6", size = 1990732, upload-time = "2026-10-10T04:18:48.252Z" }, { url = "https://files.pythonhosted.org/packages/1e/96/9f8fa839c169d19c38ce6582d9449f2d7d70dabd313ac28f15a930260c1f/black-26.10.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b5347d760f0c02bb00dd249384cab71c3bf828b4f68d5b401eb116e0390f147d", size = 2065235, upload-time = "2026-10-10T04:18:50.064Z" }, { url = "https://files.pythonhosted.org/packages/2b/ee/929eaf7de6f05a3141d6df72acf9248118f49a5e48f9375be3180e9babf7/black-26.10.1-cp312-cp312-win_amd64.whl", hash = "sha256:4d9a90516db1d99c25dbb20cc0998e0e01531dd903466c7744e56d66f864220a", size = 1621579, upload-time = "2026-10-10T04:18:51.59Z" }, { url = "https://files.pythonhosted.org/packages/90/3a/8b6a44abf9648b087311003b91298226c1763172cd8d223ece5b7cf8f795/black-26.10.1-cp312-cp312-win_arm64.whl", hash = "sha256:2ffbc023a12d0c729408823b8f10514490bd0baa301d0d4e21a7240249f9507f", size = 1469662, upload-time = "2026-10-10T04:18:53.245Z" }, { url = "https://files.pythonhosted.org/packages/99/6b/bc0d39990bd7a71457d669639fa06acf6191bec28bafe8b8103f597cce76/black-26.10.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b6272cfd7e1e8e271f5b0e0207259fe2834687e5cb9b5f620b34a44db9754993", size = 2153812, upload-time = "2026-10-10T04:18:55.041Z" }, { url = "https://files.pythonhosted.org/packages/85/0e/cc83b88a6b1a51fc051aa4e24663918f0bf4fa9fca98552ee93a125a6780/black-26.10.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:978113a40223a6aaefc17364176a809a320e6b288683841427fff04c6d7b4130", size = 1980352, upload-time = "2026-10-10T04:18:56.849Z" }, { url = "https://files.pythonhosted.org/packages/7b/0f/4dd24ea0ebadbe05e293c3ea7fbebd4389f50508f714640c9b309a0849d2/black-26.10.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03c0ddd93bb392e71209903a691767eb366fe1a76deb9509ccbaae9e1f14bb52", size = 2055268, upload-time = "2026-10-10T04:18:58.974Z" }, { url = "https://files.pythonhosted.org/packages/a2/06/221f8e81891ecf5e4df9c258a11037997f1f218533ffe4fa1dc4db37f19c/black-26.10.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6dba8138cdc99061ef07b958ac082d2aa057b6961d1936f9717c350f02bab5f", size = 1625938, upload-time = "2026-10-10T04:19:00.65Z" }, { url = "https://files.pythonhosted.org/packages/2b/3b/763d2dd073fc1e2cbf0fe5e584fe96913bf0d14cd725d1116f4b38453adf/black-26.10.1-cp313-cp313-win_arm64.whl", hash = "sha256:d42dd2fac7c342ae67e64ee99c9532e20b2a84e92c79ed3317fa2ef54c801d93", size = 1470752, upload-time = "2026-10-10T04:19:03.009Z" }, { url = "https://files.pythonhosted.org/packages/76/7b/e8d275b23f3023881c0b0d8ce3bbda0d100c79bdef87fb6518bf9e0c5040/black-26.10.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:8375962579d537364cc0efa19b1474481915d3a793f9fc0774901814c5e5b5f4", size = 2147731, upload-time = "2026-10-10T04:19:04.643Z" }, { url = "https://files.pythonhosted.org/packages/ee/62/e44f86ee5fc7b893ee0ee935a79c785932da7758d2c339749d90a20ef213/black-26.10.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d8b3a9074a680b3c5749633714e9ae3992a1e5a23343a97ad61cd9b119b444d2", size = 1981714, upload-time = "2026-10-10T04:19:06.287Z" }, { url = "https://files.pythonhosted.org/packages/89/2f/e12ca76edfcd037fe9e7e7635d9468d1d51255cf25a01b2f8f97a38f147d/black-26.10.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:289282aa2e09d3162312a3be1788ff21b08e9ea9cc4a81e656024728b32428fb", size = 2047807, upload-time = "2026-10-10T04:19:08.193Z" }, { url = "https://files.pythonhosted.org/packages/c3/af/676d3c5cbb2ab0f17c8459b3ae8ea15ebdcc5a2f1c1c586228ec85b8897f/black-26.10.1-cp314-cp314-win_amd64.whl", hash = "sha256:5cd88fd7b444ca51f3fc883b6f6657ea53a258b0b2eef6d9f2dfcfa17ce0e27b", size = 1652537, upload-time = "2026-10-10T04:19:09.817Z" }, { url = "https://files.pythonhosted.org/packages/ad/7b/860022d369fdd5fe14280b1e7109f58753c371ff91ce2311d68a056e488a/black-26.10.1-cp314-cp314-win_arm64.whl", hash = "sha256:2520037aa62f8a1454d0811b8f5c88b444445b03a4bfba480d8d220893b64c34", size = 1504226, upload-time = "2026-10-10T04:19:11.367Z" }, { url = "https://files.pythonhosted.org/packages/b1/28/9dd29175c1db777e6189e2a0bb5f37101adf19c3e509b236c8920f778d0b/black-26.10.1-py3-none-any.whl", hash = "sha256:28842f9a8207cc1df6eb983a35a14c5a0dfcd603d214fe82d84bef552afd2e3a", size = 239267, upload-time = "2026-10-10T04:13:38.808Z" }, ] [[package]] name = "certifi" version = "2026.7.22" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, ] [[package]] name = "charset-normalizer" version = "3.5.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e5/3f/143b048436775b0f76ac3eec145c019e8173ccc2885c8f20319b996d5e83/charset_normalizer-3.5.1.tar.gz", hash = "sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3", size = 171764, upload-time = "2026-08-15T08:20:44.807Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/6a/b6/034f6802e9c3f6418966cfabb7db8c9252cc2429c5098f41cc43af804149/charset_normalizer-3.5.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:eda059b6bc8bc0812d626fd91a7ce01bf583df0a61296eff390fd94141a34e30", size = 363585, upload-time = "2026-08-15T08:16:46.646Z" }, { url = "https://files.pythonhosted.org/packages/d5/fa/6a7e2a7c4b5451912b8c417732df79574354443592a88d616de03da66ae5/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa2bb0b37202dca27175591f761108b5d34096ade1191ffe4808bdf6b1571488", size = 251189, upload-time = "2026-08-15T08:16:48.287Z" }, { url = "https://files.pythonhosted.org/packages/a4/c8/ab42b07cfd82e919f427fcfaa7c41abae8242833ad1aad66d42bae40b669/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b2b1b3fa5670c127b246df1d0c059defd41f689a868a3b9d79df9b1cac42d22", size = 239724, upload-time = "2026-08-15T08:16:49.67Z" }, { url = "https://files.pythonhosted.org/packages/e7/80/b9348b5d3041209f98b4cdad7655766369233f1d533f4f4f7558e9717bec/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6e5e4d73d588ca5ed09df1b7dcd1b203d1df3c542e3f50d126c947d432b10731", size = 280078, upload-time = "2026-08-15T08:16:51.228Z" }, { url = "https://files.pythonhosted.org/packages/82/38/083a24028304bc85bb9e376fed801178423dcbb67495f73b6ea0624e1894/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b54e7e13267d49ffbfe68e25b3cbd774dab38fa37238f71265e91b36146eb21c", size = 276650, upload-time = "2026-08-15T08:16:52.625Z" }, { url = "https://files.pythonhosted.org/packages/0d/35/731ac04aa0a097fc1c97f0994c375bdb230c6c96619db794208fe664e9ce/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c7b742bf31c88566b4bb6335a7f393bb322e580b6bb98df7bd0c25e6e3519ce8", size = 262325, upload-time = "2026-08-15T08:16:54.085Z" }, { url = "https://files.pythonhosted.org/packages/f5/28/c2028e7021fb89c6e56868ed0e387b8e9aa811abdd2ab3208d6578d2c930/charset_normalizer-3.5.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6ba32c4d2abf1d2fe7cf27d280f4cca5664233b0f885549c7761719eb977f486", size = 261140, upload-time = "2026-08-15T08:16:55.604Z" }, { url = "https://files.pythonhosted.org/packages/28/f0/0c0ceec6d98b7daa62e361e418135d59685811d79ba11529aad5cdf15e84/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0722590aabf9dc6a6c0343d523c05458fa2b5047dbe6302fd526bb570600753f", size = 252791, upload-time = "2026-08-15T08:16:57.103Z" }, { url = "https://files.pythonhosted.org/packages/f0/3e/48f4cd187b1c33189d86039e9cbe4f92c05454175504b44ff81806d4d1bf/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:aa1099b956fb795e686d073568f6dc002a0bb89765ea6d5b055dd7d9bf1b116c", size = 240730, upload-time = "2026-08-15T08:16:58.418Z" }, { url = "https://files.pythonhosted.org/packages/42/85/f9e22af69af67c54cce42be9455d9c81294f918b4ccc454db01f66efcac2/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:bd6c173f04743d483881bffa1478d5a4624475b8cd1d2194956a75548e191c18", size = 280791, upload-time = "2026-08-15T08:16:59.918Z" }, { url = "https://files.pythonhosted.org/packages/fd/4c/9044135f42127630b6fa742feb51256353f6ab87a78f2fdd1de3de955a7f/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:f298e218441525d3794428b4c8b8fb8662c6d3ea79925d4807ee6b9a96a3bca5", size = 259598, upload-time = "2026-08-15T08:17:01.421Z" }, { url = "https://files.pythonhosted.org/packages/ba/ed/1dd7cfebb4e75812934c49ca3b79757d11948053f7937ab7070c151f3c55/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:6e2912d4babbc65196ac13c2f53468dc57fb8b9c25ef913e8c59ddf7c6dc0e1b", size = 278217, upload-time = "2026-08-15T08:17:02.782Z" }, { url = "https://files.pythonhosted.org/packages/bf/eb/239c84503cc9e3ba6eb34686a24bc66e84f3924efdd7e38e751a19f6bc10/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3d27167433c0d5f18dc850f07d0b3816221984fecdc405d6c157a6f0b8f8e9e6", size = 263417, upload-time = "2026-08-15T08:17:04.216Z" }, { url = "https://files.pythonhosted.org/packages/37/ab/4e4510e1e288478e2c8333131d1c1382382ba8cd2165053c79e39d1da961/charset_normalizer-3.5.1-cp311-cp311-win32.whl", hash = "sha256:ac00177c4831ffa650f8609e4bdddd5fe09c03b1c0c47acece7e6ea20421598b", size = 181774, upload-time = "2026-08-15T08:17:05.58Z" }, { url = "https://files.pythonhosted.org/packages/e3/57/32f0ccea59e8612057c61d6fd22ef2cb63cca93c9fe594094919696ac170/charset_normalizer-3.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:f9b1e28d0e8dbfa858abdba91d6b547beaf2df1a59bec6da6faae7b96a4991a9", size = 206653, upload-time = "2026-08-15T08:17:07.075Z" }, { url = "https://files.pythonhosted.org/packages/17/d4/b65c433fc521e58b5f54293982a5e51c05cb5f2dd3f1c7a6acb65b75324e/charset_normalizer-3.5.1-cp311-cp311-win_arm64.whl", hash = "sha256:ae31a1a1db2ee6cc2942fccaf695c934bc7f3db9f2133a3fef1f367cf1a4ab10", size = 185630, upload-time = "2026-08-15T08:17:08.502Z" }, { url = "https://files.pythonhosted.org/packages/30/27/78873dc8b6a56357517b74b6bb9568b80450e7bb4f6ef7e3fa9d22aa0bd7/charset_normalizer-3.5.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:5b6d1386bf0096d26d3a863dc0a487a5b4eb9aa93cf5ba69683d29dde6b9d60f", size = 344456, upload-time = "2026-08-15T08:17:10.072Z" }, { url = "https://files.pythonhosted.org/packages/9a/4c/be49ada26b1f0232d57aa89bbebf997a5cc2332a5616b6eca26ff680044d/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4582c27e8c889d64811987b5967fbd3ae0c823fe1fd933b543d55ac20bb475fa", size = 238530, upload-time = "2026-08-15T08:17:11.563Z" }, { url = "https://files.pythonhosted.org/packages/76/84/6f1290fa07ae6978d3960caa3eb1b8019bf9284ab7c2297b00c099ef4250/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1d1c7a53a6c2103925cdd6d7229f8c567379f211c869793df679f2e9f738c369", size = 230200, upload-time = "2026-08-15T08:17:12.919Z" }, { url = "https://files.pythonhosted.org/packages/e7/a0/47b18adeed31c8f16ba9700f32c1b18594cfa09f47eb672a488c273c22bf/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e6621fb2a4988d6e53eedc455e5903e2679f3967b8acb3d639f1b63c14a2e893", size = 262222, upload-time = "2026-08-15T08:17:14.571Z" }, { url = "https://files.pythonhosted.org/packages/38/fe/341861ac118dae06f3ec0eb487488af52128f2ef2faf0b11003944d22259/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7c0c10730342b0c9b35dd1d619beb8214e520bd96a1f870f452680b238aab3e0", size = 258951, upload-time = "2026-08-15T08:17:16.158Z" }, { url = "https://files.pythonhosted.org/packages/6f/89/bb5108dc6c3651dca963f2b0a3ba19bbcb370c94e1b6d3e0e844a58e6dca/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08", size = 248801, upload-time = "2026-08-15T08:17:17.683Z" }, { url = "https://files.pythonhosted.org/packages/b1/ba/ef83ae3aca816393decfa3530976f38a79812d707b80b580ac33b83f9877/charset_normalizer-3.5.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f9f8405c2c758532c74fed975dbee57be1f31a6e865c031870c79a6ed3212ada", size = 244070, upload-time = "2026-08-15T08:17:19.191Z" }, { url = "https://files.pythonhosted.org/packages/f6/0b/c5292a2462d69b7378ea89793bbb5b2b6fcf6f7dd6d1667f9619094ad553/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:96fef3e886d6a9874b14f27fc193fbdc69d5d8035783d86aa4e1cea594e695f9", size = 240110, upload-time = "2026-08-15T08:17:20.547Z" }, { url = "https://files.pythonhosted.org/packages/46/22/111e5be3b740d5c2a5bfcedb3d237b6591e5c2e82ae9d6ffcb121fe0909c/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5d8531a6569d025f68e2321e7638fb7978f23db58e5f69f56913837aae03816e", size = 232836, upload-time = "2026-08-15T08:17:21.895Z" }, { url = "https://files.pythonhosted.org/packages/f9/d2/d2aad6fe0dbb44b194bf3becb60f5a0ac48446ade999a47fe7bb41eb09a7/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:aae2ee51122d3ae968a3837d97dc24a0aeebb0dea23694422cd172bd30017cd6", size = 262712, upload-time = "2026-08-15T08:17:23.727Z" }, { url = "https://files.pythonhosted.org/packages/35/5a/337e4663a5eae6de99db940ee8066d4145caafb61327db62deda15313cce/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7235dc28fc6dd9d832ac7c7bce95367dedb85929f17368a0c2bee1e080b9acbf", size = 242977, upload-time = "2026-08-15T08:17:25.157Z" }, { url = "https://files.pythonhosted.org/packages/ca/85/f82f8a92e31c7519410e2e1afdc630f28ec47490ce2c09a11c1a43cbb459/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4abdc5f9ad448c1ecbfae2974b820535d6bc6e7eef63babbab3d81cf46968c71", size = 260207, upload-time = "2026-08-15T08:17:26.602Z" }, { url = "https://files.pythonhosted.org/packages/b7/52/643d11ffd60e9ac2fd1fb87e167a19285b9eefeff4a40e63c87cbfbeab36/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ba501e667c17d8411f98e67a022d9604ef179aff0e459b7e292c796837c13573", size = 250562, upload-time = "2026-08-15T08:17:27.971Z" }, { url = "https://files.pythonhosted.org/packages/62/16/46556278c2168d12df9da7fede5dc6fc70e60301b26a82bbeec238c9cfe3/charset_normalizer-3.5.1-cp312-cp312-win32.whl", hash = "sha256:cfa1c0cc3a8f9f53f1243a5a99ac36fd003880199383b37672e86ddda9cb07e2", size = 178507, upload-time = "2026-08-15T08:17:29.277Z" }, { url = "https://files.pythonhosted.org/packages/9d/7a/4c6c298171e6b3e745633180ff59350fc0ca0db1ffd28df1e369e0579f71/charset_normalizer-3.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:3617ac3cfd8b9888f145ad89dd6e692285834b0201c6074a5eeaad3fd4d668c2", size = 200551, upload-time = "2026-08-15T08:17:30.668Z" }, { url = "https://files.pythonhosted.org/packages/cd/d7/eb95a042f0dd22e304b0b6472b154f3546a1a039a9ee89ccb2a7f61591fc/charset_normalizer-3.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:88e85ab89cb822c1e635f51d6d32e488f94e002e70e2f492bdb8b945543f345a", size = 180700, upload-time = "2026-08-15T08:17:32.028Z" }, { url = "https://files.pythonhosted.org/packages/bc/61/2cb6ad133dbbb449fa2d37ccae973232f4827e799af258d15e589a3d1e9e/charset_normalizer-3.5.1-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:4f298bdadb8f0b9e5672877f647d1be9373ef5320c9e2f049795e26cad28b6a9", size = 211584, upload-time = "2026-08-15T08:17:33.597Z" }, { url = "https://files.pythonhosted.org/packages/18/57/a305c968be1ca13f3dd1b32f445877e97addf55d80b65c7cb35fac82b777/charset_normalizer-3.5.1-cp313-cp313-android_24_x86_64.whl", hash = "sha256:88ca277405c2d3b71c4e1c2ee0e7966e807bcba86a69d11e19ba199d18ae4491", size = 223359, upload-time = "2026-08-15T08:17:35.022Z" }, { url = "https://files.pythonhosted.org/packages/09/0a/d3646670292ce8d8f8cc11ac067d44885e697a5591f57a9221128da5e7b3/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9362dd90aa7dab48c0054a21187791ccf05473f7dba5d92b8033ae62164675e7", size = 194464, upload-time = "2026-08-15T08:17:36.452Z" }, { url = "https://files.pythonhosted.org/packages/de/93/d51ec556e01042fed6f993ea859311bc7917b466684182fbbceb6ca24762/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:977cdbd483a9cff38179bea4fd754289a6f2195c7abd414aba85410b3e66cc5e", size = 197676, upload-time = "2026-08-15T08:17:37.819Z" }, { url = "https://files.pythonhosted.org/packages/a4/a0/562247944386f7d4ef94467e84876600cc1e0f1b93239aaa9213d2bc3cbd/charset_normalizer-3.5.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e90251c0c7bdd54a100a0dce3c07b7e637278c93af29dbf78ebb89a58c4bac7d", size = 340473, upload-time = "2026-08-15T08:17:39.303Z" }, { url = "https://files.pythonhosted.org/packages/31/e7/1d994be1b93d41e9502b8b0460eaa88a1dd8df335df415db87d6c3e91ab2/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94d78ecec2605a8d0398b0f365d5f12a63248438516f5dac536a5eff7337df4a", size = 240156, upload-time = "2026-08-15T08:17:40.66Z" }, { url = "https://files.pythonhosted.org/packages/09/53/27923ce5cc6cbccb832037b27dca98882d9c53e9b69e866bbbef4aae7fc8/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d59b75732e9b6f27388e10c14b0259cc5f2e48c78627d185e6a177b58ad3cffe", size = 228246, upload-time = "2026-08-15T08:17:42.003Z" }, { url = "https://files.pythonhosted.org/packages/ce/48/5a97e84d63af1d55c07439cb80e56d99a8efb4295700eb4e18c0d1615d2c/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0d929fc574b4d6fd9e7c0f5c2ede8716a41911923aa7fa5fce38e0818aa4a1ac", size = 263660, upload-time = "2026-08-15T08:17:43.627Z" }, { url = "https://files.pythonhosted.org/packages/7a/c2/071575791dcc88316c0a9a65ce38897a82e4cfe4a325f0f7fe1b1ac47bcf/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:394fea06235c8543390050ed5f529187074b029fb027213f6c46ac11ab5d950e", size = 260354, upload-time = "2026-08-15T08:17:45.094Z" }, { url = "https://files.pythonhosted.org/packages/fb/af/63240b0c0248c075c2535a1f1bd992821d8251b9f173abc13329661d09e4/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62b55f6722735a6c472f88361cde6640608773d9443cebdbb51abf436a1fcdd3", size = 250638, upload-time = "2026-08-15T08:17:46.496Z" }, { url = "https://files.pythonhosted.org/packages/4d/66/70dfad64f15be09c15ccfee81330a7e515895dbe296dd23114e9a231268a/charset_normalizer-3.5.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fa48b1b63d639f9483e0633e092f5851e2348c352f1f9bb6c8182f87884ef876", size = 244583, upload-time = "2026-08-15T08:17:47.963Z" }, { url = "https://files.pythonhosted.org/packages/c0/24/ef36367d38b9ddd4bccbf72888c342e8de1f5ae506fa0b2dcf970e2732a1/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c71fb0d56c920c269cd3e2e3fe7c610e3f1fdb21a6ce60efa6430ff63676cea6", size = 242038, upload-time = "2026-08-15T08:17:49.481Z" }, { url = "https://files.pythonhosted.org/packages/db/ab/55e683ba0fff2e43adafc10daa3001eac90fdaa419a97227d5a7067eedde/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:485a0d363cafefcd2538a73c7c838daa2035f09b2c9f9b5e3133f80c6aeb84c2", size = 233677, upload-time = "2026-08-15T08:17:50.845Z" }, { url = "https://files.pythonhosted.org/packages/bd/67/0f40eaf8d1b6e7cf15e82382a2965efaca787fc1c2794b7021d37aaf5036/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0ea61a470e070686aa30892fed79e297d2c8d0ab46b8bcdf027d38c51da591", size = 264491, upload-time = "2026-08-15T08:17:52.61Z" }, { url = "https://files.pythonhosted.org/packages/5c/64/12b4c2a11ee8df4fcc518c78b0d93e3a92bd3d5253d1617ce74ff0e8c7ef/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:90b7481fb62fbe172c558bc6fd1c4c98d82004a54a7551f20e11ac9bf0b8708c", size = 245196, upload-time = "2026-08-15T08:17:54.023Z" }, { url = "https://files.pythonhosted.org/packages/37/2e/651d910af6d0fba325eee1cda37ec5443462ed25360e666c144166eb6091/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:35fe081843b35aad20ffeccec3eeffbe637b15d14f3fb22cc1b59cd8ec17e93c", size = 261660, upload-time = "2026-08-15T08:17:55.491Z" }, { url = "https://files.pythonhosted.org/packages/90/c6/b09e05e6db7f64338e0dc067c79577b1138da86c1e38369096851d96be88/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f", size = 252618, upload-time = "2026-08-15T08:17:57.025Z" }, { url = "https://files.pythonhosted.org/packages/76/4e/362d4f9fdcdf5556fb2aa3ce7d4a58ebce03ed1ff03aa1d9aca8d02f13f3/charset_normalizer-3.5.1-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:9d9a0dc7cbe9bec24c3f767c9122c41fe5a1bc43f47cd099d00d393e09769de4", size = 140362, upload-time = "2026-08-15T08:17:58.425Z" }, { url = "https://files.pythonhosted.org/packages/b4/d4/703be739b26acce318bd29eb3b25b7209e1b1f527f9eae3d1f1f01fdde2b/charset_normalizer-3.5.1-cp313-cp313-win32.whl", hash = "sha256:d63600d620ad0064c3a748b950ac5ea38a80190e5498532efefa4b7b3f1da1f3", size = 177755, upload-time = "2026-08-15T08:18:00.037Z" }, { url = "https://files.pythonhosted.org/packages/8a/33/56d97ade41c8db611e727168c52ae46c9224c362ec28d4b65d7e9869e8da/charset_normalizer-3.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:aea996a6aba25260827c9ea511d1addfde2da9eb686ac961838509086188b7e6", size = 199295, upload-time = "2026-08-15T08:18:01.506Z" }, { url = "https://files.pythonhosted.org/packages/5b/75/5b20dd1e6573a01a08158fe104104fa2c8abf941745596954185726cd46c/charset_normalizer-3.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0", size = 179856, upload-time = "2026-08-15T08:18:02.929Z" }, { url = "https://files.pythonhosted.org/packages/29/cd/2b812ce5e888f1ce69a5350281e58aab07ae64a958ecae8912f30865718e/charset_normalizer-3.5.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:774d157f112367ff4abd29019f38f023c24e00e56edc7829c20e358a5a913ad8", size = 212318, upload-time = "2026-08-15T08:18:04.403Z" }, { url = "https://files.pythonhosted.org/packages/9e/4a/a6ee107430768a5334e6d63f31f148a04a1a491ef161a1ac9415a73f2fa8/charset_normalizer-3.5.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:26422d45fd13551cf564c58932f7d72b4f58b93b0fcf18c35ba6be12b46bb102", size = 224897, upload-time = "2026-08-15T08:18:05.997Z" }, { url = "https://files.pythonhosted.org/packages/c3/d9/35ae3f64f29d0179c35c3baefe575904df2913dde519129c7f75995a2b1d/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:09a7bba9f739468c8e78c36a75c33768e53cb1959fc638f510454c14683f00d5", size = 194848, upload-time = "2026-08-15T08:18:07.397Z" }, { url = "https://files.pythonhosted.org/packages/74/76/f2fc7380f056cc273a53af37f50d08ad54b2c59f61078f31432edcf1c2bd/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4c9548dc78002099910abaebc0a72ac58b7d30931869e0351c09b507dff4ece3", size = 198163, upload-time = "2026-08-15T08:18:08.989Z" }, { url = "https://files.pythonhosted.org/packages/e9/40/095ce62fa078483cccc1fa2b36e6bc9580b85422a20ee9f925341c50e44f/charset_normalizer-3.5.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c428c6c31eb5f4277d7f8eccaf767fbd548ddd5ce3c8b4f4cbbfab3d96b5904c", size = 341823, upload-time = "2026-08-15T08:18:10.458Z" }, { url = "https://files.pythonhosted.org/packages/f1/5a/0e58b1c04a1596e0256f407274a92d5fb2ee21324409d1fab1da48a65b5b/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f06b7eae9dbe77fe1d644ca244dad508de8d302870a43f3c559b521270938a0", size = 242458, upload-time = "2026-08-15T08:18:11.989Z" }, { url = "https://files.pythonhosted.org/packages/22/95/b4618ce912e6db0b1aae89ba788e38e8a7eba0f3025cc66e8c0699f977b2/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b7430cf5728e68f6c462254009a6ef4086e1bea43cf2f57aa9c55fb4f50ff96", size = 226717, upload-time = "2026-08-15T08:18:13.401Z" }, { url = "https://files.pythonhosted.org/packages/8a/76/c681192bbda3d55356db5dadd64381d5202b37c6b598fcda5282e88b5d3d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab743e9bc90c1f73552ec33e10e3331315acd2c397b36065b591b0181de533cc", size = 266111, upload-time = "2026-08-15T08:18:14.961Z" }, { url = "https://files.pythonhosted.org/packages/88/be/55127bfca72c0cff6c022488d140d7c5b04c771e3b72e9bdb4836d54979d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6f7deae3feb4edfa2efaf7c574fe88cbf055038a6abdb40188e4fff66d5699f", size = 263128, upload-time = "2026-08-15T08:18:16.515Z" }, { url = "https://files.pythonhosted.org/packages/e0/91/39c3af510b0aa32bbda03374259200f28430febfd1bf5e511fe765282ce5/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15f024313246a4ed976c60f440bb8d257815513a681d212ff74fd46f7d715a90", size = 251240, upload-time = "2026-08-15T08:18:18.127Z" }, { url = "https://files.pythonhosted.org/packages/1c/a5/cbe418bbc6ecdfc3e05a0116002897c4b403a5e838d697e64c78e9f0190d/charset_normalizer-3.5.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:823f82903d189af463d7df250ef1f7f696f3cee08cc8d91deb565e8d425f6506", size = 245282, upload-time = "2026-08-15T08:18:19.625Z" }, { url = "https://files.pythonhosted.org/packages/cc/a4/689bb42e8e7cd492f3cb64907c6bc00ad247ec9a3628cd3f8eed126e8ae1/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:01e93745f7f219b703b60ba7afead36cfc4242782be5af484673fc500df12da5", size = 244597, upload-time = "2026-08-15T08:18:21.121Z" }, { url = "https://files.pythonhosted.org/packages/c1/ce/9962938e179cf9f699d3f1e7b3114b5d7642dee6a893745229f9dd04f274/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:329fc3ccb63ad22d867d84c2adea759a64079a37ba4a343433b02c7a2816871e", size = 231376, upload-time = "2026-08-15T08:18:22.57Z" }, { url = "https://files.pythonhosted.org/packages/85/54/46000450ada53bd9eac5429a2c8c54cd2d9b39c0c255f229aea9af0948a5/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:bb57753e36e4855b8ca375069482250a6246372331a3e4f3407eaebb007443f5", size = 266715, upload-time = "2026-08-15T08:18:24.235Z" }, { url = "https://files.pythonhosted.org/packages/3d/bb/618749d70f792b44252a777bf89bfb86823b9bbc1ea13fe8ce759b07f38a/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fce8cbd4997efeb450bd298b54f755dcdff18d496f7a5ddbb4867c6d7c88fdc3", size = 245848, upload-time = "2026-08-15T08:18:25.726Z" }, { url = "https://files.pythonhosted.org/packages/7e/3f/ffb64458527c7668031d5eb095d978de561958dc9f5b53f8e488a533e603/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6c9cdde8becb25a7fde49924511aa2644d6f8081cc8df8e9452724303348d8e3", size = 264521, upload-time = "2026-08-15T08:18:27.193Z" }, { url = "https://files.pythonhosted.org/packages/4f/ab/74a55fd803916a35ac461daf002708191aac19b546b80dc8cabfedc63d98/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9ac4444d8d4fd4c4bd08bf451ed3167aa9e7ec6cdb41b648794f1d1103652e36", size = 253054, upload-time = "2026-08-15T08:18:28.568Z" }, { url = "https://files.pythonhosted.org/packages/a0/2a/6a9034b7d3c60b17499afb482df5878bf9fa20b50cc3887d5ef017a833db/charset_normalizer-3.5.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:f03ac127268b43ef4fe9e6ab6794a6794b49485a0cc0c1db79876d2f33f75bc7", size = 140580, upload-time = "2026-08-15T08:18:30.214Z" }, { url = "https://files.pythonhosted.org/packages/f3/46/1d362e1a00d035d66b9869e1281eee115907f7e390a16a07824ab5737360/charset_normalizer-3.5.1-cp314-cp314-win32.whl", hash = "sha256:1f5883d77fd409a261abb5dc8ccbe335720d798b1de4abb3b1d47ccbbc76b53b", size = 180325, upload-time = "2026-08-15T08:18:31.877Z" }, { url = "https://files.pythonhosted.org/packages/7a/7c/4938c329b6a9d446f6a59aa2092ff7118f274209b5ed0e26893d1d30a63c/charset_normalizer-3.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:c658c50ac0c98cd755a2dd50b7977d3bca7df401dcc47fbdfa87db53ef7d4e8b", size = 204175, upload-time = "2026-08-15T08:18:33.466Z" }, { url = "https://files.pythonhosted.org/packages/ac/33/eeb384dbd8dec570661354592f4f2e1b2fcc92585624d146a000caf53841/charset_normalizer-3.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:4bea7f8ebe90bbd7f0e4a2de42ca6924ba23e3e76418c408ff82f1d46fabd687", size = 184123, upload-time = "2026-08-15T08:18:34.913Z" }, { url = "https://files.pythonhosted.org/packages/1c/6c/c73fa9d5a85f6ab05395de61c5f6984e0a9ff40bb5ff888d46dff02526c6/charset_normalizer-3.5.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:fbc597639158fd7c14d55e808718848319540f51b0e6746e3eefa59723a4a348", size = 381682, upload-time = "2026-08-15T08:18:36.349Z" }, { url = "https://files.pythonhosted.org/packages/30/c7/63565f860921457feba93bae6c86fb7746deb4cffeed2f375cb845318146/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e71c909f353863b2b89c83de2ebed71ea6d0df8a6ef65a128193c5e650766bef", size = 240826, upload-time = "2026-08-15T08:18:37.887Z" }, { url = "https://files.pythonhosted.org/packages/06/ae/7ae8807410dfa33f8e6f1715740adeaafa8a816cc4cb33508f54b1f7c896/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7ac76cf9afd34929d76eb7fcb63be476a4853d8a96f0dcf2d0db68a0cbdf9885", size = 227861, upload-time = "2026-08-15T08:18:39.315Z" }, { url = "https://files.pythonhosted.org/packages/e9/a3/887c1642f0da26000b0e0652d91071113c0e72cea33952e225cf589f49a9/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3a370082ce34d0612f421e15fe011c53bb1feff21a26d06ad4fb244dab5a375", size = 260758, upload-time = "2026-08-15T08:18:40.88Z" }, { url = "https://files.pythonhosted.org/packages/3e/11/e6f5b9a3d0e55b0ef7505cd3765cdd48f22db89994c947b316f52f801fd8/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:256dd4d85d9e4dc595e2bc983c980e73f62ddeb3165c58b4c3dfe78c5c8548c1", size = 259950, upload-time = "2026-08-15T08:18:42.351Z" }, { url = "https://files.pythonhosted.org/packages/1b/ee/e4e10a94d51cd1ee638aa7e00b65399e6b2a4e8376ab6d2eac9f95586671/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58d4aa13a59c969dbfdf9e6a9560e242cbfd9e8a8f50c2747714df1a423adf65", size = 249329, upload-time = "2026-08-15T08:18:43.914Z" }, { url = "https://files.pythonhosted.org/packages/c4/25/d5f4198819e6059735a84e8d0bfb72dc33976da67b97adcd3fb5a5e07ec6/charset_normalizer-3.5.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0c6dfb5ca6723eeed15aa8e564a014d69fcb8812f94eef11fe3631e0508199f5", size = 243137, upload-time = "2026-08-15T08:18:45.368Z" }, { url = "https://files.pythonhosted.org/packages/a5/e9/e925ca7569cf9fb9701fd82503fee73eea5268fdb856bdd64947092d3daa/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c010f5581d9c612804cc59fcf7b524b707fbcb72828551237ab545bb5c7034af", size = 242820, upload-time = "2026-08-15T08:18:46.842Z" }, { url = "https://files.pythonhosted.org/packages/34/17/672c251a888ed2aebcdd2fe830ad0104e25ff83c43f5c4f9c15e9fc6853c/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:52ec005752a56ae79547a05c0139ca2501a0c866390b6115008456b9f0e7cde1", size = 230504, upload-time = "2026-08-15T08:18:48.353Z" }, { url = "https://files.pythonhosted.org/packages/3f/fc/f6a85abebd42ce4da2f1db0aa56cc6a0df1995e318b3875d14401b8381d1/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2bced4061f000f7187254a02ad3433ae17eaf991747ceea2f478422590a5bba9", size = 263087, upload-time = "2026-08-15T08:18:49.859Z" }, { url = "https://files.pythonhosted.org/packages/98/66/7c42677e739ba66746b297e2046918d793078094dc239e1e72768cffccc6/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:9eea3ab2597a5e65fe65296e2d6a84570845a6b55532d90333d740d48bbc850a", size = 243269, upload-time = "2026-08-15T08:18:51.601Z" }, { url = "https://files.pythonhosted.org/packages/de/d8/a50b79237f417af10f8c2a501ce8d1ca87829a22e69117891ca4ba20a69e/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:496846868fea80e479324862fa877f02411f2fd0f83b79ccee2607aa68b2a032", size = 258766, upload-time = "2026-08-15T08:18:53.23Z" }, { url = "https://files.pythonhosted.org/packages/2e/1d/0fc91aeaeb3c83b748f532399ce67cf84604b48297405d740000f7a9e786/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:85d5855daafc240cc045c026d7a15fd198a09b0fc8ff6f5ecbb5297b509cb11e", size = 250814, upload-time = "2026-08-15T08:18:54.768Z" }, { url = "https://files.pythonhosted.org/packages/ae/10/3d8c777cf9024615295aa1b808324ad5b4a77855869c00824bad74ffaf8a/charset_normalizer-3.5.1-cp314-cp314t-win32.whl", hash = "sha256:58d3e12c88e0950bca850ae1f7c256055c097639c2edb9eb123af9807d8b15e4", size = 191074, upload-time = "2026-08-15T08:18:56.305Z" }, { url = "https://files.pythonhosted.org/packages/4d/81/ae557d3c44d1a1d688696d60563413a0866a91b7ebc50f20df838be3d8c8/charset_normalizer-3.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:acaf604462bf330b0d07e7a07c1d6e4adac79e5fb13e9c5140590542cafacc00", size = 216476, upload-time = "2026-08-15T08:18:57.889Z" }, { url = "https://files.pythonhosted.org/packages/27/e9/61c01fb8b804692569c036b3fc50495814502dcf13a60649c6055390b02c/charset_normalizer-3.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f", size = 194115, upload-time = "2026-08-15T08:18:59.418Z" }, { url = "https://files.pythonhosted.org/packages/4a/4e/8544831ef59d8f27ce92c80871380fdacc8076a8a56ed62f82e54f991333/charset_normalizer-3.5.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:9085f87b0e38a2b92b8923059b4e8789fe40d9279712d15dcc670048d77079af", size = 342048, upload-time = "2026-08-15T08:19:01.054Z" }, { url = "https://files.pythonhosted.org/packages/7f/a6/e3b46852424246065355644f4fb6dbccc0239a42a2eee27ecfc8957f0bcd/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2679de311c7946dde5d3b6f44941844133ff5c7cb86099c0061ab1e8901c20a8", size = 242997, upload-time = "2026-08-15T08:19:02.492Z" }, { url = "https://files.pythonhosted.org/packages/03/3b/0cc9a26777334ab2f2e3089b948bbf4e4fe72ea70b897715ef6415043ec8/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:baf3775a2635e5a11fbd5e4e64ee69c7e86875d224a5c72aca4c141064589a90", size = 237014, upload-time = "2026-08-15T08:19:03.943Z" }, { url = "https://files.pythonhosted.org/packages/8c/c2/027335f0aa337a2a2e121bac1ad88c4f02ba6053ea0926802784f3db11af/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ac8c94b6539074e0f40899301273ac8402b9b3e01c7b7ba269ff30340aaaf20", size = 266174, upload-time = "2026-08-15T08:19:05.598Z" }, { url = "https://files.pythonhosted.org/packages/86/d3/e367787febe4e74769dec0f406f2c3c8d1b955fce5aee1fd0f94e8367a45/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe532b3c966d1fb794e0698e4589d0444017ae77fc0b31edea13c0e35bcc449", size = 263361, upload-time = "2026-08-15T08:19:07.251Z" }, { url = "https://files.pythonhosted.org/packages/af/3d/391b193eb9f3e84b02f9314088c386debdc0debee843535aaea2e2c6715d/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5c84bec0ab5ae0c64bfe73a7d2adcb5ce73b467523fc27fd6a28ab2aa6cbe35a", size = 252143, upload-time = "2026-08-15T08:19:08.816Z" }, { url = "https://files.pythonhosted.org/packages/2e/57/de221f1745a90d418199761967e2776bfe2c275a1194220985e8c1d37833/charset_normalizer-3.5.1-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:854066be00447fa8de2ccbbe893e2ffc4b123ef16d897af794c1e18bd4a714b0", size = 252086, upload-time = "2026-08-15T08:19:10.255Z" }, { url = "https://files.pythonhosted.org/packages/c8/e3/d119f86a01f9331e8186175f24873b1d74a7ee9e2e4b4d68f9947dae5afd/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:21b82d8082f6f5e7f456ef0bd16323d08de1266efbfeb476e64b2a91d1471a4e", size = 245231, upload-time = "2026-08-15T08:19:11.807Z" }, { url = "https://files.pythonhosted.org/packages/26/de/d8e48c135ae480879539cdb179c8d3b50c7879497d75dd899b5763b69cee/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:838648accb3a7fd9803fd45c87bce8509648eb0c11bc34e216141300977244f2", size = 241546, upload-time = "2026-08-15T08:19:13.416Z" }, { url = "https://files.pythonhosted.org/packages/67/c4/217755fd1abc50d326c252922cd642002758095a81ff45010337b8b3ef65/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:195ce897c6153c0700078142cf8efe3e6454ca4cf4357499e4078dfd83396626", size = 267033, upload-time = "2026-08-15T08:19:14.981Z" }, { url = "https://files.pythonhosted.org/packages/b8/d7/34d8e404e358d2adcc5a228c2134643af00104c8fb0bf525f3688d756f05/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:978eab16f55b4ab2c2a745be9a0a840bf8f09a7f227d9c76eb30214d078865a5", size = 252045, upload-time = "2026-08-15T08:19:16.618Z" }, { url = "https://files.pythonhosted.org/packages/5e/fa/40414471acf0aa0692ca77305aa00e434fcd8288f0941c93c30e9a5f8f2f/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:cc0329df4caaceb950d2f580b5ac716a377f7059624a0bafaeaf8a218c6ed774", size = 264866, upload-time = "2026-08-15T08:19:18.101Z" }, { url = "https://files.pythonhosted.org/packages/32/90/fcc850bae791abd2e0c041847f13e270aa08692a79f3e00de6d2dce1cb50/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:687c9ca3035544b113bea2055e180af96fb63c0c476e22a9180f51925186e7b7", size = 253932, upload-time = "2026-08-15T08:19:19.734Z" }, { url = "https://files.pythonhosted.org/packages/af/af/53afe99068b3c10b4cbae592a52ef72a7c92c0188440e83ee3a078fd8f75/charset_normalizer-3.5.1-cp315-cp315-win32.whl", hash = "sha256:706bfd38730a5ac7a365793269a00f4e988178cec121391f4248d84ad8c972e9", size = 180320, upload-time = "2026-08-15T08:19:21.37Z" }, { url = "https://files.pythonhosted.org/packages/c9/bc/f46a132041b29e4a8779ed712d3df1bf112e94ca8de58b66d7ec2c0cf8b9/charset_normalizer-3.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:92caef967d287a407085d61176fce4012b1dd62daed4eb6d5ceb26d3d2538712", size = 204174, upload-time = "2026-08-15T08:19:23.088Z" }, { url = "https://files.pythonhosted.org/packages/a1/5d/9ed554480eda8e447b673648628fdc29574d23dbad01fe11837adedd1cae/charset_normalizer-3.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:5fc45d653ea8c9a20479167e11d4a0f8cb2fa3470737ab6f9c827532313187b7", size = 184126, upload-time = "2026-08-15T08:19:24.471Z" }, { url = "https://files.pythonhosted.org/packages/3b/32/9b8929bf384061ee1fe5d9c27c6f9776d3d824039ad4e14c88ec00c7808e/charset_normalizer-3.5.1-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:59171c6e45bf07d0d5cab3b0bf81d945035530f6873398b3b531c31184d46663", size = 381441, upload-time = "2026-08-15T08:19:26.038Z" }, { url = "https://files.pythonhosted.org/packages/96/10/e9aa7923d3ddac652c99a1c5f7be494e737e151566a44abe018daf757f2c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9dbdd9205662134957cf0c324f639bdc5031c0ca056e2369e238db75187c0f11", size = 241742, upload-time = "2026-08-15T08:19:27.532Z" }, { url = "https://files.pythonhosted.org/packages/28/53/a2d249ebddf47b889a100c0bdcb61a2f9dbb8bc24ef325cc062e4f476877/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4b018dc5a0eee4676e38fe84a47a427816c590b93b55d9025274ec4d6ffc2dc", size = 235298, upload-time = "2026-08-15T08:19:29.274Z" }, { url = "https://files.pythonhosted.org/packages/7d/07/469f78af590f7d5cd48e20d8dbfa3d66deeff9ba37768c04d886b5afd45c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ced3fdd71aaa83ce593746c2edb42b7a59cb4c19c8b5c407781c72e493aae55a", size = 262500, upload-time = "2026-08-15T08:19:30.955Z" }, { url = "https://files.pythonhosted.org/packages/55/66/3bb56a47f7dcba014055b1a1d33c6f08bbe9c1e74dba154cfa25f90ae885/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:19a3dd5aa73cef1c99687c4fc57db016a9c17104ae1185da88ba566a5d3bebe4", size = 258888, upload-time = "2026-08-15T08:19:32.458Z" }, { url = "https://files.pythonhosted.org/packages/ff/c1/2adc2800903fb013210349313b710a5376856578d9e33e6b9a1d8b36714a/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cc5d36d96478aa9c60654bd932525bf32964c62a7281eafdf16d85003a8d6004", size = 250243, upload-time = "2026-08-15T08:19:33.94Z" }, { url = "https://files.pythonhosted.org/packages/95/b5/a18d0dd1157ab655cc2cb14a545f4a4784bbad70ab3502412e36097502d9/charset_normalizer-3.5.1-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:04368edf83514385ffc3e1cfd4546e595f4f1272dd23ba437a93a9cc3741d47b", size = 249871, upload-time = "2026-08-15T08:19:35.413Z" }, { url = "https://files.pythonhosted.org/packages/ad/c3/525f508cd1e58d0450ac55ed40ac75bc3a97482c59def5278456a5fbf03c/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:9b5db6052055d34d41230fb78d7c439c23dc536a9896f6cb039e8dd92cfc1263", size = 243580, upload-time = "2026-08-15T08:19:36.886Z" }, { url = "https://files.pythonhosted.org/packages/7c/c1/49a91fe7e97c8140094ca5c64161ab623a70d9f636bf834eace14048acb5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:252d099029bcbea642f2a06c4ed5046bdf8b5a8150b64afa5e027e88b106e5ee", size = 239807, upload-time = "2026-08-15T08:19:38.392Z" }, { url = "https://files.pythonhosted.org/packages/d3/58/56a48c296601274c4689b864a8e2dfb209b81dfcb39472753ce95eea662b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:6199d5606e2bbf2b096cf64d03f8b6790c91081d5ac866b8e7bb6422738cc60c", size = 264083, upload-time = "2026-08-15T08:19:39.856Z" }, { url = "https://files.pythonhosted.org/packages/10/4c/dc48409274a1817ff349711d26c62aa0c597df865d4d69ef79160c859193/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:77efcff2b23071c349402ac1066667a3d011f62398d81408c9b88ad991747c9e", size = 250317, upload-time = "2026-08-15T08:19:41.53Z" }, { url = "https://files.pythonhosted.org/packages/81/58/d325912115caec62d6bdd77bbab5e0b7da5d234a9f20affdffcbcb530d0b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:a5cbd90ecf0fc62e64726917ad083b73001f0563657a87ec3c0b504e277dc90d", size = 258173, upload-time = "2026-08-15T08:19:43.07Z" }, { url = "https://files.pythonhosted.org/packages/34/f7/b13b1ccae2c8ec63980d13be1890eb73f8aeabbfce02a24aabc0908788f5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:4d26f14f041e83dd8edfd61f4cd4fa7285d31798b5bf1f28e70c367ba6c41d61", size = 251960, upload-time = "2026-08-15T08:19:44.587Z" }, { url = "https://files.pythonhosted.org/packages/1e/25/ed3f9919c5aef8cc818be1f972f565f7610d7b2076b8ebb98839516ffc3c/charset_normalizer-3.5.1-cp315-cp315t-win32.whl", hash = "sha256:ac13b004224fb341e1e25a1ed5e19d32f57cdb2a403e01f003b46f051a550f6f", size = 191186, upload-time = "2026-08-15T08:19:46.293Z" }, { url = "https://files.pythonhosted.org/packages/69/d5/43c2b3e9d8267092b913eb8b0603f0f71993c395632886bd37a7223f96cf/charset_normalizer-3.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:35aea775dc2bd5f54cd84a1cd2696cc3207c479cb9cf0bd346f0d343e4300ddb", size = 215947, upload-time = "2026-08-15T08:19:47.853Z" }, { url = "https://files.pythonhosted.org/packages/a8/76/9aad3e9c8865e5e0efa9a7f6f81c37a67635a985145ecd44528a81e088ee/charset_normalizer-3.5.1-cp315-cp315t-win_arm64.whl", hash = "sha256:fb78f6e7fcd8ad785d28cd577168bc1aaee827b25bb8755638f694794ea98f0a", size = 193909, upload-time = "2026-08-15T08:19:49.383Z" }, { url = "https://files.pythonhosted.org/packages/5b/97/fb4e82231aba271ffd775a1b4993b0defc4e3059f286ae41d9433409fe85/charset_normalizer-3.5.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2", size = 331467, upload-time = "2026-08-15T08:19:50.959Z" }, { url = "https://files.pythonhosted.org/packages/9f/2f/fe3f187327aac18e2d54e9d2b08e15d27bf9b642d9e51c219f130fc34d1a/charset_normalizer-3.5.1-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99", size = 253057, upload-time = "2026-08-15T08:19:52.654Z" }, { url = "https://files.pythonhosted.org/packages/d7/c7/9e48cee5c161fe24da823b61bf381921d77cb994a0a4de148e95018c1984/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2", size = 240930, upload-time = "2026-08-15T08:19:54.163Z" }, { url = "https://files.pythonhosted.org/packages/49/e0/716601f3cc69be7b198951150c75ead1ece33c3c8036ff6ffa46029659a0/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235", size = 230822, upload-time = "2026-08-15T08:19:55.807Z" }, { url = "https://files.pythonhosted.org/packages/d3/05/71bfc5caa0abcc45aea1f6a4d50ac68e59605ddc7666fe8494f4cd229665/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598", size = 260037, upload-time = "2026-08-15T08:19:57.312Z" }, { url = "https://files.pythonhosted.org/packages/c3/92/de7e32ed05341e7a9c4c877c318418197b7f2d66a3b68d561bf2ac57ca3e/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96", size = 255097, upload-time = "2026-08-15T08:19:59.056Z" }, { url = "https://files.pythonhosted.org/packages/f5/7b/ade0a122600319dfa0b1000ab0f9731c94a817904cf3c5de408c73a4ede7/charset_normalizer-3.5.1-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962", size = 250166, upload-time = "2026-08-15T08:20:00.612Z" }, { url = "https://files.pythonhosted.org/packages/75/9c/019fbb9f4834491a160951349b1a3714439376f66e5f7cf18b4f18f0c7aa/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3", size = 241821, upload-time = "2026-08-15T08:20:02.321Z" }, { url = "https://files.pythonhosted.org/packages/2b/b8/11d4840bfc99330cc7fbcc2681ee5a044553a6e77655508d8f9b2bff7b34/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950", size = 232529, upload-time = "2026-08-15T08:20:04.008Z" }, { url = "https://files.pythonhosted.org/packages/18/96/2b3a21492d9f65171ac75d872f5018260013d00bfa0ff70ec9f179148cbd/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8", size = 260348, upload-time = "2026-08-15T08:20:05.877Z" }, { url = "https://files.pythonhosted.org/packages/d6/aa/a69a2028e8bd052476c245460ab19d7de595de084dd968f2d75cd50c3e25/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031", size = 247234, upload-time = "2026-08-15T08:20:07.487Z" }, { url = "https://files.pythonhosted.org/packages/35/8a/3d130aeabcaf3d2466af76b7b141c08d9e89c9016ab4b7cdd0f7dc2d1c62/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072", size = 256917, upload-time = "2026-08-15T08:20:09.142Z" }, { url = "https://files.pythonhosted.org/packages/80/c2/a7379b840292d0c1ab9fbd17d1f3967aa81794dc95bc74be8999d7fedcf7/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d", size = 254846, upload-time = "2026-08-15T08:20:10.727Z" }, { url = "https://files.pythonhosted.org/packages/01/65/d43b714731bb2f40d4053dfa00ecfc1c5a301f8e3316c5db3a09af59fe94/charset_normalizer-3.5.1-cp37-abi3-win32.whl", hash = "sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc", size = 174216, upload-time = "2026-08-15T08:20:12.334Z" }, { url = "https://files.pythonhosted.org/packages/35/4f/b911ed898b26a09789eba9c9200c999aff6c61b4bafaf4838e56d1a1e1a3/charset_normalizer-3.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959", size = 199764, upload-time = "2026-08-15T08:20:13.908Z" }, { url = "https://files.pythonhosted.org/packages/f0/a7/920baf467bfd9bf689f3b318340f37aee4572a71f162bd8db51da55ba4fa/charset_normalizer-3.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e", size = 287318, upload-time = "2026-08-15T08:20:15.551Z" }, { url = "https://files.pythonhosted.org/packages/cc/61/d01fc49b8dea277640b55a9e15960dbca9fdc8c9fde18e572d39c59f4019/charset_normalizer-3.5.1-py3-none-any.whl", hash = "sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6", size = 68658, upload-time = "2026-08-15T08:20:43.306Z" }, ] [[package]] name = "click" version = "8.5.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/c7/0e/7fa0ef50764b67090eca4114772a2abf8b6148198475e54c660b97caeee6/click-8.5.0.tar.gz", hash = "sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34", size = 382235, upload-time = "2026-08-26T13:33:14.56Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" }, ] [[package]] name = "colorama" version = "0.4.6" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] [[package]] name = "ghp-import" version = "2.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "python-dateutil" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d9/29/d40217cbe2f6b1359e00c6c307bb3fc876ba74068cbab3dde77f03ca0dc4/ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343", size = 10943, upload-time = "2022-05-02T15:47:16.11Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", size = 11034, upload-time = "2022-05-02T15:47:14.552Z" }, ] [[package]] name = "griffe-inherited-docstrings" version = "1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffelib" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cb/da/fd002dc5f215cd896bfccaebe8b4aa1cdeed8ea1d9d60633685bd61ff933/griffe_inherited_docstrings-1.1.3.tar.gz", hash = "sha256:cd1f937ec9336a790e5425e7f9b92f5a5ab17f292ba86917f1c681c0704cb64e", size = 26738, upload-time = "2026-02-21T09:38:44.312Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/16/20/4bc15f242181daad1c104e0a7d33be49e712461ea89e548152be0365b9ea/griffe_inherited_docstrings-1.1.3-py3-none-any.whl", hash = "sha256:aa7f6e624515c50d9325a5cfdf4b2acac547f1889aca89092d5da7278f739695", size = 6710, upload-time = "2026-02-20T11:06:38.75Z" }, ] [[package]] name = "griffelib" version = "2.3.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/27/af/018c10bc9edd42b6ef6db2e96b09542050d5253f9b195e74bc910b2d13ab/griffelib-2.3.0.tar.gz", hash = "sha256:7b0952caf5bca6afa4bb5ee8c6a2d183fe3f21b62efc5f6c7243cb2b26d2d115", size = 234534, upload-time = "2026-09-04T15:08:17.472Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/41/63/e876e789525063c840ccfa8857febdabd6523bcef9ce7eb979b9305ea895/griffelib-2.3.0-py3-none-any.whl", hash = "sha256:1b8f9cd525681c26b1d6d574faa1371651e8459ca51d209684f50b8096ae06e0", size = 169423, upload-time = "2026-09-04T15:08:12.956Z" }, ] [[package]] name = "hypothesis" version = "6.168.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "sortedcontainers" }, ] sdist = { url = "https://files.pythonhosted.org/packages/aa/31/b54cd138ee36a32d0047d40fe7bdcb63cefd57ce098a26024161332d15fd/hypothesis-6.168.1.tar.gz", hash = "sha256:fd8acb5c67f260dfbdcc1b2dc95ff48b2b3f90708ae595c76f591bbf3f6f8eeb", size = 510823, upload-time = "2026-09-23T04:36:04.09Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/48/95/3091b9ad24baa9666d49feea433a8c4060ad47245f0d529d736af3672ddd/hypothesis-6.168.1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:5d9cc2d2a4779c92788389564fc6760df7c2909185f6431378133dea6933194f", size = 791345, upload-time = "2026-09-23T04:35:39.379Z" }, { url = "https://files.pythonhosted.org/packages/c5/b3/b954ffa69f91d111f801030c28d6721f422222af245a846571fc87874242/hypothesis-6.168.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:d0e1ec15d32af11da6784d2506de0c901e4c1abf97a3cdcfc144225457783f4e", size = 787104, upload-time = "2026-09-23T04:35:21.799Z" }, { url = "https://files.pythonhosted.org/packages/e5/dc/732df08f570d845a480d3729bfb46beccc441bfabe2804bb3a458649be73/hypothesis-6.168.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cc38519a57c4b919b2c59714a67501d20b46b0c96a0078cbdc9f86e728efce3", size = 1123848, upload-time = "2026-09-23T04:33:48.857Z" }, { url = "https://files.pythonhosted.org/packages/88/06/e1e937df42243267ba25ede5e9b29921186f290f6c0515729481cc41726e/hypothesis-6.168.1-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7e229e937e761830c7fa6437fb12cb801616a1737af0898abf5edba57d2ad691", size = 1147698, upload-time = "2026-09-23T04:35:33.439Z" }, { url = "https://files.pythonhosted.org/packages/be/d1/3b41c008dc2a49e43bc4893dcd27ea9d9fef64fef50382ca5f7107680b1f/hypothesis-6.168.1-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:569ba21e5543840895d17633cc6716a9578ac83b1193a2978b3f24cc16b93085", size = 1149280, upload-time = "2026-09-23T04:35:45.514Z" }, { url = "https://files.pythonhosted.org/packages/71/ea/d54781058fe6a487b81ec6c636f3806fff6a417b48cf3fcd2b003d80f3ce/hypothesis-6.168.1-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a232665716607c3c50c9a3aeeac5929be1d4e58703f5e7cb51db7099445a8c6", size = 1191693, upload-time = "2026-09-23T04:35:57.896Z" }, { url = "https://files.pythonhosted.org/packages/4f/f2/43bdd07450978611d25989ffdcbe1579acd46c1500a484f8838359fe7880/hypothesis-6.168.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9022fe330d8839259da9b496573c4b83ea248bf32029eba23ff589f3c25754bf", size = 1169725, upload-time = "2026-09-23T04:34:32.59Z" }, { url = "https://files.pythonhosted.org/packages/d1/72/fd4b657cba66940f208bc0eccaceaa777553009d26e86068ff3ab5e55b0b/hypothesis-6.168.1-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:69be660b831d55ab35619b30abd311a6cb8cd0068a1192210d194a776b211094", size = 1129181, upload-time = "2026-09-23T04:35:27.495Z" }, { url = "https://files.pythonhosted.org/packages/d6/3f/516a29d7a2bd31e28972de635350eab086c94faa07afcdd3ad0f06e70db8/hypothesis-6.168.1-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:42e2cccca80ee4ffe8f66edf5b2adb2a16b8cd3c5110b3a774d70f2fc75a42aa", size = 1160162, upload-time = "2026-09-23T04:35:29.333Z" }, { url = "https://files.pythonhosted.org/packages/54/09/e2c7e32f281b31d4c19cc2fa935bfdde5b31ad4848a485e5ebb8de2389da/hypothesis-6.168.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:11cbd2a2539194f0960de4ed9f980882e3a3f55c476afcc6f4cc342eec5a07d2", size = 1299714, upload-time = "2026-09-23T04:35:12.529Z" }, { url = "https://files.pythonhosted.org/packages/ba/bc/85850bdb1fc35d0fa9e299ca3b2e354f403959666c7ef4d0bfda9c308eee/hypothesis-6.168.1-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:aa1620111616c660a1db03347b1790b5914ef30f86c6e8ddd6c765785291f0b4", size = 1425335, upload-time = "2026-09-23T04:35:23.557Z" }, { url = "https://files.pythonhosted.org/packages/40/c0/66bdb24f32d32aa5636655b6037af07962118b89fae16ee74bc9a1bb982e/hypothesis-6.168.1-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:a07c8665c95bb856f99ecd60b6f50d94dcf8157415387c32dcd5bb5b5061d032", size = 1376916, upload-time = "2026-09-23T04:34:59.742Z" }, { url = "https://files.pythonhosted.org/packages/31/f0/7ad0b17bda81c473cbc539009148f3e0950880c4618028691bcf62eeb8d2/hypothesis-6.168.1-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:8a50b6335d5fa2312dd9f5d50faeb953e61475b301d2c073cd7a4c5efd7e6171", size = 1280975, upload-time = "2026-09-23T04:34:21.966Z" }, { url = "https://files.pythonhosted.org/packages/e2/01/8d04d1eecf03a6097e26400e8393eff7a81fd52c877a664ee4e97adf7880/hypothesis-6.168.1-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:4ccb0276db8ad197a63dff25c4868e11123346347df6900625d2b1f794ce7c19", size = 1300238, upload-time = "2026-09-23T04:34:52.825Z" }, { url = "https://files.pythonhosted.org/packages/f5/74/eda0c70b3845789c713fe2ac72d3d4586da83f38a34f1f7aefbda0b5b01e/hypothesis-6.168.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85ca7788574e06d0591d51e19ecbb1d09bc82d50c6eae5de0fe8317590b1e138", size = 1336088, upload-time = "2026-09-23T04:33:42.002Z" }, { url = "https://files.pythonhosted.org/packages/80/c9/43d8528e6d43eebb7688d36e530d1fc7763809b8f923b62cc8b4d95f99ff/hypothesis-6.168.1-cp310-abi3-win32.whl", hash = "sha256:77378e04aa47a22a615af80f42970dc44782f02b8a82ac105b619a2cc7f5eee2", size = 678004, upload-time = "2026-09-23T04:34:29.564Z" }, { url = "https://files.pythonhosted.org/packages/8c/8b/f7d4de5b57972f7f68a6c7fb5698f60425589eae6fcb175f285f2525e8c0/hypothesis-6.168.1-cp310-abi3-win_amd64.whl", hash = "sha256:f94d3dbc31112f8fbcb3af19a662ee5e90e380e6ae676357903caa14a1604763", size = 684722, upload-time = "2026-09-23T04:34:41.279Z" }, { url = "https://files.pythonhosted.org/packages/a5/fd/528eb221a82da0aa477e34c7af6ed2b4a5109b326638d4418b9e49435e96/hypothesis-6.168.1-cp310-abi3-win_arm64.whl", hash = "sha256:9bd7e8c6e2c3f6a5befee6b03ebd2170da2e50658075c6decad184777c72a726", size = 682711, upload-time = "2026-09-23T04:33:46.085Z" }, { url = "https://files.pythonhosted.org/packages/6d/1b/620484ba32fbb655ccbd02f3c510e7a961e840a466c3672ef0b877ab8716/hypothesis-6.168.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:9fd235c4ccfefe18fd97c181bdf247187cb3d4fdc8800a4d623322234ce4cfd3", size = 792049, upload-time = "2026-09-23T04:34:28.21Z" }, { url = "https://files.pythonhosted.org/packages/03/df/2ad250c2c4ee4d5f43d6d9bc5d852e7b81c771e4c8946212931b8cafe849/hypothesis-6.168.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0722212cf825d001a0d3b4db9303cc31af467f6814b69acab91d91c3f629c95a", size = 787932, upload-time = "2026-09-23T04:35:20.079Z" }, { url = "https://files.pythonhosted.org/packages/24/69/4f8e8bf1d5ad81352a05aa0835baef82e691e3e43933bafe6d6f2c171763/hypothesis-6.168.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94a1053575f133ff6978876fd4b4c2a153c1111f1b5f14d6cfb355c306fc4cb5", size = 1124000, upload-time = "2026-09-23T04:34:14.106Z" }, { url = "https://files.pythonhosted.org/packages/b6/e7/b72a34e0c6b434d3ad2b2e0040fa88d06a15a7880c59bddacf1e27d4902a/hypothesis-6.168.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2c74d496304574cfed537e4f657620be7fee3e287381c0aa4b481211e6ec92", size = 1170172, upload-time = "2026-09-23T04:34:30.93Z" }, { url = "https://files.pythonhosted.org/packages/b6/6c/afe1b73b58479ba0d75e5408259978d580e96fee812eb4ea2bb83a76f7ff/hypothesis-6.168.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cf4cc6819ba9689effba38dce98a3025554669f76be07e47cc1002e93e8c4918", size = 1300163, upload-time = "2026-09-23T04:35:53.549Z" }, { url = "https://files.pythonhosted.org/packages/b0/3d/aad54ffe7942c06da79c719fc98daa1dffdd8ac2d384ca64cc3611b1a833/hypothesis-6.168.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7888e0876e1d93fb92a6a4869834b0d0de18f07f6600edbf3d1cad16cf089357", size = 1336274, upload-time = "2026-09-23T04:35:10.655Z" }, { url = "https://files.pythonhosted.org/packages/0f/cb/8d2a3474e7921cfb5e59f933ac6fe1ad83b95cc98a871af8f0a68dcf946e/hypothesis-6.168.1-cp311-cp311-win_amd64.whl", hash = "sha256:4aa312b6447743a72283698292ea0e1c4d684279b38dc7f270729c4efda6c27a", size = 684479, upload-time = "2026-09-23T04:33:59.307Z" }, { url = "https://files.pythonhosted.org/packages/7e/6f/9d5ae55d81e10f0fc46868c19ab9d468e9959e82ff8cc8c26990b16114f3/hypothesis-6.168.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:dfe14a88b1ed03ce47333d800cc37949da73feb0cd9767e7ef076d2202768928", size = 793124, upload-time = "2026-09-23T04:33:52.589Z" }, { url = "https://files.pythonhosted.org/packages/14/11/54c19d259d7f92176aa2199c90f7951e4e396a26898a596dc875ccb9c389/hypothesis-6.168.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8acd000c3bf233cdcd28c1ff03f1bf56cb03821b1c742d00d57978b20fc6e6f8", size = 784674, upload-time = "2026-09-23T04:34:38.137Z" }, { url = "https://files.pythonhosted.org/packages/5f/ae/f80a9810f88bd654ab0d0ed0652c274e4c0384e22d4f9442b6753b9200a6/hypothesis-6.168.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29280d4743546ddcf74644f9acdc5726972cba9182699a86b7e77f3f8f6e5dde", size = 1122860, upload-time = "2026-09-23T04:33:51.375Z" }, { url = "https://files.pythonhosted.org/packages/f4/5d/aa5b7ffd57ec475a9252c55985c07a366823c3f85267d44648e94c1ec321/hypothesis-6.168.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd9a488d46b8c7da3103058c7569376a3a08994aee9a7d6f1d160d3d3b49b82d", size = 1168922, upload-time = "2026-09-23T04:34:44.567Z" }, { url = "https://files.pythonhosted.org/packages/65/10/4862448192d4c9cc61a3b8a620730800f56b1e83b0bfda63dbc728526eb3/hypothesis-6.168.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:451ee6c42955887a9e1850c53813e31182eab1663e4d3dacea4e535dedc45b5d", size = 1298568, upload-time = "2026-09-23T04:35:31.191Z" }, { url = "https://files.pythonhosted.org/packages/15/ad/55e73fda9afbfcf3336b2eee564833bc9a4fee1f7a32fe5f9d183742da39/hypothesis-6.168.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bc2c036f4a9534af64977e38b2976295136f83417fdb95bed816a52cc3023497", size = 1335091, upload-time = "2026-09-23T04:34:23.42Z" }, { url = "https://files.pythonhosted.org/packages/ef/99/b6340fa21a89b0088ed023fe984109e1c47a07dfd741fd6d8e17a9e2795f/hypothesis-6.168.1-cp312-cp312-win_amd64.whl", hash = "sha256:d603d7c96d030a63701b6ad9bff0bc4870e4c226335b3db509f1eed1f7fc061b", size = 682011, upload-time = "2026-09-23T04:34:34.565Z" }, { url = "https://files.pythonhosted.org/packages/f5/23/093b9dc768e89d2cec74d29115f4004fe1be64218bac65f8d817d727d30b/hypothesis-6.168.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:aebf7e9d7ba9f920e154abc65851d4affab497daeaea285474bed93840a983ea", size = 793062, upload-time = "2026-09-23T04:33:58.177Z" }, { url = "https://files.pythonhosted.org/packages/df/d7/f191686b44b3f892ae1404e238a55260152a645a2deafe8b2949c7e90fe9/hypothesis-6.168.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b26616604cac458dd83e75900fefd9190f1074e67d5d9dc4d9e321d30f295eca", size = 784518, upload-time = "2026-09-23T04:34:02.012Z" }, { url = "https://files.pythonhosted.org/packages/10/25/b7a5dca1911c9012a564d0b7aaeb44e393ade9424a695dea179ff32d79fe/hypothesis-6.168.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb6d89cff2edcd45e5af957c7b6a71ae7887419561f7b9c7055e1c50dea41017", size = 1122844, upload-time = "2026-09-23T04:35:06.897Z" }, { url = "https://files.pythonhosted.org/packages/ea/60/ddacd0d244e6719ea9a9d322d4d4b6b054fd27ada94ec7aa7dce7ef11297/hypothesis-6.168.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a4ce6310ec21000446ae75cdc6667b81d4ee0f2d226a0dfaf8baca42dd151d0", size = 1168779, upload-time = "2026-09-23T04:33:56.885Z" }, { url = "https://files.pythonhosted.org/packages/dc/07/343bab6676a41f38cb7c5912ab3ca059258bf40e41c181d1b77cc7251801/hypothesis-6.168.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a22c6e730c327d22a4d61b53f66639cdf6f0fe4a5a1b3595c36533e5b8127169", size = 1298434, upload-time = "2026-09-23T04:34:48.576Z" }, { url = "https://files.pythonhosted.org/packages/7d/bd/4b72fafa5f46a455024a9242b62505b1a53fad95ace162e831c4ffb233b9/hypothesis-6.168.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dcd66493fe180d38641b02dd0456cc1e429655f8940dd1396ab874808f16dd34", size = 1334975, upload-time = "2026-09-23T04:35:01.531Z" }, { url = "https://files.pythonhosted.org/packages/3f/ae/48643915cda400ca2dccb9bcdc363690b13bcd5c179172bc6450c3d76ce8/hypothesis-6.168.1-cp313-cp313-win_amd64.whl", hash = "sha256:a8a96f2e9fa2a57f95865aaf092943775e7ea96f72535371b2be20fe5464298c", size = 681978, upload-time = "2026-09-23T04:36:00.056Z" }, { url = "https://files.pythonhosted.org/packages/0c/0e/e6999cddbaef975ecc1b0a7e9028a3d1a1014b542fa003889d0cb3870fa9/hypothesis-6.168.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:35b6582ecc22ed682ea39bbf972f8b8b1798043a3bc624ae1dce02d42fcf3ad4", size = 793097, upload-time = "2026-09-23T04:35:03.374Z" }, { url = "https://files.pythonhosted.org/packages/d5/f3/2ff6bf4bf43ab12641b163bd8fea4b6cddc9a0f07499c4779519099034c9/hypothesis-6.168.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:48e467c73b7fb46bc072323c5d67e8e683cf32218fbba47c4e6ee0fe0d72687f", size = 784662, upload-time = "2026-09-23T04:35:08.976Z" }, { url = "https://files.pythonhosted.org/packages/10/3a/b3653de877981cdab6861476c3b68d02b7c682c2875557cff7b92dbde068/hypothesis-6.168.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c82fcc3dece367bd43f3eca5c2746cf60b73f61bba071ef9421d4cfd6188a85", size = 1123106, upload-time = "2026-09-23T04:34:26.476Z" }, { url = "https://files.pythonhosted.org/packages/49/25/24c725cb94c8953a98d8bdec83971f76c8f4b103070df1fdacd8b7f37253/hypothesis-6.168.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36031c8f8fd4bec7185cf70ed080ba54d45c106cf4f575bc8c2482c01f5f9b21", size = 1168942, upload-time = "2026-09-23T04:35:41.237Z" }, { url = "https://files.pythonhosted.org/packages/1d/e4/cbe45b9e78e4dd9c18ed2328ff95f659f59235b04ae3c73cf7e9c498adc9/hypothesis-6.168.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ddfe21791579fdc90423973c733578efa23b010cbfb685dcd5201cf7b0a928b6", size = 1298937, upload-time = "2026-09-23T04:33:50.181Z" }, { url = "https://files.pythonhosted.org/packages/93/e9/cacaa3f58ee6eb98ce29a2182483d57686011bba35c8fb8a9bc8bf981c7f/hypothesis-6.168.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ecf86b1c9c8700f6d45324becbbe25dd581c69dc03e1357a64f4bf9a1afe823f", size = 1335198, upload-time = "2026-09-23T04:36:02.021Z" }, { url = "https://files.pythonhosted.org/packages/31/d8/ca51626959f59a557656cf8024d9afda4d213257c170412fa3193d11e11f/hypothesis-6.168.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:d5ab336a7dd233858bc3066a69479e295da788cef345d7e4efd7b77dad724db0", size = 624105, upload-time = "2026-09-23T04:34:08.466Z" }, { url = "https://files.pythonhosted.org/packages/83/61/6ce96e7d58fc7a592ad7c84d8c7cf9ea36db51468b30ac8306bbf4c3e6ba/hypothesis-6.168.1-cp314-cp314-win_amd64.whl", hash = "sha256:63244ba76464202d3d24126d2e4c2c1abf36db60fffe16ebaa93292572c2a42a", size = 681859, upload-time = "2026-09-23T04:34:50.801Z" }, { url = "https://files.pythonhosted.org/packages/ff/b1/6c7345e2b37ea8a1df2520a0ecfbdc0243a897895ff950309ca7f687429a/hypothesis-6.168.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:6cec76c8cf76738ad4c0c0c5d6e4632a32d74748639ea6ce26fa672490d9bf95", size = 791685, upload-time = "2026-09-23T04:33:54.243Z" }, { url = "https://files.pythonhosted.org/packages/72/8c/4f6cfafde4b31add263d6a562f615939cd49649e87a3695ac8693506c811/hypothesis-6.168.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9da10ba2dbbcaba93c6a11eb394fd1a0cfee6e2ba21345c3d6f2fc7401d80670", size = 783237, upload-time = "2026-09-23T04:34:25.049Z" }, { url = "https://files.pythonhosted.org/packages/a0/25/646c301a9d1fe3825a2ff8362594c4aee0ceb2ce192f49469f7f2824a742/hypothesis-6.168.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3218c6d18a041c0eba0cccd88b20d74be24687e42a73338abd0a31ec5294c265", size = 1121414, upload-time = "2026-09-23T04:34:09.876Z" }, { url = "https://files.pythonhosted.org/packages/27/4f/f84ec8434ef765e6a5c4f9942bbb7652be444b23b9ff868da43e96e61e86/hypothesis-6.168.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:760cad0526a84122bbe2a0157217e16d76d649d333422142853c7f1ba43ca44d", size = 1167536, upload-time = "2026-09-23T04:33:55.443Z" }, { url = "https://files.pythonhosted.org/packages/38/74/c53334d793b0c54054ffc781cdf79ccae6ee51daf347bd9cfa7e0015cece/hypothesis-6.168.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:50a60b57b485dff68b56f99e4506b7f1b52994f801ee1aaa4cb9125a90bf7834", size = 1297122, upload-time = "2026-09-23T04:34:17.235Z" }, { url = "https://files.pythonhosted.org/packages/43/e7/b115a1faea6d40bddce6342da703b7536d803d2384fa50d277dc55170c64/hypothesis-6.168.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68017d433ca5c558c34e3308a6653fc63077749af8bf86c10850cb6bb3cb559f", size = 1334055, upload-time = "2026-09-23T04:35:37.298Z" }, { url = "https://files.pythonhosted.org/packages/1d/dd/f272620d304dd67ff7815509745e2fb8d2bf58e4e1da4b7842a83f1de2da/hypothesis-6.168.1-cp314-cp314t-win_amd64.whl", hash = "sha256:ff92099a0185ba14bf4bc2111299b456074596199148ae84590eee4d480fac2e", size = 681804, upload-time = "2026-09-23T04:35:35.374Z" }, { url = "https://files.pythonhosted.org/packages/54/e9/45595951f8bc9bd03f032325e7b176ca990a0c86bf2e09d481e8ee5a5cab/hypothesis-6.168.1-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:e9510e33467d586a7700d56b9a6ee65b44a06832c1cab3d270369d7c53381276", size = 791046, upload-time = "2026-09-23T04:35:55.724Z" }, { url = "https://files.pythonhosted.org/packages/8d/3e/718431d0ab72c4c07454f71040b4de298a63e765f20088557dc1bef0a48b/hypothesis-6.168.1-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:99970e3a75f11dde0414d3d6724bd8b7a2bd1deeb9a509ec9236c91d41a5cff0", size = 782982, upload-time = "2026-09-23T04:34:06.017Z" }, { url = "https://files.pythonhosted.org/packages/d7/05/c60db6df1456a70a3c589e42acaa446c22f56ed342c1b8dd04b8e49df8fd/hypothesis-6.168.1-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5ea78773de5bcc25266e37eb086472e90a85fc81b8d4a9ca8afc15858ee15fb", size = 1120961, upload-time = "2026-09-23T04:33:43.494Z" }, { url = "https://files.pythonhosted.org/packages/8b/fe/d943250395750ecc3b4ec95b1ab3c8119f6cbad08c1ba9169ffee7d89049/hypothesis-6.168.1-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:13f502ce2423e46dfcd2af38950a86ecf97084d94e721f7f13dd5a70afb36e0a", size = 1143877, upload-time = "2026-09-23T04:34:56.065Z" }, { url = "https://files.pythonhosted.org/packages/d3/9d/8c1f8e11f20fd81ce40caed7ad6d6a9776e24775070e87f3aa9ba3683272/hypothesis-6.168.1-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:69b4bcaa8b0e753fc8921011d5d10c8a7c3caf814c6a968bc140a08917973fdc", size = 1146503, upload-time = "2026-09-23T04:34:04.791Z" }, { url = "https://files.pythonhosted.org/packages/71/d8/ac2e9fa653d2f05a13724837007e857d376d81fdf728543ab85f6c7f7350/hypothesis-6.168.1-cp315-abi3.abi3t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7ed811908d38ca8e05c20ad25ecb61edc1e20a519667d4b668be49dd09e6bf3e", size = 1189216, upload-time = "2026-09-23T04:34:36.271Z" }, { url = "https://files.pythonhosted.org/packages/2b/7b/640952448781c7df5098880e2e2d6e7702b3a1c0badd4c0ea5fca6d6ab53/hypothesis-6.168.1-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9239c868418088d58ae18d041f9f424bd7a75dd143f209c35fde38770b65f74f", size = 1166919, upload-time = "2026-09-23T04:35:16.498Z" }, { url = "https://files.pythonhosted.org/packages/b6/de/0d46b330a28f7614850fb1a0564cf88af7c0b29cbd791e9d04ade7f66f13/hypothesis-6.168.1-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:5cac279798ecc92d3301ec4421c0458e22627f8a79fdde8a6c98ab1d4b8811f6", size = 1126652, upload-time = "2026-09-23T04:35:49.432Z" }, { url = "https://files.pythonhosted.org/packages/92/36/6adeca1724ed4966095406ffe7308fcff338cf9f5b7443d5ea34a01ab6e1/hypothesis-6.168.1-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bc660dc14d633641e22e407417d35bf2925331273ed71bc6ab5536dfca5c72d", size = 1155687, upload-time = "2026-09-23T04:35:18.234Z" }, { url = "https://files.pythonhosted.org/packages/bd/00/e9b7c7e4a0a75d1621be95a126999352e01a16d68b0a59b396695c7b4991/hypothesis-6.168.1-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:157a29e4d6bcb77ae732cb83d6546beef47265ecac88408fdd5d5b2bd325685d", size = 1296471, upload-time = "2026-09-23T04:34:12.763Z" }, { url = "https://files.pythonhosted.org/packages/8f/d6/52cb320e9e07cc67f718506617b5900344370394e39f4affada317aab313/hypothesis-6.168.1-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:b125aac89594ee2a76687b724f057b2423ea83dfee6b8ff59f54289d6eef19a1", size = 1421838, upload-time = "2026-09-23T04:34:20.451Z" }, { url = "https://files.pythonhosted.org/packages/9b/93/969c33a87cd2865beffebe491294b8115e7e7421c6840f79a67486535bfc/hypothesis-6.168.1-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:4b960c9d31b85dbc6a25888897c02bbea5f498844932841612925184fc78d9eb", size = 1373994, upload-time = "2026-09-23T04:34:00.608Z" }, { url = "https://files.pythonhosted.org/packages/34/c9/5fe4ade4bd23baa2e4cd27668bb8ac10e840ce19fbcfc74116ea529cbaa7/hypothesis-6.168.1-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:d4d76f35874471fe9b8e1bb696fb532363ef3d9588161a06d30520f638f15a39", size = 1278216, upload-time = "2026-09-23T04:34:07.253Z" }, { url = "https://files.pythonhosted.org/packages/a1/17/30b9a1ad975e90700dea74ce8b08ab07b9e58c4d83918422bc8841942c1e/hypothesis-6.168.1-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:d5147c5f7497d109eed0267c498adaf5cad6e4beeaca6fad6057f2007bae70a7", size = 1297647, upload-time = "2026-09-23T04:35:51.491Z" }, { url = "https://files.pythonhosted.org/packages/e0/2f/233a9d5adafad6e187ab8f4d9275592cf120b56e63ea1cfb02213c01021e/hypothesis-6.168.1-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:a909f89758e184c4f86418598de297a2ced5b922979c0f98c5d73ff6bafe38e5", size = 1333790, upload-time = "2026-09-23T04:34:42.828Z" }, { url = "https://files.pythonhosted.org/packages/10/d7/4c9d4d597f70316d781cd99b600adae090a6f5d675487196e10510a23afe/hypothesis-6.168.1-cp315-abi3.abi3t-win32.whl", hash = "sha256:66b9fb6f02840f8ffa2a4e6359aff6ecc1b2c6630c8a2066771044fa22956c7a", size = 675206, upload-time = "2026-09-23T04:35:14.592Z" }, { url = "https://files.pythonhosted.org/packages/b4/a8/f9fc0c0dbb9e80238fc7145cb4ce52e0897ea417a23217022911855bccda/hypothesis-6.168.1-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:9246eaa80044c33a182be661cbed7e6a7314c61359ef501ea9e9efd90d50ca3b", size = 681503, upload-time = "2026-09-23T04:34:11.223Z" }, { url = "https://files.pythonhosted.org/packages/11/b4/962ae350c10860be37c9abf98c0ef1ef23e4baca3ee7ad77208941e370e5/hypothesis-6.168.1-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:52e0c804cbf697d4760400c84444a91cbd5dfe7ae0ef1bb3228f74fb5547eb9e", size = 679199, upload-time = "2026-09-23T04:34:03.472Z" }, { url = "https://files.pythonhosted.org/packages/aa/c5/2bfdcc601b08eb9611bc524d18ca0a8cdbbdd7bd07bc68c38e0196a00668/hypothesis-6.168.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:91e3700d9c35e184dd253cff2f151f890557242bede75a2535901e9062d42975", size = 792943, upload-time = "2026-09-23T04:35:25.308Z" }, { url = "https://files.pythonhosted.org/packages/4b/52/36ae5128d9102d73dc578cd7b241c33f5483bd56e05799a2d06043a1ce7d/hypothesis-6.168.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:c19bc5da324a5899527ea749522e9ceb47f1eb54772869673abe59ae0b703aef", size = 788802, upload-time = "2026-09-23T04:34:39.655Z" }, { url = "https://files.pythonhosted.org/packages/23/34/698fc3481757503dc60f07547c29098dce97ac035a1ee134d66d88814281/hypothesis-6.168.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a4c9e46b2f349658d3013a357e1fbf114bc08abced7ee8e91cd6e0bc718b195", size = 1124766, upload-time = "2026-09-23T04:34:57.869Z" }, { url = "https://files.pythonhosted.org/packages/02/c9/1f871666106d90048027fc6b309ec505f0fb44e4519add73ddaad98bcd9a/hypothesis-6.168.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a0beee16fcbc0516951f2ef3ef1a479ad11d9726da49840139303b36ebc1d0", size = 1171766, upload-time = "2026-09-23T04:34:54.476Z" }, { url = "https://files.pythonhosted.org/packages/17/44/eff662526259ac4447dde4abe4fc285d0e8e5ff5345d35af836c6f0f12cc/hypothesis-6.168.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:4a8beb513c066fcb187b885fdfa93da4006f223d50d605b687bfb7e60aac33e9", size = 685449, upload-time = "2026-09-23T04:33:44.837Z" }, ] [[package]] name = "idna" version = "3.20" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, ] [[package]] name = "iniconfig" version = "2.3.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] [[package]] name = "jinja2" version = "3.1.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markupsafe" }, ] sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] [[package]] name = "jsonschema" version = "4.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, { name = "jsonschema-specifications" }, { name = "referencing" }, { name = "rpds-py" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, ] [[package]] name = "jsonschema-specifications" version = "2025.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "referencing" }, ] sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] [[package]] name = "markdown" version = "3.10.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/29/6f/da4c6aea59b3001f2e8c0ec7497475aadaf3b021c10cab5b2858f0f32b26/markdown-3.10.3.tar.gz", hash = "sha256:3589362618f743188b4d955b874402bc814f4f83f544dc207719f4baa7d9c45f", size = 372596, upload-time = "2026-07-30T19:05:29.005Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/64/69/4a5af2bc115a9a33fefe51709749de8262be3f9ba063d1753a837cdbc49c/markdown-3.10.3-py3-none-any.whl", hash = "sha256:fa6c92a00a4a3c98b22728c64a935ae1928250ae65058a6ded814d2cc29a4cea", size = 110757, upload-time = "2026-07-30T19:05:27.883Z" }, ] [[package]] name = "markupsafe" version = "3.0.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940, upload-time = "2025-09-27T18:36:22.249Z" }, { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887, upload-time = "2025-09-27T18:36:23.535Z" }, { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692, upload-time = "2025-09-27T18:36:24.823Z" }, { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471, upload-time = "2025-09-27T18:36:25.95Z" }, { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923, upload-time = "2025-09-27T18:36:27.109Z" }, { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572, upload-time = "2025-09-27T18:36:28.045Z" }, { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077, upload-time = "2025-09-27T18:36:29.025Z" }, { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876, upload-time = "2025-09-27T18:36:29.954Z" }, { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] [[package]] name = "mergedeep" version = "1.3.4" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/3a/41/580bb4006e3ed0361b8151a01d324fb03f420815446c7def45d02f74c270/mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8", size = 4661, upload-time = "2021-02-05T18:55:30.623Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/2c/19/04f9b178c2d8a15b076c8b5140708fa6ffc5601fb6f1e975537072df5b2a/mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307", size = 6354, upload-time = "2021-02-05T18:55:29.583Z" }, ] [[package]] name = "mkdocs" version = "1.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "ghp-import" }, { name = "jinja2" }, { name = "markdown" }, { name = "markupsafe" }, { name = "mergedeep" }, { name = "mkdocs-get-deps" }, { name = "packaging" }, { name = "pathspec" }, { name = "pyyaml" }, { name = "pyyaml-env-tag" }, { name = "watchdog" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bc/c6/bbd4f061bd16b378247f12953ffcb04786a618ce5e904b8c5a01a0309061/mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2", size = 3889159, upload-time = "2024-08-30T12:24:06.899Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/22/5b/dbc6a8cddc9cfa9c4971d59fb12bb8d42e161b7e7f8cc89e49137c5b279c/mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e", size = 3864451, upload-time = "2024-08-30T12:24:05.054Z" }, ] [[package]] name = "mkdocs-autorefs" version = "1.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown" }, { name = "markupsafe" }, { name = "mkdocs" }, ] sdist = { url = "https://files.pythonhosted.org/packages/52/c0/f641843de3f612a6b48253f39244165acff36657a91cc903633d456ae1ac/mkdocs_autorefs-1.4.4.tar.gz", hash = "sha256:d54a284f27a7346b9c38f1f852177940c222da508e66edc816a0fa55fc6da197", size = 56588, upload-time = "2026-02-10T15:23:55.105Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/28/de/a3e710469772c6a89595fc52816da05c1e164b4c866a89e3cb82fb1b67c5/mkdocs_autorefs-1.4.4-py3-none-any.whl", hash = "sha256:834ef5408d827071ad1bc69e0f39704fa34c7fc05bc8e1c72b227dfdc5c76089", size = 25530, upload-time = "2026-02-10T15:23:53.817Z" }, ] [[package]] name = "mkdocs-get-deps" version = "0.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mergedeep" }, { name = "platformdirs" }, { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ce/25/b3cccb187655b9393572bde9b09261d267c3bf2f2cdabe347673be5976a6/mkdocs_get_deps-0.2.2.tar.gz", hash = "sha256:8ee8d5f316cdbbb2834bc1df6e69c08fe769a83e040060de26d3c19fad3599a1", size = 11047, upload-time = "2026-03-10T02:46:33.632Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/88/29/744136411e785c4b0b744d5413e56555265939ab3a104c6a4b719dad33fd/mkdocs_get_deps-0.2.2-py3-none-any.whl", hash = "sha256:e7878cbeac04860b8b5e0ca31d3abad3df9411a75a32cde82f8e44b6c16ff650", size = 9555, upload-time = "2026-03-10T02:46:32.256Z" }, ] [[package]] name = "mkdocs-material" version = "9.7.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "babel" }, { name = "backrefs" }, { name = "colorama" }, { name = "jinja2" }, { name = "markdown" }, { name = "mkdocs" }, { name = "mkdocs-material-extensions" }, { name = "paginate" }, { name = "pygments" }, { name = "pymdown-extensions" }, { name = "requests" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f1/cd/c05d3a530ba7934f144fb45f7203cd236adc25c7bdcc34673d202f4b0278/mkdocs_material-9.7.7.tar.gz", hash = "sha256:c0649c065b1b0512d60aad8c10f947f8e455284475239b364b610f2deb4d0855", size = 4097923, upload-time = "2026-07-17T16:21:33.156Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ad/21/17c1bc9e6f47c972ad66fb2ac2568f99f90f1207eeb6fc3b34d094dba7b5/mkdocs_material-9.7.7-py3-none-any.whl", hash = "sha256:8ea9bb1737a5b524a5f9dcf2e1b4ebda8274ae3008aa7845720a97083bef708f", size = 9305438, upload-time = "2026-07-17T16:21:30.017Z" }, ] [[package]] name = "mkdocs-material-extensions" version = "1.3.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/79/9b/9b4c96d6593b2a541e1cb8b34899a6d021d208bb357042823d4d2cabdbe7/mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443", size = 11847, upload-time = "2023-11-22T19:09:45.208Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/5b/54/662a4743aa81d9582ee9339d4ffa3c8fd40a4965e033d77b9da9774d3960/mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31", size = 8728, upload-time = "2023-11-22T19:09:43.465Z" }, ] [[package]] name = "mkdocstrings" version = "1.0.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jinja2" }, { name = "markdown" }, { name = "markupsafe" }, { name = "mkdocs" }, { name = "mkdocs-autorefs" }, { name = "pymdown-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/71/f85bdf13355073ae15a7375f09879375a830553552e58c1c4b7e0bbc5c8b/mkdocstrings-1.0.6.tar.gz", hash = "sha256:a0b8c2bdd29a6416c80d717aa369bbf7831946bd9f23c2a66db1b1dbe7693dbd", size = 100649, upload-time = "2026-07-11T19:38:05.732Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/5d/5b/4c1902e8bdd5c4db63284e9d101dece4038d4025d6d88850ffe0a1578980/mkdocstrings-1.0.6-py3-none-any.whl", hash = "sha256:2703708697487d1b6d6d7b412e176fa436edf120c1bf81dc9e126b12d00893c7", size = 35787, upload-time = "2026-07-11T19:38:04.417Z" }, ] [[package]] name = "mkdocstrings-python" version = "2.0.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffelib" }, { name = "mkdocs-autorefs" }, { name = "mkdocstrings" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c5/65/0db919b7c91bf033b42d04f15ff16a56f4edd6afb26e84d448359660ece3/mkdocstrings_python-2.0.8.tar.gz", hash = "sha256:34545b54681cf0b212aab95394d37b0f5326a93b102f6ad1df3d965c79384c9d", size = 201771, upload-time = "2026-08-31T20:16:06.698Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0b/b7/a5a854d4e44853f26decc0c5e0d68df052399f3da344a313da4178f00816/mkdocstrings_python-2.0.8-py3-none-any.whl", hash = "sha256:e555d9ef53b0febdeae4dd603144d4f86ec90fef672276645ea3bb61232c3d24", size = 105443, upload-time = "2026-08-31T20:16:05.25Z" }, ] [[package]] name = "mypy-extensions" version = "1.1.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] [[package]] name = "packaging" version = "26.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, ] [[package]] name = "paginate" version = "0.5.7" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ec/46/68dde5b6bc00c1296ec6466ab27dddede6aec9af1b99090e1107091b3b84/paginate-0.5.7.tar.gz", hash = "sha256:22bd083ab41e1a8b4f3690544afb2c60c25e5c9a63a30fa2f483f6c60c8e5945", size = 19252, upload-time = "2024-08-25T14:17:24.139Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/90/96/04b8e52da071d28f5e21a805b19cb9390aa17a47462ac87f5e2696b9566d/paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591", size = 13746, upload-time = "2024-08-25T14:17:22.55Z" }, ] [[package]] name = "pathspec" version = "1.1.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, ] [[package]] name = "platformdirs" version = "4.11.12" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ea/dd/65804b0c2925a1c821a05502ea57517b69a073ff400d25ab9faa3a2cf012/platformdirs-4.11.12.tar.gz", hash = "sha256:e8dc1cb58f1153fd7f61db1374317770baababec2480b37b8f01c6cc25b45267", size = 40016, upload-time = "2026-09-22T01:02:36.973Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ec/2b/69de6317fbf48865bb12aa56c9cfcfff46b3add0cd45250725d96146dce0/platformdirs-4.11.12-py3-none-any.whl", hash = "sha256:b1ba966ac76153d9985a0787ffab951e7c18416189873d4e4ddb2c8a04a0a254", size = 24976, upload-time = "2026-09-22T01:02:35.742Z" }, ] [[package]] name = "pluggy" version = "1.6.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] [[package]] name = "pydantic" version = "2.13.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-types" }, { name = "pydantic-core" }, { name = "typing-extensions" }, { name = "typing-inspection" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/ef/fc4f868f4e2cee79f863883abffceff107875f569b848507319842d2a681/pydantic-2.13.5.tar.gz", hash = "sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08", size = 845750, upload-time = "2026-08-28T14:04:00.916Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl", hash = "sha256:346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73", size = 472589, upload-time = "2026-08-28T14:03:59.136Z" }, ] [[package]] name = "pydantic-core" version = "2.46.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/af/f9/8a06bea35ef8daf588f707784c973a7046e0034c8d8cfb08828eeffb8b75/pydantic_core-2.46.5.tar.gz", hash = "sha256:10416c15b8839ecc4ef4d0885da76da6fd0f67333a0eb8aff6d93c4b8f2910fc", size = 472262, upload-time = "2026-08-28T10:01:31.677Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a2/b6/81d2d19ea0be2c03664381b59f65fa72fc7969decedae00bc2c4ad835708/pydantic_core-2.46.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a1dee1b804ff4d11c663636cf15d2ea47e9f79cd56c033fb1cbf08924842a48f", size = 2074737, upload-time = "2026-08-28T09:57:57.711Z" }, { url = "https://files.pythonhosted.org/packages/0c/18/b70da8300e292df4099684ea11b1958043580d2f50d2dc8bf7e542bdd84a/pydantic_core-2.46.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d625a186a65201c23a9e3b8ed9c47e90a026e03256608cc91851c6709096844f", size = 1921751, upload-time = "2026-08-28T09:57:59.265Z" }, { url = "https://files.pythonhosted.org/packages/e7/1a/0d590341b6ffa4b4aca83508e6b8db4761aaeacfc15a25ca3815876d4797/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f8507560a9284e1370bb048ed4282012fbef4e8d109875b95e884d228552061", size = 1948231, upload-time = "2026-08-28T09:58:00.678Z" }, { url = "https://files.pythonhosted.org/packages/7d/1d/02eb35761c51f2f7b1b042d6ab4cda6600f0c8c88a2243b3f734376201e5/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f93c5fe914d75fbec9a49209b00da5f08e9e467d69da2b1510c81940cfd10be", size = 2020708, upload-time = "2026-08-28T09:58:02.267Z" }, { url = "https://files.pythonhosted.org/packages/4a/ea/f86073830e35d508cc8ddf9c3d9e6e6840fcb88d34bf726b0b4710186f27/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c767f552b21b10f774aeac128e828eafb796adfa1b666a18bf6321453c3a", size = 2194914, upload-time = "2026-08-28T09:58:03.934Z" }, { url = "https://files.pythonhosted.org/packages/bb/d7/fc36240d7791ce90939e51608568c33bfdae26202016f9770c229a487d86/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:701b2e04b560eeb4bddf7a25ab8ca476176e34fdbd9a0e18196f0d12d4685f0b", size = 2235622, upload-time = "2026-08-28T09:58:05.516Z" }, { url = "https://files.pythonhosted.org/packages/cf/bc/3fa2d76b83162820a17da7f645b28d1cba99fc8e1e5fc6517067ec450fa1/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49776eab08766a08dfff7012f8b422dcd7e25e43b316eedf0477c24fcfa84b7c", size = 2062091, upload-time = "2026-08-28T09:58:07.135Z" }, { url = "https://files.pythonhosted.org/packages/ab/9a/095d557bb492c90cd8a70a6dd048bf793d433d03d86c81c11e912e4cd049/pydantic_core-2.46.5-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:a2468d93d181667a7abd66e1b64bb9f76f361b0fef8faddf687456453576f5ee", size = 2089904, upload-time = "2026-08-28T09:58:08.814Z" }, { url = "https://files.pythonhosted.org/packages/24/98/7b76b1ad10a19a617a52aaa1d80e159115af939b095e86f8e756fd52e0df/pydantic_core-2.46.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:53feb344243bb9510a9dec7bf3cf1b64d88a98af5dc7872a5160465f8b198c8e", size = 2132244, upload-time = "2026-08-28T09:58:10.435Z" }, { url = "https://files.pythonhosted.org/packages/20/32/7d6ca365fadba186a0c8f85de1a701663bce81efd309d9479be58687622f/pydantic_core-2.46.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:cd5214352ae68f3b5e9af7768bdc5253695ee069675db3480518420b3be881f2", size = 2143901, upload-time = "2026-08-28T09:58:12.033Z" }, { url = "https://files.pythonhosted.org/packages/f8/09/eb9a6aa57f22fd1541a9c0aa2a1f3aeef3ec65347d33e10a6da2f43e0ee9/pydantic_core-2.46.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:9432f3598db432cb51c5b37fdbf29a60fcccc79e30d37a05022776a6bc4ab689", size = 2299425, upload-time = "2026-08-28T09:58:13.614Z" }, { url = "https://files.pythonhosted.org/packages/8a/f9/548a5bb9d4ba8cd26e26daf48052236f6b38bb61e7b7241fbc3c995719eb/pydantic_core-2.46.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8feeac04b5794e513e710af2f9c87d49f31a6dc47967bb264a1fed61a8989bec", size = 2318566, upload-time = "2026-08-28T09:58:15.199Z" }, { url = "https://files.pythonhosted.org/packages/4a/20/06454d18834c02c406c9133f1a3b485305fd9ee984f9636c2f730bef6a9d/pydantic_core-2.46.5-cp311-cp311-win32.whl", hash = "sha256:892a881d5f68c2b9ea304b7a6c2c60d9343df578a311b0f86b94bc8f1ffe8129", size = 1954258, upload-time = "2026-08-28T09:58:16.813Z" }, { url = "https://files.pythonhosted.org/packages/9e/c2/718b9deb4b72453b5d8c7447a3b14cb77bef36917ef5f514e0948a4096a0/pydantic_core-2.46.5-cp311-cp311-win_amd64.whl", hash = "sha256:40375c2d05acec10323e45dfe2077ac44bc74659008614af5069034e2cfc781c", size = 2041030, upload-time = "2026-08-28T09:58:18.288Z" }, { url = "https://files.pythonhosted.org/packages/67/ea/c1d1a5b72d6e1ff7f377a4d9199f6591f095beb5b409a8a5d89f7238d939/pydantic_core-2.46.5-cp311-cp311-win_arm64.whl", hash = "sha256:28a6a556cd3b6066bea827857f9d9cce027c96f776e512f544a581f9e42161f8", size = 2009234, upload-time = "2026-08-28T09:58:19.929Z" }, { url = "https://files.pythonhosted.org/packages/82/3f/76358795aa7a8c6d4f36e2cb828ad1c90ee118e1393a9281664f5aade9d4/pydantic_core-2.46.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b9fe6fb92520e3fd61f2e49000b6911b188824f089b75973ea06d6267f0b476d", size = 2076516, upload-time = "2026-08-28T09:58:21.576Z" }, { url = "https://files.pythonhosted.org/packages/db/50/26b091836076ce4cb2fac264186936acc069e0595772cfd02a563bc4761a/pydantic_core-2.46.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a39ac25a9a2fa4072efdb429833c4a4c8009a51ff9eea3eeae131713cd27991e", size = 1922874, upload-time = "2026-08-28T09:58:23.766Z" }, { url = "https://files.pythonhosted.org/packages/09/f0/2a8ce3849e299d44e2d2c196b6082643a3235565a735cb51db7a6261f614/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fdc8b93a41521988916eeaa271173fcca7fa0803d62f87675aac8dcec1c8e29", size = 1951772, upload-time = "2026-08-28T09:58:25.435Z" }, { url = "https://files.pythonhosted.org/packages/87/46/ac0dc8bdd9e6048183a14eb127764e7ad9240021c17513074a4711b0e31e/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b98134087d9de723658d17a42c7d0da8d6e2ef08015dee7dc93889047315f5e4", size = 2031832, upload-time = "2026-08-28T09:58:27.102Z" }, { url = "https://files.pythonhosted.org/packages/c4/c2/339de5bef7be36301a2231eaa52e62163742c2281f11b5f4892bc79785cd/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e652ab17569c94bff5475520f907b7148b8c24036a8ebbe5cf7cf7493d28579a", size = 2208645, upload-time = "2026-08-28T09:58:28.948Z" }, { url = "https://files.pythonhosted.org/packages/7b/a0/9ff22b797724262da14427abaed4dd1d864a139693fc5e7809114376a716/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d925f3d9afd05a8c0fb3a1031463a8d59ebe5e2afad297e29c78be19e13b4e62", size = 2265935, upload-time = "2026-08-28T09:58:30.625Z" }, { url = "https://files.pythonhosted.org/packages/c0/a4/eb9409ec0736e50aa70a412f16c204ed149516846912f7e6724d4c73ee53/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fc5be0abd4a407e200d844b404e33639a554e7bd0d448e7b9ae181be4789ac2", size = 2066284, upload-time = "2026-08-28T09:58:32.289Z" }, { url = "https://files.pythonhosted.org/packages/c0/02/7f6156ffc926857f1c37c07d9a388682865a81830ab6a1b637082c25e399/pydantic_core-2.46.5-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:816ff0a6550ffc06c098ccd2e0698600f9aa7da192a79eaa6f9af504a35db869", size = 2105889, upload-time = "2026-08-28T09:58:33.986Z" }, { url = "https://files.pythonhosted.org/packages/92/b1/e781d357ebe09fc929f995700f1b3503e8897f1cece183ecb1300d4d67e9/pydantic_core-2.46.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c7ea57fc63aa7da93a1bd2d644e6577befae10c52c4e36377635eea1056a74f5", size = 2158006, upload-time = "2026-08-28T09:58:35.647Z" }, { url = "https://files.pythonhosted.org/packages/70/0a/644597d84ab400e50609c192120b85c9681c22d3a20461b9060a79be0a7a/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:efd62a42486f1bda5d24cb4f63d15a3c7768375fe83d36f9417b4ad7a2fb20b3", size = 2158408, upload-time = "2026-08-28T09:58:37.38Z" }, { url = "https://files.pythonhosted.org/packages/1e/ee/ca3b7b3a4b3769ffe9ce9432a7c9be755de9593a46d3b0d54d0409323e44/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:2bc9419666990c06d7397831f2126a1ecc3594aaa3ff7de5bf2d066802f4e07b", size = 2309609, upload-time = "2026-08-28T09:58:39.22Z" }, { url = "https://files.pythonhosted.org/packages/ce/52/39fa1f451486019524ca685020390e7ca351832fd874530ba30c8628e6dc/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:18a09e1e1011b462f2e32774f25859ef1223d5c2b0546a633cf56654710721e0", size = 2342618, upload-time = "2026-08-28T09:58:40.89Z" }, { url = "https://files.pythonhosted.org/packages/81/5e/468fc630568c61dcef3cd47ad32ffbeed9af643f49208d1ea86ab4f890c4/pydantic_core-2.46.5-cp312-cp312-win32.whl", hash = "sha256:5cb482e9e84c851f4e623fe4acc1ced89168cf1fe18f7089db4548c8f5bbb65b", size = 1939475, upload-time = "2026-08-28T09:58:42.591Z" }, { url = "https://files.pythonhosted.org/packages/cf/c9/4c19f41b84cf6b622a72fbeed7665b25d47a187d68d47d0d430c07f23268/pydantic_core-2.46.5-cp312-cp312-win_amd64.whl", hash = "sha256:5e81740c09e310f5aa5cbd3e434a01c154d4bef93241c7877b39f211d2b78ba8", size = 2043140, upload-time = "2026-08-28T09:58:44.272Z" }, { url = "https://files.pythonhosted.org/packages/af/dd/0c1a050299147c746e5256db16d645ab5efd4f78c59937d581a0524e74a2/pydantic_core-2.46.5-cp312-cp312-win_arm64.whl", hash = "sha256:f7b0ec93a2893de856652154d73b7ba622f26fa97726487dcac373de5f4c6084", size = 1997729, upload-time = "2026-08-28T09:58:46.13Z" }, { url = "https://files.pythonhosted.org/packages/f5/37/5abe39a8372a61d3dc3c1338fc504281c01b32fdb3169cd7187153b56d3e/pydantic_core-2.46.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:b7ca9034437b6022f941f4857459562ee00a560b97e7cce8a0ec5a74fc6766e0", size = 2075885, upload-time = "2026-08-28T09:58:47.856Z" }, { url = "https://files.pythonhosted.org/packages/21/43/6323b1f8b217780454c61304bcd2b38ae4762f50754414124603ccc90bb2/pydantic_core-2.46.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f332f0e72a5a0400141f830744e141bf9f97917878dbe968669e8a7fefea78ff", size = 1922768, upload-time = "2026-08-28T09:58:49.58Z" }, { url = "https://files.pythonhosted.org/packages/0f/a3/c05ca796e1197618a774b01e596aeedfefc2f7d8c01ae3054e910b120e8a/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:193375f3548919d3f0b60936ca113ada3e38f264f91b9b8e0508efaad57be931", size = 1951241, upload-time = "2026-08-28T09:58:51.511Z" }, { url = "https://files.pythonhosted.org/packages/68/32/33bc39ac705c52cffc908e8389f9754fdb208aea5c69cceddf4eb3ce99af/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:79bdfa52f843137045b2d081cc05c120ba6665d29b7559c2c47690906f39279f", size = 2031975, upload-time = "2026-08-28T09:58:53.166Z" }, { url = "https://files.pythonhosted.org/packages/b0/70/2333e885c0f6a67bc105c5916965dac9b57f2718ee20d81d1a06a4ebdc13/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:24922243639cbdac66c75fcb6fd6495a9cb52b213d62f9a0d16f0310b1ff8038", size = 2208542, upload-time = "2026-08-28T09:58:55.017Z" }, { url = "https://files.pythonhosted.org/packages/f7/ea/296debfb4264207bbda5936133892e027c0a58875ad53ebd512fba8ec3a2/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c76fe65e607be28c7fd4d56fc3c42b1583aa058ce3408b7ad0fd540171d31f9f", size = 2264692, upload-time = "2026-08-28T09:58:56.767Z" }, { url = "https://files.pythonhosted.org/packages/d3/f2/9e4de77a6271e07a76d2d58b11c091a979c191ed2939bf80067568b369d2/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f7b393a8b3da82f5c1fc0751e6d01ac6c55b93c18226a60bdfba4a724efafd1", size = 2066633, upload-time = "2026-08-28T09:58:58.531Z" }, { url = "https://files.pythonhosted.org/packages/8d/db/f9e9d0c97445987b2084823d5c240de88087338f04fc2cfaa2df186b8049/pydantic_core-2.46.5-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:7ac031912d54f3d83ef3b3eb98dfabc1608802e2202263d25957eeed40b94761", size = 2105235, upload-time = "2026-08-28T09:59:00.421Z" }, { url = "https://files.pythonhosted.org/packages/07/c5/79169b047b3b2c3e99e04bc76372af9637e0bf6db638274fa927df96369e/pydantic_core-2.46.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:837b396ca3d7b74091ca623f6cbd8351bd42d670a79c2683e79fb089f06a2de5", size = 2157367, upload-time = "2026-08-28T09:59:02.442Z" }, { url = "https://files.pythonhosted.org/packages/26/b5/ba6057afb7c291bd449f51b867f95aef2072941c4ce4e5c31d6ffd132d3b/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:5ee239d575f80b08eca11f6e20f90c4c695de7825c67eefe6091fbf20dda648e", size = 2158420, upload-time = "2026-08-28T09:59:04.2Z" }, { url = "https://files.pythonhosted.org/packages/6e/28/2057abecaafdc22912afa819603a51f0a62d40643b7c4871c51721fea9be/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:e80675d75ae2cd14372cb65cad5400d9347a3d3f6c13000183f22dfd027283ed", size = 2309588, upload-time = "2026-08-28T09:59:06.048Z" }, { url = "https://files.pythonhosted.org/packages/71/9d/881156dc404e27479c4246128d73538464cab4a239bec61995e227644c30/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:9c4b71f10dd532fb7a5cbc8f58707779e64f03a258c2bf8bfbaecfcd9970b519", size = 2341866, upload-time = "2026-08-28T09:59:08.539Z" }, { url = "https://files.pythonhosted.org/packages/5a/38/d66f443a259f84d13babdceae568e572b0ed26da17ca5d0a649ebb110a67/pydantic_core-2.46.5-cp313-cp313-win32.whl", hash = "sha256:97bf8de4d541598c94a59344eeb988a94c08ff76b5723c41f6567ec18c7892ea", size = 1938580, upload-time = "2026-08-28T09:59:10.402Z" }, { url = "https://files.pythonhosted.org/packages/2c/1e/1d5371213f4cc9a7ed70c0bfcc7911de22311ee99a662a56077d7292d2ac/pydantic_core-2.46.5-cp313-cp313-win_amd64.whl", hash = "sha256:15f4a94963c95accac15b7b657bb177d3ad82bb90b0d0526d9a9b85079925db5", size = 2041980, upload-time = "2026-08-28T09:59:12.396Z" }, { url = "https://files.pythonhosted.org/packages/5a/48/4222d90b1c67568bace4dec6dca6271449c66de3595d72b6d098f5fde597/pydantic_core-2.46.5-cp313-cp313-win_arm64.whl", hash = "sha256:d22a945598fb91236b4dd793a6e42e4f3dd7740bb5aace5ebd7d4c08d13bb575", size = 1997213, upload-time = "2026-08-28T09:59:14.245Z" }, { url = "https://files.pythonhosted.org/packages/8e/8a/14596f2a8367da50cf7cbac48169ee5d9c8e11d486a3b527082384630c72/pydantic_core-2.46.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c1c43ad4339643d70ebb8124e1305a7dab423001eff58bb41a0f731adbc98355", size = 2074081, upload-time = "2026-08-28T09:59:16.141Z" }, { url = "https://files.pythonhosted.org/packages/ae/d5/d8a4eb6d6c7f66b91dd37c576d76e9e60fba900caf5372c17bcf949febc2/pydantic_core-2.46.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1a353f84de772f423b5ffb11d7ae352fbbef0f446f3c0b0af0f8236d7233606e", size = 1920497, upload-time = "2026-08-28T09:59:18.065Z" }, { url = "https://files.pythonhosted.org/packages/8e/26/092079428f86e927e030b2c0ced87df69dbb1c875cdeaa67bf42ea2be746/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5086029a57366b8cf81b130a43908738095c270c21a8d7f0e8bdfdb89718e2f3", size = 1952130, upload-time = "2026-08-28T09:59:20.476Z" }, { url = "https://files.pythonhosted.org/packages/08/c3/8ec0e290a9ebaebd64047bf5fda94be835c6b1551b02437e4b76778fbcd7/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:46c25dda9d092a06c08db76ffe0a197107904d0dfac653f7d5306bbcd6d6119c", size = 2026371, upload-time = "2026-08-28T09:59:22.227Z" }, { url = "https://files.pythonhosted.org/packages/01/72/4fd20ad520fb8da0157f95b27a7eb05a72790ef08138e7701ac972c342ea/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:37ea7b83c935e5b0d68c9449b82651accf78a10828b2c02b2f2d9e9496446c21", size = 2202822, upload-time = "2026-08-28T09:59:24.277Z" }, { url = "https://files.pythonhosted.org/packages/31/b0/d16e0771206b29314f0d52198b720be21e8a99ab2bf11e3bc0d7c9cebdff/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e64e88d5585bea9ce95861079de72006c7fa6d3df4e3a3b65ba31eb979c15c9f", size = 2262756, upload-time = "2026-08-28T09:59:26.608Z" }, { url = "https://files.pythonhosted.org/packages/2c/9b/59634b7ac631c63b2a37760eb6943af3e29573d6b59a4abc5e7f019d4cee/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54d510bac3ee52247af28ed4bb18a1e799f040ac60fd2bf5ccd4c92f1fbe786f", size = 2068352, upload-time = "2026-08-28T09:59:29.044Z" }, { url = "https://files.pythonhosted.org/packages/08/7c/570abb1ad2155348dc754ea91be22e5aaa18eb6d69a6068f7c6f2679a6ed/pydantic_core-2.46.5-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:a2a5e1d0ff29adddc9f6d6821a66302e4493f8ca898b715b6b1182c2c201ea0a", size = 2104777, upload-time = "2026-08-28T09:59:30.95Z" }, { url = "https://files.pythonhosted.org/packages/8e/25/5bf74adc65a1ac5b7be3f6cb0bcb5433615c1598a801c19d830d84c98ded/pydantic_core-2.46.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:03b9666e41e35d8909852ba191a0607520f81b74eaf12ccf8737005dbb313821", size = 2156312, upload-time = "2026-08-28T09:59:32.604Z" }, { url = "https://files.pythonhosted.org/packages/90/6a/2ef38830675e050121040618135564ed56b860b45433b02d9b4ebece46f3/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:a91c17edf6eea2402cb5457b4c89e99bc5ed1004aa34c4adf1d4258c1a5c22c2", size = 2150067, upload-time = "2026-08-28T09:59:34.453Z" }, { url = "https://files.pythonhosted.org/packages/90/ef/a7dbb03a14a64c2a4621f989c615ed9a892535a6cad938fc27079f919d80/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b49924c73a235e969511bf2aabdff3beebf9820931f646c80274d5d780010c47", size = 2304516, upload-time = "2026-08-28T09:59:36.194Z" }, { url = "https://files.pythonhosted.org/packages/68/f8/6bb4c4b80e8a6fde1904c64a51c62a1d04fcdfa3ea521a66b2ddefa1d885/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:2cbd9a5eff05e51c447c34dfa4632145b26b09120cf04bd0c871e44c1a5e1c9a", size = 2335223, upload-time = "2026-08-28T09:59:37.931Z" }, { url = "https://files.pythonhosted.org/packages/2a/80/f46b8c681195190b2c1f1c7c0a81abce60663e987613e09ef64d433dd96b/pydantic_core-2.46.5-cp314-cp314-win32.whl", hash = "sha256:2d5d76654becf5efd62c9e51c3756c67b49498b0c9a40884934c40807adbd074", size = 1934827, upload-time = "2026-08-28T09:59:39.836Z" }, { url = "https://files.pythonhosted.org/packages/f7/3c/60674207246bc0a4009d2391b7c7251c7159f279c8d2ab8aae8ef46f3dee/pydantic_core-2.46.5-cp314-cp314-win_amd64.whl", hash = "sha256:fa10ef4112775900e7a0661068635eb67b2ab824fbde764de6e0e21982a93db0", size = 2042648, upload-time = "2026-08-28T09:59:41.792Z" }, { url = "https://files.pythonhosted.org/packages/69/0c/117c562c7c1babdf44576b72a5e496906506c93690387ecfbca7c729ae2e/pydantic_core-2.46.5-cp314-cp314-win_arm64.whl", hash = "sha256:045ab3b6d308439e32b81cc173bba5b9018bc6ed896afd0c65b3b009b1699af5", size = 1989652, upload-time = "2026-08-28T09:59:43.702Z" }, { url = "https://files.pythonhosted.org/packages/e8/66/9336ae58f9eb68c41d121894e52c4c89eccb07eb8f602a04ee9c3f37736a/pydantic_core-2.46.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8816f3d218beb4b787de5c9759c259b8fa61f9dec42dc7811f320a33771778b7", size = 2065829, upload-time = "2026-08-28T09:59:45.364Z" }, { url = "https://files.pythonhosted.org/packages/c5/02/bc19b47a96c2d3109760711acf22369e56bd7e405ca52f7ade164d2ead57/pydantic_core-2.46.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:bce57638e08ac148e5778cce7feb968307a727d66f8e2274a543d0cf0c9ad6a3", size = 1905716, upload-time = "2026-08-28T09:59:47.18Z" }, { url = "https://files.pythonhosted.org/packages/52/a4/70b47c0509923dd98ccfed04fb3e32ea3849c82a0ff2205bb41009b43c00/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:976e1128455aa595ea04c79ccfedff1aaeab96ee013fcc916bed120c4f0ad94f", size = 1934216, upload-time = "2026-08-28T09:59:49.241Z" }, { url = "https://files.pythonhosted.org/packages/52/ab/aa03b65f7bb198585edf806b906c3223ecf1795543e39e23aec4cce27ad2/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b891faeedeafba41b2983e5001a81b6a915b69544c7e7570d1989ce1c36ac7", size = 2010635, upload-time = "2026-08-28T09:59:51.692Z" }, { url = "https://files.pythonhosted.org/packages/3c/8b/0da06343f30b84ec549aafd309c6456223d5dc8bd36af504c573faad561d/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5f194189415698233dd1114a093a9b56e61e2c57e11b469be3b0506f46f0771c", size = 2209369, upload-time = "2026-08-28T09:59:53.582Z" }, { url = "https://files.pythonhosted.org/packages/d6/5b/844c4defaa34a3df66eb9257087d121d70c201298b96abdf9f492fc2f1bf/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82a36973cf8a2ef5406f4fe2edbf8ed0c99629535d959e0b100c76a32535a111", size = 2253238, upload-time = "2026-08-28T09:59:55.484Z" }, { url = "https://files.pythonhosted.org/packages/f4/64/a4e536cb16d7f61a7fd3120b46c577fc7fa7325992f69c4f52bc786d77d8/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdbb78909f52b981d3b2d56b97328d71eb0b974c36bd77c920123a7ebb192829", size = 2065740, upload-time = "2026-08-28T09:59:58.038Z" }, { url = "https://files.pythonhosted.org/packages/5f/75/aaa38c6bc2d085f6605b34eabdc6a8a4e0b2e61fc9c8e6e52b28e97b3125/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:52e24eacdb536cade636aa90fb851835222becff8484b7001fdc78cb0290f2aa", size = 2087425, upload-time = "2026-08-28T09:59:59.898Z" }, { url = "https://files.pythonhosted.org/packages/55/ae/fcab4cfc39aba3689e1d20c8b5250ad280957022c09af2ed9cd585602a5e/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:37ae34309d7bd8c0d61ab839668058f2a7962ea1fc51d105d2db228fe0618034", size = 2139306, upload-time = "2026-08-28T10:00:03.057Z" }, { url = "https://files.pythonhosted.org/packages/2d/f4/f1d03a4bc9d9acbc62f4d742b8a319af52f71885079868b2ff8e48a651ee/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:0cdbada856a1c69a7624a64d3d9aefe79300bd6ef827b43a4f265010b9b55184", size = 2144589, upload-time = "2026-08-28T10:00:05.645Z" }, { url = "https://files.pythonhosted.org/packages/83/f3/7a53bb1356de514a4cd295f25b6ac39237895620c0462d2592b76c16e114/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:545f26c504b27c3758439a5e6d9349931f0a04f855668d5fe323c89e82300a38", size = 2288882, upload-time = "2026-08-28T10:00:07.931Z" }, { url = "https://files.pythonhosted.org/packages/cd/94/5a81583660c175c59d49ffb09f4b3a44debeaf86a19fca664ae1cdd9ee32/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:ff218293c9c806138dca139765e3b067621be52bcd93cdc14c7711be7ddc90a9", size = 2335210, upload-time = "2026-08-28T10:00:10.177Z" }, { url = "https://files.pythonhosted.org/packages/5a/9f/5d685c2693b972d1a59c998586e8823712b66603aeff47ee60a4bdaafd37/pydantic_core-2.46.5-cp314-cp314t-win32.whl", hash = "sha256:97cf3eb53a8cccacf9d46686a0926186c9bfb5574f2ed66d3639d5fe117cd3a9", size = 1921180, upload-time = "2026-08-28T10:00:12.35Z" }, { url = "https://files.pythonhosted.org/packages/70/12/5c94ee16d65a37a15f9e869f5e6256df111154491173801a4c5e800ab548/pydantic_core-2.46.5-cp314-cp314t-win_amd64.whl", hash = "sha256:d2f9fc07a8042a8f95925b35c4f04f469707c981fc33245b6ca187cf5d2dd290", size = 2020515, upload-time = "2026-08-28T10:00:14.774Z" }, { url = "https://files.pythonhosted.org/packages/63/19/67830dda664e6bdf9285ee2e40f355d0d7d6b92aa0c42e8d217bb8d33d36/pydantic_core-2.46.5-cp314-cp314t-win_arm64.whl", hash = "sha256:acf8a67ba51f4ca9ddbd0e6b3000a65ac51ab734661778b3e7ba64d99a710f2f", size = 1989276, upload-time = "2026-08-28T10:00:16.984Z" }, { url = "https://files.pythonhosted.org/packages/af/1e/ecca01fce348f7e8afa9572441ff6f7d1cc70d21e4859f33944d10877e1e/pydantic_core-2.46.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:c14ad3bdc85ee7f318742c457ca3968a92126d144b15721c759033bfb06296c2", size = 2075342, upload-time = "2026-08-28T10:00:51.353Z" }, { url = "https://files.pythonhosted.org/packages/1f/4c/af80c7a8032dfc897040ad5cb772bebde529a381186499e6e29987f23f8c/pydantic_core-2.46.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0bddb4020d8f04175865ccd17eff3040874fc11fb593f424edb452653b4b947c", size = 1907219, upload-time = "2026-08-28T10:00:53.438Z" }, { url = "https://files.pythonhosted.org/packages/be/3e/54d89e2b092e778716bf6153634ef479e955f48c261090be23aa1e0fb0b5/pydantic_core-2.46.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2471fd51c61c610e1dcf7de44d7299283661654d11264ab4802b303368d69c47", size = 1953393, upload-time = "2026-08-28T10:00:55.58Z" }, { url = "https://files.pythonhosted.org/packages/ea/89/828ee90cda28ce17bdefaa3a6eaf74fe430e113295a10e6126beca559d6c/pydantic_core-2.46.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b10ec717381bdbfafef34607824db4c91de69ff085e4fca3b2af91b4fa17e68a", size = 2099024, upload-time = "2026-08-28T10:00:57.794Z" }, { url = "https://files.pythonhosted.org/packages/df/dd/053c2e4303f791f3b8f8a14ab0b22008e8eb21d868c0c90b4f9be705b76a/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:013d6f3483d81e02e7c328831808f336c8596ee33b4bd4026b9ffb1e960b8942", size = 2062540, upload-time = "2026-08-28T10:01:00.318Z" }, { url = "https://files.pythonhosted.org/packages/d7/dd/a18df751a5e37dd51bfad7f68e766999125bebe68c9e1d10a493ad01bd63/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:e9c134bb666dd54b778b9fc0d2b50cbb7f979b9e3716f26a88c9ab3b6fc1dd0f", size = 1902040, upload-time = "2026-08-28T10:01:02.529Z" }, { url = "https://files.pythonhosted.org/packages/b7/13/01d40f9d07ce8a779fd6e0bd8ad4fba91309500dd67b869e2e219d261a6d/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:347ec774390c87326a2e4929d58d3f7e8763a104d5d35f4cd595a4c952366433", size = 1967479, upload-time = "2026-08-28T10:01:05.004Z" }, { url = "https://files.pythonhosted.org/packages/fa/04/c81d4841331c2178b6fb09ae225425e110ed72d990c9fe556c4ec03d1013/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e24d8f05fa2d28513d94e877e9c75ad66175376209b3977f916e240e623193c", size = 2111034, upload-time = "2026-08-28T10:01:07.345Z" }, { url = "https://files.pythonhosted.org/packages/20/21/22102e9950b3049526d20e811b95396508377d87651edd2b80d2b3d28659/pydantic_core-2.46.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ab4b66edffb32d9e951efb3814bd104b8367a7501b81b955cacb5726d897389f", size = 2071333, upload-time = "2026-08-28T10:01:09.636Z" }, { url = "https://files.pythonhosted.org/packages/d8/18/87aefa427d191e6d3ab1447f1efc1cdcac86af1069239b133e8a0fd7f7c9/pydantic_core-2.46.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:337639ba62a11acde6ef3aeb08c8ea755f8ef1fe5e513356c0f36a2b0d7568b0", size = 1912713, upload-time = "2026-08-28T10:01:12.285Z" }, { url = "https://files.pythonhosted.org/packages/1f/93/fd89e9ad49b1805ca94d24ce1088b7d305f05c35ffafcedb9819d03588a0/pydantic_core-2.46.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:413a717a410d0c817ef5b786a059415550b3794e1d0c2abffd9efb93a3d9f7b4", size = 2090926, upload-time = "2026-08-28T10:01:15.19Z" }, { url = "https://files.pythonhosted.org/packages/6f/45/8e59dab6acf8d35f02f0a958980074f31038968bdb2c983fcae9d1efee03/pydantic_core-2.46.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1e449def1945a462c464331254e5a44fca7c3b4f9aedf59ec2f50f8066dd8e25", size = 2131303, upload-time = "2026-08-28T10:01:17.937Z" }, { url = "https://files.pythonhosted.org/packages/d5/a5/e1d4dc5180dd887a9522efc1f8716b8692b7606b1d3273d7862eaf66be44/pydantic_core-2.46.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:a445486499897b88a7d6c310c88ed64dd37b1b59bfd7ae9107490bbb362f47d6", size = 2145128, upload-time = "2026-08-28T10:01:20.694Z" }, { url = "https://files.pythonhosted.org/packages/c2/d7/ad493864a7fb21c0c4df98f965e2db430cb25a9d7369b5778d5016c09fd9/pydantic_core-2.46.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:2d330aaba8621b1edcec8ae2c4050f63b84ccf6d98723a8f212e9684713abf0e", size = 2294560, upload-time = "2026-08-28T10:01:23.495Z" }, { url = "https://files.pythonhosted.org/packages/02/8e/b41c84c913f29973a268e6c2b5bbf13c95adb9956c126d10da11ba3b2bef/pydantic_core-2.46.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b6acfb46a814762367fb7ba0828b0a17d441b92ce249a0e007474c9072662dda", size = 2317531, upload-time = "2026-08-28T10:01:26.334Z" }, { url = "https://files.pythonhosted.org/packages/db/1d/068464f23075f66a8f1b806935e9cd9363ee446636ea70d2c22ee8659dbf/pydantic_core-2.46.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d0a24b40877af2de4950252be9d21eaf7fb07660f3c2cae1f56c6b599ada5266", size = 2140686, upload-time = "2026-08-28T10:01:28.947Z" }, ] [[package]] name = "pygments" version = "2.21.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, ] [[package]] name = "pymdown-extensions" version = "11.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown" }, { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/21/a9/5f0c535ba3b08fe09270c16808e053a968868242ecbd5676d4e3a488bf28/pymdown_extensions-11.0.1.tar.gz", hash = "sha256:dd2905ae6fc5b75582fafb139a1266ffc754705efa902aa50067fa7ff4f94ec0", size = 857113, upload-time = "2026-07-02T17:59:22.955Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d6/54/da572c98c0b77626a91b5d3b89f0231d8bff5125c225420908632f8b342d/pymdown_extensions-11.0.1-py3-none-any.whl", hash = "sha256:db3943a62bab7e03af1364f0c4083e64b91fb097675a4b6cceccfbe9a77e5eb2", size = 269455, upload-time = "2026-07-02T17:59:21.271Z" }, ] [[package]] name = "pytest" version = "9.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] [[package]] name = "pytest-examples" version = "0.0.18" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "black" }, { name = "pytest" }, { name = "ruff" }, ] sdist = { url = "https://files.pythonhosted.org/packages/af/71/4ae972fd95f474454aa450108ee1037830e7ba11840363e981b8d48fd16a/pytest_examples-0.0.18.tar.gz", hash = "sha256:9a464f007f805b113677a15e2f8942ebb92d7d3eb5312e9a405d018478ec9801", size = 21237, upload-time = "2025-05-06T07:46:10.705Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/09/52/7bbfb6e987d9a8a945f22941a8da63e3529465f1b106ef0e26f5df7c780d/pytest_examples-0.0.18-py3-none-any.whl", hash = "sha256:86c195b98c4e55049a0df3a0a990ca89123b7280473ab57608eecc6c47bcfe9c", size = 18169, upload-time = "2025-05-06T07:46:09.349Z" }, ] [[package]] name = "python-dateutil" version = "2.9.0.post0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "six" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] [[package]] name = "pytokens" version = "0.4.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/b6/34/b4e015b99031667a7b960f888889c5bd34ef585c85e1cb56a594b92836ac/pytokens-0.4.1.tar.gz", hash = "sha256:292052fe80923aae2260c073f822ceba21f3872ced9a68bb7953b348e561179a", size = 23015, upload-time = "2026-01-30T01:03:45.924Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/3d/92/790ebe03f07b57e53b10884c329b9a1a308648fc083a6d4a39a10a28c8fc/pytokens-0.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d70e77c55ae8380c91c0c18dea05951482e263982911fc7410b1ffd1dadd3440", size = 160864, upload-time = "2026-01-30T01:02:57.882Z" }, { url = "https://files.pythonhosted.org/packages/13/25/a4f555281d975bfdd1eba731450e2fe3a95870274da73fb12c40aeae7625/pytokens-0.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a58d057208cb9075c144950d789511220b07636dd2e4708d5645d24de666bdc", size = 248565, upload-time = "2026-01-30T01:02:59.912Z" }, { url = "https://files.pythonhosted.org/packages/17/50/bc0394b4ad5b1601be22fa43652173d47e4c9efbf0044c62e9a59b747c56/pytokens-0.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b49750419d300e2b5a3813cf229d4e5a4c728dae470bcc89867a9ad6f25a722d", size = 260824, upload-time = "2026-01-30T01:03:01.471Z" }, { url = "https://files.pythonhosted.org/packages/4e/54/3e04f9d92a4be4fc6c80016bc396b923d2a6933ae94b5f557c939c460ee0/pytokens-0.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d9907d61f15bf7261d7e775bd5d7ee4d2930e04424bab1972591918497623a16", size = 264075, upload-time = "2026-01-30T01:03:04.143Z" }, { url = "https://files.pythonhosted.org/packages/d1/1b/44b0326cb5470a4375f37988aea5d61b5cc52407143303015ebee94abfd6/pytokens-0.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6", size = 103323, upload-time = "2026-01-30T01:03:05.412Z" }, { url = "https://files.pythonhosted.org/packages/41/5d/e44573011401fb82e9d51e97f1290ceb377800fb4eed650b96f4753b499c/pytokens-0.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:140709331e846b728475786df8aeb27d24f48cbcf7bcd449f8de75cae7a45083", size = 160663, upload-time = "2026-01-30T01:03:06.473Z" }, { url = "https://files.pythonhosted.org/packages/f0/e6/5bbc3019f8e6f21d09c41f8b8654536117e5e211a85d89212d59cbdab381/pytokens-0.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d6c4268598f762bc8e91f5dbf2ab2f61f7b95bdc07953b602db879b3c8c18e1", size = 255626, upload-time = "2026-01-30T01:03:08.177Z" }, { url = "https://files.pythonhosted.org/packages/bf/3c/2d5297d82286f6f3d92770289fd439956b201c0a4fc7e72efb9b2293758e/pytokens-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24afde1f53d95348b5a0eb19488661147285ca4dd7ed752bbc3e1c6242a304d1", size = 269779, upload-time = "2026-01-30T01:03:09.756Z" }, { url = "https://files.pythonhosted.org/packages/20/01/7436e9ad693cebda0551203e0bf28f7669976c60ad07d6402098208476de/pytokens-0.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ad948d085ed6c16413eb5fec6b3e02fa00dc29a2534f088d3302c47eb59adf9", size = 268076, upload-time = "2026-01-30T01:03:10.957Z" }, { url = "https://files.pythonhosted.org/packages/2e/df/533c82a3c752ba13ae7ef238b7f8cdd272cf1475f03c63ac6cf3fcfb00b6/pytokens-0.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:3f901fe783e06e48e8cbdc82d631fca8f118333798193e026a50ce1b3757ea68", size = 103552, upload-time = "2026-01-30T01:03:12.066Z" }, { url = "https://files.pythonhosted.org/packages/cb/dc/08b1a080372afda3cceb4f3c0a7ba2bde9d6a5241f1edb02a22a019ee147/pytokens-0.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8bdb9d0ce90cbf99c525e75a2fa415144fd570a1ba987380190e8b786bc6ef9b", size = 160720, upload-time = "2026-01-30T01:03:13.843Z" }, { url = "https://files.pythonhosted.org/packages/64/0c/41ea22205da480837a700e395507e6a24425151dfb7ead73343d6e2d7ffe/pytokens-0.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5502408cab1cb18e128570f8d598981c68a50d0cbd7c61312a90507cd3a1276f", size = 254204, upload-time = "2026-01-30T01:03:14.886Z" }, { url = "https://files.pythonhosted.org/packages/e0/d2/afe5c7f8607018beb99971489dbb846508f1b8f351fcefc225fcf4b2adc0/pytokens-0.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29d1d8fb1030af4d231789959f21821ab6325e463f0503a61d204343c9b355d1", size = 268423, upload-time = "2026-01-30T01:03:15.936Z" }, { url = "https://files.pythonhosted.org/packages/68/d4/00ffdbd370410c04e9591da9220a68dc1693ef7499173eb3e30d06e05ed1/pytokens-0.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:970b08dd6b86058b6dc07efe9e98414f5102974716232d10f32ff39701e841c4", size = 266859, upload-time = "2026-01-30T01:03:17.458Z" }, { url = "https://files.pythonhosted.org/packages/a7/c9/c3161313b4ca0c601eeefabd3d3b576edaa9afdefd32da97210700e47652/pytokens-0.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:9bd7d7f544d362576be74f9d5901a22f317efc20046efe2034dced238cbbfe78", size = 103520, upload-time = "2026-01-30T01:03:18.652Z" }, { url = "https://files.pythonhosted.org/packages/8f/a7/b470f672e6fc5fee0a01d9e75005a0e617e162381974213a945fcd274843/pytokens-0.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4a14d5f5fc78ce85e426aa159489e2d5961acf0e47575e08f35584009178e321", size = 160821, upload-time = "2026-01-30T01:03:19.684Z" }, { url = "https://files.pythonhosted.org/packages/80/98/e83a36fe8d170c911f864bfded690d2542bfcfacb9c649d11a9e6eb9dc41/pytokens-0.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f50fd18543be72da51dd505e2ed20d2228c74e0464e4262e4899797803d7fa", size = 254263, upload-time = "2026-01-30T01:03:20.834Z" }, { url = "https://files.pythonhosted.org/packages/0f/95/70d7041273890f9f97a24234c00b746e8da86df462620194cef1d411ddeb/pytokens-0.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc74c035f9bfca0255c1af77ddd2d6ae8419012805453e4b0e7513e17904545d", size = 268071, upload-time = "2026-01-30T01:03:21.888Z" }, { url = "https://files.pythonhosted.org/packages/da/79/76e6d09ae19c99404656d7db9c35dfd20f2086f3eb6ecb496b5b31163bad/pytokens-0.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324", size = 271716, upload-time = "2026-01-30T01:03:23.633Z" }, { url = "https://files.pythonhosted.org/packages/79/37/482e55fa1602e0a7ff012661d8c946bafdc05e480ea5a32f4f7e336d4aa9/pytokens-0.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:b35d7e5ad269804f6697727702da3c517bb8a5228afa450ab0fa787732055fc9", size = 104539, upload-time = "2026-01-30T01:03:24.788Z" }, { url = "https://files.pythonhosted.org/packages/30/e8/20e7db907c23f3d63b0be3b8a4fd1927f6da2395f5bcc7f72242bb963dfe/pytokens-0.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8fcb9ba3709ff77e77f1c7022ff11d13553f3c30299a9fe246a166903e9091eb", size = 168474, upload-time = "2026-01-30T01:03:26.428Z" }, { url = "https://files.pythonhosted.org/packages/d6/81/88a95ee9fafdd8f5f3452107748fd04c24930d500b9aba9738f3ade642cc/pytokens-0.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79fc6b8699564e1f9b521582c35435f1bd32dd06822322ec44afdeba666d8cb3", size = 290473, upload-time = "2026-01-30T01:03:27.415Z" }, { url = "https://files.pythonhosted.org/packages/cf/35/3aa899645e29b6375b4aed9f8d21df219e7c958c4c186b465e42ee0a06bf/pytokens-0.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d31b97b3de0f61571a124a00ffe9a81fb9939146c122c11060725bd5aea79975", size = 303485, upload-time = "2026-01-30T01:03:28.558Z" }, { url = "https://files.pythonhosted.org/packages/52/a0/07907b6ff512674d9b201859f7d212298c44933633c946703a20c25e9d81/pytokens-0.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:967cf6e3fd4adf7de8fc73cd3043754ae79c36475c1c11d514fc72cf5490094a", size = 306698, upload-time = "2026-01-30T01:03:29.653Z" }, { url = "https://files.pythonhosted.org/packages/39/2a/cbbf9250020a4a8dd53ba83a46c097b69e5eb49dd14e708f496f548c6612/pytokens-0.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:584c80c24b078eec1e227079d56dc22ff755e0ba8654d8383b2c549107528918", size = 116287, upload-time = "2026-01-30T01:03:30.912Z" }, { url = "https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de", size = 13729, upload-time = "2026-01-30T01:03:45.029Z" }, ] [[package]] name = "pyyaml" version = "6.0.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] [[package]] name = "pyyaml-env-tag" version = "1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/2e/79c822141bfd05a853236b504869ebc6b70159afc570e1d5a20641782eaa/pyyaml_env_tag-1.1.tar.gz", hash = "sha256:2eb38b75a2d21ee0475d6d97ec19c63287a7e140231e4214969d0eac923cd7ff", size = 5737, upload-time = "2025-05-13T15:24:01.64Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/04/11/432f32f8097b03e3cd5fe57e88efb685d964e2e5178a48ed61e841f7fdce/pyyaml_env_tag-1.1-py3-none-any.whl", hash = "sha256:17109e1a528561e32f026364712fee1264bc2ea6715120891174ed1b980d2e04", size = 4722, upload-time = "2025-05-13T15:23:59.629Z" }, ] [[package]] name = "referencing" version = "0.37.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, { name = "rpds-py" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, ] [[package]] name = "requests" version = "2.34.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, { name = "charset-normalizer" }, { name = "idna" }, { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] [[package]] name = "rpds-py" version = "2026.6.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/94/1f/a2dca5ffdbf1d475ffc4e80e4d5d720ff3a00f691795910116960ee12511/rpds_py-2026.6.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7b689145a1485c335569bd056464f3243a29af7ed3871c7be31ad624ba239bc7", size = 342174, upload-time = "2026-06-30T07:14:54.821Z" }, { url = "https://files.pythonhosted.org/packages/4d/dc/323d08583c0832911768663d1944f0107fcd4088704858d84b5e06d105a0/rpds_py-2026.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:db08f45aecde626498fb3df07bcf6d2ec040af42e859a4f5040d79c200342911", size = 345513, upload-time = "2026-06-30T07:14:56.515Z" }, { url = "https://files.pythonhosted.org/packages/0b/2a/e31989834d18d2f26ec1d2774c5b1eb3331df4ea8ada525175294c94b48a/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acc992ab27b15f852c76755eb2ab7dce86585ddadba6fa5946e58556088845b4", size = 373783, upload-time = "2026-06-30T07:14:57.736Z" }, { url = "https://files.pythonhosted.org/packages/87/fe/e80107ee3639585c9941c17d6a42cd65325022f656c023191fce78c324c8/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f88d653e7b3b779d71ae7454e20dcc9b6bae903f33c269db9f2be41bda3f261", size = 378316, upload-time = "2026-06-30T07:14:59.077Z" }, { url = "https://files.pythonhosted.org/packages/22/6f/81e3adf81acfb6fa694de2a6e4e7d8863121e3e0799e0a7725e6cf5679c4/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e52655eaf81e32593abedaa4bfe33170c8cfedf3365ed9be6e11e07f148f0278", size = 499423, upload-time = "2026-06-30T07:15:00.488Z" }, { url = "https://files.pythonhosted.org/packages/2d/9a/41263969df0ce3d9af2a96d5005a288200af1989aed3354bfceb5fc0b21f/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dfcc8b909769d19db55c7cc9541eb64b9b774b1057ffffb4f1048070475bb9f9", size = 386077, upload-time = "2026-06-30T07:15:01.911Z" }, { url = "https://files.pythonhosted.org/packages/5e/19/7e98f468bd50346faff5b10e5297374b443bfdddacc8e9fbc65984539597/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c1255b302953c86a486b81d330d5ee1d5bd937691ce271b6be0ef0e299eaab7", size = 371315, upload-time = "2026-06-30T07:15:03.317Z" }, { url = "https://files.pythonhosted.org/packages/99/3c/2b973b4d371906a134b03decfea7f5d9835a2c6d263454392e15b64b5b18/rpds_py-2026.6.3-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:8d2294a31386bfa251d8c8a39472beee17db67d4f1a6eabea665d35c9a4461c3", size = 383502, upload-time = "2026-06-30T07:15:04.627Z" }, { url = "https://files.pythonhosted.org/packages/98/2a/12e2799500af0a307bca76b63361c51f9fe479223561489c29eea1f2ee41/rpds_py-2026.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f8f23ead891a3b762f35ab3b04623da7056545b48aa60d59957e6789914545da", size = 402673, upload-time = "2026-06-30T07:15:05.856Z" }, { url = "https://files.pythonhosted.org/packages/2d/e3/21e5872d165fe08be4f229e3d5ee9d90019c0bf0e5538de60dbd54009450/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:421aba32367055614287a4292b6a17f1939c9452299f7a0209c117e990b646d4", size = 549964, upload-time = "2026-06-30T07:15:07.159Z" }, { url = "https://files.pythonhosted.org/packages/1a/d0/5ee0fe36844297de8123bee27bc12078c1a7416ad9f1b8a8ca18d6b0c0ac/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1e5822dfc2f0d4ab7e745eaa6d85945069329beeccef965af3f3bb26058fcab6", size = 615446, upload-time = "2026-06-30T07:15:08.531Z" }, { url = "https://files.pythonhosted.org/packages/b1/80/1ea5873cb683f2fbe5f21b23ea1f6d179ead19f3c5b249b7eb5dca568ef2/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:83e35b57523816c8613fd0776b40cd8bb9f596b37ddd2692eb4a6bb5ab2f8c93", size = 576975, upload-time = "2026-06-30T07:15:09.97Z" }, { url = "https://files.pythonhosted.org/packages/c9/e1/90ef639217a5ddb15b7f4f61b1c33911fd044ad03c311bafdd2bcab85582/rpds_py-2026.6.3-cp311-cp311-win32.whl", hash = "sha256:de3eceba0b683bcbb1ab93da016d0270df1f9ae7be716b40214c5dafac6ea45a", size = 204453, upload-time = "2026-06-30T07:15:11.324Z" }, { url = "https://files.pythonhosted.org/packages/f2/b7/b7a1695d7af36f521fb11e80d6d3adbd744f73b921859bd3c2a2c0dc706f/rpds_py-2026.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:2c54a076ca4d370980ab57bc0e31df57bbe8d41340436a90ef8b1219a3cbb127", size = 223219, upload-time = "2026-06-30T07:15:12.476Z" }, { url = "https://files.pythonhosted.org/packages/d7/a2/145afacf796e4506062825941176ad9445c2dcf2b3b6a1f13d3030a15e19/rpds_py-2026.6.3-cp311-cp311-win_arm64.whl", hash = "sha256:168c733a7112e071bb7a66460e667edfcff06c017a3c523f7a8a8e08d0140804", size = 219137, upload-time = "2026-06-30T07:15:13.631Z" }, { url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" }, { url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" }, { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, { url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" }, { url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" }, { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, { url = "https://files.pythonhosted.org/packages/a4/9e/b818ee580026ec578138e961027a68820c40afeb1ec8f6819b54fb99e196/rpds_py-2026.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223", size = 343012, upload-time = "2026-06-30T07:15:36.005Z" }, { url = "https://files.pythonhosted.org/packages/f3/6b/686d9dc4359a8f163cfbbf89ee0b4e586431de22fe8248edb63a8cf50d49/rpds_py-2026.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f", size = 338203, upload-time = "2026-06-30T07:15:37.462Z" }, { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, { url = "https://files.pythonhosted.org/packages/85/ed/adab103321c0a6565d5ae1c2998349bc3ee175b82ccc5ae8fc04cc413075/rpds_py-2026.6.3-cp313-cp313-win32.whl", hash = "sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0", size = 201710, upload-time = "2026-06-30T07:15:53.894Z" }, { url = "https://files.pythonhosted.org/packages/7b/ed/a03b09668e74e5dabbf2e211f6468e1820c0552f7b0500082da31841bf7b/rpds_py-2026.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80", size = 219454, upload-time = "2026-06-30T07:15:55.25Z" }, { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, { url = "https://files.pythonhosted.org/packages/b4/9c/f0d19ac587fd0e4ab6b72cda355e9c5a6166b01ef7e064e437aef8eb9fef/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4cf2d36a2357e4d07bb5a4f98801265327b48256867816cfd2ceb001e9754a8f", size = 349791, upload-time = "2026-06-30T07:17:33.315Z" }, { url = "https://files.pythonhosted.org/packages/38/c7/1d49d204c9fd2ee6c537601dc4c1ba921e03363ca576bfab94a00254ac9a/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:30c6dc199b24a5e3e81d50da0f00858c5bbdb2617a750395687f4339c5818171", size = 352842, upload-time = "2026-06-30T07:17:34.897Z" }, { url = "https://files.pythonhosted.org/packages/ac/e5/c0b5dc93cd0d4c06ce1f438907649514e2ea077bcd911e3154a51e96c38e/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9891e594296ab9dada6551c8e7b387b2721f27a67eecd528412e8906247a7b90", size = 382094, upload-time = "2026-06-30T07:17:36.514Z" }, { url = "https://files.pythonhosted.org/packages/0d/54/ec0e907b4ca8d541112db352409bd15f871c9b243e0c92c9b5a46ae96f01/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5c2dc92304aa48a4a60443b548bb12f12e119d4b72f314015e67b9e1be97fca", size = 388662, upload-time = "2026-06-30T07:17:38.235Z" }, { url = "https://files.pythonhosted.org/packages/d3/f4/921c22a4fd0f1c1ac13a3996ffbf0aa67951e2c8ad0d1d9574938a2932e8/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:127e08c0642d880cf32ca47ec2a4a77b901f7e2dd1ad9762adb13955d72ffcc9", size = 504896, upload-time = "2026-06-30T07:17:39.689Z" }, { url = "https://files.pythonhosted.org/packages/0b/1b/a114b972cefa1ab1cdb3c7bb177cd3844a12826c507c722d3a73516dbbaf/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8bb68f03f395eb793220b45c097bd4d8c32944393da0fad8b999efac0868fc8c", size = 391545, upload-time = "2026-06-30T07:17:41.336Z" }, { url = "https://files.pythonhosted.org/packages/4e/98/af9b3db77d47fcbe6c8c1f36e2c2147ec70292819e99c325f871584a1c11/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3450b693fde92133e9f51060568a4c31fcca76d5e53bbd611e689ca446517e9", size = 380059, upload-time = "2026-06-30T07:17:42.857Z" }, { url = "https://files.pythonhosted.org/packages/c9/ba/0efd8668b97c1d26a61566386c636a7a7a09829e474fdf807caa15a2c844/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:5e8d07bddee435a2ff6f1920e18feff28d0bc4533e42f4bf6927fbd073312c41", size = 393235, upload-time = "2026-06-30T07:17:44.637Z" }, { url = "https://files.pythonhosted.org/packages/62/90/8c139ee9690f73b0829f32647de6f40d826f8f443af6fa72644f96351aac/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3a83ae6c67b7676b9878378547ca8e93ed77a580037bcbcd1d32f739e1e6089c", size = 413008, upload-time = "2026-06-30T07:17:46.225Z" }, { url = "https://files.pythonhosted.org/packages/9c/97/0043896fdd7828ce09a1d9a8b06433714d0960fc4ff3fc4aa72b666b764e/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2bfd04c19ddbd6640de0b51894d764bd2758854d5b75bd102d2ef10cb9c293a9", size = 558118, upload-time = "2026-06-30T07:17:47.759Z" }, { url = "https://files.pythonhosted.org/packages/f6/40/02355f0e134f783a8f9814c4680a1bd311d37671577a5964ea838573ff37/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:ca6546b66be9dc4738b1b043d5ebd5488c66c578c5ff0fd0e8065313fe3afb76", size = 623138, upload-time = "2026-06-30T07:17:49.355Z" }, { url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" }, ] [[package]] name = "ruff" version = "0.16.8" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ba/78/449cb84790bd5cc3823b2652ee405a4558856e5c4195aee3a16bf7b3eb5d/ruff-0.16.8.tar.gz", hash = "sha256:9247bf92b5f04d825c8639a4fe423ec2e4222acd9222e58412b0dab7e442798b", size = 4938814, upload-time = "2026-09-16T15:54:46.688Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ac/25/6071aabc530e9be7e2c195e8fe3f7aea2735405b6cf447212832d7811831/ruff-0.16.8-py3-none-linux_armv6l.whl", hash = "sha256:6ffbd6d87383c1edf5f6fa890f10200950240d7c1a16052a19a09d3a2307dd38", size = 10048966, upload-time = "2026-09-16T15:53:57.605Z" }, { url = "https://files.pythonhosted.org/packages/54/98/07f90ecbc74dd5fb5764f11f2bc774d6a7cffef92d2ff5f5b4e9e23c754e/ruff-0.16.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:42ed6b878ed61e3acca92f2730a17acff39286944ea82398544696366a6f925e", size = 10165498, upload-time = "2026-09-16T15:54:01.14Z" }, { url = "https://files.pythonhosted.org/packages/fe/1f/e6a712e3b47cad4a40600134105ed193cb773f618a42eb7ba323cb812cc0/ruff-0.16.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7ea781c7f2afba8c6a505ea0fb3f994020249e0c450635f5381286fea6b46170", size = 9830004, upload-time = "2026-09-16T15:54:03.998Z" }, { url = "https://files.pythonhosted.org/packages/23/f2/311a08776d75d81c7676e20b6b020ae63cbe881fcdc7a8dd64e6e18bdd93/ruff-0.16.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8efeae3bbe414a5efefda11a792dfb51ef90ac48d50c4830de2f644caf3e8659", size = 9986558, upload-time = "2026-09-16T15:54:06.804Z" }, { url = "https://files.pythonhosted.org/packages/f3/ed/37b6cb3d3ba8c73e68ae3eb1d502383beb5aa05a582bb7bb3a922f929f54/ruff-0.16.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a79b795469fef7fc6e908b218eed2eb17332afd85031db6480dc864560e69b2", size = 9877332, upload-time = "2026-09-16T15:54:09.552Z" }, { url = "https://files.pythonhosted.org/packages/22/cc/40873a8f36ad084cc540d55fcca7077264d5b13b24659e9180c176fb2b08/ruff-0.16.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3fdc5563cdc50555e6fba39322850860e9267c1b3d12c26a74729d8604c3c812", size = 10507125, upload-time = "2026-09-16T15:54:12.152Z" }, { url = "https://files.pythonhosted.org/packages/c3/e4/fc91a642b78ccbab6b9477720f3644ae7a10a9bcce69a934679cd64f62bc/ruff-0.16.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:34508983c70665578dab88f5223d8e6228307e1135398ca8bfc8b7e9501e282b", size = 11336694, upload-time = "2026-09-16T15:54:15.489Z" }, { url = "https://files.pythonhosted.org/packages/c2/3d/bbd2a9a600a4e73dc3e7548a249c8d1671273464b55822c6fae50f602dff/ruff-0.16.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:644bb578569e0ffc575741232bd385dacdd6fbe123f1a729e7a225f54aa3957f", size = 10774448, upload-time = "2026-09-16T15:54:18.16Z" }, { url = "https://files.pythonhosted.org/packages/1a/41/d83af9879a7b6e8bf5fe16b1da0b134049d2f5d3afac12defb0897cb84bd/ruff-0.16.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15e7d226246961db9235098333caa13063906d3851136b84c2900b82f5daa1df", size = 10323796, upload-time = "2026-09-16T15:54:20.743Z" }, { url = "https://files.pythonhosted.org/packages/f5/2c/cefd07bfe914b84943ea769ade8d607bd22750b965d3228eefd7cebd15d0/ruff-0.16.8-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:a2bf6bc3e9ebdd4449abc6f06cf64b98051a2c61cf94d2fe9596518c881f1a1e", size = 10514115, upload-time = "2026-09-16T15:54:23.497Z" }, { url = "https://files.pythonhosted.org/packages/f3/9d/76a2e26c79a23be6e6e3664c57bec9e9fc8de155cfb9e4b67ea91b64f9d7/ruff-0.16.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6ca111ba0849539165e9e59d2b442542f3c1e8060ebbdea82494f1ffbccb1e1f", size = 10072582, upload-time = "2026-09-16T15:54:26.185Z" }, { url = "https://files.pythonhosted.org/packages/2e/d4/f42edddb39668af1a559ceafa3823aedd65633a48dc9768e775485faa2c1/ruff-0.16.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:359a1e5b495448ee1e91018064382ebc86f90e8aac2fed222c7d0e4e8df85fd2", size = 9879644, upload-time = "2026-09-16T15:54:29.278Z" }, { url = "https://files.pythonhosted.org/packages/f8/d4/913e3195d95e0378786c6656945c865f534a3560e29139da4882aff630d1/ruff-0.16.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:59e8f5681349474110b24d62e93cfda6593f5fa3473446ca3705200cac1a08b9", size = 10231569, upload-time = "2026-09-16T15:54:32.036Z" }, { url = "https://files.pythonhosted.org/packages/2b/c4/8aa6ea0bdcedbd1bf87397e2fc4ed8406448ea5842f8660bc6e5f163039d/ruff-0.16.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:efa3e7a16d1baaa79957888dfdf8be9ef2e44db81cb032af06d76632ab59e773", size = 10663666, upload-time = "2026-09-16T15:54:34.838Z" }, { url = "https://files.pythonhosted.org/packages/3d/02/7f10ef4700bc223c30a3fdd10631a29830c45524b810a3c7ed947af64591/ruff-0.16.8-py3-none-win32.whl", hash = "sha256:55793ba85c69921e89be061426d91a78652d6e50317c962240922747a4eb713f", size = 10093472, upload-time = "2026-09-16T15:54:37.47Z" }, { url = "https://files.pythonhosted.org/packages/1e/5d/a509c07d714b6da88f2c518b4637cf6f1d46b074be8f0f1e5fb9ff5126fe/ruff-0.16.8-py3-none-win_amd64.whl", hash = "sha256:a6b85621fd3c81e31fc5f5add09c9c078b430db3595ca632efafdec9e64ebfaa", size = 10586899, upload-time = "2026-09-16T15:54:40.488Z" }, { url = "https://files.pythonhosted.org/packages/fe/a0/50787329e4f20bf9dc9f6230015d46ec69c51a97ace5bc202dae4755365d/ruff-0.16.8-py3-none-win_arm64.whl", hash = "sha256:d075e820af612102ce217f07cc93e69f9490b10ec13ea85fa87bd03d996cef8a", size = 10386316, upload-time = "2026-09-16T15:54:43.332Z" }, ] [[package]] name = "six" version = "1.17.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] [[package]] name = "sortedcontainers" version = "2.4.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, ] [[package]] name = "typing-extensions" version = "4.16.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] [[package]] name = "typing-inspection" version = "0.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, ] [[package]] name = "urllib3" version = "2.8.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, ] [[package]] name = "watchdog" version = "6.0.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220, upload-time = "2024-11-01T14:07:13.037Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e0/24/d9be5cd6642a6aa68352ded4b4b10fb0d7889cb7f45814fb92cecd35f101/watchdog-6.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6eb11feb5a0d452ee41f824e271ca311a09e250441c262ca2fd7ebcf2461a06c", size = 96393, upload-time = "2024-11-01T14:06:31.756Z" }, { url = "https://files.pythonhosted.org/packages/63/7a/6013b0d8dbc56adca7fdd4f0beed381c59f6752341b12fa0886fa7afc78b/watchdog-6.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ef810fbf7b781a5a593894e4f439773830bdecb885e6880d957d5b9382a960d2", size = 88392, upload-time = "2024-11-01T14:06:32.99Z" }, { url = "https://files.pythonhosted.org/packages/d1/40/b75381494851556de56281e053700e46bff5b37bf4c7267e858640af5a7f/watchdog-6.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:afd0fe1b2270917c5e23c2a65ce50c2a4abb63daafb0d419fde368e272a76b7c", size = 89019, upload-time = "2024-11-01T14:06:34.963Z" }, { url = "https://files.pythonhosted.org/packages/39/ea/3930d07dafc9e286ed356a679aa02d777c06e9bfd1164fa7c19c288a5483/watchdog-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdd4e6f14b8b18c334febb9c4425a878a2ac20efd1e0b231978e7b150f92a948", size = 96471, upload-time = "2024-11-01T14:06:37.745Z" }, { url = "https://files.pythonhosted.org/packages/12/87/48361531f70b1f87928b045df868a9fd4e253d9ae087fa4cf3f7113be363/watchdog-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c7c15dda13c4eb00d6fb6fc508b3c0ed88b9d5d374056b239c4ad1611125c860", size = 88449, upload-time = "2024-11-01T14:06:39.748Z" }, { url = "https://files.pythonhosted.org/packages/5b/7e/8f322f5e600812e6f9a31b75d242631068ca8f4ef0582dd3ae6e72daecc8/watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0", size = 89054, upload-time = "2024-11-01T14:06:41.009Z" }, { url = "https://files.pythonhosted.org/packages/68/98/b0345cabdce2041a01293ba483333582891a3bd5769b08eceb0d406056ef/watchdog-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:490ab2ef84f11129844c23fb14ecf30ef3d8a6abafd3754a6f75ca1e6654136c", size = 96480, upload-time = "2024-11-01T14:06:42.952Z" }, { url = "https://files.pythonhosted.org/packages/85/83/cdf13902c626b28eedef7ec4f10745c52aad8a8fe7eb04ed7b1f111ca20e/watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:76aae96b00ae814b181bb25b1b98076d5fc84e8a53cd8885a318b42b6d3a5134", size = 88451, upload-time = "2024-11-01T14:06:45.084Z" }, { url = "https://files.pythonhosted.org/packages/fe/c4/225c87bae08c8b9ec99030cd48ae9c4eca050a59bf5c2255853e18c87b50/watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a175f755fc2279e0b7312c0035d52e27211a5bc39719dd529625b1930917345b", size = 89057, upload-time = "2024-11-01T14:06:47.324Z" }, { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079, upload-time = "2024-11-01T14:06:59.472Z" }, { url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078, upload-time = "2024-11-01T14:07:01.431Z" }, { url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076, upload-time = "2024-11-01T14:07:02.568Z" }, { url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077, upload-time = "2024-11-01T14:07:03.893Z" }, { url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078, upload-time = "2024-11-01T14:07:05.189Z" }, { url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077, upload-time = "2024-11-01T14:07:06.376Z" }, { url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078, upload-time = "2024-11-01T14:07:07.547Z" }, { url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065, upload-time = "2024-11-01T14:07:09.525Z" }, { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070, upload-time = "2024-11-01T14:07:10.686Z" }, { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, ] [[package]] name = "zarr-metadata" source = { editable = "." } dependencies = [ { name = "annotated-types" }, { name = "typing-extensions" }, ] [package.dev-dependencies] docs = [ { name = "griffe-inherited-docstrings" }, { name = "mkdocs" }, { name = "mkdocs-material" }, { name = "mkdocstrings" }, { name = "mkdocstrings-python" }, { name = "pymdown-extensions" }, { name = "ruff" }, ] test = [ { name = "hypothesis" }, { name = "jsonschema" }, { name = "pydantic" }, { name = "pytest" }, { name = "pytest-examples" }, ] [package.metadata] requires-dist = [ { name = "annotated-types", specifier = ">=0.6" }, { name = "typing-extensions", specifier = ">=4.16" }, ] [package.metadata.requires-dev] docs = [ { name = "griffe-inherited-docstrings", specifier = "==1.1.3" }, { name = "mkdocs", specifier = "==1.6.1" }, { name = "mkdocs-material", specifier = "==9.7.7" }, { name = "mkdocstrings", specifier = "==1.0.6" }, { name = "mkdocstrings-python", specifier = "==2.0.8" }, { name = "pymdown-extensions", specifier = "==11.0.1" }, { name = "ruff", specifier = "==0.16.8" }, ] test = [ { name = "hypothesis" }, { name = "jsonschema" }, { name = "pydantic", specifier = ">=2.13" }, { name = "pytest" }, { name = "pytest-examples", specifier = ">=0.0.18" }, ] zarr-developers-zarr-python-a2f2415/pyproject.toml000066400000000000000000000376471527014137600224220ustar00rootroot00000000000000[build-system] requires = ["hatchling>=1.29.0", "hatch-vcs", "setuptools-scm>=9"] build-backend = "hatchling.build" # An allowlist, not a blocklist: anything new — a subpackage under `packages/`, a # config file in the repository root — stays out of the sdist unless it is named # here. Beyond `src` and `tests`, the entries are what keeps the shipped test # suite and docs build runnable from an unpacked sdist: `tests/test_docs.py` # walks `docs/` and `testpaths` collects `docs/user-guide`; the pages under # `docs/user-guide/examples/` pull their source out of `examples/` via pymdownx # snippet includes; `mkdocs.yml` and `mkdocs_hooks.py` let `mkdocs build` run # too. `pyproject.toml`, `README.md`, `LICENSE.txt` and `.gitignore` are added by # hatchling itself. [tool.hatch.build.targets.sdist] include = [ "/src", "/tests", "/docs", "/examples", "/mkdocs.yml", "/mkdocs_hooks.py", "/Justfile", "/ci", ] [project] name = "zarr" description = "An implementation of chunked, compressed, N-dimensional arrays for Python" readme = { file = "README.md", content-type = "text/markdown" } authors = [ { name = "Alistair Miles", email = "alimanfoo@googlemail.com" }, ] maintainers = [ { name = "Davis Bennett", email = "davis.v.bennett@gmail.com" }, { name = "jakirkham" }, { name = "Josh Moore", email = "josh@openmicroscopy.org" }, { name = "Joe Hamman", email = "joe@earthmover.io" }, { name = "Juan Nunez-Iglesias", email = "juan.nunez-iglesias@monash.edu" }, { name = "Martin Durant", email = "mdurant@anaconda.com" }, { name = "Norman Rzepka" }, { name = "Ryan Abernathey" }, { name = "David Stansby" }, { name = "Tom Augspurger", email = "tom.w.augspurger@gmail.com" }, { name = "Deepak Cherian" } ] requires-python = ">=3.12" dependencies = [ 'packaging>=22.0', 'numpy>=2', # 0.16.6 and 0.17.1 refuse to unpickle untrusted data (GHSA-jqc8-qjfv-5m6r); 0.17.0 does not. 'numcodecs>=0.16.6,!=0.17.0', 'google-crc32c>=1.5', 'typing_extensions>=4.14', 'donfig>=0.8', 'msgspec>=0.19', ] dynamic = [ "version", ] classifiers = [ 'Development Status :: 6 - Mature', 'Intended Audience :: Developers', 'Intended Audience :: Information Technology', 'Intended Audience :: Science/Research', 'Programming Language :: Python', 'Topic :: Software Development :: Libraries :: Python Modules', 'Operating System :: Unix', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: 3.14', ] license = "MIT" license-files = ["LICENSE.txt"] keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"] [project.optional-dependencies] # User-facing extras (shipped in package metadata) remote = [ "fsspec>=2023.10.0", "obstore>=0.5.1", ] gpu = [ "cupy-cuda12x; sys_platform != 'darwin'", ] cast-value-rs = ["cast-value-rs>=0.4.2"] cli = ["typer"] optional = ["universal-pathlib"] [project.scripts] zarr = "zarr._cli.cli:app" [project.urls] issues = "https://github.com/zarr-developers/zarr-python/issues" changelog = "https://zarr.readthedocs.io/en/stable/release-notes.html" Discussions = "https://github.com/zarr-developers/zarr-python/discussions" documentation = "https://zarr.readthedocs.io/" homepage = "https://github.com/zarr-developers/zarr-python" # Dev *tooling* is pinned to exact versions for reproducible CI: the hatch envs # (see `tool.hatch.envs.*`) and bare `uv run` resolve these groups fresh from # PyPI and do NOT consult uv.lock, so an unrelated tooling release can break CI # without any change on our side (e.g. the pytest 9.1.0 `duplicate # parametrization` regression). Runtime/integration deps (fsspec, obstore, s3fs, # botocore, numcodecs, universal-pathlib) are intentionally left floating so the # `optional` test matrix keeps exercising their latest releases; their floor and # bleeding edge are covered by the `min_deps` and `upstream` hatch envs. Bump the # pins deliberately, e.g. via dependabot or `uv lock --upgrade`. [dependency-groups] test = [ "coverage==7.16.2", "pytest==9.1.1", "pytest-asyncio==1.4.0", "pytest-cov==7.1.0", "pytest-accept==0.3.0", "numpydoc==1.11.0", "hypothesis==6.168.3", "pytest-reportlog==1.0.0", "pytest-xdist==3.8.0", "pytest-benchmark==5.3.0", "pytest-codspeed==5.0.3", "tomlkit==0.15.1", "uv==0.13.4", ] remote-tests = [ {include-group = "test"}, "fsspec>=2023.10.0", "obstore>=0.5.1", "botocore", "s3fs>=2023.10.0", "moto[s3,server]==5.2.3", "requests==2.34.2", ] release = [ "towncrier==26.9.0", ] docs = [ # Doc building "mkdocs-material[imaging]==9.7.7", "mkdocs==1.6.1", "mkdocstrings==1.0.6", "mkdocstrings-python==2.0.9", "mike==2.2.0", "mkdocs-redirects==1.2.3", "markdown-exec[ansi]==1.12.3", "griffe-inherited-docstrings==1.1.3", "ruff==0.16.9", # Changelog generation {include-group = "release"}, # Optional dependencies to run examples "numcodecs[msgpack]", "s3fs>=2023.10.0", "astroid==4.3.3", "pytest==9.1.1", ] dev = [ {include-group = "test"}, {include-group = "remote-tests"}, {include-group = "docs"}, "universal-pathlib", "mypy==2.3.1", ] [tool.coverage.report] exclude_also = [ 'if TYPE_CHECKING:', ] [tool.coverage.run] omit = [ "bench/compress_normal.py", ] [tool.hatch] version.source = "vcs" # Only consider zarr-python's own `v*` tags when deriving the version. Without # this filter `git describe` matches the most recent tag of any shape, # including the `zarr_metadata-v*` and `zarr_http_server-v*` tags used to # release the subpackages under `packages/` — which would make a from-source # build report e.g. a `0.2.x` version instead of `3.x`. version.raw-options = { scm = { git = { describe_command = "git describe --dirty --tags --long --match v*" } } } [tool.hatch.build] hooks.vcs.version-file = "src/zarr/_version.py" [tool.hatch.envs.dev] dependency-groups = ["dev"] [tool.hatch.envs.test] dependency-groups = ["test"] [tool.hatch.envs.test.env-vars] [[tool.hatch.envs.test.matrix]] python = ["3.12", "3.13", "3.14"] deps = ["minimal", "optional"] [tool.hatch.envs.test.overrides] matrix.deps.features = [ {value = "remote", if = ["optional"]}, {value = "optional", if = ["optional"]}, {value = "cli", if = ["optional"]}, {value = "cast-value-rs", if = ["optional"]}, ] matrix.deps.dependency-groups = [ {value = "remote-tests", if = ["optional"]}, ] [tool.hatch.envs.gputest] template = "test" extra-dependencies = [ "universal_pathlib", # Needed so tests/test_docs.py is collectable under `pytest -m gpu`; otherwise its # module-level importorskip("pytest_examples") skips the whole module and the gpu # docs example is never executed on GPU hardware. "pytest-examples", ] features = ["gpu"] [[tool.hatch.envs.gputest.matrix]] python = ["3.12", "3.13"] [tool.hatch.envs.upstream] template = 'test' python = "3.14" extra-dependencies = [ 'packaging @ git+https://github.com/pypa/packaging', 'numpy', # from scientific-python-nightly-wheels 'numcodecs @ git+https://github.com/zarr-developers/numcodecs', 's3fs @ git+https://github.com/fsspec/s3fs', 'universal_pathlib @ git+https://github.com/fsspec/universal_pathlib', 'typing_extensions @ git+https://github.com/python/typing_extensions', 'donfig @ git+https://github.com/pytroll/donfig', 'obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore', ] [tool.hatch.envs.upstream.env-vars] PIP_INDEX_URL = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/" PIP_EXTRA_INDEX_URL = "https://pypi.org/simple/" PIP_PRE = "1" [tool.hatch.envs.min_deps] description = """Test environment for minimum supported dependencies See Spec 0000 for details and drop schedule: https://scientific-python.org/specs/spec-0000/ """ template = "test" python = "3.12" features = ["remote"] dependency-groups = ["remote-tests"] extra-dependencies = [ 'packaging==22.*', 'numpy==2.0.*', 'numcodecs==0.16.*', 'fsspec==2023.10.0', 's3fs==2023.10.0', 'universal_pathlib==0.2.0', 'typing_extensions==4.14.*', 'donfig==0.8.*', 'obstore==0.5.*', 'msgspec==0.19.*', ] [tool.hatch.envs.default] installer = "uv" [tool.hatch.envs.doctest] description = "Test environment for validating executable code blocks in documentation" features = ['remote'] dependency-groups = ['remote-tests'] extra-dependencies = [ "pytest-examples", ] [tool.ruff] line-length = 100 force-exclude = true extend-exclude = [ ".bzr", ".direnv", ".eggs", ".git", ".mypy_cache", ".nox", ".pants.d", ".ruff_cache", ".venv", "__pypackages__", "_build", "buck-out", "build", "dist", "venv", "docs", "tests/test_regression/scripts/", # these are scripts that use a different version of python "src/zarr/v2/", "tests/v2/", ] [tool.ruff.lint] extend-select = [ "ANN", # flake8-annotations "B", # flake8-bugbear "C4", # flake8-comprehensions "EXE", # flake8-executable "FA", # flake8-future-annotations "FLY", # flynt "FURB", # refurb "G", # flake8-logging-format "I", # isort "ISC", # flake8-implicit-str-concat "LOG", # flake8-logging "PERF", # Perflint "PIE", # flake8-pie "PGH", # pygrep-hooks "PT", # flake8-pytest-style "PYI", # flake8-pyi "RET", # flake8-return "RSE", # flake8-raise "RUF", "S101", # flake8-bandit: no assert in runtime code "SIM", # flake8-simplify "SLOT", # flake8-slots "TC", # flake8-type-checking "TRY", # tryceratops "UP", # pyupgrade "W", # pycodestyle warnings ] ignore = [ "ANN401", "PT011", # TODO: apply this rule "RET505", "RET506", "RUF005", "RUF043", "SIM108", "TRY003", # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules "W191", "E111", "E114", "E117", "D206", "D300", "Q000", "Q001", "Q002", "Q003", "COM812", "COM819", "TC006", ] [tool.ruff.lint.extend-per-file-ignores] "tests/**" = ["ANN001", "ANN201", "RUF029", "SIM117", "SIM300"] # S101 (no assert) is for runtime code only. Tests, examples, and test-support # packages (store conformance suite, hypothesis strategies) assert on purpose. # The globs cover the root package and every package under packages/. "**/tests/**" = ["S101"] "**/examples/**" = ["S101"] "**/testing/**" = ["S101"] [tool.mypy] files = ["src", "tests"] python_version = "3.12" ignore_missing_imports = true namespace_packages = false pretty = true show_error_code_links = true show_error_context = true strict = true warn_unreachable = true enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool", "truthy-iterable"] [[tool.mypy.overrides]] module = [ "tests.package_with_entrypoint.*", "zarr.testing.stateful", "tests.test_codecs.test_transpose", "tests.test_config", "tests.test_store.test_zip", "tests.test_store.test_local", "tests.test_store.test_memory", "tests.test_codecs.test_codecs", "tests.test_metadata.*", "tests.test_store.test_core", "tests.test_store.test_logging", "tests.test_store.test_object", "tests.test_store.test_stateful", "tests.test_store.test_wrapper", ] strict = false # TODO: Move the next modules up to the strict = false section # and fix the errors [[tool.mypy.overrides]] module = [ "tests.test_store.test_fsspec", "tests.test_group", "tests.test_indexing", "tests.test_properties", "tests.test_sync", "tests.test_regression.scripts.*" ] ignore_errors = true [tool.pytest.ini_options] minversion = "9" testpaths = ["src", "tests", "docs/user-guide"] log_cli_level = "INFO" log_level = "INFO" # Enables strict_config, strict_markers, strict_xfail, strict_parametrization_ids, and # any strictness options added in future pytest releases. Note that the equivalent # `--strict-config`/`--strict-markers` flags were silently ignored when passed via # addopts before pytest 9.1 (pytest#14442), so this option is the reliable spelling. strict = true # Turn deadlocks into loud failures: if a single test exceeds this many seconds, dump # every thread's traceback and kill the run (exit_on_timeout is new in pytest 9). Sized # far above the slowest legitimate test (~20s locally; slower under coverage/Windows/ # nightly stateful-hypothesis runs) so only a genuine hang can trip it. faulthandler_timeout = 600 faulthandler_exit_on_timeout = true asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" doctest_optionflags = [ "NORMALIZE_WHITESPACE", "ELLIPSIS", "IGNORE_EXCEPTION_DETAIL", ] addopts = [ "--benchmark-columns", "min,mean,stddev,outliers,rounds,iterations", "--benchmark-disable", # benchmark routines run as tests without benchmarking instrumentation "--durations", "10", "-ra", "--doctest-modules", "--ignore=tests/test_regression/scripts", "--ignore=src/zarr/_cli", ] filterwarnings = [ "error", "ignore:Unclosed client session None: """ Print version info for use in bug reports. """ import platform from importlib.metadata import version def print_packages(packages: list[str]) -> None: not_installed = [] for package in packages: try: print(f"{package}: {version(package)}") except ModuleNotFoundError: not_installed.append(package) if not_installed: print("\n**Not Installed:**") for package in not_installed: print(package) required = [ "packaging", "numpy", "numcodecs", "typing_extensions", "donfig", ] optional = [ "botocore", "cupy-cuda12x", "fsspec", "numcodecs", "s3fs", "gcsfs", "universal-pathlib", "obstore", ] print(f"platform: {platform.platform()}") print(f"python: {platform.python_version()}") print(f"zarr: {__version__}\n") print("**Required dependencies:**") print_packages(required) print("\n**Optional dependencies:**") print_packages(optional) # The decorator ensures this always returns the same handler (and it is only # attached once). @functools.cache def _ensure_handler() -> logging.Handler: """ The first time this function is called, attach a `StreamHandler` using the same format as `logging.basicConfig` to the Zarr-Python root logger. Return this handler every time this function is called. """ handler = logging.StreamHandler() handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) _logger.addHandler(handler) return handler def set_log_level( level: Literal["NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], ) -> None: """Set the logging level for Zarr-Python. Zarr-Python uses the standard library `logging` framework under the root logger 'zarr'. This is a helper function to: - set Zarr-Python's root logger level - set the root logger handler's level, creating the handler if it does not exist yet Parameters ---------- level : str The logging level to set. """ _logger.setLevel(level) _ensure_handler().setLevel(level) def set_format(log_format: str) -> None: """Set the format of logging messages from Zarr-Python. Zarr-Python uses the standard library `logging` framework under the root logger 'zarr'. This sets the format of log messages from the root logger's StreamHandler. Parameters ---------- log_format : str A string determining the log format (as defined in the standard library's `logging` module for logging.Formatter) """ _ensure_handler().setFormatter(logging.Formatter(fmt=log_format)) __all__ = [ "Array", "AsyncArray", "AsyncGroup", "Group", "__version__", "array", "config", "consolidate_metadata", "copy", "copy_all", "copy_store", "create", "create_array", "create_group", "create_hierarchy", "empty", "empty_like", "from_array", "full", "full_like", "group", "load", "ones", "ones_like", "open", "open_array", "open_consolidated", "open_group", "open_like", "print_debug_info", "save", "save_array", "save_group", "tree", "zeros", "zeros_like", ] zarr-developers-zarr-python-a2f2415/src/zarr/_cli/000077500000000000000000000000001527014137600221405ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/_cli/__init__.py000066400000000000000000000000001527014137600242370ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/_cli/cli.py000066400000000000000000000133341527014137600232650ustar00rootroot00000000000000import logging from enum import StrEnum from typing import Annotated, Literal, cast import typer import zarr import zarr.metadata.migrate_v3 as migrate_metadata from zarr.core.common import ZarrFormat from zarr.core.sync import sync from zarr.storage._common import make_store app = typer.Typer() logger = logging.getLogger(__name__) def _set_logging_level(*, verbose: bool) -> None: if verbose: lvl = "INFO" else: lvl = "WARNING" zarr.set_log_level(cast(Literal["INFO", "WARNING"], lvl)) zarr.set_format("%(message)s") class CLIZarrFormat(StrEnum): v2 = "v2" v3 = "v3" class CLIZarrFormatV3(StrEnum): """Limit CLI choice to only v3""" v3 = "v3" @app.command() # type: ignore[untyped-decorator] def migrate( zarr_format: Annotated[ CLIZarrFormatV3, typer.Argument( help="Zarr format to migrate to. Currently only 'v3' is supported.", ), ], input_store: Annotated[ str, typer.Argument( help=( "Input Zarr to migrate - should be a store, path to directory in file system or name of zip file " "e.g. 'data/example-1.zarr', 's3://example-bucket/example'..." ) ), ], output_store: Annotated[ str | None, typer.Argument( help=( "Output location to write generated metadata (no array data will be copied). If not provided, " "metadata will be written to input_store. Should be a store, path to directory in file system " "or name of zip file e.g. 'data/example-1.zarr', 's3://example-bucket/example'..." ) ), ] = None, dry_run: Annotated[ bool, typer.Option( help="Enable a dry-run: files that would be converted are logged, but no new files are created or changed." ), ] = False, overwrite: Annotated[ bool, typer.Option( help="Remove any existing v3 metadata at the output location, before migration starts." ), ] = False, force: Annotated[ bool, typer.Option( help=( "Only used when --overwrite is given. Allows v3 metadata to be removed when no valid " "v2 metadata exists at the output location." ) ), ] = False, remove_v2_metadata: Annotated[ bool, typer.Option( help="Remove v2 metadata (if any) from the output location, after migration is complete." ), ] = False, ) -> None: """Migrate all v2 metadata in a zarr hierarchy to v3. This will create a zarr.json file for each level (every group / array). v2 files (.zarray, .zattrs etc.) will be left as-is. """ if dry_run: _set_logging_level(verbose=True) logger.info( "Dry run enabled - no new files will be created or changed. Log of files that would be created on a real run:" ) input_zarr_store = sync(make_store(input_store, mode="r+")) if output_store is not None: output_zarr_store = sync(make_store(output_store, mode="w-")) write_store = output_zarr_store else: output_zarr_store = None write_store = input_zarr_store if overwrite: sync(migrate_metadata.remove_metadata(write_store, 3, force=force, dry_run=dry_run)) migrate_metadata.migrate_v2_to_v3( input_store=input_zarr_store, output_store=output_zarr_store, dry_run=dry_run ) if remove_v2_metadata: # There should always be valid v3 metadata at the output location after migration, so force=False sync(migrate_metadata.remove_metadata(write_store, 2, force=False, dry_run=dry_run)) @app.command() # type: ignore[untyped-decorator] def remove_metadata( zarr_format: Annotated[ CLIZarrFormat, typer.Argument(help="Which format's metadata to remove - v2 or v3."), ], store: Annotated[ str, typer.Argument( help="Store or path to directory in file system or name of zip file e.g. 'data/example-1.zarr', 's3://example-bucket/example'..." ), ], force: Annotated[ bool, typer.Option( help=( "Allow metadata to be deleted when no valid alternative exists e.g. allow deletion of v2 metadata, " "when no v3 metadata is present." ) ), ] = False, dry_run: Annotated[ bool, typer.Option( help="Enable a dry-run: files that would be deleted are logged, but no files are removed or changed." ), ] = False, ) -> None: """Remove all v2 (.zarray, .zattrs, .zgroup, .zmetadata) or v3 (zarr.json) metadata files from the given Zarr. Note - this will remove metadata files at all levels of the hierarchy (every group and array). """ if dry_run: _set_logging_level(verbose=True) logger.info( "Dry run enabled - no files will be deleted or changed. Log of files that would be deleted on a real run:" ) input_zarr_store = sync(make_store(store, mode="r+")) sync( migrate_metadata.remove_metadata( store=input_zarr_store, zarr_format=cast(ZarrFormat, int(zarr_format[1:])), force=force, dry_run=dry_run, ) ) @app.callback() # type: ignore[untyped-decorator] def main( verbose: Annotated[ bool, typer.Option( help="enable verbose logging - will print info about metadata files being deleted / saved." ), ] = False, ) -> None: """ See available commands below - access help for individual commands with zarr COMMAND --help. """ _set_logging_level(verbose=verbose) if __name__ == "__main__": app() zarr-developers-zarr-python-a2f2415/src/zarr/_compat.py000066400000000000000000000066771527014137600232460ustar00rootroot00000000000000import warnings from collections.abc import Callable from functools import wraps from inspect import Parameter, signature from typing import TYPE_CHECKING, Any import numpy as np from packaging.version import Version from zarr.errors import ZarrFutureWarning if TYPE_CHECKING: from numpy.typing import NDArray # Based off https://github.com/scikit-learn/scikit-learn/blob/e87b32a81c70abed8f2e97483758eb64df8255e9/sklearn/utils/validation.py#L63 def _deprecate_positional_args[T]( func: Callable[..., T] | None = None, *, version: str = "3.1.0" ) -> Callable[..., T]: """Decorator for methods that issues warnings for positional arguments. Using the keyword-only argument syntax in pep 3102, arguments after the * will issue a warning when passed as a positional argument. Parameters ---------- func : callable, default=None Function to check arguments on. version : callable, default="3.1.0" The version when positional arguments will result in error. """ def _inner_deprecate_positional_args(f: Callable[..., T]) -> Callable[..., T]: sig = signature(f) kwonly_args = [] all_args = [] for name, param in sig.parameters.items(): if param.kind == Parameter.POSITIONAL_OR_KEYWORD: all_args.append(name) elif param.kind == Parameter.KEYWORD_ONLY: kwonly_args.append(name) @wraps(f) def inner_f(*args: Any, **kwargs: Any) -> T: extra_args = len(args) - len(all_args) if extra_args <= 0: return f(*args, **kwargs) # extra_args > 0 args_msg = [ f"{name}={arg}" for name, arg in zip(kwonly_args[:extra_args], args[-extra_args:], strict=False) ] formatted_args_msg = ", ".join(args_msg) warnings.warn( ( f"Pass {formatted_args_msg} as keyword args. From version " f"{version} passing these as positional arguments " "will result in an error" ), ZarrFutureWarning, stacklevel=2, ) kwargs.update(zip(sig.parameters, args, strict=False)) return f(**kwargs) return inner_f if func is not None: return _inner_deprecate_positional_args(func) return _inner_deprecate_positional_args # type: ignore[return-value] def _reshape_view(arr: "NDArray[Any]", shape: tuple[int, ...]) -> "NDArray[Any]": """Reshape an array without copying data. This function provides compatibility across NumPy versions for reshaping arrays as views. On NumPy >= 2.1, it uses ``reshape(copy=False)`` which explicitly fails if a view cannot be created. On older versions, it uses direct shape assignment which has the same behavior but is deprecated in 2.5+. Parameters ---------- arr : NDArray The array to reshape. shape : tuple of int The new shape. Returns ------- NDArray A reshaped view of the array. Raises ------ AttributeError If a view cannot be created (the array is not contiguous) on NumPy < 2.1. ValueError If a view cannot be created (the array is not contiguous) on NumPy >= 2.1. """ if Version(np.__version__) >= Version("2.1"): return arr.reshape(shape, copy=False) else: arr.shape = shape return arr zarr-developers-zarr-python-a2f2415/src/zarr/abc/000077500000000000000000000000001527014137600217575ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/abc/__init__.py000066400000000000000000000000001527014137600240560ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/abc/buffer.py000066400000000000000000000003041527014137600235770ustar00rootroot00000000000000from zarr.core.buffer.core import ArrayLike, Buffer, BufferPrototype, NDArrayLike, NDBuffer __all__ = [ "ArrayLike", "Buffer", "BufferPrototype", "NDArrayLike", "NDBuffer", ] zarr-developers-zarr-python-a2f2415/src/zarr/abc/codec.py000066400000000000000000000426621527014137600234200ustar00rootroot00000000000000from __future__ import annotations from abc import abstractmethod from collections.abc import Mapping from typing import TYPE_CHECKING, Literal, Protocol, TypeGuard, runtime_checkable from typing_extensions import ReadOnly, TypedDict from zarr.abc.metadata import Metadata from zarr.core.buffer import Buffer, NDBuffer from zarr.core.common import NamedConfig, concurrent_map from zarr.core.config import config if TYPE_CHECKING: from collections.abc import Awaitable, Callable, Iterable from typing import Self from zarr.abc.store import ByteGetter, ByteSetter, Store from zarr.core.array_spec import ArraySpec from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType from zarr.core.indexing import SelectorTuple from zarr.core.metadata import ArrayMetadata from zarr.core.metadata.v3 import ChunkGridMetadata __all__ = [ "ArrayArrayCodec", "ArrayBytesCodec", "ArrayBytesCodecPartialDecodeMixin", "ArrayBytesCodecPartialEncodeMixin", "BaseCodec", "BytesBytesCodec", "CodecInput", "CodecOutput", "CodecPipeline", "GetResult", "SupportsSyncCodec", ] class GetResult(TypedDict): """Metadata about a store get operation.""" status: Literal["present", "missing"] type CodecInput = NDBuffer | Buffer type CodecOutput = NDBuffer | Buffer class CodecJSON_V2[TName: str](TypedDict): """The JSON representation of a codec for Zarr V2""" id: ReadOnly[TName] def _check_codecjson_v2(data: object) -> TypeGuard[CodecJSON_V2[str]]: return isinstance(data, Mapping) and "id" in data and isinstance(data["id"], str) CodecJSON_V3 = str | NamedConfig[str, Mapping[str, object]] """The JSON representation of a codec for Zarr V3.""" # The widest type we will *accept* for a codec JSON # This covers v2 and v3 CodecJSON = str | Mapping[str, object] """The widest type of JSON-like input that could specify a codec.""" @runtime_checkable class SupportsSyncCodec[CI: CodecInput, CO: CodecOutput](Protocol): """Protocol for codecs that support synchronous encode/decode. Codecs implementing this protocol provide `_decode_sync` and `_encode_sync` methods that perform encoding/decoding without requiring an async event loop. The type parameters mirror `BaseCodec`: `CI` is the decoded type and `CO` is the encoded type. """ def _decode_sync(self, chunk_data: CO, chunk_spec: ArraySpec) -> CI: ... def _encode_sync(self, chunk_data: CI, chunk_spec: ArraySpec) -> CO | None: ... def _codec_supports_sync(codec: object) -> bool: """Whether `codec` can actually run on a synchronous (no event loop) path. Structural membership in `SupportsSyncCodec` is necessary but not always sufficient: a codec can provide `_decode_sync`/`_encode_sync` whose ability to run depends on runtime configuration the type system cannot see. `ShardingCodec` is the canonical case — its sync methods delegate to its configured inner and index codec chains, so they only work when every codec in those chains is itself sync-capable. Such codecs opt out dynamically via a `_sync_capable` attribute/property (absent means capable). """ return isinstance(codec, SupportsSyncCodec) and getattr(codec, "_sync_capable", True) class BaseCodec[CI: CodecInput, CO: CodecOutput](Metadata): """Generic base class for codecs. Codecs can be registered via zarr.codecs.registry. Warnings -------- This class is not intended to be directly, please use ArrayArrayCodec, ArrayBytesCodec or BytesBytesCodec for subclassing. """ # Whether this codec's encoded output is a fixed size given a fixed input # size. Defaults to False (the conservative answer): a codec that does not # explicitly opt in is treated as variable-size, which disables # size-dependent fast paths (e.g. the sharding bulk-decode) and makes it # ineligible as a sharding index codec. Codecs with genuinely fixed-size output (BytesCodec, # TransposeCodec, ...) override this with True. The default also keeps # third-party / variable-length codecs (VLenUTF8, numcodecs wrappers) that # never set the attribute from raising AttributeError where it is read. is_fixed_size: bool = False @abstractmethod def compute_encoded_size(self, input_byte_length: int, chunk_spec: ArraySpec) -> int: """Given an input byte length, this method returns the output byte length. Raises a NotImplementedError for codecs with variable-sized outputs (e.g. compressors). Parameters ---------- input_byte_length : int chunk_spec : ArraySpec Returns ------- int """ ... def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: """Computed the spec of the chunk after it has been encoded by the codec. This is important for codecs that change the shape, data type or fill value of a chunk. The spec will then be used for subsequent codecs in the pipeline. Parameters ---------- chunk_spec : ArraySpec Returns ------- ArraySpec """ return chunk_spec def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: """Fills in codec configuration parameters that can be automatically inferred from the array metadata. Parameters ---------- array_spec : ArraySpec Returns ------- Self """ return self def validate( self, *, shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGridMetadata, ) -> None: """Validates that the codec configuration is compatible with the array metadata. Raises errors when the codec configuration is not compatible. Parameters ---------- shape : tuple[int, ...] The array shape dtype : np.dtype[Any] The array data type chunk_grid : ChunkGridMetadata The array chunk grid metadata """ async def _decode_single(self, chunk_data: CO, chunk_spec: ArraySpec) -> CI: raise NotImplementedError # pragma: no cover async def decode( self, chunks_and_specs: Iterable[tuple[CO | None, ArraySpec]], ) -> Iterable[CI | None]: """Decodes a batch of chunks. Chunks can be None in which case they are ignored by the codec. Parameters ---------- chunks_and_specs : Iterable[tuple[CodecOutput | None, ArraySpec]] Ordered set of encoded chunks with their accompanying chunk spec. Returns ------- Iterable[CI | None] """ return await _batching_helper(self._decode_single, chunks_and_specs) async def _encode_single(self, chunk_data: CI, chunk_spec: ArraySpec) -> CO | None: raise NotImplementedError # pragma: no cover async def encode( self, chunks_and_specs: Iterable[tuple[CI | None, ArraySpec]], ) -> Iterable[CO | None]: """Encodes a batch of chunks. Chunks can be None in which case they are ignored by the codec. Parameters ---------- chunks_and_specs : Iterable[tuple[CI | None, ArraySpec]] Ordered set of to-be-encoded chunks with their accompanying chunk spec. Returns ------- Iterable[CodecOutput | None] """ return await _batching_helper(self._encode_single, chunks_and_specs) class ArrayArrayCodec(BaseCodec[NDBuffer, NDBuffer]): """Base class for array-to-array codecs.""" class ArrayBytesCodec(BaseCodec[NDBuffer, Buffer]): """Base class for array-to-bytes codecs.""" class BytesBytesCodec(BaseCodec[Buffer, Buffer]): """Base class for bytes-to-bytes codecs.""" Codec = ArrayArrayCodec | ArrayBytesCodec | BytesBytesCodec class ArrayBytesCodecPartialDecodeMixin: """Mixin for array-to-bytes codecs that implement partial decoding.""" async def _decode_partial_single( self, byte_getter: ByteGetter, selection: SelectorTuple, chunk_spec: ArraySpec ) -> NDBuffer | None: raise NotImplementedError async def decode_partial( self, batch_info: Iterable[tuple[ByteGetter, SelectorTuple, ArraySpec]], ) -> Iterable[NDBuffer | None]: """Partially decodes a batch of chunks. This method determines parts of a chunk from the slice selection, fetches these parts from the store (via ByteGetter) and decodes them. Parameters ---------- batch_info : Iterable[tuple[ByteGetter, SelectorTuple, ArraySpec]] Ordered set of information about slices of encoded chunks. The slice selection determines which parts of the chunk will be fetched. The ByteGetter is used to fetch the necessary bytes. The chunk spec contains information about the construction of an array from the bytes. Returns ------- Iterable[NDBuffer | None] """ return await concurrent_map( list(batch_info), self._decode_partial_single, config.get("async.concurrency"), ) class ArrayBytesCodecPartialEncodeMixin: """Mixin for array-to-bytes codecs that implement partial encoding.""" async def _encode_partial_single( self, byte_setter: ByteSetter, chunk_array: NDBuffer, selection: SelectorTuple, chunk_spec: ArraySpec, ) -> None: raise NotImplementedError # pragma: no cover async def encode_partial( self, batch_info: Iterable[tuple[ByteSetter, NDBuffer, SelectorTuple, ArraySpec]], ) -> None: """Partially encodes a batch of chunks. This method determines parts of a chunk from the slice selection, encodes them and writes these parts to the store (via ByteSetter). If merging with existing chunk data in the store is necessary, this method will read from the store first and perform the merge. Parameters ---------- batch_info : Iterable[tuple[ByteSetter, NDBuffer, SelectorTuple, ArraySpec]] Ordered set of information about slices of to-be-encoded chunks. The slice selection determines which parts of the chunk will be encoded. The ByteSetter is used to write the necessary bytes and fetch bytes for existing chunk data. The chunk spec contains information about the chunk. """ await concurrent_map( list(batch_info), self._encode_partial_single, config.get("async.concurrency"), ) class CodecPipeline: """Base class for implementing CodecPipeline. A CodecPipeline implements the read and write paths for chunk data. On the read path, it is responsible for fetching chunks from a store (via ByteGetter), decoding them and assembling an output array. On the write path, it encodes the chunks and writes them to a store (via ByteSetter).""" @abstractmethod def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: """Fills in codec configuration parameters that can be automatically inferred from the array metadata. Parameters ---------- array_spec : ArraySpec Returns ------- Self """ ... @classmethod @abstractmethod def from_codecs(cls, codecs: Iterable[Codec]) -> Self: """Creates a codec pipeline from an iterable of codecs. Parameters ---------- codecs : Iterable[Codec] Returns ------- Self """ ... @classmethod def from_array_metadata_and_store(cls, array_metadata: ArrayMetadata, store: Store) -> Self: """Creates a codec pipeline from array metadata and a store path. Raises NotImplementedError by default, indicating the CodecPipeline must be created with from_codecs instead. Parameters ---------- array_metadata : ArrayMetadata store : Store Returns ------- Self """ raise NotImplementedError( f"'{type(cls).__name__}' does not implement CodecPipeline.from_array_metadata_and_store." ) @property @abstractmethod def supports_partial_decode(self) -> bool: ... @property @abstractmethod def supports_partial_encode(self) -> bool: ... @abstractmethod def validate( self, *, shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGridMetadata, ) -> None: """Validates that all codec configurations are compatible with the array metadata. Raises errors when a codec configuration is not compatible. Parameters ---------- shape : tuple[int, ...] The array shape dtype : np.dtype[Any] The array data type chunk_grid : ChunkGridMetadata The array chunk grid metadata """ ... @abstractmethod def compute_encoded_size(self, byte_length: int, array_spec: ArraySpec) -> int: """Given an input byte length, this method returns the output byte length. Raises a NotImplementedError for codecs with variable-sized outputs (e.g. compressors). Parameters ---------- byte_length : int array_spec : ArraySpec Returns ------- int """ ... @abstractmethod async def decode( self, chunk_bytes_and_specs: Iterable[tuple[Buffer | None, ArraySpec]], ) -> Iterable[NDBuffer | None]: """Decodes a batch of chunks. Chunks can be None in which case they are ignored by the codec. Parameters ---------- chunk_bytes_and_specs : Iterable[tuple[Buffer | None, ArraySpec]] Ordered set of encoded chunks with their accompanying chunk spec. Returns ------- Iterable[NDBuffer | None] """ ... @abstractmethod async def encode( self, chunk_arrays_and_specs: Iterable[tuple[NDBuffer | None, ArraySpec]], ) -> Iterable[Buffer | None]: """Encodes a batch of chunks. Chunks can be None in which case they are ignored by the codec. Parameters ---------- chunk_arrays_and_specs : Iterable[tuple[NDBuffer | None, ArraySpec]] Ordered set of to-be-encoded chunks with their accompanying chunk spec. Returns ------- Iterable[Buffer | None] """ ... @abstractmethod async def read( self, batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], out: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> tuple[GetResult, ...]: """Reads chunk data from the store, decodes it and writes it into an output array. Partial decoding may be utilized if the codecs and stores support it. Parameters ---------- batch_info : Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]] Ordered set of information about the chunks. The first slice selection determines which parts of the chunk will be fetched. The second slice selection determines where in the output array the chunk data will be written. The ByteGetter is used to fetch the necessary bytes. The chunk spec contains information about the construction of an array from the bytes. If the Store returns ``None`` for a chunk, then the chunk was not written and the implementation must set the values of that chunk (or ``out``) to the fill value for the array. out : NDBuffer Returns ------- tuple[GetResult, ...] One result per chunk in ``batch_info``. """ ... @abstractmethod async def write( self, batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], value: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: """Encodes chunk data and writes it to the store. Merges with existing chunk data by reading first, if necessary. Partial encoding may be utilized if the codecs and stores support it. Parameters ---------- batch_info : Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]] Ordered set of information about the chunks. The first slice selection determines which parts of the chunk will be encoded. The second slice selection determines where in the value array the chunk data is located. The ByteSetter is used to fetch and write the necessary bytes. The chunk spec contains information about the chunk. value : NDBuffer """ ... async def _batching_helper[CI: CodecInput, CO: CodecOutput]( func: Callable[[CI, ArraySpec], Awaitable[CO | None]], batch_info: Iterable[tuple[CI | None, ArraySpec]], ) -> list[CO | None]: return await concurrent_map( list(batch_info), _noop_for_none(func), config.get("async.concurrency"), ) def _noop_for_none[CI: CodecInput, CO: CodecOutput]( func: Callable[[CI, ArraySpec], Awaitable[CO | None]], ) -> Callable[[CI | None, ArraySpec], Awaitable[CO | None]]: async def wrap(chunk: CI | None, chunk_spec: ArraySpec) -> CO | None: if chunk is None: return None return await func(chunk, chunk_spec) return wrap zarr-developers-zarr-python-a2f2415/src/zarr/abc/metadata.py000066400000000000000000000025751527014137600241220ustar00rootroot00000000000000from __future__ import annotations from collections.abc import Sequence from typing import TYPE_CHECKING if TYPE_CHECKING: from typing import Self from zarr.core.common import JSON from dataclasses import dataclass, fields __all__ = ["Metadata"] @dataclass(frozen=True) class Metadata: def to_dict(self) -> dict[str, JSON]: """ Recursively serialize this model to a dictionary. This method inspects the fields of self and calls `x.to_dict()` for any fields that are instances of `Metadata`. Sequences of `Metadata` are similarly recursed into, and the output of that recursion is collected in a list. """ out_dict = {} for field in fields(self): key = field.name value = getattr(self, key) if isinstance(value, Metadata): out_dict[field.name] = getattr(self, field.name).to_dict() elif isinstance(value, str): out_dict[key] = value elif isinstance(value, Sequence): out_dict[key] = tuple(v.to_dict() if isinstance(v, Metadata) else v for v in value) else: out_dict[key] = value return out_dict @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: """ Create an instance of the model from a dictionary """ return cls(**data) zarr-developers-zarr-python-a2f2415/src/zarr/abc/numcodec.py000066400000000000000000000053401527014137600241300ustar00rootroot00000000000000from typing import Any, Protocol, Self, TypeGuard class Numcodec(Protocol): """ A protocol that models the ``numcodecs.abc.Codec`` interface. This protocol should be considered experimental. Expect the type annotations for ``buf`` and ``out`` to narrow in the future. """ codec_id: str def encode(self, buf: Any) -> Any: """Encode data from ``buf``. Parameters ---------- buf : Any Data to be encoded. Returns ------- enc: Any Encoded data. """ ... def decode(self, buf: Any, out: Any | None = None) -> Any: """ Decode data in ``buf``. Parameters ---------- buf : Any Encoded data. out : Any Writeable buffer to store decoded data. If provided, this buffer must be exactly the right size to store the decoded data. Returns ------- dec : Any Decoded data. """ ... def get_config(self) -> Any: """ Return a JSON-serializable configuration dictionary for this codec. Must include an ``'id'`` field with the codec identifier. """ ... @classmethod def from_config(cls, config: Any) -> Self: """ Instantiate a codec from a configuration dictionary. Parameters ---------- config : Any A configuration dictionary for this codec. """ ... def _is_numcodec_cls(obj: object) -> TypeGuard[type[Numcodec]]: """ Check if the given object is a class implements the Numcodec protocol. The @runtime_checkable decorator does not allow issubclass checks for protocols with non-method members (i.e., attributes), so we use this function to manually check for the presence of the required attributes and methods on a given object. """ return ( isinstance(obj, type) and hasattr(obj, "codec_id") and isinstance(obj.codec_id, str) and hasattr(obj, "encode") and callable(obj.encode) and hasattr(obj, "decode") and callable(obj.decode) and hasattr(obj, "get_config") and callable(obj.get_config) and hasattr(obj, "from_config") and callable(obj.from_config) ) def _is_numcodec(obj: object) -> TypeGuard[Numcodec]: """ Check if the given object implements the Numcodec protocol. The @runtime_checkable decorator does not allow issubclass checks for protocols with non-method members (i.e., attributes), so we use this function to manually check for the presence of the required attributes and methods on a given object. """ return _is_numcodec_cls(type(obj)) zarr-developers-zarr-python-a2f2415/src/zarr/abc/store.py000066400000000000000000000600141527014137600234660ustar00rootroot00000000000000from __future__ import annotations import asyncio from abc import ABC, abstractmethod from dataclasses import dataclass from functools import partial from itertools import starmap from typing import TYPE_CHECKING, Literal, Protocol, runtime_checkable if TYPE_CHECKING: from collections.abc import AsyncGenerator, AsyncIterator, Iterable, Sequence from types import TracebackType from typing import Any, Self from zarr.core.buffer import Buffer, BufferPrototype __all__ = [ "ByteGetter", "ByteSetter", "Store", "SupportsDeleteSync", "SupportsGetSync", "SupportsSetSync", "SupportsSyncStore", "SyncByteGetter", "SyncByteSetter", "set_or_delete", ] @dataclass(frozen=True, slots=True) class RangeByteRequest: """Request a specific byte range""" start: int """The start of the byte range request (inclusive).""" end: int """The end of the byte range request (exclusive).""" @dataclass(frozen=True, slots=True) class OffsetByteRequest: """Request all bytes starting from a given byte offset""" offset: int """The byte offset for the offset range request.""" @dataclass(frozen=True, slots=True) class SuffixByteRequest: """Request up to the last `n` bytes""" suffix: int """The number of bytes from the suffix to request.""" type ByteRequest = RangeByteRequest | OffsetByteRequest | SuffixByteRequest class Store(ABC): """ Abstract base class for Zarr stores. """ _read_only: bool _is_open: bool def __init__(self, *, read_only: bool = False) -> None: self._is_open = False self._read_only = read_only @classmethod async def open(cls, *args: Any, **kwargs: Any) -> Self: """ Create and open the store. Parameters ---------- *args : Any Positional arguments to pass to the store constructor. **kwargs : Any Keyword arguments to pass to the store constructor. Returns ------- Store The opened store instance. """ store = cls(*args, **kwargs) await store._open() return store def with_read_only(self, read_only: bool = False) -> Store: """ Return a new store with a new read_only setting. The new store points to the same location with the specified new read_only state. The returned Store is not automatically opened, and this store is not automatically closed. Parameters ---------- read_only If True, the store will be created in read-only mode. Defaults to False. Returns ------- A new store of the same type with the new read only attribute. """ raise NotImplementedError( f"with_read_only is not implemented for the {type(self)} store type." ) def __enter__(self) -> Self: """Enter a context manager that will close the store upon exiting.""" return self def __exit__( self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None, ) -> None: """Close the store.""" self.close() async def _open(self) -> None: """ Open the store. Raises ------ ValueError If the store is already open. """ if self._is_open: raise ValueError("store is already open") self._is_open = True async def _ensure_open(self) -> None: """Open the store if it is not already open.""" if not self._is_open: await self._open() async def is_empty(self, prefix: str) -> bool: """ Check if the directory is empty. Parameters ---------- prefix : str Prefix of keys to check. Returns ------- bool True if the store is empty, False otherwise. """ if not self.supports_listing: raise NotImplementedError if prefix != "" and not prefix.endswith("/"): prefix += "/" async for _ in self.list_prefix(prefix): return False return True async def clear(self) -> None: """ Clear the store. Remove all keys and values from the store. """ if not self.supports_deletes: raise NotImplementedError if not self.supports_listing: raise NotImplementedError self._check_writable() await self.delete_dir("") @property def read_only(self) -> bool: """Is the store read-only?""" return self._read_only def _check_writable(self) -> None: """Raise an exception if the store is not writable.""" if self.read_only: raise ValueError("store was opened in read-only mode and does not support writing") @abstractmethod def __eq__(self, value: object) -> bool: """Equality comparison.""" ... @abstractmethod async def get( self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None, ) -> Buffer | None: """Retrieve the value associated with a given key. Parameters ---------- key : str prototype : BufferPrototype The prototype of the output buffer. Stores may support a default buffer prototype. byte_range : ByteRequest, optional ByteRequest may be one of the following. If not provided, all data associated with the key is retrieved. - RangeByteRequest(int, int): Request a specific range of bytes in the form (start, end). The end is exclusive. If the given range is zero-length or starts after the end of the object, an error will be returned. Additionally, if the range ends after the end of the object, the entire remainder of the object will be returned. Otherwise, the exact requested range will be returned. - OffsetByteRequest(int): Request all bytes starting from a given byte offset. This is equivalent to bytes={int}- as an HTTP header. - SuffixByteRequest(int): Request the last int bytes. Note that here, int is the size of the request, not the byte offset. This is equivalent to bytes=-{int} as an HTTP header. Returns ------- Buffer """ ... @abstractmethod async def get_partial_values( self, prototype: BufferPrototype, key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: """Retrieve possibly partial values from given key_ranges. Parameters ---------- prototype : BufferPrototype The prototype of the output buffer. Stores may support a default buffer prototype. key_ranges : Iterable[tuple[str, tuple[int | None, int | None]]] Ordered set of key, range pairs, a key may occur multiple times with different ranges Returns ------- list of values, in the order of the key_ranges, may contain null/none for missing keys """ ... @abstractmethod async def exists(self, key: str) -> bool: """Check if a key exists in the store. Parameters ---------- key : str Returns ------- bool """ ... @property @abstractmethod def supports_writes(self) -> bool: """Does the store support writes?""" ... @abstractmethod async def set(self, key: str, value: Buffer) -> None: """Store a (key, value) pair. Parameters ---------- key : str value : Buffer """ ... async def set_if_not_exists(self, key: str, value: Buffer) -> None: """ Store a key to ``value`` if the key is not already present. Parameters ---------- key : str value : Buffer """ # Note for implementers: the default implementation provided here # is not safe for concurrent writers. There's a race condition between # the `exists` check and the `set` where another writer could set some # value at `key` or delete `key`. if not await self.exists(key): await self.set(key, value) async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None: """ Insert multiple (key, value) pairs into storage. """ await asyncio.gather(*starmap(self.set, values)) @property def supports_consolidated_metadata(self) -> bool: """ Does the store support consolidated metadata?. If it doesn't an error will be raised on requests to consolidate the metadata. Returning `False` can be useful for stores which implement their own consolidation mechanism outside of the zarr-python implementation. """ return True @property @abstractmethod def supports_deletes(self) -> bool: """Does the store support deletes?""" ... @abstractmethod async def delete(self, key: str) -> None: """Remove a key from the store Parameters ---------- key : str """ ... @property def supports_partial_writes(self) -> Literal[False]: """Does the store support partial writes? Partial writes are no longer used by Zarr, so this is always false. """ return False @property @abstractmethod def supports_listing(self) -> bool: """Does the store support listing?""" ... @abstractmethod def list(self) -> AsyncIterator[str]: """Retrieve all keys in the store. Returns ------- AsyncIterator[str] """ # This method should be async, like overridden methods in child classes. # However, that's not straightforward: # https://stackoverflow.com/questions/68905848 @abstractmethod def list_prefix(self, prefix: str) -> AsyncIterator[str]: """ Retrieve all keys in the store that begin with a given prefix. Keys are returned relative to the root of the store. Parameters ---------- prefix : str Returns ------- AsyncIterator[str] """ # This method should be async, like overridden methods in child classes. # However, that's not straightforward: # https://stackoverflow.com/questions/68905848 @abstractmethod def list_dir(self, prefix: str) -> AsyncIterator[str]: """ Retrieve all keys and prefixes with a given prefix and which do not contain the character “/” after the given prefix. Parameters ---------- prefix : str Returns ------- AsyncIterator[str] """ # This method should be async, like overridden methods in child classes. # However, that's not straightforward: # https://stackoverflow.com/questions/68905848 async def delete_dir(self, prefix: str) -> None: """ Remove all keys and prefixes in the store that begin with a given prefix. """ if not self.supports_deletes: raise NotImplementedError if not self.supports_listing: raise NotImplementedError self._check_writable() if prefix != "" and not prefix.endswith("/"): prefix += "/" async for key in self.list_prefix(prefix): await self.delete(key) def close(self) -> None: """Close the store.""" self._is_open = False async def _get_many( self, requests: Iterable[tuple[str, BufferPrototype, ByteRequest | None]] ) -> AsyncGenerator[tuple[str, Buffer | None], None]: """ Retrieve a collection of objects from storage. In general this method does not guarantee that objects will be retrieved in the order in which they were requested, so this method yields tuple[str, Buffer | None] instead of just Buffer | None """ for req in requests: yield (req[0], await self.get(*req)) async def get_ranges( self, key: str, byte_ranges: Sequence[ByteRequest | None], *, prototype: BufferPrototype, max_concurrency: int = 10, max_gap_bytes: int = 1 << 20, # 1 MiB max_coalesced_bytes: int = 16 << 20, # 16 MiB ) -> AsyncIterator[Sequence[tuple[int, Buffer | None]]]: """Read many byte ranges from `key`. Yields one batch per underlying I/O operation, each a sequence of `(input_index, Buffer | None)` tuples. Batches across yields arrive in completion order, not input order. The default implementation built into `Store` runs the coalescer over `self.get`, so subclasses get a working implementation for free; stores that have a more efficient backend (e.g. ranged HTTP, S3 byte-range fetches) should override. Parameters ---------- key Storage key to read from. byte_ranges Input ranges. `None` means "the whole value". prototype Buffer prototype, forwarded to `self.get`. max_concurrency Maximum number of merged fetches in flight at once. max_gap_bytes Two `RangeByteRequest`s separated by at most this many bytes may be merged into one fetch. max_coalesced_bytes Upper bound on the size of a single merged fetch. Raises ------ BaseExceptionGroup Failures from underlying fetches are reported as a `BaseExceptionGroup` (PEP 654) and should be handled with `except*`. Inner exceptions include `FileNotFoundError` if any fetch returns `None` (i.e. `key` is absent), and any exception raised by `self.get` for the corresponding range. Pending fetches are cancelled as soon as one task fails, so the group typically contains a single non-`CancelledError` exception even under high concurrency. """ # Local import: zarr.core._coalesce imports symbols from this module. from zarr.core._coalesce import coalesced_get fetch = partial(self.get, key, prototype) async for group in coalesced_get( fetch, byte_ranges, max_concurrency=max_concurrency, max_gap_bytes=max_gap_bytes, max_coalesced_bytes=max_coalesced_bytes, ): yield group def get_ranges_sync( self, key: str, byte_ranges: Sequence[ByteRequest | None], *, prototype: BufferPrototype, max_gap_bytes: int = 1 << 20, # 1 MiB max_coalesced_bytes: int = 16 << 20, # 16 MiB ) -> Sequence[tuple[int, Buffer | None]]: """Synchronous, coalescing counterpart of `get_ranges`. Plans merged fetches with the same `coalesce_ranges` policy as the async path, then issues one synchronous `get_sync` per merged group (or per uncoalescable request) and slices results back into per-input buffers. Used by the sync codec pipeline's partial-shard reads so they get the same byte-range coalescing as the async path, without an event loop. Returns a list of `(input_index, Buffer | None)`. Raises `BaseExceptionGroup` containing a `FileNotFoundError` if the key is absent (matching `get_ranges`), so callers can handle a deleted shard uniformly across the sync and async paths. Requires the store to implement `get_sync` (`SupportsGetSync`). """ from zarr.core._coalesce import coalesce_ranges if not isinstance(self, SupportsGetSync): raise TypeError(f"{type(self).__name__} does not support synchronous reads") groups, uncoalescable = coalesce_ranges( byte_ranges, max_gap_bytes=max_gap_bytes, max_coalesced_bytes=max_coalesced_bytes ) results: list[tuple[int, Buffer | None]] = [] errors: list[BaseException] = [] def _get(req: ByteRequest | None) -> Buffer | None: return self.get_sync(key, prototype=prototype, byte_range=req) for idx, req in uncoalescable: buf = _get(req) if buf is None: errors.append(FileNotFoundError(key)) else: results.append((idx, buf)) for members in groups: if len(members) == 1: solo_idx, solo_req = members[0] buf = _get(solo_req) if buf is None: errors.append(FileNotFoundError(key)) else: results.append((solo_idx, buf)) continue start = members[0][1].start end = max(r.end for _, r in members) big = _get(RangeByteRequest(start, end)) if big is None: errors.append(FileNotFoundError(key)) continue for member_idx, r in members: results.append((member_idx, big[r.start - start : r.end - start])) if errors: raise BaseExceptionGroup("chunk read failed", errors) return results async def getsize(self, key: str) -> int: """ Return the size, in bytes, of a value in a Store. Parameters ---------- key : str Returns ------- nbytes : int The size of the value (in bytes). Raises ------ FileNotFoundError When the given key does not exist in the store. """ # Note to implementers: this default implementation is very inefficient since # it requires reading the entire object. Many systems will have ways to get the # size of an object without reading it. # avoid circular import from zarr.core.buffer.core import default_buffer_prototype value = await self.get(key, prototype=default_buffer_prototype()) if value is None: raise FileNotFoundError(key) return len(value) async def getsize_prefix(self, prefix: str) -> int: """ Return the size, in bytes, of all values under a prefix. Parameters ---------- prefix : str The prefix of the directory to measure. Returns ------- nbytes : int The sum of the sizes of the values in the directory (in bytes). See Also -------- zarr.Array.nbytes_stored Store.getsize Notes ----- ``getsize_prefix`` is just provided as a potentially faster alternative to listing all the keys under a prefix calling [`Store.getsize`][zarr.abc.store.Store.getsize] on each. In general, ``prefix`` should be the path of an Array or Group in the Store. Implementations may differ on the behavior when some other ``prefix`` is provided. """ # TODO: Overlap listing keys with getsize calls. # Currently, we load the list of keys into memory and only then move # on to getting sizes. Ideally we would overlap those two, which should # improve tail latency and might reduce memory pressure (since not all keys # would be in memory at once). # avoid circular import from zarr.core.common import concurrent_map from zarr.core.config import config if prefix != "" and not prefix.endswith("/"): prefix += "/" keys = [(x,) async for x in self.list_prefix(prefix)] limit = config.get("async.concurrency") sizes = await concurrent_map(keys, self.getsize, limit=limit) return sum(sizes) @runtime_checkable class ByteGetter(Protocol): async def get( self, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: ... @runtime_checkable class ByteSetter(Protocol): async def get( self, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: ... async def set(self, value: Buffer) -> None: ... async def delete(self) -> None: ... async def set_if_not_exists(self, default: Buffer) -> None: ... @runtime_checkable class SyncByteGetter(Protocol): """A `ByteGetter` that can also fetch synchronously, without an event loop. Non-StorePath byte getters (e.g. the sharding codec's in-memory `_ShardingByteGetter`) implement this so a synchronous codec pipeline can take its sync fast path on them instead of scheduling one coroutine per chunk. Note that `StorePath` also *has* a `get_sync` method (so it matches this protocol structurally) but it only works when its store supports synchronous IO — callers gate `StorePath` on the store's `SupportsGetSync` instead of on this protocol. """ def get_sync( self, prototype: BufferPrototype | None = None, byte_range: ByteRequest | None = None ) -> Buffer | None: ... @runtime_checkable class SyncByteSetter(SyncByteGetter, Protocol): """A `ByteSetter` that can also write synchronously. See `SyncByteGetter`.""" def set_sync(self, value: Buffer) -> None: ... def delete_sync(self) -> None: ... @runtime_checkable class SupportsGetSync(Protocol): """Store protocol for synchronous reads (`get_sync`). The store sync surface is all-or-nothing: a store implementing any of the `*_sync` methods must implement all of them (`SupportsSyncStore`), because consumers mix sync reads, writes, and deletes within one operation. Capability-gated callers consult `_store_supports_sync_io` rather than the individual protocols. """ def get_sync( self, key: str, *, prototype: BufferPrototype | None = None, byte_range: ByteRequest | None = None, ) -> Buffer | None: ... @runtime_checkable class SupportsSetSync(Protocol): """Store protocol for synchronous writes (`set_sync`). See `SupportsGetSync` for the all-or-nothing contract on the store sync surface. """ def set_sync(self, key: str, value: Buffer) -> None: ... @runtime_checkable class SupportsDeleteSync(Protocol): """Store protocol for synchronous deletes (`delete_sync`). See `SupportsGetSync` for the all-or-nothing contract on the store sync surface. """ def delete_sync(self, key: str) -> None: ... @runtime_checkable class SupportsSyncStore(SupportsGetSync, SupportsSetSync, SupportsDeleteSync, Protocol): """The full store sync surface: `get_sync`, `set_sync`, and `delete_sync`.""" def _store_supports_sync_io(store: object) -> bool: """Whether `store` can serve the full synchronous IO surface right now. Structural membership in `SupportsSyncStore` is necessary but not always sufficient: a store can present the `*_sync` methods while its ability to run them depends on runtime state the type system cannot see. Wrapper stores are the canonical case — `WrapperStore` delegates the sync methods to the store it wraps, so they only work when the wrapped store is itself sync-capable. Such stores opt out dynamically via a `_supports_sync_io` attribute/property (absent means capable). This is an interim, private convention pending a formal sync/async store architecture — the store-side twin of the codec-side `_sync_capable` convention consulted by `zarr.abc.codec._codec_supports_sync`. Synchronous IO is all-or-nothing: consumers such as the fused codec pipeline mix synchronous reads, writes, and deletes within one batch (e.g. a partial-chunk write reads existing bytes and an all-fill chunk is deleted), so a partial sync surface never satisfies this predicate. """ return isinstance(store, SupportsSyncStore) and getattr(store, "_supports_sync_io", True) async def set_or_delete(byte_setter: ByteSetter, value: Buffer | None) -> None: """Set or delete a value in a byte setter Parameters ---------- byte_setter : ByteSetter value : Buffer | None Notes ----- If value is None, the key will be deleted. """ if value is None: await byte_setter.delete() else: await byte_setter.set(value) zarr-developers-zarr-python-a2f2415/src/zarr/api/000077500000000000000000000000001527014137600220035ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/api/__init__.py000066400000000000000000000000001527014137600241020ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/api/asynchronous.py000066400000000000000000001516261527014137600251230ustar00rootroot00000000000000from __future__ import annotations import asyncio import dataclasses import warnings from typing import TYPE_CHECKING, Any, Literal, NotRequired, TypedDict, cast import numpy as np import numpy.typing as npt from typing_extensions import deprecated from zarr.abc.store import Store from zarr.core.array import ( DEFAULT_FILL_VALUE, Array, AsyncArray, CompressorLike, create_array, from_array, get_array_metadata, ) from zarr.core.array_spec import ArrayConfigLike, parse_array_config from zarr.core.buffer import NDArrayLike from zarr.core.common import ( JSON, AccessModeLiteral, ChunksLike, DimensionNamesLike, MemoryOrder, ZarrFormat, _default_zarr_format, _warn_write_empty_chunks_kwarg, ) from zarr.core.dtype import ZDTypeLike, get_data_type_from_native_dtype from zarr.core.group import ( AsyncGroup, ConsolidatedMetadata, GroupMetadata, create_hierarchy, ) from zarr.core.metadata import ArrayMetadataDict, ArrayV2Metadata from zarr.errors import ( ArrayNotFoundError, GroupNotFoundError, NodeTypeValidationError, ZarrDeprecationWarning, ZarrRuntimeWarning, ZarrUserWarning, ) from zarr.storage import StorePath from zarr.storage._common import make_store_path if TYPE_CHECKING: from collections.abc import Iterable from zarr.abc.codec import Codec from zarr.abc.numcodec import Numcodec from zarr.core.buffer import NDArrayLikeOrScalar from zarr.core.chunk_key_encodings import ChunkKeyEncoding from zarr.core.metadata.v2 import CompressorLikev2 from zarr.storage import StoreLike from zarr.types import AnyArray, AnyAsyncArray # TODO: this type could use some more thought type ArrayLike = AnyAsyncArray | AnyArray | npt.NDArray[Any] PathLike = str __all__ = [ "array", "consolidate_metadata", "copy", "copy_all", "copy_store", "create", "create_array", "create_hierarchy", "empty", "empty_like", "from_array", "full", "full_like", "group", "load", "ones", "ones_like", "open", "open_array", "open_consolidated", "open_group", "open_like", "save", "save_array", "save_group", "tree", "zeros", "zeros_like", ] _READ_MODES: tuple[AccessModeLiteral, ...] = ("r", "r+", "a") _CREATE_MODES: tuple[AccessModeLiteral, ...] = ("a", "w", "w-") _OVERWRITE_MODES: tuple[AccessModeLiteral, ...] = ("w",) def _infer_overwrite(mode: AccessModeLiteral) -> bool: """ Check that an `AccessModeLiteral` is compatible with overwriting an existing Zarr node. """ return mode in _OVERWRITE_MODES def _warn_unimplemented_kwargs(kwargs: dict[str, Any]) -> None: """ Emit a "not yet implemented" warning for each provided keyword argument that is not None. `kwargs` maps a keyword argument name to its supplied value. The `stacklevel` is chosen so the warning points at the caller of the public API function (the same location as an inline `warnings.warn(..., stacklevel=2)` would). """ for name, value in kwargs.items(): if value is not None: warnings.warn(f"{name} is not yet implemented", ZarrRuntimeWarning, stacklevel=3) def _get_shape_chunks(a: ArrayLike | Any) -> tuple[tuple[int, ...] | None, tuple[int, ...] | None]: """Helper function to get the shape and chunks from an array-like object""" shape = None chunks = None if hasattr(a, "shape") and isinstance(a.shape, tuple): shape = a.shape if hasattr(a, "chunks") and isinstance(a.chunks, tuple) and (len(a.chunks) == len(a.shape)): chunks = a.chunks elif hasattr(a, "chunklen"): # bcolz carray chunks = (a.chunklen,) + a.shape[1:] return shape, chunks class _LikeArgs(TypedDict): shape: NotRequired[tuple[int, ...]] chunks: NotRequired[tuple[int, ...]] dtype: NotRequired[np.dtype[np.generic]] order: NotRequired[Literal["C", "F"]] filters: NotRequired[tuple[Numcodec, ...] | None] compressor: NotRequired[CompressorLikev2] codecs: NotRequired[tuple[Codec, ...]] fill_value: NotRequired[Any] zarr_format: NotRequired[ZarrFormat] def _like_args(a: ArrayLike, zarr_format: ZarrFormat | None) -> _LikeArgs: """ Arguments for creating an array like `a` in `zarr_format`. If `a` is a zarr array, the new array has the zarr format of `a` unless `zarr_format` requests another. The storage settings of `a` (memory order and codecs) are specific to its zarr format, so they are copied only when the new array has the format of `a`. Otherwise the new array uses the defaults of its own format. """ new: _LikeArgs = {} shape, chunks = _get_shape_chunks(a) if shape is not None: new["shape"] = shape if chunks is not None: new["chunks"] = chunks if hasattr(a, "dtype"): new["dtype"] = a.dtype if isinstance(a, AsyncArray | Array): new["fill_value"] = a.metadata.fill_value if zarr_format is None: zarr_format = a.metadata.zarr_format if a.metadata.zarr_format == zarr_format: if isinstance(a.metadata, ArrayV2Metadata): new["order"] = a.order new["compressor"] = a.metadata.compressor new["filters"] = a.metadata.filters else: # TODO: Remove type: ignore statement when type inference improves. # mypy cannot correctly infer the type of a.metadata here for some reason. new["codecs"] = a.metadata.codecs else: # TODO: set default values compressor/codecs # to do this, we may need to evaluate if this is a v2 or v3 array # new["compressor"] = "default" pass if zarr_format is not None: new["zarr_format"] = zarr_format return new async def consolidate_metadata( store: StoreLike, path: str | None = None, zarr_format: ZarrFormat | None = None, ) -> AsyncGroup: """ Consolidate the metadata of all nodes in a hierarchy. Upon completion, the metadata of the root node in the Zarr hierarchy will be updated to include all the metadata of child nodes. For Stores that do not support consolidated metadata, this operation raises a `TypeError`. Parameters ---------- store : StoreLike The store-like object whose metadata you wish to consolidate. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. path : str, optional A path to a group in the store to consolidate at. Only children below that group will be consolidated. By default, the root node is used so all the metadata in the store is consolidated. zarr_format : {2, 3, None}, optional The zarr format of the hierarchy. By default the zarr format is inferred. Returns ------- group: AsyncGroup The group, with the `consolidated_metadata` field set to include the metadata of each child node. If the Store doesn't support consolidated metadata, this function raises a `TypeError`. See `Store.supports_consolidated_metadata`. """ store_path = await make_store_path(store, path=path) if not store_path.store.supports_consolidated_metadata: store_name = type(store_path.store).__name__ raise TypeError( f"The Zarr Store in use ({store_name}) doesn't support consolidated metadata", ) group = await AsyncGroup.open(store_path, zarr_format=zarr_format, use_consolidated=False) group.store_path.store._check_writable() members_metadata = { k: v.metadata async for k, v in group.members(max_depth=None, use_consolidated_for_children=False) } # While consolidating, we want to be explicit about when child groups # are empty by inserting an empty dict for consolidated_metadata.metadata for k, v in members_metadata.items(): if isinstance(v, GroupMetadata) and v.consolidated_metadata is None: v = dataclasses.replace(v, consolidated_metadata=ConsolidatedMetadata(metadata={})) members_metadata[k] = v if any(m.zarr_format == 3 for m in members_metadata.values()): warnings.warn( "Consolidated metadata is currently not part in the Zarr format 3 specification. It " "may not be supported by other zarr implementations and may change in the future.", category=ZarrUserWarning, stacklevel=1, ) ConsolidatedMetadata._flat_to_nested(members_metadata) consolidated_metadata = ConsolidatedMetadata(metadata=members_metadata) metadata = dataclasses.replace(group.metadata, consolidated_metadata=consolidated_metadata) group = dataclasses.replace( group, metadata=metadata, ) await group._save_metadata() return group async def copy(*args: Any, **kwargs: Any) -> tuple[int, int, int]: """ Not implemented. """ raise NotImplementedError async def copy_all(*args: Any, **kwargs: Any) -> tuple[int, int, int]: """ Not implemented. """ raise NotImplementedError async def copy_store(*args: Any, **kwargs: Any) -> tuple[int, int, int]: """ Not implemented. """ raise NotImplementedError async def load( *, store: StoreLike, path: str | None = None, zarr_format: ZarrFormat | None = None, ) -> NDArrayLikeOrScalar | dict[str, NDArrayLikeOrScalar]: """Load data from an array or group into memory. Parameters ---------- store : StoreLike StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. path : str or None, optional The path within the store from which to load. Returns ------- out If the path contains an array, out will be a numpy array. If the path contains a group, out will be a dict-like object where keys are array names and values are numpy arrays. See Also -------- save, open Notes ----- If loading data from a group of arrays, data will not be immediately loaded into memory. Rather, arrays will be loaded into memory as they are requested. Unlike [`open`][zarr.open], which returns a lazy [`Array`][zarr.Array] or [`Group`][zarr.Group] backed by the store, `load` eagerly reads the data and returns it as an in-memory array (or a dict of arrays for a group). The array type is NumPy by default, but follows the configured buffer prototype (for example, CuPy for GPU use cases). Use `open` when you want to read or write data incrementally without loading it all into memory. """ obj = await open(store=store, path=path, zarr_format=zarr_format) if isinstance(obj, AsyncArray): return await obj.getitem(Ellipsis) else: raise NotImplementedError("loading groups not yet supported") async def open( *, store: StoreLike | None = None, mode: AccessModeLiteral | None = None, zarr_format: ZarrFormat | None = None, path: str | None = None, storage_options: dict[str, Any] | None = None, config: ArrayConfigLike | None = None, **kwargs: Any, # TODO: type kwargs as valid args to open_array ) -> AnyAsyncArray | AsyncGroup: """Convenience function to open a group or array using file-mode-like semantics. Parameters ---------- store : StoreLike or None, default=None StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. mode : {'r', 'r+', 'a', 'w', 'w-'}, optional Persistence mode: 'r' means read only (must exist); 'r+' means read/write (must exist); 'a' means read/write (create if doesn't exist); 'w' means create (overwrite if exists); 'w-' means create (fail if exists). On a store that cannot delete keys, 'w' raises an error instead of replacing an existing node. If the store is read-only, the default is 'r'; otherwise, it is 'a'. zarr_format : {2, 3, None}, optional The zarr format to use when saving. path : str or None, optional The path within the store to open. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. config : ArrayConfigLike or None, default=None Runtime configuration for the array, whether it is opened or created. Keys not specified are taken from the global configuration. Groups have no runtime configuration, so passing `config` when a group is opened or created raises a `TypeError`. **kwargs Additional parameters are passed through to `zarr.open_array` or `zarr.open_group`. Returns ------- z : array or group Return type depends on what exists in the given store. See Also -------- load Notes ----- `open` returns a lazy [`Array`][zarr.Array] or [`Group`][zarr.Group] backed by the store, so data is read and written incrementally. Use [`load`][zarr.load] instead when you want the data eagerly read into an in-memory array (a NumPy array by default). """ if mode is None: if isinstance(store, (Store, StorePath)) and store.read_only: mode = "r" else: mode = "a" store_path = await make_store_path(store, mode=mode, path=path, storage_options=storage_options) # TODO: the mode check below seems wrong! if "shape" not in kwargs and mode in (*_READ_MODES, "w"): # mode "w" replaces any existing node, so there is nothing to open if mode in _READ_MODES: try: metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format) # TODO: remove this cast when we fix typing for array metadata dicts _metadata_dict = cast("ArrayMetadataDict", metadata_dict) # for v2, the above would already have raised an exception if not an array zarr_format = _metadata_dict["zarr_format"] is_v3_array = zarr_format == 3 and _metadata_dict.get("node_type") == "array" if is_v3_array or zarr_format == 2: return AsyncArray(store_path=store_path, metadata=_metadata_dict, config=config) except (FileNotFoundError, NodeTypeValidationError): pass _check_no_group_config(config, store_path) return await open_group(store=store_path, zarr_format=zarr_format, mode=mode, **kwargs) try: return await open_array( store=store_path, zarr_format=zarr_format, mode=mode, config=config, **kwargs ) except (KeyError, NodeTypeValidationError): # KeyError for a missing key # NodeTypeValidationError for failing to parse node metadata as an array when it's # actually a group _check_no_group_config(config, store_path) return await open_group(store=store_path, zarr_format=zarr_format, mode=mode, **kwargs) def _check_no_group_config(config: ArrayConfigLike | None, store_path: StorePath) -> None: """Raise if `config` is given for a node that `open` resolves to a group.""" if config is not None: raise TypeError( f"config is the runtime configuration of an array, but {store_path} resolves to a " "group. Groups have no runtime configuration." ) async def open_consolidated( *args: Any, use_consolidated: Literal[True] = True, **kwargs: Any ) -> AsyncGroup: """ Alias for [`open_group`][zarr.api.asynchronous.open_group] with `use_consolidated=True`. """ if use_consolidated is not True: raise TypeError( "'use_consolidated' must be 'True' in 'open_consolidated'. Use 'open' with " "'use_consolidated=False' to bypass consolidated metadata." ) return await open_group(*args, use_consolidated=use_consolidated, **kwargs) async def save( store: StoreLike, *args: NDArrayLike, zarr_format: ZarrFormat | None = None, path: str | None = None, storage_options: dict[str, Any] | None = None, **kwargs: Any, # TODO: type kwargs as valid args to save ) -> None: """Convenience function to save an array or group of arrays to a store. Parameters ---------- store : StoreLike StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. *args : ndarray NumPy arrays with data to save. zarr_format : {2, 3, None}, optional The zarr format to use when saving. path : str or None, optional The path within the group where the arrays will be saved. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. **kwargs NumPy arrays with data to save. """ if len(args) == 0 and len(kwargs) == 0: raise ValueError("at least one array must be provided") if len(args) == 1 and len(kwargs) == 0: await save_array( store, args[0], zarr_format=zarr_format, path=path, storage_options=storage_options ) else: await save_group( store, *args, zarr_format=zarr_format, path=path, storage_options=storage_options, **kwargs, ) async def save_array( store: StoreLike, arr: NDArrayLike, *, zarr_format: ZarrFormat | None = None, path: str | None = None, storage_options: dict[str, Any] | None = None, **kwargs: Any, # TODO: type kwargs as valid args to create ) -> None: """Convenience function to save a NumPy array to a store, following a similar API to the NumPy save() function. Parameters ---------- store : StoreLike StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. arr : ndarray NumPy array with data to save. zarr_format : {2, 3, None}, optional The zarr format to use when saving. The default is `None`, which will use the default Zarr format defined in the global configuration object. path : str or None, optional The path within the store where the array will be saved. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. **kwargs Passed through to [`create`][zarr.api.asynchronous.create], e.g., compressor. """ if zarr_format is None: zarr_format = _default_zarr_format() if not isinstance(arr, NDArrayLike): raise TypeError("arr argument must be numpy or other NDArrayLike array") mode = kwargs.pop("mode", "a") store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) if np.isscalar(arr): arr = np.array(arr) shape = arr.shape chunks = getattr(arr, "chunks", None) # for array-likes with chunks attribute overwrite = kwargs.pop("overwrite", None) or _infer_overwrite(mode) zarr_dtype = get_data_type_from_native_dtype(arr.dtype) new = await AsyncArray._create( store_path, zarr_format=zarr_format, shape=shape, dtype=zarr_dtype, chunks=chunks, overwrite=overwrite, **kwargs, ) await new.setitem(Ellipsis, arr) async def save_group( store: StoreLike, *args: NDArrayLike, zarr_format: ZarrFormat | None = None, path: str | None = None, storage_options: dict[str, Any] | None = None, **kwargs: NDArrayLike, ) -> None: """Convenience function to save several NumPy arrays to a store, following a similar API to the NumPy savez()/savez_compressed() functions. Parameters ---------- store : StoreLike StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. *args : ndarray NumPy arrays with data to save. zarr_format : {2, 3, None}, optional The zarr format to use when saving. path : str or None, optional Path within the store where the group will be saved. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. **kwargs NumPy arrays with data to save. """ if zarr_format is None: zarr_format = _default_zarr_format() for arg in args: if not isinstance(arg, NDArrayLike): raise TypeError( "All arguments must be numpy or other NDArrayLike arrays (except store, path, storage_options, and zarr_format)" ) for k, v in kwargs.items(): if not isinstance(v, NDArrayLike): raise TypeError(f"Keyword argument '{k}' must be a numpy or other NDArrayLike array") if len(args) == 0 and len(kwargs) == 0: raise ValueError("at least one array must be provided") # Resolve every data type before anything is deleted, so an array that Zarr cannot # store raises while the existing node is still intact. for arr in (*args, *kwargs.values()): get_data_type_from_native_dtype(arr.dtype) store_path = await make_store_path(store, path=path, mode="w", storage_options=storage_options) # The group replaces whatever is stored under the path, now that the arguments are known # to be valid. await AsyncGroup.from_store(store_path, zarr_format=zarr_format, overwrite=True) aws = [] # `store_path` already consumed `storage_options`, so passing them on again would # make `make_store_path` reject them as unused. for i, arr in enumerate(args): aws.append(save_array(store_path, arr, zarr_format=zarr_format, path=f"arr_{i}")) for k, arr in kwargs.items(): aws.append(save_array(store_path, arr, zarr_format=zarr_format, path=k)) await asyncio.gather(*aws) @deprecated("Use AsyncGroup.tree instead.", category=ZarrDeprecationWarning) async def tree(grp: AsyncGroup, expand: bool | None = None, level: int | None = None) -> Any: """Provide a rich display of the hierarchy. !!! warning "Deprecated" `zarr.tree()` is deprecated since v3.0.0 and will be removed in a future release. Use `group.tree()` instead. Parameters ---------- grp : Group Zarr or h5py group. expand : bool, optional Only relevant for HTML representation. If True, tree will be fully expanded. level : int, optional Maximum depth to descend into hierarchy. Returns ------- TreeRepr A pretty-printable object displaying the hierarchy. """ return await grp.tree(expand=expand, level=level) async def array(data: npt.ArrayLike | AnyArray, **kwargs: Any) -> AnyAsyncArray: """Create an array filled with `data`. Parameters ---------- data : array_like The data to fill the array with. **kwargs Passed through to [`create`][zarr.api.asynchronous.create]. Returns ------- array : array The new array. """ if isinstance(data, Array): return await from_array(data=data, **kwargs) # ensure data is array-like if not hasattr(data, "shape") or not hasattr(data, "dtype"): data = np.asanyarray(data) # setup dtype kw_dtype = kwargs.get("dtype") if kw_dtype is None and hasattr(data, "dtype"): kwargs["dtype"] = data.dtype else: kwargs["dtype"] = kw_dtype # setup shape and chunks data_shape, data_chunks = _get_shape_chunks(data) kwargs["shape"] = data_shape kw_chunks = kwargs.get("chunks") if kw_chunks is None: kwargs["chunks"] = data_chunks else: kwargs["chunks"] = kw_chunks read_only = kwargs.pop("read_only", False) if read_only: raise ValueError("read_only=True is no longer supported when creating new arrays") # instantiate array z = await create(**kwargs) # fill with data await z.setitem(Ellipsis, data) return z async def group( *, # Note: this is a change from v2 store: StoreLike | None = None, overwrite: bool = False, chunk_store: StoreLike | None = None, # not used cache_attrs: bool | None = None, # not used, default changed synchronizer: Any | None = None, # not used path: str | None = None, zarr_format: ZarrFormat | None = None, meta_array: Any | None = None, # not used attributes: dict[str, JSON] | None = None, storage_options: dict[str, Any] | None = None, ) -> AsyncGroup: """Create a group. Parameters ---------- store : StoreLike or None, default=None StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. overwrite : bool, optional If True, delete any pre-existing data in `store` at `path` before creating the group. chunk_store : StoreLike or None, default=None Separate storage for chunks. Not implemented. cache_attrs : bool, optional If True (default), user attributes will be cached for attribute read operations. If False, user attributes are reloaded from the store prior to all attribute read operations. synchronizer : object, optional Array synchronizer. path : str, optional Group path within store. meta_array : array-like, optional An array instance to use for determining arrays to create and return to users. Use `numpy.empty(())` by default. zarr_format : {2, 3, None}, optional The zarr format to use when saving. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. Returns ------- g : group The new group. """ mode: AccessModeLiteral if overwrite: mode = "w" else: mode = "a" return await open_group( store=store, mode=mode, chunk_store=chunk_store, cache_attrs=cache_attrs, synchronizer=synchronizer, path=path, zarr_format=zarr_format, meta_array=meta_array, attributes=attributes, storage_options=storage_options, ) async def create_group( *, store: StoreLike, path: str | None = None, overwrite: bool = False, zarr_format: ZarrFormat | None = None, attributes: dict[str, Any] | None = None, storage_options: dict[str, Any] | None = None, ) -> AsyncGroup: """Create a group. Parameters ---------- store : StoreLike StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. path : str, optional Group path within store. overwrite : bool, optional If True, pre-existing data at `path` will be deleted before creating the group. zarr_format : {2, 3, None}, optional The zarr format to use when saving. If no `zarr_format` is provided, the default format will be used. This default can be changed by modifying the value of `default_zarr_format` in [`zarr.config`][zarr.config]. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. Returns ------- AsyncGroup The new group. """ if zarr_format is None: zarr_format = _default_zarr_format() mode: Literal["a"] = "a" store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) return await AsyncGroup.from_store( store=store_path, zarr_format=zarr_format, overwrite=overwrite, attributes=attributes, ) async def open_group( store: StoreLike | None = None, *, # Note: this is a change from v2 mode: AccessModeLiteral = "a", cache_attrs: bool | None = None, # not used, default changed synchronizer: Any = None, # not used path: str | None = None, chunk_store: StoreLike | None = None, # not used storage_options: dict[str, Any] | None = None, zarr_format: ZarrFormat | None = None, meta_array: Any | None = None, # not used attributes: dict[str, JSON] | None = None, use_consolidated: bool | str | None = None, ) -> AsyncGroup: """Open a group using file-mode-like semantics. Parameters ---------- store : StoreLike or None, default=None StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. mode : {'r', 'r+', 'a', 'w', 'w-'}, optional Persistence mode: 'r' means read only (must exist); 'r+' means read/write (must exist); 'a' means read/write (create if doesn't exist); 'w' means create (overwrite if exists); 'w-' means create (fail if exists). On a store that cannot delete keys, 'w' raises an error instead of replacing an existing node. cache_attrs : bool, optional If True (default), user attributes will be cached for attribute read operations. If False, user attributes are reloaded from the store prior to all attribute read operations. synchronizer : object, optional Array synchronizer. path : str, optional Group path within store. chunk_store : StoreLike or None, default=None Separate storage for chunks. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. meta_array : array-like, optional An array instance to use for determining arrays to create and return to users. Use `numpy.empty(())` by default. attributes : dict A dictionary of JSON-serializable values with user-defined attributes. use_consolidated : bool or str, default None Whether to use consolidated metadata. By default, consolidated metadata is used if it's present in the store (in the `zarr.json` for Zarr format 3 and in the `.zmetadata` file for Zarr format 2). To explicitly require consolidated metadata, set `use_consolidated=True`, which will raise an exception if consolidated metadata is not found. To explicitly *not* use consolidated metadata, set `use_consolidated=False`, which will fall back to using the regular, non consolidated metadata. Zarr format 2 allowed configuring the key storing the consolidated metadata (`.zmetadata` by default). Specify the custom key as `use_consolidated` to load consolidated metadata from a non-default key. Returns ------- g : group The new group. """ _warn_unimplemented_kwargs( { "cache_attrs": cache_attrs, "synchronizer": synchronizer, "meta_array": meta_array, "chunk_store": chunk_store, } ) store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path) if attributes is None: attributes = {} try: if mode in _READ_MODES: return await AsyncGroup.open( store_path, zarr_format=zarr_format, use_consolidated=use_consolidated ) except (KeyError, FileNotFoundError): pass if mode in _CREATE_MODES: overwrite = _infer_overwrite(mode) _zarr_format = zarr_format or _default_zarr_format() return await AsyncGroup.from_store( store_path, zarr_format=_zarr_format, overwrite=overwrite, attributes=attributes, ) msg = f"No group found in store {store!r} at path {store_path.path!r}" raise GroupNotFoundError(msg) async def create( shape: tuple[int, ...] | int, *, # Note: this is a change from v2 chunks: ChunksLike | None = None, dtype: ZDTypeLike | None = None, compressor: CompressorLike = "auto", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, store: StoreLike | None = None, synchronizer: Any | None = None, overwrite: bool = False, path: PathLike | None = None, chunk_store: StoreLike | None = None, filters: Iterable[dict[str, JSON] | Numcodec] | None = None, cache_metadata: bool | None = None, cache_attrs: bool | None = None, read_only: bool | None = None, object_codec: Codec | None = None, # TODO: type has changed dimension_separator: Literal[".", "/"] | None = None, write_empty_chunks: bool | None = None, zarr_format: ZarrFormat | None = None, meta_array: Any | None = None, # TODO: need type attributes: dict[str, JSON] | None = None, # v3 only chunk_shape: ChunksLike | None = None, chunk_key_encoding: ( ChunkKeyEncoding | tuple[Literal["default"], Literal[".", "/"]] | tuple[Literal["v2"], Literal[".", "/"]] | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: DimensionNamesLike = None, storage_options: dict[str, Any] | None = None, config: ArrayConfigLike | None = None, mode: AccessModeLiteral | None = None, data: npt.ArrayLike | None = None, ) -> AnyAsyncArray: """Create an array. Parameters ---------- shape : int or tuple of ints Array shape. chunks : ChunksLike, optional Chunk shape. If None (the default), it is guessed from `shape` and `dtype`. If False, it is set to `shape`, i.e., a single chunk for the whole array. If an int, the chunk size in each dimension is given by the value of `chunks`. `True` is not a chunk shape and raises a `ValueError`. dtype : str or dtype, optional NumPy dtype. compressor : Codec, optional Primary compressor to compress chunk data. Zarr format 2 only. Zarr format 3 arrays should use `codecs` instead. If neither `compressor` nor `filters` are provided, the default compressor [`zarr.codecs.ZstdCodec`][] is used. If `compressor` is set to `None`, no compression is used. fill_value : Any, optional Fill value for the array. order : {'C', 'F'}, optional Deprecated in favor of the `config` keyword argument. Pass `{'order': }` to `create` instead of using this parameter. Memory layout to be used within each chunk. If not specified, the `array.order` parameter in the global config will be used. store : StoreLike or None, default=None StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. synchronizer : object, optional Array synchronizer. overwrite : bool, optional If True, delete all pre-existing data in `store` at `path` before creating the array. path : str, optional Path under which array is stored. chunk_store : StoreLike or None, default=None Separate storage for chunks. If not provided, `store` will be used for storage of both chunks and metadata. filters : Iterable[Codec] | Literal["auto"], optional Iterable of filters to apply to each chunk of the array, in order, before serializing that chunk to bytes. For Zarr format 3, a "filter" is a codec that takes an array and returns an array, and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the order of your filters is consistent with the behavior of each filter. The default value of `"auto"` instructs Zarr to use a default based on the data type of the array and the Zarr format specified. For all data types in Zarr V3, and most data types in Zarr V2, the default filters are empty. The only cases where default filters are not empty is when the Zarr format is 2, and the data type is a variable-length data type like [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, the default filters contains a single element which is a codec specific to that particular data type. To create an array with no filters, provide an empty iterable or the value `None`. cache_metadata : bool, optional If True, array configuration metadata will be cached for the lifetime of the object. If False, array metadata will be reloaded prior to all data access and modification operations (may incur overhead depending on storage and data access pattern). cache_attrs : bool, optional If True (default), user attributes will be cached for attribute read operations. If False, user attributes are reloaded from the store prior to all attribute read operations. read_only : bool, optional True if array should be protected against modification. object_codec : Codec, optional A codec to encode object arrays, only needed if dtype=object. dimension_separator : {'.', '/'}, optional Separator placed between the dimensions of a chunk. Zarr format 2 only. Zarr format 3 arrays should use `chunk_key_encoding` instead. write_empty_chunks : bool, optional Deprecated in favor of the `config` keyword argument. Pass `{'write_empty_chunks': }` to `create` instead of using this parameter. If True, all chunks will be stored regardless of their contents. If False, each chunk is compared to the array's fill value prior to storing. If a chunk is uniformly equal to the fill value, then that chunk is not be stored, and the store entry for that chunk's key is deleted. zarr_format : {2, 3, None}, optional The Zarr format to use when creating an array. The default is `None`, which instructs Zarr to choose the default Zarr format value defined in the runtime configuration. meta_array : array-like, optional Not implemented. attributes : dict[str, JSON], optional A dictionary of user attributes to store with the array. chunk_shape : ChunksLike, optional The shape of the Array's chunks (default is None). Zarr format 3 only. Zarr format 2 arrays should use `chunks` instead. chunk_key_encoding : ChunkKeyEncoding, optional A specification of how the chunk keys are represented in storage. Zarr format 3 only. Zarr format 2 arrays should use `dimension_separator` instead. Default is `("default", "/")`. codecs : Sequence of Codecs or dicts, optional An iterable of Codec or dict serializations of Codecs. Zarr V3 only. The elements of `codecs` specify the transformation from array values to stored bytes. Zarr format 3 only. Zarr format 2 arrays should use `filters` and `compressor` instead. The [Zarr format 3 specification](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-encoding) requires, in order, zero or more array -> array codecs ([`zarr.abc.codec.ArrayArrayCodec`][]), exactly one array -> bytes codec ([`zarr.abc.codec.ArrayBytesCodec`][]), and zero or more bytes -> bytes codecs ([`zarr.abc.codec.BytesBytesCodec`][]). Elsewhere in Zarr-Python these are called filters, the serializer, and compressors; those names are Zarr-Python aliases, not terms defined in the specification. If no codecs are provided, default codecs will be used based on the data type of the array. For most data types, the default codecs are the tuple `(BytesCodec(), ZstdCodec())`; data types that require a special [`zarr.abc.codec.ArrayBytesCodec`][], like variable-length strings or bytes, will use the [`zarr.abc.codec.ArrayBytesCodec`][] required for the data type instead of [`zarr.codecs.BytesCodec`][]. dimension_names : Iterable[str | None] | None = None An iterable of dimension names. Zarr format 3 only. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. config : ArrayConfigLike, optional Runtime configuration of the array. If provided, will override the default values from `zarr.config.array`. mode : {'r', 'r+', 'a', 'w', 'w-'}, optional Legacy way to control overwriting, kept for compatibility with Zarr-Python 2. Prefer `overwrite`. The access mode used to open `store`; the default, `None`, is `'a'`. - `'a'` and `'r+'` create the array and fail if a node exists at `path`, unless `overwrite` is `True`. - `'w'` replaces anything stored under `path` (the whole store if `path` is not set), even if `overwrite` is `False`. On a store that cannot delete keys, `'w'` raises an error instead of replacing an existing node. - `'w-'` fails if anything is stored under `path`, even if `overwrite` is `True`. - `'r'` always fails. If `store` is a `StorePath`, `mode` is not validated against it: `'w'` still sets `overwrite`, and the other modes have no effect. data : array-like, optional Values written into the new array after it is created. Unlike the `data` parameter of `create_array`, it does not set the shape or data type of the array. To create an array from existing data, use `create_array(data=...)`. A Zarr array is not supported as `data`. Returns ------- z : array The array. """ if zarr_format is None: zarr_format = _default_zarr_format() _warn_unimplemented_kwargs( { "synchronizer": synchronizer, "chunk_store": chunk_store, "cache_metadata": cache_metadata, "cache_attrs": cache_attrs, "object_codec": object_codec, "read_only": read_only, "meta_array": meta_array, } ) if write_empty_chunks is not None: _warn_write_empty_chunks_kwarg() if mode is None: mode = "a" overwrite = overwrite or _infer_overwrite(mode) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) config_parsed = parse_array_config(config) if write_empty_chunks is not None: if config is not None: msg = ( "Both write_empty_chunks and config keyword arguments are set. " "This is redundant. When both are set, write_empty_chunks will be used instead " "of the value in config." ) warnings.warn(ZarrUserWarning(msg), stacklevel=1) config_parsed = dataclasses.replace(config_parsed, write_empty_chunks=write_empty_chunks) return await AsyncArray._create( store_path, shape=shape, chunks=chunks, # Legacy v2 behavior: an unspecified dtype defaults to float64. dtype="float64" if dtype is None else dtype, compressor=compressor, fill_value=fill_value, overwrite=overwrite, filters=filters, dimension_separator=dimension_separator, order=order, zarr_format=zarr_format, chunk_shape=chunk_shape, chunk_key_encoding=chunk_key_encoding, codecs=codecs, dimension_names=dimension_names, attributes=attributes, config=config_parsed, data=data, ) async def empty(shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: """Create an empty array with the specified shape. The contents will be filled with the specified fill value or zeros if no fill value is provided. Parameters ---------- shape : int or tuple of int Shape of the empty array. **kwargs Keyword arguments passed to [`create`][zarr.api.asynchronous.create]. Notes ----- The contents of an empty Zarr array are not defined. On attempting to retrieve data from an empty Zarr array, any values may be returned, and these are not guaranteed to be stable from one access to the next. """ return await create(shape=shape, **kwargs) async def empty_like( a: ArrayLike, *, zarr_format: ZarrFormat | None = None, **kwargs: Any ) -> AnyAsyncArray: """Create an empty array like `a`. The contents will be filled with the array's fill value or zeros if no fill value is provided. Parameters ---------- a : array-like The array to create an empty array like. zarr_format : {2, 3, None}, optional The zarr format of the new array. If `None` (default), the zarr format of `a` if it is a zarr array, otherwise the default zarr format. **kwargs Keyword arguments passed to [`create`][zarr.api.asynchronous.create]. Returns ------- Array The new array. Notes ----- The contents of an empty Zarr array are not defined. On attempting to retrieve data from an empty Zarr array, any values may be returned, and these are not guaranteed to be stable from one access to the next. """ like_kwargs = _like_args(a, zarr_format) | kwargs return await empty(**like_kwargs) # type: ignore[arg-type] # TODO: add type annotations for fill_value and kwargs async def full(shape: tuple[int, ...], fill_value: Any, **kwargs: Any) -> AnyAsyncArray: """Create an array, with `fill_value` being used as the default value for uninitialized portions of the array. Parameters ---------- shape : int or tuple of int Shape of the empty array. fill_value : scalar Fill value. **kwargs Keyword arguments passed to [`create`][zarr.api.asynchronous.create]. Returns ------- Array The new array. """ return await create(shape=shape, fill_value=fill_value, **kwargs) # TODO: add type annotations for kwargs async def full_like( a: ArrayLike, *, zarr_format: ZarrFormat | None = None, **kwargs: Any ) -> AnyAsyncArray: """Create a filled array like `a`. Parameters ---------- a : array-like The array to create an empty array like. zarr_format : {2, 3, None}, optional The zarr format of the new array. If `None` (default), the zarr format of `a` if it is a zarr array, otherwise the default zarr format. **kwargs Keyword arguments passed to [`zarr.api.asynchronous.create`][]. Returns ------- Array The new array. """ like_kwargs = _like_args(a, zarr_format) | kwargs return await full(**like_kwargs) # type: ignore[arg-type] async def ones(shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: """Create an array, with one being used as the default value for uninitialized portions of the array. Parameters ---------- shape : int or tuple of int Shape of the empty array. **kwargs Keyword arguments passed to [`zarr.api.asynchronous.create`][]. Returns ------- Array The new array. """ return await create(shape=shape, fill_value=1, **kwargs) async def ones_like( a: ArrayLike, *, zarr_format: ZarrFormat | None = None, **kwargs: Any ) -> AnyAsyncArray: """Create an array of ones like `a`. Parameters ---------- a : array-like The array to create an empty array like. zarr_format : {2, 3, None}, optional The zarr format of the new array. If `None` (default), the zarr format of `a` if it is a zarr array, otherwise the default zarr format. **kwargs Keyword arguments passed to [`zarr.api.asynchronous.create`][]. Returns ------- Array The new array. """ like_args = _like_args(a, zarr_format) # `ones` supplies its own fill_value, so drop any inherited from `a`. like_args.pop("fill_value", None) like_kwargs = like_args | kwargs return await ones(**like_kwargs) # type: ignore[arg-type] async def open_array( *, # note: this is a change from v2 store: StoreLike | None = None, zarr_format: ZarrFormat | None = None, path: PathLike = "", storage_options: dict[str, Any] | None = None, config: ArrayConfigLike | None = None, **kwargs: Any, # TODO: type kwargs as valid args to save ) -> AnyAsyncArray: """Open an array using file-mode-like semantics. Parameters ---------- store : StoreLike StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. zarr_format : {2, 3, None}, optional The zarr format to use when saving. path : str, optional Path in store to array. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. config : ArrayConfigLike or None, default=None Runtime configuration for the array, whether it is opened or created. Keys not specified are taken from the global configuration. **kwargs Any keyword arguments to pass to [`create`][zarr.api.asynchronous.create]. Returns ------- AsyncArray The opened array. """ mode = kwargs.pop("mode", None) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) if "write_empty_chunks" in kwargs: _warn_write_empty_chunks_kwarg() # mode "w" replaces any existing array, so there is nothing to open if mode != "w": try: return await AsyncArray.open(store_path, zarr_format=zarr_format, config=config) except FileNotFoundError as err: if store_path.read_only or mode not in _CREATE_MODES: msg = f"No array found in store {store_path.store} at path {store_path.path}" raise ArrayNotFoundError(msg) from err return await create( store=store_path, zarr_format=zarr_format or _default_zarr_format(), overwrite=_infer_overwrite(mode), config=config, **kwargs, ) async def open_like( a: ArrayLike, path: str, *, zarr_format: ZarrFormat | None = None, **kwargs: Any ) -> AnyAsyncArray: """Open a persistent array like `a`. Parameters ---------- a : Array The shape and data-type of a define these same attributes of the returned array. path : str The path to the new array. zarr_format : {2, 3, None}, optional The zarr format of the array to open or create. If `None` (default), an existing array of either format is opened, and a missing one is created in the default zarr format. The zarr format of `a` is not inherited. **kwargs Additional keyword arguments passed to `open_array`. If `mode` is omitted or `None`, it defaults to `"a"`. Pass `mode="r"` when opening an existing array from a read-only store. Returns ------- AsyncArray The opened array. """ # The zarr format of `a` is not inherited: `open_array` would then look for an existing # array in only that format. like_args = _like_args(a, zarr_format or _default_zarr_format()) like_args.pop("zarr_format") like_kwargs = like_args | kwargs if like_kwargs.get("mode") is None: like_kwargs["mode"] = "a" return await open_array(path=path, zarr_format=zarr_format, **like_kwargs) # type: ignore[arg-type] async def zeros(shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: """Create an array, with zero being used as the default value for uninitialized portions of the array. Parameters ---------- shape : int or tuple of int Shape of the empty array. **kwargs Keyword arguments passed to [`zarr.api.asynchronous.create`][]. Returns ------- Array The new array. """ return await create(shape=shape, fill_value=0, **kwargs) async def zeros_like( a: ArrayLike, *, zarr_format: ZarrFormat | None = None, **kwargs: Any ) -> AnyAsyncArray: """Create an array of zeros like `a`. Parameters ---------- a : array-like The array to create an empty array like. zarr_format : {2, 3, None}, optional The zarr format of the new array. If `None` (default), the zarr format of `a` if it is a zarr array, otherwise the default zarr format. **kwargs Keyword arguments passed to [`create`][zarr.api.asynchronous.create]. Returns ------- Array The new array. """ like_args = _like_args(a, zarr_format) # `zeros` supplies its own fill_value, so drop any inherited from `a`. like_args.pop("fill_value", None) like_kwargs = like_args | kwargs return await zeros(**like_kwargs) # type: ignore[arg-type] zarr-developers-zarr-python-a2f2415/src/zarr/api/synchronous.py000066400000000000000000001650621527014137600247610ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING, Any, Literal from typing_extensions import deprecated import zarr.api.asynchronous as async_api import zarr.core.array from zarr.core.array import DEFAULT_FILL_VALUE, Array, AsyncArray, CompressorLike from zarr.core.group import Group from zarr.core.sync import sync from zarr.core.sync_group import create_hierarchy from zarr.errors import ZarrDeprecationWarning if TYPE_CHECKING: from collections.abc import Iterable import numpy as np import numpy.typing as npt from zarr.abc.codec import Codec from zarr.abc.numcodec import Numcodec from zarr.api.asynchronous import ArrayLike, PathLike from zarr.core.array import ( CompressorsLike, FiltersLike, SerializerLike, ShardsLike, ) from zarr.core.array_spec import ArrayConfigLike from zarr.core.buffer import NDArrayLike, NDArrayLikeOrScalar from zarr.core.chunk_key_encodings import ChunkKeyEncoding, ChunkKeyEncodingLike from zarr.core.common import ( JSON, AccessModeLiteral, ChunksLike, DimensionNamesLike, MemoryOrder, ShapeLike, ZarrFormat, ) from zarr.core.dtype import ZDTypeLike from zarr.storage import StoreLike from zarr.types import AnyArray __all__ = [ "array", "consolidate_metadata", "copy", "copy_all", "copy_store", "create", "create_array", "create_hierarchy", "empty", "empty_like", "from_array", "full", "full_like", "group", "load", "ones", "ones_like", "open", "open_array", "open_consolidated", "open_group", "open_like", "save", "save_array", "save_group", "tree", "zeros", "zeros_like", ] def consolidate_metadata( store: StoreLike, path: str | None = None, zarr_format: ZarrFormat | None = None, ) -> Group: """ Consolidate the metadata of all nodes in a hierarchy. Upon completion, the metadata of the root node in the Zarr hierarchy will be updated to include all the metadata of child nodes. For Stores that do not use consolidated metadata, this operation raises a `TypeError`. Parameters ---------- store : StoreLike The store-like object whose metadata you wish to consolidate. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. path : str, optional A path to a group in the store to consolidate at. Only children below that group will be consolidated. By default, the root node is used so all the metadata in the store is consolidated. zarr_format : {2, 3, None}, optional The zarr format of the hierarchy. By default the zarr format is inferred. Returns ------- group: Group The group, with the `consolidated_metadata` field set to include the metadata of each child node. If the Store doesn't support consolidated metadata, this function raises a `TypeError`. See `Store.supports_consolidated_metadata`. """ return Group(sync(async_api.consolidate_metadata(store, path=path, zarr_format=zarr_format))) def copy(*args: Any, **kwargs: Any) -> tuple[int, int, int]: """ Not implemented. """ return sync(async_api.copy(*args, **kwargs)) def copy_all(*args: Any, **kwargs: Any) -> tuple[int, int, int]: """ Not implemented. """ return sync(async_api.copy_all(*args, **kwargs)) def copy_store(*args: Any, **kwargs: Any) -> tuple[int, int, int]: """ Not implemented. """ return sync(async_api.copy_store(*args, **kwargs)) def load( store: StoreLike, path: str | None = None, zarr_format: ZarrFormat | None = None, ) -> NDArrayLikeOrScalar | dict[str, NDArrayLikeOrScalar]: """Load data from an array or group into memory. Parameters ---------- store : StoreLike StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. path : str or None, optional The path within the store from which to load. Returns ------- out If the path contains an array, out will be a numpy array. If the path contains a group, out will be a dict-like object where keys are array names and values are numpy arrays. See Also -------- save, savez, open Notes ----- If loading data from a group of arrays, data will not be immediately loaded into memory. Rather, arrays will be loaded into memory as they are requested. Unlike [`open`][zarr.open], which returns a lazy [`Array`][zarr.Array] or [`Group`][zarr.Group] backed by the store, `load` eagerly reads the data and returns it as an in-memory array (or a dict of arrays for a group). The array type is NumPy by default, but follows the configured buffer prototype (for example, CuPy for GPU use cases). Use `open` when you want to read or write data incrementally without loading it all into memory. """ return sync(async_api.load(store=store, zarr_format=zarr_format, path=path)) def open( store: StoreLike | None = None, *, mode: AccessModeLiteral | None = None, zarr_format: ZarrFormat | None = None, path: str | None = None, storage_options: dict[str, Any] | None = None, config: ArrayConfigLike | None = None, **kwargs: Any, # TODO: type kwargs as valid args to async_api.open ) -> AnyArray | Group: """Open a group or array using file-mode-like semantics. Parameters ---------- store : StoreLike or None, default=None StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. mode : {'r', 'r+', 'a', 'w', 'w-'}, optional Persistence mode: 'r' means read only (must exist); 'r+' means read/write (must exist); 'a' means read/write (create if doesn't exist); 'w' means create (overwrite if exists); 'w-' means create (fail if exists). On a store that cannot delete keys, 'w' raises an error instead of replacing an existing node. If the store is read-only, the default is 'r'; otherwise, it is 'a'. zarr_format : {2, 3, None}, optional The zarr format to use when saving. path : str or None, optional The path within the store to open. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. config : ArrayConfigLike or None, default=None Runtime configuration for the array, whether it is opened or created. Keys not specified are taken from the global configuration. Groups have no runtime configuration, so passing `config` when a group is opened or created raises a `TypeError`. **kwargs Additional parameters are passed through to `zarr.open_array` or `zarr.open_group`. Returns ------- z : array or group Return type depends on what exists in the given store. See Also -------- load Notes ----- `open` returns a lazy [`Array`][zarr.Array] or [`Group`][zarr.Group] backed by the store, so data is read and written incrementally. Use [`load`][zarr.load] instead when you want the data eagerly read into an in-memory array (a NumPy array by default). """ obj = sync( async_api.open( store=store, mode=mode, zarr_format=zarr_format, path=path, storage_options=storage_options, config=config, **kwargs, ) ) if isinstance(obj, AsyncArray): return Array(obj) else: return Group(obj) def open_consolidated(*args: Any, use_consolidated: Literal[True] = True, **kwargs: Any) -> Group: """ Alias for [`open_group`][zarr.api.synchronous.open_group] with `use_consolidated=True`. """ return Group( sync(async_api.open_consolidated(*args, use_consolidated=use_consolidated, **kwargs)) ) def save( store: StoreLike, *args: NDArrayLike, zarr_format: ZarrFormat | None = None, path: str | None = None, storage_options: dict[str, Any] | None = None, **kwargs: Any, # TODO: type kwargs as valid args to async_api.save ) -> None: """Save an array or group of arrays to a store. Parameters ---------- store : StoreLike StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. *args : ndarray NumPy arrays with data to save. zarr_format : {2, 3, None}, optional The zarr format to use when saving. path : str or None, optional The path within the group where the arrays will be saved. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. **kwargs NumPy arrays with data to save. """ return sync( async_api.save( store, *args, zarr_format=zarr_format, path=path, storage_options=storage_options, **kwargs, ) ) def save_array( store: StoreLike, arr: NDArrayLike, *, zarr_format: ZarrFormat | None = None, path: str | None = None, storage_options: dict[str, Any] | None = None, **kwargs: Any, # TODO: type kwargs as valid args to async_api.save_array ) -> None: """Save a NumPy array to a store. Follows a similar API to the NumPy save() function. Parameters ---------- store : StoreLike StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. arr : ndarray NumPy array with data to save. zarr_format : {2, 3, None}, optional The zarr format to use when saving. The default is `None`, which will use the default Zarr format defined in the global configuration object. path : str or None, optional The path within the store where the array will be saved. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. **kwargs Passed through to [`create`][zarr.api.asynchronous.create], e.g., compressor. """ return sync( async_api.save_array( store=store, arr=arr, zarr_format=zarr_format, path=path, storage_options=storage_options, **kwargs, ) ) def save_group( store: StoreLike, *args: NDArrayLike, zarr_format: ZarrFormat | None = None, path: str | None = None, storage_options: dict[str, Any] | None = None, **kwargs: NDArrayLike, ) -> None: """Save several NumPy arrays to a store. Follows a similar API to the NumPy savez()/savez_compressed() functions. Parameters ---------- store : StoreLike StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. *args : ndarray NumPy arrays with data to save. zarr_format : {2, 3, None}, optional The zarr format to use when saving. path : str or None, optional Path within the store where the group will be saved. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. **kwargs NumPy arrays with data to save. """ return sync( async_api.save_group( store, *args, zarr_format=zarr_format, path=path, storage_options=storage_options, **kwargs, ) ) @deprecated("Use Group.tree instead.", category=ZarrDeprecationWarning) def tree(grp: Group, expand: bool | None = None, level: int | None = None) -> Any: """Provide a rich display of the hierarchy. !!! warning "Deprecated" `zarr.tree()` is deprecated since v3.0.0 and will be removed in a future release. Use `group.tree()` instead. Parameters ---------- grp : Group Zarr or h5py group. expand : bool, optional Only relevant for HTML representation. If True, tree will be fully expanded. level : int, optional Maximum depth to descend into hierarchy. Returns ------- TreeRepr A pretty-printable object displaying the hierarchy. """ return sync(async_api.tree(grp._async_group, expand=expand, level=level)) # TODO: add type annotations for kwargs def array(data: npt.ArrayLike | AnyArray, **kwargs: Any) -> AnyArray: """Create an array filled with `data`. Parameters ---------- data : array_like The data to fill the array with. **kwargs Passed through to [`create`][zarr.api.asynchronous.create]. Returns ------- array : Array The new array. """ return Array(sync(async_api.array(data=data, **kwargs))) def group( store: StoreLike | None = None, *, overwrite: bool = False, chunk_store: StoreLike | None = None, # not used cache_attrs: bool | None = None, # not used, default changed synchronizer: Any | None = None, # not used path: str | None = None, zarr_format: ZarrFormat | None = None, meta_array: Any | None = None, # not used attributes: dict[str, JSON] | None = None, storage_options: dict[str, Any] | None = None, ) -> Group: """Create a group. Parameters ---------- store : StoreLike or None, default=None StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. overwrite : bool, optional If True, delete any pre-existing data in `store` at `path` before creating the group. chunk_store : StoreLike or None, default=None Separate storage for chunks. Not implemented. cache_attrs : bool, optional If True (default), user attributes will be cached for attribute read operations. If False, user attributes are reloaded from the store prior to all attribute read operations. synchronizer : object, optional Array synchronizer. path : str, optional Group path within store. meta_array : array-like, optional An array instance to use for determining arrays to create and return to users. Use `numpy.empty(())` by default. zarr_format : {2, 3, None}, optional The zarr format to use when saving. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. Returns ------- g : Group The new group. """ return Group( sync( async_api.group( store=store, overwrite=overwrite, chunk_store=chunk_store, cache_attrs=cache_attrs, synchronizer=synchronizer, path=path, zarr_format=zarr_format, meta_array=meta_array, attributes=attributes, storage_options=storage_options, ) ) ) def open_group( store: StoreLike | None = None, *, mode: AccessModeLiteral = "a", cache_attrs: bool | None = None, # default changed, not used in async api synchronizer: Any = None, # not used in async api path: str | None = None, chunk_store: StoreLike | None = None, # not used in async api storage_options: dict[str, Any] | None = None, # not used in async api zarr_format: ZarrFormat | None = None, meta_array: Any | None = None, # not used in async api attributes: dict[str, JSON] | None = None, use_consolidated: bool | str | None = None, ) -> Group: """Open a group using file-mode-like semantics. Parameters ---------- store : StoreLike or None, default=None StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. mode : {'r', 'r+', 'a', 'w', 'w-'}, optional Persistence mode: 'r' means read only (must exist); 'r+' means read/write (must exist); 'a' means read/write (create if doesn't exist); 'w' means create (overwrite if exists); 'w-' means create (fail if exists). On a store that cannot delete keys, 'w' raises an error instead of replacing an existing node. cache_attrs : bool, optional If True (default), user attributes will be cached for attribute read operations. If False, user attributes are reloaded from the store prior to all attribute read operations. synchronizer : object, optional Array synchronizer. path : str, optional Group path within store. chunk_store : StoreLike or None, default=None Separate storage for chunks. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. meta_array : array-like, optional An array instance to use for determining arrays to create and return to users. Use `numpy.empty(())` by default. attributes : dict A dictionary of JSON-serializable values with user-defined attributes. use_consolidated : bool or str, default None Whether to use consolidated metadata. By default, consolidated metadata is used if it's present in the store (in the `zarr.json` for Zarr format 3 and in the `.zmetadata` file for Zarr format 2). To explicitly require consolidated metadata, set `use_consolidated=True`, which will raise an exception if consolidated metadata is not found. To explicitly *not* use consolidated metadata, set `use_consolidated=False`, which will fall back to using the regular, non consolidated metadata. Zarr format 2 allowed configuring the key storing the consolidated metadata (`.zmetadata` by default). Specify the custom key as `use_consolidated` to load consolidated metadata from a non-default key. Returns ------- g : Group The new group. """ return Group( sync( async_api.open_group( store=store, mode=mode, cache_attrs=cache_attrs, synchronizer=synchronizer, path=path, chunk_store=chunk_store, storage_options=storage_options, zarr_format=zarr_format, meta_array=meta_array, attributes=attributes, use_consolidated=use_consolidated, ) ) ) def create_group( store: StoreLike, *, path: str | None = None, zarr_format: ZarrFormat | None = None, overwrite: bool = False, attributes: dict[str, Any] | None = None, storage_options: dict[str, Any] | None = None, ) -> Group: """Create a group. Parameters ---------- store : StoreLike StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. path : str, optional Group path within store. overwrite : bool, optional If True, pre-existing data at `path` will be deleted before creating the group. zarr_format : {2, 3, None}, optional The zarr format to use when saving. If no `zarr_format` is provided, the default format will be used. This default can be changed by modifying the value of `default_zarr_format` in [`zarr.config`][zarr.config]. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. Returns ------- Group The new group. """ return Group( sync( async_api.create_group( store=store, path=path, overwrite=overwrite, storage_options=storage_options, zarr_format=zarr_format, attributes=attributes, ) ) ) def create( shape: tuple[int, ...] | int, *, # Note: this is a change from v2 chunks: ChunksLike | None = None, dtype: ZDTypeLike | None = None, compressor: CompressorLike = "auto", fill_value: Any | None = DEFAULT_FILL_VALUE, # TODO: need type order: MemoryOrder | None = None, store: StoreLike | None = None, synchronizer: Any | None = None, overwrite: bool = False, path: PathLike | None = None, chunk_store: StoreLike | None = None, filters: Iterable[dict[str, JSON] | Numcodec] | None = None, cache_metadata: bool | None = None, cache_attrs: bool | None = None, read_only: bool | None = None, object_codec: Codec | None = None, # TODO: type has changed dimension_separator: Literal[".", "/"] | None = None, write_empty_chunks: bool | None = None, # TODO: default has changed zarr_format: ZarrFormat | None = None, meta_array: Any | None = None, # TODO: need type attributes: dict[str, JSON] | None = None, # v3 only chunk_shape: ChunksLike | None = None, chunk_key_encoding: ( ChunkKeyEncoding | tuple[Literal["default"], Literal[".", "/"]] | tuple[Literal["v2"], Literal[".", "/"]] | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: DimensionNamesLike = None, storage_options: dict[str, Any] | None = None, config: ArrayConfigLike | None = None, mode: AccessModeLiteral | None = None, data: npt.ArrayLike | None = None, ) -> AnyArray: """Create an array. Parameters ---------- shape : int or tuple of ints Array shape. chunks : ChunksLike, optional Chunk shape. If None (the default), it is guessed from `shape` and `dtype`. If False, it is set to `shape`, i.e., a single chunk for the whole array. If an int, the chunk size in each dimension is given by the value of `chunks`. `True` is not a chunk shape and raises a `ValueError`. dtype : str or dtype, optional NumPy dtype. compressor : Codec, optional Primary compressor to compress chunk data. Zarr format 2 only. Zarr format 3 arrays should use `codecs` instead. If neither `compressor` nor `filters` are provided, the default compressor [`zarr.codecs.ZstdCodec`][] is used. If `compressor` is set to `None`, no compression is used. fill_value : Any, optional Fill value for the array. order : {'C', 'F'}, optional Deprecated in favor of the `config` keyword argument. Pass `{'order': }` to `create` instead of using this parameter. Memory layout to be used within each chunk. If not specified, the `array.order` parameter in the global config will be used. store : StoreLike or None, default=None StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. synchronizer : object, optional Array synchronizer. overwrite : bool, optional If True, delete all pre-existing data in `store` at `path` before creating the array. path : str, optional Path under which array is stored. chunk_store : StoreLike or None, default=None Separate storage for chunks. If not provided, `store` will be used for storage of both chunks and metadata. filters : Iterable[Codec] | Literal["auto"], optional Iterable of filters to apply to each chunk of the array, in order, before serializing that chunk to bytes. For Zarr format 3, a "filter" is a codec that takes an array and returns an array, and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the order of your filters is consistent with the behavior of each filter. The default value of `"auto"` instructs Zarr to use a default based on the data type of the array and the Zarr format specified. For all data types in Zarr V3, and most data types in Zarr V2, the default filters are empty. The only cases where default filters are not empty is when the Zarr format is 2, and the data type is a variable-length data type like [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, the default filters contains a single element which is a codec specific to that particular data type. To create an array with no filters, provide an empty iterable or the value `None`. cache_metadata : bool, optional If True, array configuration metadata will be cached for the lifetime of the object. If False, array metadata will be reloaded prior to all data access and modification operations (may incur overhead depending on storage and data access pattern). cache_attrs : bool, optional If True (default), user attributes will be cached for attribute read operations. If False, user attributes are reloaded from the store prior to all attribute read operations. read_only : bool, optional True if array should be protected against modification. object_codec : Codec, optional A codec to encode object arrays, only needed if dtype=object. dimension_separator : {'.', '/'}, optional Separator placed between the dimensions of a chunk. Zarr format 2 only. Zarr format 3 arrays should use `chunk_key_encoding` instead. write_empty_chunks : bool, optional Deprecated in favor of the `config` keyword argument. Pass `{'write_empty_chunks': }` to `create` instead of using this parameter. If True, all chunks will be stored regardless of their contents. If False, each chunk is compared to the array's fill value prior to storing. If a chunk is uniformly equal to the fill value, then that chunk is not be stored, and the store entry for that chunk's key is deleted. zarr_format : {2, 3, None}, optional The Zarr format to use when creating an array. The default is `None`, which instructs Zarr to choose the default Zarr format value defined in the runtime configuration. meta_array : array-like, optional Not implemented. attributes : dict[str, JSON], optional A dictionary of user attributes to store with the array. chunk_shape : ChunksLike, optional The shape of the Array's chunks (default is None). Zarr format 3 only. Zarr format 2 arrays should use `chunks` instead. chunk_key_encoding : ChunkKeyEncoding, optional A specification of how the chunk keys are represented in storage. Zarr format 3 only. Zarr format 2 arrays should use `dimension_separator` instead. Default is `("default", "/")`. codecs : Sequence of Codecs or dicts, optional An iterable of Codec or dict serializations of Codecs. Zarr V3 only. The elements of `codecs` specify the transformation from array values to stored bytes. Zarr format 3 only. Zarr format 2 arrays should use `filters` and `compressor` instead. The [Zarr format 3 specification](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-encoding) requires, in order, zero or more array -> array codecs ([`zarr.abc.codec.ArrayArrayCodec`][]), exactly one array -> bytes codec ([`zarr.abc.codec.ArrayBytesCodec`][]), and zero or more bytes -> bytes codecs ([`zarr.abc.codec.BytesBytesCodec`][]). Elsewhere in Zarr-Python these are called filters, the serializer, and compressors; those names are Zarr-Python aliases, not terms defined in the specification. If no codecs are provided, default codecs will be used based on the data type of the array. For most data types, the default codecs are the tuple `(BytesCodec(), ZstdCodec())`; data types that require a special [`zarr.abc.codec.ArrayBytesCodec`][], like variable-length strings or bytes, will use the [`zarr.abc.codec.ArrayBytesCodec`][] required for the data type instead of [`zarr.codecs.BytesCodec`][]. dimension_names : Iterable[str | None] | None = None An iterable of dimension names. Zarr format 3 only. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. config : ArrayConfigLike, optional Runtime configuration of the array. If provided, will override the default values from `zarr.config.array`. mode : {'r', 'r+', 'a', 'w', 'w-'}, optional Legacy way to control overwriting, kept for compatibility with Zarr-Python 2. Prefer `overwrite`. The access mode used to open `store`; the default, `None`, is `'a'`. - `'a'` and `'r+'` create the array and fail if a node exists at `path`, unless `overwrite` is `True`. - `'w'` replaces anything stored under `path` (the whole store if `path` is not set), even if `overwrite` is `False`. On a store that cannot delete keys, `'w'` raises an error instead of replacing an existing node. - `'w-'` fails if anything is stored under `path`, even if `overwrite` is `True`. - `'r'` always fails. If `store` is a `StorePath`, `mode` is not validated against it: `'w'` still sets `overwrite`, and the other modes have no effect. data : array-like, optional Values written into the new array after it is created. Unlike the `data` parameter of `create_array`, it does not set the shape or data type of the array. To create an array from existing data, use `create_array(data=...)`. A Zarr array is not supported as `data`. Returns ------- z : Array The array. """ return Array( sync( async_api.create( shape=shape, chunks=chunks, dtype=dtype, compressor=compressor, fill_value=fill_value, order=order, store=store, synchronizer=synchronizer, overwrite=overwrite, path=path, chunk_store=chunk_store, filters=filters, cache_metadata=cache_metadata, cache_attrs=cache_attrs, read_only=read_only, object_codec=object_codec, dimension_separator=dimension_separator, write_empty_chunks=write_empty_chunks, zarr_format=zarr_format, meta_array=meta_array, attributes=attributes, chunk_shape=chunk_shape, chunk_key_encoding=chunk_key_encoding, codecs=codecs, dimension_names=dimension_names, storage_options=storage_options, config=config, mode=mode, data=data, ) ) ) def create_array( store: StoreLike, *, name: str | None = None, shape: ShapeLike | None = None, dtype: ZDTypeLike | None = None, data: np.ndarray[Any, np.dtype[Any]] | None = None, chunks: ChunksLike | Literal["auto"] = "auto", shards: ShardsLike | None = None, filters: FiltersLike = "auto", compressors: CompressorsLike = "auto", serializer: SerializerLike = "auto", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, zarr_format: ZarrFormat | None = 3, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfigLike | None = None, write_data: bool = True, ) -> AnyArray: """Create an array. This function wraps [zarr.core.array.create_array][]. Parameters ---------- store : StoreLike StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. name : str or None, optional The name of the array within the store. If `name` is `None`, the array will be located at the root of the store. shape : ShapeLike, optional Shape of the array. Must be `None` if `data` is provided. dtype : ZDTypeLike | None Data type of the array. Must be `None` if `data` is provided. data : np.ndarray, optional Array-like data to use for initializing the array. If this parameter is provided, the `shape` and `dtype` parameters must be `None`. chunks : tuple[int, ...] | Sequence[Sequence[int]] | Literal["auto"], default="auto" Chunk shape of the array. If chunks is "auto", a chunk shape is guessed based on the shape of the array and the dtype. A nested list of per-dimension edge sizes creates a rectilinear grid. Rectilinear chunk grids are experimental and must be explicitly enabled with `zarr.config.set({'array.rectilinear_chunks': True})` while the feature is stabilizing. shards : tuple[int, ...], optional Shard shape of the array. The default value of `None` results in no sharding at all. filters : Iterable[Codec] | Literal["auto"], optional Iterable of filters to apply to each chunk of the array, in order, before serializing that chunk to bytes. For Zarr format 3, a "filter" is a codec that takes an array and returns an array, and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the order of your filters is consistent with the behavior of each filter. The default value of `"auto"` instructs Zarr to use a default based on the data type of the array and the Zarr format specified. For all data types in Zarr V3, and most data types in Zarr V2, the default filters are empty. The only cases where default filters are not empty is when the Zarr format is 2, and the data type is a variable-length data type like [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, the default filters contains a single element which is a codec specific to that particular data type. To create an array with no filters, provide an empty iterable or the value `None`. compressors : Iterable[Codec], optional List of compressors to apply to the array. Compressors are applied in order, and after any filters are applied (if any are specified) and the data is serialized into bytes. For Zarr format 3, a "compressor" is a codec that takes a bytestream, and returns another bytestream. These values must be instances of [`zarr.abc.codec.BytesBytesCodec`][], or dict representations of [`zarr.abc.codec.BytesBytesCodec`][]. Multiple compressors may be provided for Zarr format 3. Codecs that take an array and return bytes are serializers and must be supplied with `serializer` instead. If no `compressors` are provided, the default for Zarr format 3 is `(ZstdCodec(),)`. Use `None` to omit default compressors. For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may be provided for Zarr format 2. If no `compressor` is provided, a default compressor will be used. This default can be changed by modifying the value of `array.v2_default_compressor` in [`zarr.config`][zarr.config]. Use `None` to omit the default compressor. serializer : dict[str, JSON] | ArrayBytesCodec, optional Array-to-bytes codec to use for encoding the array data. Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. Codecs that are instances of [`zarr.abc.codec.ArrayBytesCodec`][] must be supplied here, not with `compressors`. If no `serializer` is provided, a default serializer is chosen based on the data type: `VLenUTF8Codec` for variable-length strings, `VLenBytesCodec` for variable-length bytes, and `BytesCodec` otherwise. fill_value : Any, optional Fill value for the array. order : {"C", "F"}, optional The memory order of the array (default is "C"). For Zarr format 2, this parameter sets the memory order of the array. For Zarr format 3, this parameter is deprecated, because memory order is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. If no `order` is provided, a default order will be used. This default can be changed by modifying the value of `array.order` in [`zarr.config`][zarr.config]. zarr_format : {2, 3}, optional The zarr format to use when saving. attributes : dict, optional Attributes for the array. chunk_key_encoding : ChunkKeyEncodingLike, optional A specification of how the chunk keys are represented in storage. For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. dimension_names : Iterable[str], optional The names of the dimensions (default is None). Zarr format 3 only. Zarr format 2 arrays should not use this parameter. storage_options : dict, optional If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. overwrite : bool, default False Whether to overwrite an array with the same name in the store, if one exists. If `True`, any existing keys under that path are deleted first. config : ArrayConfigLike, optional Runtime configuration for the array. write_data : bool If a pre-existing array-like object was provided to this function via the `data` parameter then `write_data` determines whether the values in that array-like object should be written to the Zarr array created by this function. If `write_data` is `False`, then the array will be left empty. Returns ------- Array The array. Examples -------- ```python import zarr store = zarr.storage.MemoryStore() arr = zarr.create_array( store=store, shape=(100,100), chunks=(10,10), dtype='i4', fill_value=0) # ``` """ return Array( sync( zarr.core.array.create_array( store, name=name, shape=shape, dtype=dtype, data=data, chunks=chunks, shards=shards, filters=filters, compressors=compressors, serializer=serializer, fill_value=fill_value, order=order, zarr_format=zarr_format, attributes=attributes, chunk_key_encoding=chunk_key_encoding, dimension_names=dimension_names, storage_options=storage_options, overwrite=overwrite, config=config, write_data=write_data, ) ) ) def from_array( store: StoreLike, *, data: AnyArray | npt.ArrayLike, write_data: bool = True, name: str | None = None, chunks: ChunksLike | Literal["auto", "keep"] = "keep", shards: ShardsLike | Literal["keep"] | None = "keep", filters: FiltersLike | Literal["keep"] = "keep", compressors: CompressorsLike | Literal["keep"] = "keep", serializer: SerializerLike | Literal["keep"] = "keep", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, zarr_format: ZarrFormat | None = None, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfigLike | None = None, ) -> AnyArray: """Create an array from an existing array or array-like. Parameters ---------- store : StoreLike StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. data : Array | array-like The array to copy. write_data : bool, default True Whether to copy the data from the input array to the new array. If `write_data` is `False`, the new array will be created with the same metadata as the input array, but without any data. name : str or None, optional The name of the array within the store. If `name` is `None`, the array will be located at the root of the store. chunks : tuple[int, ...] or Sequence[Sequence[int]] or "auto" or "keep", optional Chunk shape of the array. Following values are supported: - "auto": Automatically determine the chunk shape based on the array's shape and dtype. - "keep": Retain the chunk grid of the data array if it is a zarr Array. - tuple[int, ...]: A tuple of integers representing the chunk shape (regular grid). - Sequence[Sequence[int]]: Per-dimension chunk edge lists (rectilinear grid). Rectilinear chunk grids are experimental and must be explicitly enabled with `zarr.config.set({'array.rectilinear_chunks': True})` while the feature is stabilizing. If not specified, defaults to "keep" if data is a zarr Array, otherwise "auto". shards : tuple[int, ...], optional Shard shape of the array. Following values are supported: - "auto": Automatically determine the shard shape based on the array's shape and chunk shape. - "keep": Retain the shard shape of the data array if it is a zarr Array. - tuple[int, ...]: A tuple of integers representing the shard shape. - None: No sharding. If not specified, defaults to "keep" if data is a zarr Array, otherwise None. filters : Iterable[Codec] | Literal["auto", "keep"], optional Iterable of filters to apply to each chunk of the array, in order, before serializing that chunk to bytes. For Zarr format 3, a "filter" is a codec that takes an array and returns an array, and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the order of your filters is consistent with the behavior of each filter. The default value of `"keep"` instructs Zarr to infer `filters` from `data`. If that inference is not possible, Zarr will fall back to the behavior specified by `"auto"`, which is to choose default filters based on the data type of the array and the Zarr format specified. For all data types in Zarr V3, and most data types in Zarr V2, the default filters are the empty tuple `()`. The only cases where default filters are not empty is when the Zarr format is 2, and the data type is a variable-length data type like [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthUTF8`][]. In these cases, the default filters is a tuple with a single element which is a codec specific to that particular data type. To create an array with no filters, provide an empty iterable or the value `None`. compressors : Iterable[Codec] or "auto" or "keep", optional List of compressors to apply to the array. Compressors are applied in order, and after any filters are applied (if any are specified) and the data is serialized into bytes. For Zarr format 3, a "compressor" is a codec that takes a bytestream, and returns another bytestream. Multiple compressors may be provided for Zarr format 3. For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may be provided for Zarr format 2. Following values are supported: - Iterable[Codec]: List of compressors to apply to the array. - "auto": Automatically determine the compressors based on the array's dtype. - "keep": Retain the compressors of the input array if it is a zarr Array. If no `compressors` are provided, defaults to "keep" if data is a zarr Array, otherwise "auto". serializer : dict[str, JSON] | ArrayBytesCodec or "auto" or "keep", optional Array-to-bytes codec to use for encoding the array data. Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. Following values are supported: - dict[str, JSON]: A dict representation of an `ArrayBytesCodec`. - ArrayBytesCodec: An instance of `ArrayBytesCodec`. - "auto": a default serializer will be used. These defaults can be changed by modifying the value of `array.v3_default_serializer` in [`zarr.config`][zarr.config]. - "keep": Retain the serializer of the input array if it is a zarr Array. fill_value : Any, optional Fill value for the array. If not specified, defaults to the fill value of the data array. Pass `None` explicitly to use the default scalar of the data type (Zarr format 3) or a null fill value (Zarr format 2) instead. order : {"C", "F"}, optional The memory order of the array (default is "C"). For Zarr format 2, this parameter sets the memory order of the array. For Zarr format 3, this parameter is deprecated, because memory order is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. If not specified, defaults to the memory order of the data array. zarr_format : {2, 3}, optional The zarr format to use when saving. If not specified, defaults to the zarr format of the data array. attributes : dict, optional Attributes for the array. If not specified, the source Zarr array's attributes are deep-copied so nested containers are independent. Deeply nested attributes can raise `RecursionError` during copying even if they can be stored and reopened; the threshold depends on Python's recursion limit and the current call stack. Pass an empty dict to create the array with no attributes. chunk_key_encoding : ChunkKeyEncoding, optional A specification of how the chunk keys are represented in storage. For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. If not specified and the data array has the same zarr format as the target array, the chunk key encoding of the data array is used. dimension_names : Iterable[str | None] | None The names of the dimensions (default is None). Zarr format 3 only. Zarr format 2 arrays should not use this parameter. If not specified, defaults to the dimension names of the data array. storage_options : dict, optional If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. overwrite : bool, default False Whether to overwrite an array with the same name in the store, if one exists. config : ArrayConfig or ArrayConfigLike, optional Runtime configuration for the array. Returns ------- Array The array. Examples -------- Create an array from an existing Array: >>> import asyncio >>> import zarr >>> store = zarr.storage.LocalStore("example_from_array.zarr") >>> arr = zarr.create_array( ... store={}, ... shape=(100,100), ... chunks=(10,10), ... dtype="int32", ... fill_value=0 ... ) >>> arr2 = zarr.from_array(store, data=arr, overwrite=True) >>> arr2 >>> asyncio.run(store.clear()) # Remove files generated by test Create an array from an existing NumPy array: >>> import numpy as np >>> zarr.from_array({}, data=np.arange(10000, dtype="i4").reshape(100, 100)) Create an array from any array-like object: >>> arr3 = zarr.from_array({}, data=[[1, 2], [3, 4]]) >>> arr3 >>> arr3[...] array([[1, 2], [3, 4]]) Create an array from an existing Array without copying the data: >>> arr4 = zarr.from_array({}, data=[[1, 2], [3, 4]]) >>> arr5 = zarr.from_array({}, data=arr4, write_data=False) >>> arr5 >>> arr5[...] array([[0, 0], [0, 0]]) """ return Array( sync( zarr.core.array.from_array( store, data=data, write_data=write_data, name=name, chunks=chunks, shards=shards, filters=filters, compressors=compressors, serializer=serializer, fill_value=fill_value, order=order, zarr_format=zarr_format, attributes=attributes, chunk_key_encoding=chunk_key_encoding, dimension_names=dimension_names, storage_options=storage_options, overwrite=overwrite, config=config, ) ) ) # TODO: add type annotations for kwargs def empty(shape: tuple[int, ...], **kwargs: Any) -> AnyArray: """Create an empty array with the specified shape. The contents will be filled with the array's fill value or zeros if no fill value is provided. Parameters ---------- shape : int or tuple of int Shape of the empty array. **kwargs Keyword arguments passed to [`create`][zarr.api.asynchronous.create]. Returns ------- Array The new array. Notes ----- The contents of an empty Zarr array are not defined. On attempting to retrieve data from an empty Zarr array, any values may be returned, and these are not guaranteed to be stable from one access to the next. """ return Array(sync(async_api.empty(shape, **kwargs))) # TODO: move ArrayLike to common module # TODO: add type annotations for kwargs def empty_like(a: ArrayLike, *, zarr_format: ZarrFormat | None = None, **kwargs: Any) -> AnyArray: """Create an empty array like another array. The contents will be filled with the array's fill value or zeros if no fill value is provided. Parameters ---------- a : array-like The array to create an empty array like. zarr_format : {2, 3, None}, optional The zarr format of the new array. If `None` (default), the zarr format of `a` if it is a zarr array, otherwise the default zarr format. **kwargs Keyword arguments passed to [`create`][zarr.api.asynchronous.create]. Returns ------- Array The new array. Notes ----- The contents of an empty Zarr array are not defined. On attempting to retrieve data from an empty Zarr array, any values may be returned, and these are not guaranteed to be stable from one access to the next. """ return Array(sync(async_api.empty_like(a, zarr_format=zarr_format, **kwargs))) # TODO: add type annotations for kwargs and fill_value def full(shape: tuple[int, ...], fill_value: Any, **kwargs: Any) -> AnyArray: """Create an array with a default fill value. Parameters ---------- shape : int or tuple of int Shape of the empty array. fill_value : scalar Fill value. **kwargs Keyword arguments passed to [`create`][zarr.api.asynchronous.create]. Returns ------- Array The new array. """ return Array(sync(async_api.full(shape=shape, fill_value=fill_value, **kwargs))) # TODO: move ArrayLike to common module # TODO: add type annotations for kwargs def full_like(a: ArrayLike, *, zarr_format: ZarrFormat | None = None, **kwargs: Any) -> AnyArray: """Create a filled array like another array. Parameters ---------- a : array-like The array to create an empty array like. zarr_format : {2, 3, None}, optional The zarr format of the new array. If `None` (default), the zarr format of `a` if it is a zarr array, otherwise the default zarr format. **kwargs Keyword arguments passed to [`zarr.api.asynchronous.create`][]. Returns ------- Array The new array. """ return Array(sync(async_api.full_like(a, zarr_format=zarr_format, **kwargs))) # TODO: add type annotations for kwargs def ones(shape: tuple[int, ...], **kwargs: Any) -> AnyArray: """Create an array with a fill value of one. Parameters ---------- shape : int or tuple of int Shape of the empty array. **kwargs Keyword arguments passed to [`zarr.api.asynchronous.create`][]. Returns ------- Array The new array. """ return Array(sync(async_api.ones(shape, **kwargs))) # TODO: add type annotations for kwargs def ones_like(a: ArrayLike, *, zarr_format: ZarrFormat | None = None, **kwargs: Any) -> AnyArray: """Create an array of ones like another array. Parameters ---------- a : array-like The array to create an empty array like. zarr_format : {2, 3, None}, optional The zarr format of the new array. If `None` (default), the zarr format of `a` if it is a zarr array, otherwise the default zarr format. **kwargs Keyword arguments passed to [`zarr.api.asynchronous.create`][]. Returns ------- Array The new array. """ return Array(sync(async_api.ones_like(a, zarr_format=zarr_format, **kwargs))) # TODO: update this once async_api.open_array is fully implemented def open_array( store: StoreLike | None = None, *, zarr_format: ZarrFormat | None = None, path: PathLike = "", storage_options: dict[str, Any] | None = None, config: ArrayConfigLike | None = None, **kwargs: Any, ) -> AnyArray: """Open an array using file-mode-like semantics. Parameters ---------- store : StoreLike StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. zarr_format : {2, 3, None}, optional The zarr format to use when saving. path : str, optional Path in store to array. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. config : ArrayConfigLike or None, default=None Runtime configuration for the array, whether it is opened or created. Keys not specified are taken from the global configuration. **kwargs Any keyword arguments to pass to [`create`][zarr.api.asynchronous.create]. Returns ------- AsyncArray The opened array. """ return Array( sync( async_api.open_array( store=store, zarr_format=zarr_format, path=path, storage_options=storage_options, config=config, **kwargs, ) ) ) # TODO: add type annotations for kwargs def open_like( a: ArrayLike, path: str, *, zarr_format: ZarrFormat | None = None, **kwargs: Any ) -> AnyArray: """Open a persistent array like another array. Parameters ---------- a : Array The shape and data-type of a define these same attributes of the returned array. path : str The path to the new array. zarr_format : {2, 3, None}, optional The zarr format of the array to open or create. If `None` (default), an existing array of either format is opened, and a missing one is created in the default zarr format. The zarr format of `a` is not inherited. **kwargs Additional keyword arguments passed to `open_array`. If `mode` is omitted or `None`, it defaults to `"a"`. Pass `mode="r"` when opening an existing array from a read-only store. Returns ------- Array The opened array. """ return Array(sync(async_api.open_like(a, path=path, zarr_format=zarr_format, **kwargs))) # TODO: add type annotations for kwargs def zeros(shape: tuple[int, ...], **kwargs: Any) -> AnyArray: """Create an array with a fill value of zero. Parameters ---------- shape : int or tuple of int Shape of the empty array. **kwargs Keyword arguments passed to [`zarr.api.asynchronous.create`][]. Returns ------- Array The new array. """ return Array(sync(async_api.zeros(shape=shape, **kwargs))) # TODO: add type annotations for kwargs def zeros_like(a: ArrayLike, *, zarr_format: ZarrFormat | None = None, **kwargs: Any) -> AnyArray: """Create an array of zeros like another array. Parameters ---------- a : array-like The array to create an empty array like. zarr_format : {2, 3, None}, optional The zarr format of the new array. If `None` (default), the zarr format of `a` if it is a zarr array, otherwise the default zarr format. **kwargs Keyword arguments passed to [`create`][zarr.api.asynchronous.create]. Returns ------- Array The new array. """ return Array(sync(async_api.zeros_like(a, zarr_format=zarr_format, **kwargs))) zarr-developers-zarr-python-a2f2415/src/zarr/buffer/000077500000000000000000000000001527014137600225035ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/buffer/__init__.py000066400000000000000000000004331527014137600246140ustar00rootroot00000000000000""" Implementations of the Zarr Buffer interface. See Also ======== zarr.abc.buffer: Abstract base class for the Zarr Buffer interface. """ from zarr.buffer import cpu, gpu from zarr.core.buffer import default_buffer_prototype __all__ = ["cpu", "default_buffer_prototype", "gpu"] zarr-developers-zarr-python-a2f2415/src/zarr/buffer/cpu.py000066400000000000000000000004161527014137600236450ustar00rootroot00000000000000from zarr.core.buffer.cpu import ( Buffer, NDBuffer, as_numpy_array_wrapper, buffer_prototype, numpy_buffer_prototype, ) __all__ = [ "Buffer", "NDBuffer", "as_numpy_array_wrapper", "buffer_prototype", "numpy_buffer_prototype", ] zarr-developers-zarr-python-a2f2415/src/zarr/buffer/gpu.py000066400000000000000000000002111527014137600236420ustar00rootroot00000000000000from zarr.core.buffer.gpu import Buffer, NDBuffer, buffer_prototype __all__ = [ "Buffer", "NDBuffer", "buffer_prototype", ] zarr-developers-zarr-python-a2f2415/src/zarr/codecs/000077500000000000000000000000001527014137600224725ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/codecs/__init__.py000066400000000000000000000071261527014137600246110ustar00rootroot00000000000000from __future__ import annotations from zarr.codecs.blosc import BloscCname, BloscCodec, BloscShuffle from zarr.codecs.bytes import BytesCodec, Endian from zarr.codecs.cast_value import CastValue from zarr.codecs.crc32c_ import Crc32cCodec from zarr.codecs.gzip import GzipCodec from zarr.codecs.numcodecs import ( BZ2, CRC32, CRC32C, LZ4, LZMA, ZFPY, Adler32, AsType, BitRound, Blosc, Delta, FixedScaleOffset, Fletcher32, GZip, JenkinsLookup3, PackBits, PCodec, Quantize, Shuffle, Zlib, Zstd, ) from zarr.codecs.scale_offset import ScaleOffset from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation, SubchunkWriteOrder from zarr.codecs.transpose import TransposeCodec from zarr.codecs.vlen_utf8 import VLenBytesCodec, VLenUTF8Codec from zarr.codecs.zstd import ZstdCodec from zarr.registry import register_codec __all__ = [ "BloscCname", "BloscCodec", "BloscShuffle", "BytesCodec", "CastValue", "Crc32cCodec", "Endian", "GzipCodec", "ScaleOffset", "ShardingCodec", "ShardingCodecIndexLocation", "SubchunkWriteOrder", "TransposeCodec", "VLenBytesCodec", "VLenUTF8Codec", "ZstdCodec", ] register_codec("blosc", BloscCodec) register_codec("cast_value", CastValue) register_codec("bytes", BytesCodec) # compatibility with earlier versions of ZEP1 register_codec("endian", BytesCodec) register_codec("crc32c", Crc32cCodec) register_codec("gzip", GzipCodec) register_codec("scale_offset", ScaleOffset) register_codec("sharding_indexed", ShardingCodec) register_codec("zstd", ZstdCodec) register_codec("vlen-utf8", VLenUTF8Codec) register_codec("vlen-bytes", VLenBytesCodec) register_codec("transpose", TransposeCodec) # Register all the codecs formerly contained in numcodecs.zarr3 register_codec("numcodecs.bz2", BZ2, qualname="zarr.codecs.numcodecs.BZ2") register_codec("numcodecs.crc32", CRC32, qualname="zarr.codecs.numcodecs.CRC32") register_codec("numcodecs.crc32c", CRC32C, qualname="zarr.codecs.numcodecs.CRC32C") register_codec("numcodecs.lz4", LZ4, qualname="zarr.codecs.numcodecs.LZ4") register_codec("numcodecs.lzma", LZMA, qualname="zarr.codecs.numcodecs.LZMA") register_codec("numcodecs.zfpy", ZFPY, qualname="zarr.codecs.numcodecs.ZFPY") register_codec("numcodecs.adler32", Adler32, qualname="zarr.codecs.numcodecs.Adler32") register_codec("numcodecs.astype", AsType, qualname="zarr.codecs.numcodecs.AsType") register_codec("numcodecs.bitround", BitRound, qualname="zarr.codecs.numcodecs.BitRound") register_codec("numcodecs.blosc", Blosc, qualname="zarr.codecs.numcodecs.Blosc") register_codec("numcodecs.delta", Delta, qualname="zarr.codecs.numcodecs.Delta") register_codec( "numcodecs.fixedscaleoffset", FixedScaleOffset, qualname="zarr.codecs.numcodecs.FixedScaleOffset", ) register_codec("numcodecs.fletcher32", Fletcher32, qualname="zarr.codecs.numcodecs.Fletcher32") register_codec("numcodecs.gzip", GZip, qualname="zarr.codecs.numcodecs.GZip") register_codec( "numcodecs.jenkins_lookup3", JenkinsLookup3, qualname="zarr.codecs.numcodecs.JenkinsLookup3" ) register_codec("numcodecs.pcodec", PCodec, qualname="zarr.codecs.numcodecs.PCodec") register_codec("numcodecs.packbits", PackBits, qualname="zarr.codecs.numcodecs.PackBits") register_codec("numcodecs.quantize", Quantize, qualname="zarr.codecs.numcodecs.Quantize") register_codec("numcodecs.shuffle", Shuffle, qualname="zarr.codecs.numcodecs.Shuffle") register_codec("numcodecs.zlib", Zlib, qualname="zarr.codecs.numcodecs.Zlib") register_codec("numcodecs.zstd", Zstd, qualname="zarr.codecs.numcodecs.Zstd") zarr-developers-zarr-python-a2f2415/src/zarr/codecs/_deprecated_enum.py000066400000000000000000000044521527014137600263340ustar00rootroot00000000000000"""Helpers for deprecating string-valued enums in favor of literal strings. See PR #3963 for context on the deprecation pattern. """ from __future__ import annotations import warnings from enum import Enum class _DeprecatedStrEnumMeta(type): """ Metaclass for legacy enum-like classes. Accessing a member name on the class (e.g. `LegacyShim.foo`) emits a `DeprecationWarning` and returns the equivalent string. Members are declared by setting a `_members` class attribute mapping each member name to its string value. """ _members: dict[str, str] def __getattr__(cls, name: str) -> str: members: dict[str, str] = type.__getattribute__(cls, "_members") if name in members: warnings.warn( f"{cls.__name__}.{name} is deprecated; pass the string {members[name]!r} instead.", DeprecationWarning, stacklevel=2, ) return members[name] raise AttributeError(name) def _coerce_enum_input(value: object, param_name: str, codec_name: str) -> object: """ If `value` is a real `enum.Enum` instance, emit a deprecation warning naming `codec_name` and return `value.value`. Otherwise return `value` unchanged. The third argument lets the warning text name the actual codec (e.g. `BloscCodec`, `BytesCodec`, `ShardingCodec`). Note that zarr's own legacy classes (e.g. `ShardingCodecIndexLocation`) never reach the `Enum` branch here: they no longer inherit from `Enum`, and member access on them already returns a plain string (with its own warning) via `_DeprecatedStrEnumMeta`. This branch exists for enum instances defined *outside* zarr — in particular `str`-mixin enums that downstream code defined to mirror zarr's old enums, which the old `parse_enum`-based codepath accepted because they are `str` instances. Coercing them to `value.value` keeps the stored attribute a plain string and gives those callers a migration warning. """ if isinstance(value, Enum): warnings.warn( f"Passing an enum to {codec_name}(..., {param_name}=...) is deprecated; " "pass the equivalent literal string instead.", DeprecationWarning, stacklevel=3, ) return value.value return value zarr-developers-zarr-python-a2f2415/src/zarr/codecs/_v2.py000066400000000000000000000125601527014137600235360ustar00rootroot00000000000000from __future__ import annotations import asyncio from dataclasses import dataclass from typing import TYPE_CHECKING import numpy as np from numcodecs.compat import ensure_bytes, ensure_ndarray_like from zarr.abc.codec import ArrayBytesCodec from zarr.core.config import config from zarr.errors import PickleCodecDisabledError from zarr.registry import get_ndbuffer_class if TYPE_CHECKING: from zarr.abc.numcodec import Numcodec from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer, NDBuffer def _check_pickle_allowed(codecs: tuple[Numcodec | None, ...]) -> None: """ Raise if any of ``codecs`` is the pickle codec and reading pickled data has not been enabled. Unpickling can run arbitrary code, so data from an untrusted store must not be decoded with the pickle codec unless the user has explicitly opted in. """ if not any(getattr(codec, "codec_id", None) == "pickle" for codec in codecs): return # Compare with ``True`` rather than testing truthiness: donfig keeps an environment # variable it cannot parse as a Python literal (e.g. ``false``) as a non-empty string. if config.get("array.allow_unsafe_pickle_decode") is not True: raise PickleCodecDisabledError( "Refusing to decode data with the 'pickle' codec, because unpickling data " "from an untrusted source can execute arbitrary code. If you trust this data, " "enable the pickle codec with zarr.config.set({'array.allow_unsafe_pickle_decode': True}) " "or set the environment variable ZARR_ARRAY__ALLOW_UNSAFE_PICKLE_DECODE=True. numcodecs has " "its own opt-in, which must be enabled as well: set " "numcodecs.pickles.ALLOW_UNSAFE_PICKLE_DECODE = True, or set the environment " "variable NUMCODECS_ALLOW_UNSAFE_PICKLE_DECODE=1 before importing numcodecs." ) @dataclass(frozen=True) class V2Codec(ArrayBytesCodec): filters: tuple[Numcodec, ...] | None compressor: Numcodec | None is_fixed_size = False def _decode_sync( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> NDBuffer: _check_pickle_allowed((self.compressor, *(self.filters or ()))) cdata = chunk_bytes.as_array_like() # decompress if self.compressor: chunk = self.compressor.decode(cdata) else: chunk = cdata # apply filters if self.filters: for f in reversed(self.filters): chunk = f.decode(chunk) # view as numpy array with correct dtype chunk = ensure_ndarray_like(chunk) # special case object dtype, because incorrect handling can lead to # segfaults and other bad things happening if chunk_spec.dtype.dtype_cls is not np.dtypes.ObjectDType: try: chunk = chunk.view(chunk_spec.dtype.to_native_dtype()) except TypeError: # this will happen if the dtype of the chunk # does not match the dtype of the array spec i.g. if # the dtype of the chunk_spec is a string dtype, but the chunk # is an object array. In this case, we need to convert the object # array to the correct dtype. chunk = np.array(chunk).astype(chunk_spec.dtype.to_native_dtype()) elif chunk.dtype != object: # If we end up here, someone must have hacked around with the filters. # We cannot deal with object arrays unless there is an object # codec in the filter chain, i.e., a filter that converts from object # array to something else during encoding, and converts back to object # array during decoding. raise RuntimeError("cannot read object array without object codec") # ensure correct chunk shape chunk = chunk.reshape(-1, order="A") chunk = chunk.reshape(chunk_spec.shape, order=chunk_spec.order) return get_ndbuffer_class().from_ndarray_like(chunk) def _encode_sync( self, chunk_array: NDBuffer, chunk_spec: ArraySpec, ) -> Buffer | None: chunk = chunk_array.as_ndarray_like() # ensure contiguous and correct order chunk = chunk.astype(chunk_spec.dtype.to_native_dtype(), order=chunk_spec.order, copy=False) # apply filters if self.filters: for f in self.filters: chunk = f.encode(chunk) # check object encoding if ensure_ndarray_like(chunk).dtype == object: raise RuntimeError("cannot write object array without object codec") # compress if self.compressor: cdata = self.compressor.encode(chunk) else: cdata = chunk cdata = ensure_bytes(cdata) return chunk_spec.prototype.buffer.from_bytes(cdata) async def _decode_single( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> NDBuffer: return await asyncio.to_thread(self._decode_sync, chunk_bytes, chunk_spec) async def _encode_single( self, chunk_array: NDBuffer, chunk_spec: ArraySpec, ) -> Buffer | None: return await asyncio.to_thread(self._encode_sync, chunk_array, chunk_spec) def compute_encoded_size(self, _input_byte_length: int, _chunk_spec: ArraySpec) -> int: raise NotImplementedError zarr-developers-zarr-python-a2f2415/src/zarr/codecs/blosc.py000066400000000000000000000271521527014137600241550ustar00rootroot00000000000000from __future__ import annotations import asyncio from dataclasses import dataclass, field, replace from functools import cached_property from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, TypedDict import numcodecs from numcodecs.blosc import Blosc from packaging.version import Version from zarr.abc.codec import BytesBytesCodec from zarr.codecs._deprecated_enum import _coerce_enum_input, _DeprecatedStrEnumMeta from zarr.core.buffer.cpu import as_numpy_array_wrapper from zarr.core.common import JSON, NamedRequiredConfig, parse_named_configuration from zarr.core.dtype.common import HasItemSize from zarr.core.json_parse import parse_field if TYPE_CHECKING: from typing import Self from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer BloscShuffleLiteral = Literal["noshuffle", "shuffle", "bitshuffle"] """The shuffle values permitted for the blosc codec""" BLOSC_SHUFFLE: Final = ("noshuffle", "shuffle", "bitshuffle") BloscCnameLiteral = Literal["lz4", "lz4hc", "blosclz", "snappy", "zlib", "zstd"] """The codec identifiers used in the blosc codec""" BLOSC_CNAME: Final = ("lz4", "lz4hc", "blosclz", "snappy", "zlib", "zstd") class BloscConfigV2(TypedDict): """Configuration for the V2 Blosc codec""" cname: BloscCnameLiteral clevel: int shuffle: int blocksize: int typesize: NotRequired[int] class BloscConfigV3(TypedDict): """Configuration for the V3 Blosc codec""" cname: BloscCnameLiteral clevel: int shuffle: BloscShuffleLiteral blocksize: int typesize: int class BloscJSON_V3(NamedRequiredConfig[Literal["blosc"], BloscConfigV3]): """ The JSON form of the Blosc codec in Zarr V3. """ class BloscShuffle(metaclass=_DeprecatedStrEnumMeta): """ Deprecated. Pass a literal string (`"noshuffle"`, `"shuffle"`, or `"bitshuffle"`) directly to `BloscCodec` instead. """ _members: ClassVar[dict[str, str]] = { "noshuffle": "noshuffle", "shuffle": "shuffle", "bitshuffle": "bitshuffle", } @staticmethod def from_int(num: int) -> BloscShuffleLiteral: mapping: dict[int, BloscShuffleLiteral] = { 0: "noshuffle", 1: "shuffle", 2: "bitshuffle", } if num not in mapping: raise ValueError(f"Value must be between 0 and 2. Got {num}.") return mapping[num] class BloscCname(metaclass=_DeprecatedStrEnumMeta): """ Deprecated. Pass a literal string (one of `"lz4"`, `"lz4hc"`, `"blosclz"`, `"snappy"`, `"zlib"`, `"zstd"`) directly to `BloscCodec` instead. """ _members: ClassVar[dict[str, str]] = { "lz4": "lz4", "lz4hc": "lz4hc", "blosclz": "blosclz", "snappy": "snappy", "zstd": "zstd", "zlib": "zlib", } # See https://zarr.readthedocs.io/en/stable/user-guide/performance.html#configuring-blosc numcodecs.blosc.use_threads = False def parse_typesize(data: JSON) -> int: parsed: int = parse_field(data, int, "typesize", error=TypeError) if parsed > 0: return parsed else: raise ValueError( f"Value must be greater than 0. Got {parsed}, which is less or equal to 0." ) # todo: real validation def parse_clevel(data: JSON) -> int: parsed: int = parse_field(data, int, "clevel", error=TypeError) return parsed def parse_blocksize(data: JSON) -> int: parsed: int = parse_field(data, int, "blocksize", error=TypeError) return parsed def _parse_cname(data: object) -> BloscCnameLiteral: if isinstance(data, str) and data in BLOSC_CNAME: return data # type: ignore[return-value] raise ValueError(f"cname must be one of {list(BLOSC_CNAME)!r}. Got {data!r}.") def _parse_shuffle(data: object) -> BloscShuffleLiteral: if isinstance(data, str) and data in BLOSC_SHUFFLE: return data # type: ignore[return-value] raise ValueError(f"shuffle must be one of {list(BLOSC_SHUFFLE)!r}. Got {data!r}.") @dataclass(frozen=True) class BloscCodec(BytesBytesCodec): """ Blosc compression codec for zarr. Blosc is a high-performance compressor optimized for binary data. It uses a combination of blocking, shuffling, and fast compression algorithms to achieve excellent compression ratios and speed. Attributes ---------- is_fixed_size : bool Always False for Blosc codec, as compression produces variable-sized output. typesize : int The data type size in bytes used for shuffle filtering. cname : BloscCnameLiteral The compression algorithm being used; one of "lz4", "lz4hc", "blosclz", "snappy", "zlib", or "zstd". clevel : int The compression level (0-9). shuffle : BloscShuffleLiteral The shuffle filter mode; one of "noshuffle", "shuffle", or "bitshuffle". blocksize : int The size of compressed blocks in bytes (0 for automatic). Parameters ---------- typesize : int, optional The data type size in bytes. This affects how the shuffle filter processes the data. If None, defaults to 1 and the attribute is marked as tunable. Default: 1. cname : BloscCnameLiteral, optional The compression algorithm to use; one of "lz4", "lz4hc", "blosclz", "snappy", "zlib", or "zstd". Default is "zstd". Passing a `BloscCname` enum is deprecated. clevel : int, optional The compression level, from 0 (no compression) to 9 (maximum compression). Higher values provide better compression at the cost of speed. Default: 5. shuffle : BloscShuffleLiteral or None, optional The shuffle filter to apply before compression; one of "noshuffle", "shuffle", or "bitshuffle": - 'noshuffle': No shuffling - 'shuffle': Byte shuffling (better for typesize > 1) - 'bitshuffle': Bit shuffling (better for typesize == 1) If None, defaults to 'bitshuffle' and the attribute is marked as tunable. Default: 'bitshuffle'. blocksize : int, optional The requested size of compressed blocks in bytes. A value of 0 means automatic block size selection. Default: 0. Notes ----- **Tunable attributes**: If `typesize` or `shuffle` are set to None during initialization, they are marked as tunable attributes. This means they can be adjusted later based on the data type of the array being compressed. **Thread Safety**: This codec sets `numcodecs.blosc.use_threads = False` at module import time to avoid threading issues in asyncio contexts. Examples -------- Create a Blosc codec with default settings: >>> codec = BloscCodec() >>> codec.typesize 1 >>> codec.shuffle 'bitshuffle' Create a codec with specific compression settings: >>> codec = BloscCodec(cname='zstd', clevel=9, shuffle='shuffle') >>> codec.cname 'zstd' """ # This attribute tracks parameters were set to None at init time, and thus tunable _tunable_attrs: set[Literal["typesize", "shuffle"]] = field(init=False) is_fixed_size = False typesize: int cname: BloscCnameLiteral clevel: int shuffle: BloscShuffleLiteral blocksize: int def __init__( self, *, typesize: int | None = None, cname: BloscCname | BloscCnameLiteral = "zstd", clevel: int = 5, shuffle: BloscShuffle | BloscShuffleLiteral | None = None, blocksize: int = 0, ) -> None: object.__setattr__(self, "_tunable_attrs", set()) if typesize is None: typesize = 1 self._tunable_attrs.update({"typesize"}) if shuffle is None: shuffle = "bitshuffle" self._tunable_attrs.update({"shuffle"}) cname = _coerce_enum_input(cname, "cname", "BloscCodec") # type: ignore[assignment] shuffle = _coerce_enum_input(shuffle, "shuffle", "BloscCodec") # type: ignore[assignment] typesize_parsed = parse_typesize(typesize) cname_parsed = _parse_cname(cname) clevel_parsed = parse_clevel(clevel) shuffle_parsed = _parse_shuffle(shuffle) blocksize_parsed = parse_blocksize(blocksize) object.__setattr__(self, "typesize", typesize_parsed) object.__setattr__(self, "cname", cname_parsed) object.__setattr__(self, "clevel", clevel_parsed) object.__setattr__(self, "shuffle", shuffle_parsed) object.__setattr__(self, "blocksize", blocksize_parsed) @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: _, configuration_parsed = parse_named_configuration(data, "blosc") return cls(**configuration_parsed) # type: ignore[arg-type] def to_dict(self) -> dict[str, JSON]: result: BloscJSON_V3 = { "name": "blosc", "configuration": { "typesize": self.typesize, "cname": self.cname, "clevel": self.clevel, "shuffle": self.shuffle, "blocksize": self.blocksize, }, } return result # type: ignore[return-value] def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: """ Create a new codec with typesize and shuffle parameters adjusted according to the size of each element in the data type associated with array_spec. Parameters are only updated if they were set to None when self.__init__ was called. """ item_size = 1 if isinstance(array_spec.dtype, HasItemSize): item_size = array_spec.dtype.item_size new_codec = self if "typesize" in self._tunable_attrs: new_codec = replace(new_codec, typesize=item_size) if "shuffle" in self._tunable_attrs: new_codec = replace( new_codec, shuffle=("bitshuffle" if item_size == 1 else "shuffle"), ) return new_codec @cached_property def _blosc_codec(self) -> Blosc: map_shuffle_str_to_int: dict[BloscShuffleLiteral, int] = { "noshuffle": 0, "shuffle": 1, "bitshuffle": 2, } config_dict: BloscConfigV2 = { "cname": self.cname, "clevel": self.clevel, "shuffle": map_shuffle_str_to_int[self.shuffle], "blocksize": self.blocksize, } # See https://github.com/zarr-developers/numcodecs/pull/713 if Version(numcodecs.__version__) >= Version("0.16.0"): config_dict["typesize"] = self.typesize return Blosc.from_config(config_dict) def _decode_sync( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer: return as_numpy_array_wrapper(self._blosc_codec.decode, chunk_bytes, chunk_spec.prototype) async def _decode_single( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer: return await asyncio.to_thread(self._decode_sync, chunk_bytes, chunk_spec) def _encode_sync( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer | None: # Since blosc only support host memory, we convert the input and output of the encoding # between numpy array and buffer return chunk_spec.prototype.buffer.from_bytes( self._blosc_codec.encode(chunk_bytes.as_numpy_array()) ) async def _encode_single( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer | None: return await asyncio.to_thread(self._encode_sync, chunk_bytes, chunk_spec) def compute_encoded_size(self, _input_byte_length: int, _chunk_spec: ArraySpec) -> int: raise NotImplementedError zarr-developers-zarr-python-a2f2415/src/zarr/codecs/bytes.py000066400000000000000000000145371527014137600242040ustar00rootroot00000000000000from __future__ import annotations import sys import warnings from dataclasses import dataclass, replace from typing import TYPE_CHECKING, ClassVar, Final, Literal from zarr.abc.codec import ArrayBytesCodec from zarr.codecs._deprecated_enum import _coerce_enum_input, _DeprecatedStrEnumMeta from zarr.core.common import JSON, parse_named_configuration from zarr.core.dtype.common import HasEndianness from zarr.core.dtype.npy.structured import Struct if TYPE_CHECKING: from typing import Self from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer, NDBuffer EndianLiteral = Literal["little", "big"] """Byte order of multi-byte numeric data.""" ENDIAN: Final = ("little", "big") class Endian(metaclass=_DeprecatedStrEnumMeta): """ Deprecated. Pass a literal string (`"little"` or `"big"`) directly to `BytesCodec` instead. """ _members: ClassVar[dict[str, str]] = {"little": "little", "big": "big"} def _parse_endian(data: object) -> EndianLiteral: if isinstance(data, str) and data in ENDIAN: return data # type: ignore[return-value] raise ValueError(f"endian must be one of {list(ENDIAN)!r}. Got {data!r}.") @dataclass(frozen=True) class BytesCodec(ArrayBytesCodec): """bytes codec""" is_fixed_size = True endian: EndianLiteral | None def __init__(self, *, endian: Endian | EndianLiteral | None = sys.byteorder) -> None: if endian is None: endian_parsed: EndianLiteral | None = None else: coerced = _coerce_enum_input(endian, "endian", "BytesCodec") endian_parsed = _parse_endian(coerced) object.__setattr__(self, "endian", endian_parsed) @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: _, configuration_parsed = parse_named_configuration( data, "bytes", require_configuration=False ) configuration_parsed = configuration_parsed or {} configuration_parsed.setdefault("endian", None) return cls(**configuration_parsed) # type: ignore[arg-type] def to_dict(self) -> dict[str, JSON]: if self.endian is None: return {"name": "bytes"} else: return {"name": "bytes", "configuration": {"endian": self.endian}} def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: if isinstance(array_spec.dtype, Struct): if array_spec.dtype.has_multi_byte_fields(): if self.endian is None: warnings.warn( "Missing 'endian' for structured dtype with multi-byte fields. " "Assuming little-endian for legacy compatibility.", UserWarning, stacklevel=2, ) return replace(self, endian="little") else: if self.endian is not None: return replace(self, endian=None) elif not isinstance(array_spec.dtype, HasEndianness): if self.endian is not None: return replace(self, endian=None) elif self.endian is None: raise ValueError( "The `endian` configuration needs to be specified for multi-byte data types." ) return self def _decode_sync( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> NDBuffer: endian_str = self.endian dtype = chunk_spec.dtype.to_native_dtype() # The byte order of the stored data is set by this codec's `endian` # configuration; the byte order of the decoded array is set by the array's # data type. The two are independent: the raw bytes are viewed with a dtype # in the stored byte order, then converted to the declared dtype if needed. if isinstance(chunk_spec.dtype, HasEndianness): view_dtype = replace(chunk_spec.dtype, endianness=endian_str).to_native_dtype() # type: ignore[call-arg] elif isinstance(chunk_spec.dtype, Struct) and endian_str is not None: # Per the struct data type spec, all multi-byte fields are stored in the # byte order configured on this codec. view_dtype = dtype.newbyteorder(endian_str) else: view_dtype = dtype as_array_like = chunk_bytes.as_array_like() chunk_array = chunk_spec.prototype.nd_buffer.from_ndarray_like( as_array_like.view(dtype=view_dtype) # type: ignore[attr-defined] ) if view_dtype != dtype: # This byte-swapping conversion copies the chunk. The dtype inequality # guard keeps the common case, where the stored and declared byte orders # already match, on the zero-copy view path above. chunk_array = chunk_array.astype(dtype) # ensure correct chunk shape if chunk_array.shape != chunk_spec.shape: chunk_array = chunk_array.reshape( chunk_spec.shape, ) return chunk_array async def _decode_single( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> NDBuffer: return self._decode_sync(chunk_bytes, chunk_spec) def _encode_sync( self, chunk_array: NDBuffer, chunk_spec: ArraySpec, ) -> Buffer | None: if chunk_array.dtype.itemsize > 1 and self.endian is not None: # Compare full dtypes rather than the top-level byteorder: numpy reports # byteorder '|' for structured dtypes even when their fields are # byte-order-sensitive, so newbyteorder is the only reliable way to # detect (and normalize) a byte-order mismatch. new_dtype = chunk_array.dtype.newbyteorder(self.endian) if new_dtype != chunk_array.dtype: chunk_array = chunk_array.astype(new_dtype) nd_array = chunk_array.as_ndarray_like() # Flatten the nd-array (only copy if needed) and reinterpret as bytes nd_array = nd_array.ravel().view(dtype="B") return chunk_spec.prototype.buffer.from_array_like(nd_array) async def _encode_single( self, chunk_array: NDBuffer, chunk_spec: ArraySpec, ) -> Buffer | None: return self._encode_sync(chunk_array, chunk_spec) def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int: return input_byte_length zarr-developers-zarr-python-a2f2415/src/zarr/codecs/cast_value.py000066400000000000000000000375701527014137600252060ustar00rootroot00000000000000"""Cast-value array-to-array codec. Value-converts array elements to a new data type during encoding, and back to the original data type during decoding, with configurable rounding, out-of-range handling, and explicit scalar mappings. Requires the optional ``cast-value-rs`` package for the actual casting logic. Install it with: ``pip install 'cast-value-rs>=0.4.2'``. """ from __future__ import annotations from collections.abc import Mapping from dataclasses import dataclass, replace from importlib.metadata import PackageNotFoundError, version from typing import TYPE_CHECKING, Final, Literal, TypedDict, cast import numpy as np from packaging.version import Version from zarr.abc.codec import ArrayArrayCodec from zarr.core.common import JSON, parse_named_configuration from zarr.core.dtype import get_data_type_from_json if TYPE_CHECKING: from typing import NotRequired, Self from zarr.core.array_spec import ArraySpec from zarr.core.buffer import NDBuffer from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType from zarr.core.metadata.v3 import ChunkGridMetadata class ScalarMapJSON(TypedDict): encode: NotRequired[list[tuple[object, object]]] decode: NotRequired[list[tuple[object, object]]] RoundingMode = Literal[ "nearest-even", "towards-zero", "towards-positive", "towards-negative", "nearest-away", ] OutOfRangeMode = Literal["clamp", "wrap"] class ScalarMap(TypedDict, total=False): """ The normalized, in-memory form of a scalar map. """ encode: Mapping[str | float | int, str | float | int] decode: Mapping[str | float | int, str | float | int] # see https://github.com/zarr-developers/zarr-extensions/tree/main/codecs/cast_value CAST_VALUE_INT_DTYPES: Final[set[str]] = { # signed "int2", "int4", "int8", "int16", "int32", "int64", # unsigned "uint2", "uint4", "uint8", "uint16", "uint32", "uint64", } """Integer dtype identifiers permitted as the source or target of `cast_value`. Membership in this set drives the `out_of_range="wrap"` rule, which the spec restricts to integral targets that use two's-complement representation for modular arithmetic. """ CAST_VALUE_FLOAT_DTYPES: Final[set[str]] = { "float4_e2m1fn", "float6_e2m3fn", "float6_e3m2fn", "float8_e3m4", "float8_e4m3", "float8_e4m3b11fnuz", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz", "float8_e8m0fnu", "bfloat16", "float16", "float32", "float64", } """Floating-point dtype identifiers permitted as the source or target of `cast_value`.""" PERMITTED_DATA_TYPE_NAMES: Final[set[str]] = CAST_VALUE_INT_DTYPES | CAST_VALUE_FLOAT_DTYPES """All dtype identifiers the `cast_value` codec is defined for.""" def parse_scalar_map(obj: ScalarMapJSON | ScalarMap) -> ScalarMap: """ Parse a scalar map into its normalized dict-of-dicts form. Accepts either the JSON form (lists of tuples) or an already-normalized form (dicts). For example, ``{"encode": [("NaN", 0)]}`` becomes ``{"encode": {"NaN": 0}}``. """ result: ScalarMap = {} for direction in ("encode", "decode"): if direction in obj: entries = obj[direction] if entries is not None: if isinstance(entries, Mapping): result[direction] = entries else: result[direction] = dict(entries) # type: ignore[arg-type] return result # --------------------------------------------------------------------------- # Backend: cast-value-rs # --------------------------------------------------------------------------- # Versions below this silently transpose input arrays that are not row-major - # the layout `transpose` hands to the next codec - writing corrupted data with # no error. Keep in sync with the `cast-value-rs` extra in pyproject.toml. CAST_VALUE_RS_MIN_VERSION: Final = "0.4.2" _INSTALL_HINT: Final = f"Install it with: pip install 'cast-value-rs>={CAST_VALUE_RS_MIN_VERSION}'" def _check_backend_version() -> str | None: """Return a message describing an unusable backend version, or `None` if it is usable.""" try: installed = version("cast-value-rs") except PackageNotFoundError: # Importable but without distribution metadata, e.g. a `maturin develop` # build. There is no version to compare, so let it through. return None if Version(installed) < Version(CAST_VALUE_RS_MIN_VERSION): return ( f"The cast_value codec requires cast-value-rs >= {CAST_VALUE_RS_MIN_VERSION}, " f"but version {installed} is installed. Earlier versions silently corrupt data " f"when the input array is not row-major. {_INSTALL_HINT}" ) return None # Set once at import; raised from `_do_cast`, so an unusable backend does not # make `import zarr` fail for users who never touch this codec. _BACKEND_ERROR: str | None try: from cast_value_rs import cast_array as cast_array_rs except ModuleNotFoundError: _BACKEND_ERROR = f"The cast_value codec requires the 'cast-value-rs' package. {_INSTALL_HINT}" else: _BACKEND_ERROR = _check_backend_version() def _check_representable( value: JSON, zdtype: ZDType[TBaseDType, TBaseScalar], label: str, ) -> None: """Raise ``ValueError`` if *value* cannot be parsed by *zdtype*.""" try: zdtype.from_json_scalar(value, zarr_format=3) except (TypeError, ValueError, OverflowError) as e: raise ValueError( f"{label} {value!r} is not representable in dtype {zdtype.to_native_dtype()}." ) from e # --------------------------------------------------------------------------- # Codec # --------------------------------------------------------------------------- @dataclass(frozen=True) class CastValue(ArrayArrayCodec): """Cast-value array-to-array codec. Value-converts array elements to a new data type during encoding, and back to the original data type during decoding. Requires the `cast-value-rs` package for the actual casting logic. Parameters ---------- data_type : str or ZDType Target zarr v3 data type. Strings are looked up by spec name (e.g. "uint8", "float32"); a `ZDType` instance is used as-is. rounding : RoundingMode How to round when exact representation is impossible. Default is "nearest-even". out_of_range : OutOfRangeMode or None What to do when a value is outside the target's range. `None` means error; "clamp" clips to range; "wrap" uses modular arithmetic (only valid for integer types). Default is `None`. scalar_map : ScalarMap, ScalarMapJSON, or None Explicit mapping from input scalars to output scalars. Default is `None`. Attributes ---------- dtype : ZDType Resolved target data type (a `ZDType` instance, regardless of whether the constructor received a string or a `ZDType`). rounding : RoundingMode The rounding mode, as supplied to the constructor. out_of_range : OutOfRangeMode or None The out-of-range behaviour, as supplied to the constructor. scalar_map : ScalarMap or None Parsed scalar map (always normalized to `ScalarMap` form). References ---------- - The `cast_value` codec spec: https://github.com/zarr-developers/zarr-extensions/tree/main/codecs/cast_value """ is_fixed_size = True dtype: ZDType[TBaseDType, TBaseScalar] rounding: RoundingMode out_of_range: OutOfRangeMode | None scalar_map: ScalarMap | None def __init__( self, *, data_type: str | ZDType[TBaseDType, TBaseScalar], rounding: RoundingMode = "nearest-even", out_of_range: OutOfRangeMode | None = None, scalar_map: ScalarMapJSON | ScalarMap | None = None, ) -> None: if isinstance(data_type, str): zdtype = get_data_type_from_json(data_type, zarr_format=3) else: zdtype = data_type if zdtype.to_json(zarr_format=3) not in PERMITTED_DATA_TYPE_NAMES: raise ValueError( f"Invalid target data type {data_type!r}. " f"cast_value codec only supports integer and floating-point data types. " f"Got {zdtype}." ) object.__setattr__(self, "dtype", zdtype) object.__setattr__(self, "rounding", rounding) object.__setattr__(self, "out_of_range", out_of_range) if scalar_map is not None: parsed = parse_scalar_map(scalar_map) else: parsed = None object.__setattr__(self, "scalar_map", parsed) @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: _, configuration_parsed = parse_named_configuration( data, "cast_value", require_configuration=True ) return cls(**configuration_parsed) # type: ignore[arg-type] def to_dict(self) -> dict[str, JSON]: config: dict[str, JSON] = {"data_type": cast("JSON", self.dtype.to_json(zarr_format=3))} if self.rounding != "nearest-even": config["rounding"] = self.rounding if self.out_of_range is not None: config["out_of_range"] = self.out_of_range if self.scalar_map is not None: json_map: dict[str, list[tuple[object, object]]] = {} for direction in ("encode", "decode"): if direction in self.scalar_map: json_map[direction] = [(k, v) for k, v in self.scalar_map[direction].items()] config["scalar_map"] = cast("JSON", json_map) return {"name": "cast_value", "configuration": config} def validate( self, *, shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGridMetadata, ) -> None: # `dtype` is the source (the array's dtype); `self.dtype` is the # cast target. The spec requires both to be permitted, and rules # like `out_of_range="wrap"` apply to the target. source_name = dtype.to_json(zarr_format=3) target_name = self.dtype.to_json(zarr_format=3) for role, name in (("source", source_name), ("target", target_name)): if name not in PERMITTED_DATA_TYPE_NAMES: raise ValueError( f"The cast_value codec only supports integer and floating-point data types. " f"Got {role} dtype {name}." ) if self.out_of_range == "wrap" and target_name not in CAST_VALUE_INT_DTYPES: raise ValueError( f"out_of_range='wrap' is only valid for integer target types. " f"Got target dtype {target_name}." ) if self.scalar_map is not None: self._validate_scalar_map(dtype, self.dtype) def _validate_scalar_map( self, source_zdtype: ZDType[TBaseDType, TBaseScalar], target_zdtype: ZDType[TBaseDType, TBaseScalar], ) -> None: """Validate that scalar map entries are compatible with source/target dtypes.""" scalar_map = self.scalar_map if scalar_map is None: return # For encode: keys are source values, values are target values. # For decode: keys are target values, values are source values. direction_dtypes: dict[ str, tuple[ZDType[TBaseDType, TBaseScalar], ZDType[TBaseDType, TBaseScalar]] ] = { "encode": (source_zdtype, target_zdtype), "decode": (target_zdtype, source_zdtype), } for direction, (key_zdtype, val_zdtype) in direction_dtypes.items(): if direction not in scalar_map: continue sub_map = scalar_map[direction] # type: ignore[literal-required] for k, v in sub_map.items(): _check_representable(k, key_zdtype, f"scalar_map {direction} key") _check_representable(v, val_zdtype, f"scalar_map {direction} value") def _do_cast( self, arr: np.ndarray, *, target_dtype: np.dtype, scalar_map: Mapping[str | float | int, str | float | int] | None, ) -> np.ndarray: if _BACKEND_ERROR is not None: raise ImportError(_BACKEND_ERROR) scalar_map_entries: dict[float | int, float | int] | None = None if scalar_map is not None: src_dtype = arr.dtype to_src = int if np.issubdtype(src_dtype, np.integer) else float to_tgt = int if np.issubdtype(target_dtype, np.integer) else float scalar_map_entries = {to_src(k): to_tgt(v) for k, v in scalar_map.items()} return cast_array_rs( # type: ignore[no-any-return] arr, target_dtype=target_dtype, rounding_mode=self.rounding, out_of_range_mode=self.out_of_range, scalar_map_entries=scalar_map_entries, ) def _get_scalar_map( self, direction: str ) -> Mapping[str | float | int, str | float | int] | None: """Extract the encode or decode mapping from scalar_map, or None.""" if self.scalar_map is None: return None return self.scalar_map.get(direction) # type: ignore[return-value] def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: """ Update the fill value of the output spec by applying casting procedure. """ target_zdtype = self.dtype target_native = target_zdtype.to_native_dtype() source_native = chunk_spec.dtype.to_native_dtype() fill = chunk_spec.fill_value fill_arr = np.array([fill], dtype=source_native) new_fill_arr = self._do_cast( fill_arr, target_dtype=target_native, scalar_map=self._get_scalar_map("encode") ) new_fill = target_native.type(new_fill_arr[0]) return replace(chunk_spec, dtype=target_zdtype, fill_value=new_fill) def _encode_sync( self, chunk_array: NDBuffer, _chunk_spec: ArraySpec, ) -> NDBuffer | None: arr = chunk_array.as_ndarray_like() target_native = self.dtype.to_native_dtype() result = self._do_cast( np.asarray(arr), target_dtype=target_native, scalar_map=self._get_scalar_map("encode") ) return chunk_array.__class__.from_ndarray_like(result) async def _encode_single( self, chunk_data: NDBuffer, chunk_spec: ArraySpec, ) -> NDBuffer | None: return self._encode_sync(chunk_data, chunk_spec) def _decode_sync( self, chunk_array: NDBuffer, chunk_spec: ArraySpec, ) -> NDBuffer: arr = chunk_array.as_ndarray_like() target_native = chunk_spec.dtype.to_native_dtype() result = self._do_cast( np.asarray(arr), target_dtype=target_native, scalar_map=self._get_scalar_map("decode") ) return chunk_array.__class__.from_ndarray_like(result) async def _decode_single( self, chunk_data: NDBuffer, chunk_spec: ArraySpec, ) -> NDBuffer: return self._decode_sync(chunk_data, chunk_spec) def compute_encoded_size(self, input_byte_length: int, chunk_spec: ArraySpec) -> int: dtype_name = chunk_spec.dtype.to_json(zarr_format=3) if dtype_name not in PERMITTED_DATA_TYPE_NAMES: raise ValueError( "cast_value codec only supports fixed-size integer and floating-point data types. " f"Got source dtype: {chunk_spec.dtype}." ) source_itemsize = chunk_spec.dtype.to_native_dtype().itemsize target_itemsize = self.dtype.to_native_dtype().itemsize if source_itemsize == 0 or target_itemsize == 0: raise ValueError( "cast_value codec requires fixed-size data types. " f"Got source itemsize={source_itemsize}, target itemsize={target_itemsize}." ) num_elements = input_byte_length // source_itemsize return num_elements * target_itemsize zarr-developers-zarr-python-a2f2415/src/zarr/codecs/crc32c_.py000066400000000000000000000047251527014137600242720ustar00rootroot00000000000000from __future__ import annotations from collections.abc import Buffer as ABCBuffer from dataclasses import dataclass from typing import TYPE_CHECKING, cast import google_crc32c import numpy as np from zarr.abc.codec import BytesBytesCodec from zarr.core.common import JSON, parse_named_configuration if TYPE_CHECKING: from typing import Self from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer @dataclass(frozen=True) class Crc32cCodec(BytesBytesCodec): """crc32c codec""" is_fixed_size = True @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: parse_named_configuration(data, "crc32c", require_configuration=False) return cls() def to_dict(self) -> dict[str, JSON]: return {"name": "crc32c"} def _decode_sync( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer: data = chunk_bytes.as_numpy_array() crc32_bytes = data[-4:] inner_bytes = data[:-4] # Need to do a manual cast until https://github.com/numpy/numpy/issues/26783 is resolved computed_checksum = np.uint32(google_crc32c.value(cast(ABCBuffer, inner_bytes))).tobytes() stored_checksum = bytes(crc32_bytes) if computed_checksum != stored_checksum: raise ValueError( f"Stored and computed checksum do not match. Stored: {stored_checksum!r}. Computed: {computed_checksum!r}." ) return chunk_spec.prototype.buffer.from_array_like(inner_bytes) async def _decode_single( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer: return self._decode_sync(chunk_bytes, chunk_spec) def _encode_sync( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer | None: data = chunk_bytes.as_numpy_array() # Calculate the checksum and "cast" it to a numpy array checksum = np.array([google_crc32c.value(cast(ABCBuffer, data))], dtype=np.uint32) # Append the checksum (as bytes) to the data return chunk_spec.prototype.buffer.from_array_like(np.append(data, checksum.view("B"))) async def _encode_single( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer | None: return self._encode_sync(chunk_bytes, chunk_spec) def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int: return input_byte_length + 4 zarr-developers-zarr-python-a2f2415/src/zarr/codecs/gzip.py000066400000000000000000000047361527014137600240270ustar00rootroot00000000000000from __future__ import annotations import asyncio from dataclasses import dataclass from functools import cached_property from typing import TYPE_CHECKING from numcodecs.gzip import GZip from zarr.abc.codec import BytesBytesCodec from zarr.core.buffer.cpu import as_numpy_array_wrapper from zarr.core.common import JSON, parse_named_configuration from zarr.core.json_parse import parse_field if TYPE_CHECKING: from typing import Self from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer def parse_gzip_level(data: JSON) -> int: parsed: int = parse_field(data, int, "level", error=TypeError) if parsed not in range(10): raise ValueError( f"Expected an integer from the inclusive range (0, 9). Got {parsed} instead." ) return parsed @dataclass(frozen=True) class GzipCodec(BytesBytesCodec): """gzip codec""" is_fixed_size = False level: int = 5 def __init__(self, *, level: int = 5) -> None: level_parsed = parse_gzip_level(level) object.__setattr__(self, "level", level_parsed) @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: _, configuration_parsed = parse_named_configuration(data, "gzip") return cls(**configuration_parsed) # type: ignore[arg-type] def to_dict(self) -> dict[str, JSON]: return {"name": "gzip", "configuration": {"level": self.level}} @cached_property def _gzip_codec(self) -> GZip: return GZip(self.level) def _decode_sync( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer: return as_numpy_array_wrapper(self._gzip_codec.decode, chunk_bytes, chunk_spec.prototype) async def _decode_single( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer: return await asyncio.to_thread(self._decode_sync, chunk_bytes, chunk_spec) def _encode_sync( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer | None: return as_numpy_array_wrapper(self._gzip_codec.encode, chunk_bytes, chunk_spec.prototype) async def _encode_single( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer | None: return await asyncio.to_thread(self._encode_sync, chunk_bytes, chunk_spec) def compute_encoded_size( self, _input_byte_length: int, _chunk_spec: ArraySpec, ) -> int: raise NotImplementedError zarr-developers-zarr-python-a2f2415/src/zarr/codecs/numcodecs/000077500000000000000000000000001527014137600244525ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/codecs/numcodecs/__init__.py000066400000000000000000000016051527014137600265650ustar00rootroot00000000000000from __future__ import annotations from zarr.codecs.numcodecs._codecs import ( BZ2, CRC32, CRC32C, LZ4, LZMA, ZFPY, Adler32, AsType, BitRound, Blosc, Delta, FixedScaleOffset, Fletcher32, GZip, JenkinsLookup3, PackBits, PCodec, Quantize, Shuffle, Zlib, Zstd, _NumcodecsArrayArrayCodec, _NumcodecsArrayBytesCodec, _NumcodecsBytesBytesCodec, _NumcodecsCodec, ) __all__ = [ "BZ2", "CRC32", "CRC32C", "LZ4", "LZMA", "ZFPY", "Adler32", "AsType", "BitRound", "Blosc", "Delta", "FixedScaleOffset", "Fletcher32", "GZip", "JenkinsLookup3", "PCodec", "PackBits", "Quantize", "Shuffle", "Zlib", "Zstd", "_NumcodecsArrayArrayCodec", "_NumcodecsArrayBytesCodec", "_NumcodecsBytesBytesCodec", "_NumcodecsCodec", ] zarr-developers-zarr-python-a2f2415/src/zarr/codecs/numcodecs/_codecs.py000066400000000000000000000306161527014137600264310ustar00rootroot00000000000000""" This module provides compatibility for [numcodecs][] in Zarr version 3. These codecs were previously defined in [numcodecs][], and have now been moved to `zarr`. ```python import numpy as np import zarr import zarr.codecs.numcodecs as numcodecs store = zarr.storage.MemoryStore() array = zarr.create_array( store=store, shape=(1024, 1024), chunks=(64, 64), dtype="uint32", filters=[numcodecs.Delta(dtype="uint32")], compressors=[numcodecs.BZ2(level=5)], overwrite=True ) array[:] = np.arange(np.prod(array.shape), dtype=array.dtype).reshape(*array.shape) ``` !!! note Please note that the codecs in [zarr.codecs.numcodecs][] are not part of the Zarr version 3 specification. Using these codecs might cause interoperability issues with other Zarr implementations. """ from __future__ import annotations import asyncio import math from dataclasses import dataclass, replace from functools import cached_property from typing import TYPE_CHECKING, Any, Self import numpy as np from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec from zarr.abc.metadata import Metadata from zarr.core.buffer.cpu import as_numpy_array_wrapper from zarr.core.common import JSON, parse_named_configuration, product from zarr.dtype import UInt8, ZDType, parse_dtype from zarr.registry import get_numcodec if TYPE_CHECKING: from zarr.abc.numcodec import Numcodec from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer, NDBuffer CODEC_PREFIX = "numcodecs." def _expect_name_prefix(codec_name: str) -> str: if not codec_name.startswith(CODEC_PREFIX): raise ValueError( f"Expected name to start with '{CODEC_PREFIX}'. Got {codec_name} instead." ) # pragma: no cover return codec_name.removeprefix(CODEC_PREFIX) def _parse_codec_configuration(data: dict[str, JSON]) -> dict[str, JSON]: parsed_name, parsed_configuration = parse_named_configuration(data) if not parsed_name.startswith(CODEC_PREFIX): raise ValueError( f"Expected name to start with '{CODEC_PREFIX}'. Got {parsed_name} instead." ) # pragma: no cover id = _expect_name_prefix(parsed_name) return {"id": id, **parsed_configuration} @dataclass(frozen=True) class _NumcodecsCodec(Metadata): codec_name: str codec_config: dict[str, JSON] def __init_subclass__(cls, *, codec_name: str | None = None, **kwargs: Any) -> None: """To be used only when creating the actual public-facing codec class.""" super().__init_subclass__(**kwargs) if codec_name is not None: namespace = codec_name cls_name = f"{CODEC_PREFIX}{namespace}.{cls.__name__}" cls.codec_name = f"{CODEC_PREFIX}{namespace}" cls.__doc__ = f""" See [{cls_name}][] for more details and parameters. """ def __init__(self, **codec_config: JSON) -> None: if not self.codec_name: raise ValueError( "The codec name needs to be supplied through the `codec_name` attribute." ) # pragma: no cover unprefixed_codec_name = _expect_name_prefix(self.codec_name) if "id" not in codec_config: codec_config = {"id": unprefixed_codec_name, **codec_config} elif codec_config["id"] != unprefixed_codec_name: raise ValueError( f"Codec id does not match {unprefixed_codec_name}. Got: {codec_config['id']}." ) # pragma: no cover object.__setattr__(self, "codec_config", codec_config) @cached_property def _codec(self) -> Numcodec: return get_numcodec(self.codec_config) # type: ignore[arg-type] @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: codec_config = _parse_codec_configuration(data) return cls(**codec_config) def to_dict(self) -> dict[str, JSON]: codec_config = self.codec_config.copy() codec_config.pop("id", None) return { "name": self.codec_name, "configuration": codec_config, } def compute_encoded_size(self, input_byte_length: int, chunk_spec: ArraySpec) -> int: raise NotImplementedError # pragma: no cover # Override __repr__ because dynamically constructed classes don't seem to work otherwise def __repr__(self) -> str: codec_config = self.codec_config.copy() codec_config.pop("id", None) return f"{self.__class__.__name__}(codec_name={self.codec_name!r}, codec_config={codec_config!r})" class _NumcodecsBytesBytesCodec(_NumcodecsCodec, BytesBytesCodec): def __init__(self, **codec_config: JSON) -> None: super().__init__(**codec_config) def _decode_sync(self, chunk_data: Buffer, chunk_spec: ArraySpec) -> Buffer: return as_numpy_array_wrapper(self._codec.decode, chunk_data, chunk_spec.prototype) def _encode_sync(self, chunk_data: Buffer, chunk_spec: ArraySpec) -> Buffer: encoded = self._codec.encode(chunk_data.as_array_like()) if isinstance(encoded, np.ndarray): # Required for checksum codecs return chunk_spec.prototype.buffer.from_bytes(encoded.tobytes()) return chunk_spec.prototype.buffer.from_bytes(encoded) async def _decode_single(self, chunk_data: Buffer, chunk_spec: ArraySpec) -> Buffer: return await asyncio.to_thread(self._decode_sync, chunk_data, chunk_spec) async def _encode_single(self, chunk_data: Buffer, chunk_spec: ArraySpec) -> Buffer: return await asyncio.to_thread(self._encode_sync, chunk_data, chunk_spec) class _NumcodecsArrayArrayCodec(_NumcodecsCodec, ArrayArrayCodec): def __init__(self, **codec_config: JSON) -> None: super().__init__(**codec_config) def _decode_sync(self, chunk_data: NDBuffer, chunk_spec: ArraySpec) -> NDBuffer: chunk_ndarray = chunk_data.as_ndarray_like() out = self._codec.decode(chunk_ndarray) return chunk_spec.prototype.nd_buffer.from_ndarray_like(out.reshape(chunk_spec.shape)) def _encode_sync(self, chunk_data: NDBuffer, chunk_spec: ArraySpec) -> NDBuffer: # numcodecs codecs flatten with order="A", so an F-contiguous chunk # would be encoded in transposed element order (gh-3558) chunk_ndarray = np.ascontiguousarray(chunk_data.as_ndarray_like()) out = self._codec.encode(chunk_ndarray) return chunk_spec.prototype.nd_buffer.from_ndarray_like(out) async def _encode_single(self, chunk_data: NDBuffer, chunk_spec: ArraySpec) -> NDBuffer: return await asyncio.to_thread(self._encode_sync, chunk_data, chunk_spec) async def _decode_single(self, chunk_data: NDBuffer, chunk_spec: ArraySpec) -> NDBuffer: return await asyncio.to_thread(self._decode_sync, chunk_data, chunk_spec) class _NumcodecsArrayBytesCodec(_NumcodecsCodec, ArrayBytesCodec): def __init__(self, **codec_config: JSON) -> None: super().__init__(**codec_config) def _decode_sync(self, chunk_data: Buffer, chunk_spec: ArraySpec) -> NDBuffer: chunk_bytes = chunk_data.to_bytes() out = self._codec.decode(chunk_bytes) return chunk_spec.prototype.nd_buffer.from_ndarray_like(out.reshape(chunk_spec.shape)) def _encode_sync(self, chunk_data: NDBuffer, chunk_spec: ArraySpec) -> Buffer: # numcodecs codecs flatten with order="A", so an F-contiguous chunk # would be encoded in transposed element order (gh-3558) chunk_ndarray = np.ascontiguousarray(chunk_data.as_ndarray_like()) out = self._codec.encode(chunk_ndarray) return chunk_spec.prototype.buffer.from_bytes(out) async def _encode_single(self, chunk_data: NDBuffer, chunk_spec: ArraySpec) -> Buffer: return await asyncio.to_thread(self._encode_sync, chunk_data, chunk_spec) async def _decode_single(self, chunk_data: Buffer, chunk_spec: ArraySpec) -> NDBuffer: return await asyncio.to_thread(self._decode_sync, chunk_data, chunk_spec) # bytes-to-bytes codecs class Blosc(_NumcodecsBytesBytesCodec, codec_name="blosc"): pass class LZ4(_NumcodecsBytesBytesCodec, codec_name="lz4"): pass class Zstd(_NumcodecsBytesBytesCodec, codec_name="zstd"): pass class Zlib(_NumcodecsBytesBytesCodec, codec_name="zlib"): pass class GZip(_NumcodecsBytesBytesCodec, codec_name="gzip"): pass class BZ2(_NumcodecsBytesBytesCodec, codec_name="bz2"): pass class LZMA(_NumcodecsBytesBytesCodec, codec_name="lzma"): pass class Shuffle(_NumcodecsBytesBytesCodec, codec_name="shuffle"): def evolve_from_array_spec(self, array_spec: ArraySpec) -> Shuffle: if self.codec_config.get("elementsize") is None: dtype = array_spec.dtype.to_native_dtype() return Shuffle(**{**self.codec_config, "elementsize": dtype.itemsize}) return self # pragma: no cover # array-to-array codecs ("filters") class Delta(_NumcodecsArrayArrayCodec, codec_name="delta"): def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: if astype := self.codec_config.get("astype"): dtype = parse_dtype(np.dtype(astype), zarr_format=3) # type: ignore[call-overload] return replace(chunk_spec, dtype=dtype) return chunk_spec class BitRound(_NumcodecsArrayArrayCodec, codec_name="bitround"): pass class FixedScaleOffset(_NumcodecsArrayArrayCodec, codec_name="fixedscaleoffset"): def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: if astype := self.codec_config.get("astype"): dtype = parse_dtype(np.dtype(astype), zarr_format=3) # type: ignore[call-overload] return replace(chunk_spec, dtype=dtype) return chunk_spec def evolve_from_array_spec(self, array_spec: ArraySpec) -> FixedScaleOffset: if self.codec_config.get("dtype") is None: dtype = array_spec.dtype.to_native_dtype() return FixedScaleOffset(**{**self.codec_config, "dtype": str(dtype)}) return self class Quantize(_NumcodecsArrayArrayCodec, codec_name="quantize"): def __init__(self, **codec_config: JSON) -> None: super().__init__(**codec_config) def evolve_from_array_spec(self, array_spec: ArraySpec) -> Quantize: if self.codec_config.get("dtype") is None: dtype = array_spec.dtype.to_native_dtype() return Quantize(**{**self.codec_config, "dtype": str(dtype)}) return self class PackBits(_NumcodecsArrayArrayCodec, codec_name="packbits"): def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: return replace( chunk_spec, shape=(1 + math.ceil(product(chunk_spec.shape) / 8),), dtype=UInt8(), ) # todo: remove this type: ignore when this class can be defined w.r.t. # a single zarr dtype API def validate(self, *, dtype: ZDType[Any, Any], **_kwargs: Any) -> None: # this is bugged and will fail _dtype = dtype.to_native_dtype() if _dtype != np.dtype("bool"): raise ValueError(f"Packbits filter requires bool dtype. Got {dtype}.") class AsType(_NumcodecsArrayArrayCodec, codec_name="astype"): def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: dtype = parse_dtype(np.dtype(self.codec_config["encode_dtype"]), zarr_format=3) # type: ignore[arg-type] return replace(chunk_spec, dtype=dtype) def evolve_from_array_spec(self, array_spec: ArraySpec) -> AsType: if self.codec_config.get("decode_dtype") is None: # TODO: remove these coverage exemptions the correct way, i.e. with tests dtype = array_spec.dtype.to_native_dtype() # pragma: no cover return AsType(**{**self.codec_config, "decode_dtype": str(dtype)}) # pragma: no cover return self # bytes-to-bytes checksum codecs class _NumcodecsChecksumCodec(_NumcodecsBytesBytesCodec): is_fixed_size = True def compute_encoded_size(self, input_byte_length: int, chunk_spec: ArraySpec) -> int: return input_byte_length + 4 # pragma: no cover class CRC32(_NumcodecsChecksumCodec, codec_name="crc32"): pass class CRC32C(_NumcodecsChecksumCodec, codec_name="crc32c"): pass class Adler32(_NumcodecsChecksumCodec, codec_name="adler32"): pass class Fletcher32(_NumcodecsChecksumCodec, codec_name="fletcher32"): pass class JenkinsLookup3(_NumcodecsChecksumCodec, codec_name="jenkins_lookup3"): pass # array-to-bytes codecs class PCodec(_NumcodecsArrayBytesCodec, codec_name="pcodec"): pass class ZFPY(_NumcodecsArrayBytesCodec, codec_name="zfpy"): pass zarr-developers-zarr-python-a2f2415/src/zarr/codecs/scale_offset.py000066400000000000000000000464751527014137600255210ustar00rootroot00000000000000from __future__ import annotations from dataclasses import dataclass, replace from typing import TYPE_CHECKING, Any, cast import numpy as np import numpy.typing as npt from zarr.abc.codec import ArrayArrayCodec from zarr.core.common import JSON, parse_named_configuration if TYPE_CHECKING: from typing import Self from zarr.core.array_spec import ArraySpec from zarr.core.buffer import NDBuffer from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType from zarr.core.metadata.v3 import ChunkGridMetadata _WIDE_INT = np.dtype(np.int64) def _encode_fits_natively(dtype: np.dtype[Any], offset: int, scale: int) -> bool: """Static range proof: is ``(x - offset) * scale`` always in range for every ``x`` in dtype? Uses Python ints (unbounded) to avoid overflow in the proof itself. """ info = np.iinfo(dtype) d_lo = int(info.min) - offset d_hi = int(info.max) - offset # Taking min/max of both products handles negative scale without a sign branch. products = (d_lo * scale, d_hi * scale) lo, hi = min(products), max(products) return info.min <= lo and hi <= info.max def _decode_fits_natively(dtype: np.dtype[Any], offset: int, scale: int) -> bool: """Static range proof for decode: is ``x // scale + offset`` always in range?""" info = np.iinfo(dtype) # x // scale is bounded by the extremes of x / scale (integer division stays within that range) if scale > 0: q_lo, q_hi = int(info.min) // scale, int(info.max) // scale else: q_lo, q_hi = int(info.max) // scale, int(info.min) // scale lo, hi = q_lo + offset, q_hi + offset return info.min <= lo and hi <= info.max def _check_int_range( values: npt.NDArray[np.integer[Any]], target: np.dtype[np.integer[Any]] ) -> None: """Raise if any value is outside the representable range of ``target``. Uses a single min/max pass instead of two ``np.any`` passes. """ info = np.iinfo(target) lo, hi = values.min(), values.max() if lo < info.min or hi > info.max: raise ValueError( f"scale_offset produced a value outside the range of dtype {target} " f"[{info.min}, {info.max}]." ) def _check_exact_division( arr: npt.NDArray[np.integer[Any]], scale: np.integer[Any], scale_repr: object ) -> None: """Raise ValueError if ``arr`` has any element not exactly divisible by ``scale``.""" if np.any(arr % scale): raise ValueError( f"scale_offset decode produced a non-zero remainder when dividing by " f"scale={scale_repr!r}; result is not exactly representable in dtype {arr.dtype}." ) def _encode_int_native( arr: npt.NDArray[np.integer[Any]], offset: np.integer[Any], scale: np.integer[Any] ) -> npt.NDArray[np.integer[Any]]: """Compute ``(arr - offset) * scale`` directly in ``arr.dtype``. This is the fast path; it exists only as a separate function to make the contract with ``_encode_fits_natively`` explicit: the caller must have already proved that no ``x`` in ``arr.dtype``'s range can overflow, so we can skip widening and range-checking entirely. Using it without that proof would silently wrap on overflow. """ return cast("npt.NDArray[np.integer[Any]]", (arr - offset) * scale) def _encode_int_widened( arr: npt.NDArray[np.integer[Any]], offset: np.integer[Any], scale: np.integer[Any] ) -> npt.NDArray[np.integer[Any]]: """Overflow-checked integer encode for int8..int64 and uint8..uint32. Exists because numpy integer arithmetic silently wraps on overflow, which the spec forbids. We widen to int64, perform the arithmetic there (int64 holds the product of any two values from these dtypes), range-check against the target dtype, then cast back. uint64 cannot use this path because its range exceeds int64 — see ``_encode_uint64``. """ wide_arr = arr.astype(_WIDE_INT, copy=False) result = (wide_arr - _WIDE_INT.type(offset)) * _WIDE_INT.type(scale) _check_int_range(result, arr.dtype) return result.astype(arr.dtype, copy=False) def _encode_float( arr: npt.NDArray[np.floating[Any]], offset: np.floating[Any], scale: np.floating[Any] ) -> npt.NDArray[np.floating[Any]]: """Encode float arrays in-dtype, guarding only against silent promotion. Float arithmetic doesn't need widening — float64 is already the widest supported dtype, and ``inf``/``nan`` from overflow are representable IEEE 754 values, so no range check is required by the spec. The one thing that can still go wrong is numpy promoting the result to a wider float dtype (e.g. float32 * float64 scalar -> float64), which would violate the spec's "arithmetic semantics of the input array's data type" clause. """ result = cast("npt.NDArray[np.floating[Any]]", (arr - offset) * scale) if result.dtype != arr.dtype: raise ValueError( f"scale_offset changed dtype from {arr.dtype} to {result.dtype}. " f"Arithmetic must preserve the data type." ) return result def _check_py_int_range( result: np.ndarray[tuple[Any, ...], np.dtype[Any]], target: np.dtype[np.unsignedinteger[Any]], ) -> None: """Range-check an ``object``-dtype ndarray holding Python ints against ``target``'s iinfo. Exists as a uint64-specific counterpart to ``_check_int_range``. That one compares numpy integers against ``iinfo``; here the values are unbounded Python ints produced by ``_encode_uint64`` / ``_decode_uint64``, so we rely on Python's arbitrary-precision comparison to detect values outside the target dtype's range. """ info = np.iinfo(target) # np.min/np.max on an object array returns a Python int (which compares correctly with iinfo). # Works uniformly for 0-d arrays where .flat iteration is awkward. lo = np.min(result) hi = np.max(result) if lo < int(info.min) or hi > int(info.max): raise ValueError( f"scale_offset produced a value outside the range of dtype {target} " f"[{info.min}, {info.max}]." ) def _encode_uint64( arr: npt.NDArray[np.unsignedinteger[Any]], offset: int, scale: int ) -> npt.NDArray[np.unsignedinteger[Any]]: """Encode uint64 via Python-int arithmetic in an ``object``-dtype array. Exists because uint64's range [0, 2**64) exceeds int64, so the int64 widening used by ``_encode_int_widened`` would itself overflow. Python ints are unbounded, so computing via ``object`` dtype is correct by construction. The trade-off is speed: object-dtype arithmetic is interpreted per element and is roughly 10x slower than ufunc paths. """ obj = arr.astype(object, copy=False) # np.asarray restores ndarray-ness in the 0-d/scalar edge case. result = np.asarray((obj - offset) * scale, dtype=object) _check_py_int_range(result, arr.dtype) return cast("npt.NDArray[np.unsignedinteger[Any]]", result.astype(arr.dtype, copy=False)) def _decode_uint64( arr: npt.NDArray[np.unsignedinteger[Any]], offset: int, scale: int ) -> npt.NDArray[np.unsignedinteger[Any]]: """Decode uint64 via Python-int arithmetic. See ``_encode_uint64`` for why.""" obj = arr.astype(object, copy=False) result = np.asarray((obj // scale) + offset, dtype=object) _check_py_int_range(result, arr.dtype) return cast("npt.NDArray[np.unsignedinteger[Any]]", result.astype(arr.dtype, copy=False)) def _decode_int_native( arr: npt.NDArray[np.integer[Any]], offset: np.integer[Any], scale: np.integer[Any] ) -> npt.NDArray[np.integer[Any]]: """Compute ``arr // scale + offset`` directly in ``arr.dtype``. Fast-path counterpart to ``_encode_int_native``; same contract. Caller must have proved via ``_decode_fits_natively`` that the result can't overflow. Divisibility is checked upstream in ``_decode`` before this is called, so ``//`` is exact here. """ return cast("npt.NDArray[np.integer[Any]]", (arr // scale) + offset) def _decode_int_widened( arr: npt.NDArray[np.integer[Any]], offset: np.integer[Any], scale: np.integer[Any] ) -> npt.NDArray[np.integer[Any]]: """Overflow-checked integer decode for int8..int64 and uint8..uint32. Counterpart to ``_encode_int_widened``. Widens to int64 so the addition of ``offset`` after division can't silently wrap, then range-checks against the target dtype. """ wide_arr = arr.astype(_WIDE_INT, copy=False) result = (wide_arr // _WIDE_INT.type(scale)) + _WIDE_INT.type(offset) _check_int_range(result, arr.dtype) return result.astype(arr.dtype, copy=False) def _decode_float( arr: npt.NDArray[np.floating[Any]], offset: np.floating[Any], scale: np.floating[Any] ) -> npt.NDArray[np.floating[Any]]: """Decode float arrays in-dtype, guarding only against silent promotion. Counterpart to ``_encode_float``; same reasoning. ``arr / scale`` is true division and always well-defined for floats (including ``0/0 = nan`` and ``x/0 = ±inf``), so no range or exactness check is needed. """ result = cast("npt.NDArray[np.floating[Any]]", (arr / scale) + offset) if result.dtype != arr.dtype: raise ValueError( f"scale_offset changed dtype from {arr.dtype} to {result.dtype}. " f"Arithmetic must preserve the data type." ) return result def _in_native_byte_order( arr: np.ndarray[tuple[Any, ...], np.dtype[Any]], ) -> np.ndarray[tuple[Any, ...], np.dtype[Any]]: """View-or-copy `arr` in the host byte order; a no-op for native input. The arithmetic below must see native arrays: numpy returns native results, so a byte-swapped input would trip the dtype-preservation check, and a byte-swapped dtype compares unequal to its native twin (`>u8 != uint64` on a little-endian host). """ return arr.astype(arr.dtype.newbyteorder("="), copy=False) def _encode( arr: np.ndarray[tuple[Any, ...], np.dtype[Any]], offset: np.generic, scale: np.generic, ) -> np.ndarray[tuple[Any, ...], np.dtype[Any]]: """Compute ``(arr - offset) * scale`` without silent overflow, returning ``arr.dtype``.""" return _encode_native(_in_native_byte_order(arr), offset, scale).astype(arr.dtype, copy=False) def _encode_native( arr: np.ndarray[tuple[Any, ...], np.dtype[Any]], offset: np.generic, scale: np.generic, ) -> np.ndarray[tuple[Any, ...], np.dtype[Any]]: """`_encode` for an `arr` in the host byte order.""" # uint64 is split out first because its full range (up to 2**64-1) doesn't fit in int64, # so the widening strategy used for every other integer dtype would itself overflow. if arr.dtype == np.uint64: u_arr = cast("npt.NDArray[np.unsignedinteger[Any]]", arr) return _encode_uint64(u_arr, int(offset), int(scale)) if np.issubdtype(arr.dtype, np.integer): i_arr = cast("npt.NDArray[np.integer[Any]]", arr) i_offset = cast("np.integer[Any]", offset) i_scale = cast("np.integer[Any]", scale) # Fast path: if a static proof shows no ``x`` in the dtype's range can overflow, # skip the int64 widening and run the arithmetic directly in the input dtype. if _encode_fits_natively(arr.dtype, int(offset), int(scale)): return _encode_int_native(i_arr, i_offset, i_scale) return _encode_int_widened(i_arr, i_offset, i_scale) # Float path: arithmetic stays in-dtype (no widening); only guard against numpy # silently promoting a narrower float to a wider one via scalar type mismatch. f_arr = cast("npt.NDArray[np.floating[Any]]", arr) f_offset = cast("np.floating[Any]", offset) f_scale = cast("np.floating[Any]", scale) return _encode_float(f_arr, f_offset, f_scale) def _decode( arr: np.ndarray[tuple[Any, ...], np.dtype[Any]], offset: np.generic, scale: np.generic, *, scale_repr: object, ) -> np.ndarray[tuple[Any, ...], np.dtype[Any]]: """Compute ``arr / scale + offset`` without silent overflow, returning ``arr.dtype``.""" native = _in_native_byte_order(arr) return _decode_native(native, offset, scale, scale_repr=scale_repr).astype( arr.dtype, copy=False ) def _decode_native( arr: np.ndarray[tuple[Any, ...], np.dtype[Any]], offset: np.generic, scale: np.generic, *, scale_repr: object, ) -> np.ndarray[tuple[Any, ...], np.dtype[Any]]: """`_decode` for an `arr` in the host byte order.""" # uint64: same reasoning as _encode — its range exceeds int64, so the Python-int path is the # only correct option. Exactness check runs first so non-divisible inputs fail before the # slower object-dtype arithmetic. if arr.dtype == np.uint64: u_arr = cast("npt.NDArray[np.unsignedinteger[Any]]", arr) _check_exact_division(u_arr, cast("np.integer[Any]", scale), scale_repr) return _decode_uint64(u_arr, int(offset), int(scale)) if np.issubdtype(arr.dtype, np.integer): i_arr = cast("npt.NDArray[np.integer[Any]]", arr) i_offset = cast("np.integer[Any]", offset) i_scale = cast("np.integer[Any]", scale) # The spec requires decode to use true division and error if the result isn't # representable. For integers that means the remainder must be zero; if any element # isn't exactly divisible we fail here rather than silently truncating via //. _check_exact_division(i_arr, i_scale, scale_repr) # Fast path mirrors _encode: static proof that ``x // scale + offset`` stays in dtype. if _decode_fits_natively(arr.dtype, int(offset), int(scale)): return _decode_int_native(i_arr, i_offset, i_scale) return _decode_int_widened(i_arr, i_offset, i_scale) # Float path: division is well-defined; only guard against dtype promotion. f_arr = cast("npt.NDArray[np.floating[Any]]", arr) f_offset = cast("np.floating[Any]", offset) f_scale = cast("np.floating[Any]", scale) return _decode_float(f_arr, f_offset, f_scale) @dataclass(frozen=True) class ScaleOffset(ArrayArrayCodec): """Scale-offset array-to-array codec. Encodes values with `out = (in - offset) * scale` and decodes with `out = (in / scale) + offset`, using the input array's data type semantics. Intermediate or final values that are not representable in that dtype are reported as errors (integer overflow, unsigned underflow, non-exact integer division). Parameters ---------- offset : int, float, or str Value subtracted during encoding. Strings preserve the exact JSON representation when round-tripping metadata. Default is 0. scale : int, float, or str Value multiplied during encoding (after offset subtraction). Strings preserve the exact JSON representation when round-tripping metadata. Default is 1. Attributes ---------- offset : int, float, or str The offset value, as supplied to the constructor. scale : int, float, or str The scale value, as supplied to the constructor. References ---------- - The `scale_offset` codec spec: https://github.com/zarr-developers/zarr-extensions/tree/main/codecs/scale_offset """ is_fixed_size = True offset: int | float | str scale: int | float | str def __init__(self, *, offset: object = 0, scale: object = 1) -> None: if not isinstance(offset, int | float | str): raise TypeError(f"offset must be a number or string, got {type(offset).__name__}") if not isinstance(scale, int | float | str): raise TypeError(f"scale must be a number or string, got {type(scale).__name__}") object.__setattr__(self, "offset", offset) object.__setattr__(self, "scale", scale) @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: _, configuration_parsed = parse_named_configuration( data, "scale_offset", require_configuration=False ) configuration_parsed = configuration_parsed or {} return cls(**configuration_parsed) def to_dict(self) -> dict[str, JSON]: if self.offset == 0 and self.scale == 1: return {"name": "scale_offset"} config: dict[str, JSON] = {} if self.offset != 0: config["offset"] = self.offset if self.scale != 1: config["scale"] = self.scale return {"name": "scale_offset", "configuration": config} def validate( self, *, shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGridMetadata, ) -> None: native = dtype.to_native_dtype() if not np.issubdtype(native, np.integer) and not np.issubdtype(native, np.floating): raise ValueError( f"scale_offset codec only supports integer and floating-point data types. " f"Got {dtype}." ) parsed: dict[str, Any] = {} for name, value in [("offset", self.offset), ("scale", self.scale)]: try: parsed[name] = dtype.from_json_scalar(value, zarr_format=3) except (TypeError, ValueError, OverflowError) as e: raise ValueError( f"scale_offset {name} value {value!r} is not representable in dtype {native}." ) from e # ``scale`` may be given as a string, and no string is ever ``== 0``, so this has to # compare the parsed scalar rather than the value as supplied. if parsed["scale"] == 0: raise ValueError("scale_offset scale must be non-zero.") def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: zdtype = chunk_spec.dtype fill = np.asarray(zdtype.cast_scalar(chunk_spec.fill_value)) offset = cast("np.generic", zdtype.from_json_scalar(self.offset, zarr_format=3)) scale = cast("np.generic", zdtype.from_json_scalar(self.scale, zarr_format=3)) new_fill = _encode(fill, offset, scale) return replace(chunk_spec, fill_value=new_fill.reshape(()).item()) def _decode_sync( self, chunk_array: NDBuffer, chunk_spec: ArraySpec, ) -> NDBuffer: arr = cast("np.ndarray[tuple[Any, ...], np.dtype[Any]]", chunk_array.as_ndarray_like()) zdtype = chunk_spec.dtype offset = cast("np.generic", zdtype.from_json_scalar(self.offset, zarr_format=3)) scale = cast("np.generic", zdtype.from_json_scalar(self.scale, zarr_format=3)) result = _decode(arr, offset, scale, scale_repr=self.scale) return chunk_spec.prototype.nd_buffer.from_ndarray_like(result) async def _decode_single( self, chunk_array: NDBuffer, chunk_spec: ArraySpec, ) -> NDBuffer: return self._decode_sync(chunk_array, chunk_spec) def _encode_sync( self, chunk_array: NDBuffer, chunk_spec: ArraySpec, ) -> NDBuffer | None: arr = cast("np.ndarray[tuple[Any, ...], np.dtype[Any]]", chunk_array.as_ndarray_like()) zdtype = chunk_spec.dtype offset = cast("np.generic", zdtype.from_json_scalar(self.offset, zarr_format=3)) scale = cast("np.generic", zdtype.from_json_scalar(self.scale, zarr_format=3)) result = _encode(arr, offset, scale) return chunk_spec.prototype.nd_buffer.from_ndarray_like(result) async def _encode_single( self, chunk_array: NDBuffer, _chunk_spec: ArraySpec, ) -> NDBuffer | None: return self._encode_sync(chunk_array, _chunk_spec) def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int: return input_byte_length zarr-developers-zarr-python-a2f2415/src/zarr/codecs/sharding.py000066400000000000000000002354431527014137600246560ustar00rootroot00000000000000from __future__ import annotations from collections.abc import Iterable, Mapping, MutableMapping, Sequence from dataclasses import dataclass, replace from functools import lru_cache from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple import numpy as np import numpy.typing as npt from zarr.abc.codec import ( ArrayBytesCodec, ArrayBytesCodecPartialDecodeMixin, ArrayBytesCodecPartialEncodeMixin, Codec, CodecPipeline, _codec_supports_sync, ) from zarr.abc.store import ( ByteGetter, ByteRequest, ByteSetter, RangeByteRequest, Store, SuffixByteRequest, _store_supports_sync_io, ) from zarr.codecs._deprecated_enum import _coerce_enum_input, _DeprecatedStrEnumMeta from zarr.codecs.bytes import BytesCodec from zarr.codecs.crc32c_ import Crc32cCodec from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer import ( Buffer, BufferPrototype, NDBuffer, default_buffer_prototype, numpy_buffer_prototype, ) from zarr.core.chunk_grids import ChunkGrid from zarr.core.chunk_utils import ( ChunkTransform, decode_and_scatter_chunk, encode_or_elide_chunk, evolve_codecs, merge_and_encode_chunk, ) from zarr.core.common import ( ShapeLike, parse_chunk_shape, parse_named_configuration, parse_shapelike, product, ) from zarr.core.config import config as zarr_config from zarr.core.dtype.common import HasEndianness from zarr.core.dtype.npy.int import UInt64 from zarr.core.dtype.npy.structured import Struct from zarr.core.indexing import ( BasicIndexer, ChunkProjection, CoordinateIndexer, SelectorTuple, SliceDimIndexer, _lexicographic_order, colexicographic_order_coords, get_indexer, lexicographic_order_coords, morton_order_coords, ) from zarr.core.metadata.v3 import ( ChunkGridMetadata, RectilinearChunkGridMetadata, RegularChunkGridMetadata, parse_codecs, ) from zarr.registry import get_ndbuffer_class, get_pipeline_class from zarr.storage._common import StorePath from zarr.storage._utils import _normalize_byte_range_index if TYPE_CHECKING: from collections.abc import Iterator from typing import Self from zarr.core.common import JSON from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType MAX_UINT_64 = 2**64 - 1 ShardMapping = Mapping[tuple[int, ...], Buffer | None] ShardMutableMapping = MutableMapping[tuple[int, ...], Buffer | None] IndexLocation = Literal["start", "end"] """Position of the shard index within the encoded shard.""" INDEX_LOCATION: Final = ("start", "end") class ShardingCodecIndexLocation(metaclass=_DeprecatedStrEnumMeta): """ Deprecated. Pass a literal string (`"start"` or `"end"`) directly to `ShardingCodec` instead. """ _members: ClassVar[dict[str, str]] = {"start": "start", "end": "end"} SubchunkWriteOrder = Literal["morton", "unordered", "lexicographic", "colexicographic"] SUBCHUNK_WRITE_ORDER: Final[tuple[str, str, str, str]] = ( "morton", "unordered", "lexicographic", "colexicographic", ) def _is_identity_full_read(indexer: Any, shard_shape: tuple[int, ...]) -> bool: """True when `indexer` selects every element of a `shard_shape` array in natural order: one whole-dimension, step-1 `SliceDimIndexer` per dimension. Structural on purpose, not `isinstance(indexer, BasicIndexer)`: a full `arr[:]` read reaches the shard as an `OrthogonalIndexer`, so a type gate would silently disable the bulk fast path for the most common case. Any gather (integer-array / boolean / coordinate selection), subset, strided, or integer-scalar selection fails the per-dimension check — output shape alone is not enough, because a reordering or duplicating selection can have the same shape as the shard while requiring `chunk_selection` / `out_selection` to be honored. """ dim_indexers = getattr(indexer, "dim_indexers", None) if dim_indexers is None or len(dim_indexers) != len(shard_shape): return False return all( isinstance(dim_indexer, SliceDimIndexer) and dim_indexer.dim_len == dim_len and dim_indexer.start == 0 and dim_indexer.stop == dim_len and dim_indexer.step == 1 for dim_indexer, dim_len in zip(dim_indexers, shard_shape, strict=True) ) def _parse_index_location(data: object) -> IndexLocation: if isinstance(data, str) and data in INDEX_LOCATION: return data # type: ignore[return-value] raise ValueError(f"index_location must be one of {list(INDEX_LOCATION)!r}. Got {data!r}.") @dataclass(frozen=True) class _ShardingByteGetter(ByteGetter): """In-memory byte getter for one inner chunk of a shard. Implements `SyncByteGetter` (dict access needs no event loop), so the synchronous codec pipeline takes its sync fast path on inner chunks instead of scheduling one coroutine per chunk; the async `get` simply delegates to `get_sync`. """ shard_dict: ShardMapping chunk_coords: tuple[int, ...] def get_sync( self, prototype: BufferPrototype | None = None, byte_range: ByteRequest | None = None ) -> Buffer | None: if prototype is not None and prototype != default_buffer_prototype(): raise ValueError( "Non-default buffer prototypes are not supported within shards. " f"Got {prototype}, expected {default_buffer_prototype()}." ) value = self.shard_dict.get(self.chunk_coords) if value is None: return None if byte_range is None: return value start, stop = _normalize_byte_range_index(value, byte_range) return value[start:stop] async def get( self, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: return self.get_sync(prototype, byte_range) @dataclass(frozen=True) class _ShardingByteSetter(_ShardingByteGetter, ByteSetter): """In-memory byte setter for one inner chunk of a shard. Implements `SyncByteSetter`; the async methods delegate to the sync ones. """ shard_dict: ShardMutableMapping def set_sync(self, value: Buffer) -> None: self.shard_dict[self.chunk_coords] = value def delete_sync(self) -> None: del self.shard_dict[self.chunk_coords] async def set(self, value: Buffer, byte_range: ByteRequest | None = None) -> None: if byte_range is not None: raise NotImplementedError("byte_range is not supported within shards.") self.set_sync(value) async def delete(self) -> None: self.delete_sync() async def set_if_not_exists(self, default: Buffer) -> None: self.shard_dict.setdefault(self.chunk_coords, default) class _ShardIndex(NamedTuple): # the chunk grid shape of a single shard chunks_per_shard: tuple[int, ...] # dtype uint64, shape (chunks_per_shard_0, chunks_per_shard_1, ..., 2) offsets_and_lengths: npt.NDArray[np.uint64] def _localize_chunk(self, chunk_coords: tuple[int, ...]) -> tuple[int, ...]: return tuple( chunk_i % shard_i for chunk_i, shard_i in zip(chunk_coords, self.chunks_per_shard, strict=False) ) def is_all_empty(self) -> bool: return bool(np.array_equiv(self.offsets_and_lengths, MAX_UINT_64)) def get_full_chunk_map(self) -> npt.NDArray[np.bool_]: return np.not_equal(self.offsets_and_lengths[..., 0], MAX_UINT_64) def is_dense(self, chunk_byte_length: int, *, data_section_start: int) -> bool: """True when the chunk payloads exactly tile the shard's data section. Every chunk must be present with length `chunk_byte_length`, and the sorted offsets must be exactly `data_section_start + i * chunk_byte_length` for `i` in `0..n_chunks-1`: no gaps, no overlaps, and nothing outside the data section (a corrupt index could otherwise point chunks into the index region or out of the blob). Used to gate the vectorized whole-shard decode: a dense fixed-size shard is a regular grid of equal-length payloads, so it can be reshaped/scattered in bulk rather than decoded chunk-by-chunk. """ offsets = self.offsets_and_lengths[..., 0].reshape(-1) lengths = self.offsets_and_lengths[..., 1].reshape(-1) # all present if bool(np.any(offsets == MAX_UINT_64)): return False # all the same fixed length if not bool(np.all(lengths == chunk_byte_length)): return False expected = np.uint64(data_section_start) + np.arange( offsets.size, dtype=np.uint64 ) * np.uint64(chunk_byte_length) return bool(np.array_equal(np.sort(offsets), expected)) def get_chunk_slice(self, chunk_coords: tuple[int, ...]) -> tuple[int, int] | None: localized_chunk = self._localize_chunk(chunk_coords) chunk_start, chunk_len = self.offsets_and_lengths[localized_chunk] if (chunk_start, chunk_len) == (MAX_UINT_64, MAX_UINT_64): return None else: return (int(chunk_start), int(chunk_start + chunk_len)) def get_chunk_slices_vectorized( self, chunk_coords_array: npt.NDArray[np.integer[Any]] ) -> tuple[npt.NDArray[np.uint64], npt.NDArray[np.uint64], npt.NDArray[np.bool_]]: """Get chunk slices for multiple coordinates at once. Parameters ---------- chunk_coords_array : ndarray of shape (n_chunks, n_dims) Array of chunk coordinates to look up. Returns ------- starts : ndarray of shape (n_chunks,) Start byte positions for each chunk. ends : ndarray of shape (n_chunks,) End byte positions for each chunk. valid : ndarray of shape (n_chunks,) Boolean mask indicating which chunks are non-empty. """ # Handle 0-dimensional arrays (n_dims == 0): the shard holds a single # chunk, so every coordinate maps to the same flat entry. if chunk_coords_array.shape[1] == 0: offsets_and_lengths = self.offsets_and_lengths.reshape(-1, 2) offsets_and_lengths = np.broadcast_to( offsets_and_lengths, (chunk_coords_array.shape[0], 2) ) else: # Localize coordinates via modulo (vectorized) shard_shape = np.array(self.chunks_per_shard, dtype=np.uint64) localized = chunk_coords_array.astype(np.uint64) % shard_shape # Build index tuple for advanced indexing index_tuple = tuple(localized[:, i] for i in range(localized.shape[1])) # Fetch all offsets and lengths at once offsets_and_lengths = self.offsets_and_lengths[index_tuple] starts = offsets_and_lengths[:, 0] lengths = offsets_and_lengths[:, 1] # Check for valid (non-empty) chunks valid = starts != MAX_UINT_64 # Compute end positions ends = starts + lengths return starts, ends, valid def set_chunk_slice(self, chunk_coords: tuple[int, ...], chunk_slice: slice | None) -> None: localized_chunk = self._localize_chunk(chunk_coords) if chunk_slice is None: self.offsets_and_lengths[localized_chunk] = (MAX_UINT_64, MAX_UINT_64) else: self.offsets_and_lengths[localized_chunk] = ( chunk_slice.start, chunk_slice.stop - chunk_slice.start, ) @classmethod def create_empty(cls, chunks_per_shard: tuple[int, ...]) -> _ShardIndex: offsets_and_lengths = np.zeros(chunks_per_shard + (2,), dtype=" _ShardReader: shard_index_size = codec._shard_index_size(chunks_per_shard) obj = cls() obj.buf = buf if codec.index_location == "start": shard_index_bytes = obj.buf[:shard_index_size] else: shard_index_bytes = obj.buf[-shard_index_size:] obj.index = await codec._decode_shard_index(shard_index_bytes, chunks_per_shard) return obj @classmethod def create_empty( cls, chunks_per_shard: tuple[int, ...], buffer_prototype: BufferPrototype | None = None ) -> _ShardReader: if buffer_prototype is None: buffer_prototype = default_buffer_prototype() index = _ShardIndex.create_empty(chunks_per_shard) obj = cls() obj.buf = buffer_prototype.buffer.create_zero_length() obj.index = index return obj def __getitem__(self, chunk_coords: tuple[int, ...]) -> Buffer: chunk_byte_slice = self.index.get_chunk_slice(chunk_coords) if chunk_byte_slice is not None: return self.buf[chunk_byte_slice[0] : chunk_byte_slice[1]] raise KeyError def __len__(self) -> int: return int(self.index.offsets_and_lengths.size / 2) def __iter__(self) -> Iterator[tuple[int, ...]]: return iter(lexicographic_order_coords(self.index.chunks_per_shard)) def to_dict_vectorized(self) -> dict[tuple[int, ...], Buffer | None]: """Build a dict of chunk coordinates to buffers using vectorized lookup. The full per-shard chunk coordinate grid (both the array used for the vectorized index lookup and the plain tuples used as dict keys) is cached on `chunks_per_shard`, so neither is rebuilt on every call. For a shard with tens of thousands of chunks this avoids reconstructing that many tuples on every partial write. Returns ------- dict mapping chunk coordinate tuples to Buffer or None """ chunks_per_shard = self.index.chunks_per_shard # The same chunk-grid coordinates are needed in two forms, and neither can # stand in for the other: # - `chunk_coords_array`: an (n_chunks, n_dims) numpy array, fed to the # vectorized index lookup, which does modulo + advanced indexing on it. # A list of tuples can't be used for that without first being arrayified. # - `chunk_coords_keys`: the same coordinates as hashable Python tuples, # used as the result dict's keys. numpy array rows are unhashable # (mutable), so they can't key a dict. # Both are cached per shape (see indexing.py), so neither is rebuilt here; # row i of the array and key i refer to the same chunk. chunk_coords_array = _lexicographic_order(chunks_per_shard) chunk_coords_keys = lexicographic_order_coords(chunks_per_shard) starts, ends, valid = self.index.get_chunk_slices_vectorized(chunk_coords_array) result: dict[tuple[int, ...], Buffer | None] = {} for i, coords in enumerate(chunk_coords_keys): if valid[i]: result[coords] = self.buf[int(starts[i]) : int(ends[i])] else: result[coords] = None return result def _drops_only_unit_axes(shape: tuple[int, ...], full: tuple[int, ...]) -> bool: """Return whether ``shape`` is ``full`` with zero or more length-1 axes removed. ``(2, 2)`` is ``(2, 1, 2)`` minus its unit axis, and ``(1, 2)`` is ``(1, 2, 1)`` minus its last; ``(2, 2)`` is not ``(4,)``, and ``(3, 2, 1)`` is not ``(3, 2)`` because it adds an axis. """ remaining = iter(full) for size in shape: for full_size in remaining: if full_size == size: break if full_size != 1: return False else: return False return all(full_size == 1 for full_size in remaining) def _check_index_codecs_fixed_size(index_codecs: tuple[Codec, ...]) -> None: """Reject index codecs whose encoded size is not fixed. The spec forbids variable-size codecs (such as compressors) in `index_codecs`: the shard index is located by its byte size, which must be known before the index is read. """ variable_size = [codec for codec in index_codecs if not codec.is_fixed_size] if variable_size: raise ValueError( "Sharding `index_codecs` must produce a fixed-size encoding, but these codecs " f"do not: {variable_size}. Compression codecs cannot be used for the shard index." ) def _parse_inner_chunk_shape(data: ShapeLike) -> tuple[int, ...]: """Parse the inner chunk shape of a sharding codec: a shape (see `parse_shapelike`) whose every size is a chunk edge length, so at least 1 (see `parse_chunk_shape`).""" return parse_chunk_shape(parse_shapelike(data)) @dataclass(frozen=True) class ShardingCodec( ArrayBytesCodec, ArrayBytesCodecPartialDecodeMixin, ArrayBytesCodecPartialEncodeMixin ): """Sharding codec. `subchunk_write_order` controls the physical order of subchunks within a shard. It is a write-time setting only: it is not stored in array metadata, so reopening a sharded array does not recover it (the setting reverts to the `morton` default per codec instance). """ is_fixed_size = False chunk_shape: tuple[int, ...] codecs: tuple[Codec, ...] index_codecs: tuple[Codec, ...] index_location: IndexLocation = "end" subchunk_write_order: SubchunkWriteOrder = "morton" def __init__( self, *, chunk_shape: ShapeLike, codecs: Iterable[Codec | dict[str, JSON]] = (BytesCodec(endian="little"),), index_codecs: Iterable[Codec | dict[str, JSON]] = ( BytesCodec(endian="little"), Crc32cCodec(), ), index_location: ShardingCodecIndexLocation | IndexLocation = "end", subchunk_write_order: SubchunkWriteOrder = "morton", ) -> None: chunk_shape_parsed = _parse_inner_chunk_shape(chunk_shape) codecs_parsed = parse_codecs(codecs) index_codecs_parsed = parse_codecs(index_codecs) _check_index_codecs_fixed_size(index_codecs_parsed) index_location_coerced = _coerce_enum_input( index_location, "index_location", "ShardingCodec" ) index_location_parsed = _parse_index_location(index_location_coerced) if subchunk_write_order not in SUBCHUNK_WRITE_ORDER: raise ValueError( f"Unrecognized subchunk write order: {subchunk_write_order}. Only {SUBCHUNK_WRITE_ORDER} are allowed." ) object.__setattr__(self, "chunk_shape", chunk_shape_parsed) object.__setattr__(self, "codecs", codecs_parsed) object.__setattr__(self, "index_codecs", index_codecs_parsed) object.__setattr__(self, "index_location", index_location_parsed) object.__setattr__(self, "subchunk_write_order", subchunk_write_order) # Use instance-local lru_cache to avoid memory leaks object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec)) object.__setattr__(self, "_get_index_chunk_spec", lru_cache()(self._get_index_chunk_spec)) object.__setattr__(self, "_get_chunks_per_shard", lru_cache()(self._get_chunks_per_shard)) object.__setattr__(self, "_shard_index_size", lru_cache()(self._shard_index_size)) object.__setattr__( self, "_get_inner_chunk_transform", lru_cache()(self._get_inner_chunk_transform) ) object.__setattr__( self, "_get_index_chunk_transform", lru_cache()(self._get_index_chunk_transform) ) # todo: typedict return type def __getstate__(self) -> dict[str, Any]: # `subchunk_write_order` is not part of codec metadata (`to_dict`), so carry it # explicitly to survive a pickle round-trip (otherwise it reverts to `morton`). return {"subchunk_write_order": self.subchunk_write_order, **self.to_dict()} def __setstate__(self, state: dict[str, Any]) -> None: config = state["configuration"] object.__setattr__(self, "chunk_shape", _parse_inner_chunk_shape(config["chunk_shape"])) object.__setattr__(self, "codecs", parse_codecs(config["codecs"])) object.__setattr__(self, "index_codecs", parse_codecs(config["index_codecs"])) object.__setattr__(self, "index_location", _parse_index_location(config["index_location"])) object.__setattr__(self, "subchunk_write_order", state["subchunk_write_order"]) # Use instance-local lru_cache to avoid memory leaks object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec)) object.__setattr__(self, "_get_index_chunk_spec", lru_cache()(self._get_index_chunk_spec)) object.__setattr__(self, "_get_chunks_per_shard", lru_cache()(self._get_chunks_per_shard)) object.__setattr__(self, "_shard_index_size", lru_cache()(self._shard_index_size)) object.__setattr__( self, "_get_inner_chunk_transform", lru_cache()(self._get_inner_chunk_transform) ) object.__setattr__( self, "_get_index_chunk_transform", lru_cache()(self._get_index_chunk_transform) ) @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: _, configuration_parsed = parse_named_configuration(data, "sharding_indexed") return cls(**configuration_parsed) # type: ignore[arg-type] @property def codec_pipeline(self) -> CodecPipeline: return get_pipeline_class().from_codecs(self.codecs) def _get_inner_pipeline(self, shard_spec: ArraySpec) -> CodecPipeline: """The nested pipeline for inner-chunk IO, evolved against the inner chunk spec. Evolving matters for two reasons: it threads the spec through the inner codec chain (spec-changing codecs see the spec they will actually operate on), and — for a synchronous pipeline — it builds the sync transform, so inner-chunk IO over the (sync-capable) sharding byte getters takes the pipeline's sync fast path instead of scheduling one coroutine per inner chunk. The bare `codec_pipeline` property returns an unevolved pipeline, which a synchronous pipeline can only run through its async fallback. Memoized per (pipeline class, batch size, shard_spec): evolving builds a ChunkTransform, which is wasteful to redo on every shard operation. The pipeline class and batch size participate in the key so the `codec_pipeline.path` and `codec_pipeline.batch_size` configs are still honored after the first use (`from_codecs` captures batch_size at construction). A benign construction race between threads is possible (last writer wins) — same as the other caches here. """ cache: dict[tuple[type[CodecPipeline], int, ArraySpec], CodecPipeline] | None = getattr( self, "_inner_pipeline_cache", None ) if cache is None: cache = {} object.__setattr__(self, "_inner_pipeline_cache", cache) key = (get_pipeline_class(), zarr_config.get("codec_pipeline.batch_size"), shard_spec) pipeline = cache.get(key) if pipeline is None: chunk_spec = self._get_chunk_spec(shard_spec) pipeline = self.codec_pipeline.evolve_from_array_spec(chunk_spec) cache[key] = pipeline return pipeline def to_dict(self) -> dict[str, JSON]: return { "name": "sharding_indexed", "configuration": { "chunk_shape": self.chunk_shape, "codecs": tuple(s.to_dict() for s in self.codecs), "index_codecs": tuple(s.to_dict() for s in self.index_codecs), "index_location": self.index_location, }, } def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: """Thread the spec through the inner chain. Each codec is evolved against the spec produced by the previous one. Evolving every codec against the same unthreaded spec is the bug shape that strips `BytesCodec.endian` behind a dtype-changing codec — and this method runs on the real array-creation path, baking the damaged chain into the evolved instance before the transform builders ever run. Parameters ---------- array_spec The base spec to be evolved as we thread it. Returns ------- This codec with the evolved code chain. """ from zarr.core.chunk_utils import evolve_codecs shard_spec = self._get_chunk_spec(array_spec) evolved_codecs = evolve_codecs(self.codecs, shard_spec) if evolved_codecs != self.codecs: return replace(self, codecs=evolved_codecs) return self def validate( self, *, shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGridMetadata, ) -> None: if len(self.chunk_shape) != len(shape): raise ValueError( "The shard's `chunk_shape` and array's `shape` need to have the same number of dimensions." ) if isinstance(chunk_grid, RegularChunkGridMetadata): edges_per_dim: tuple[tuple[int, ...], ...] = tuple((s,) for s in chunk_grid.chunk_shape) elif isinstance(chunk_grid, RectilinearChunkGridMetadata): edges_per_dim = tuple( (s,) if isinstance(s, int) else s for s in chunk_grid.chunk_shapes ) else: raise TypeError( f"Sharding is only compatible with regular and rectilinear chunk grids, " f"got {type(chunk_grid)}" ) for i, (edges, inner) in enumerate(zip(edges_per_dim, self.chunk_shape, strict=False)): for edge in set(edges): if edge % inner != 0: raise ValueError( f"Chunk edge length {edge} in dimension {i} is not " f"divisible by the shard's inner chunk size {inner}." ) self._validate_inner_codecs(dtype) def _validate_inner_codecs(self, dtype: ZDType[TBaseDType, TBaseScalar]) -> None: """Validate the codecs that encode each inner chunk, as array metadata validates its codecs: each against the chunk the codecs before it leave it, a regular grid of one inner chunk. A sharding codec among them is thereby checked against the chunk it splits, which `validate` of the outer codec alone does not see. """ spec = ArraySpec( shape=self.chunk_shape, dtype=dtype, fill_value=dtype.default_scalar(), config=ArrayConfig.from_dict({}), prototype=default_buffer_prototype(), ) for codec in self.codecs: codec.validate( shape=spec.shape, dtype=spec.dtype, chunk_grid=RegularChunkGridMetadata(chunk_shape=spec.shape), ) spec = codec.resolve_metadata(spec) def _get_inner_chunk_transform(self, shard_spec: ArraySpec) -> Any: """The synchronous transform for the inner codec chain. Memoized by the instance-local `lru_cache` wrapping installed in `__init__`/`__setstate__` (the single cache mechanism for these builders — do not add another layer inside the body). Codecs are evolved with the spec THREADED forward (`evolve_codecs`): each inner codec is evolved against the spec produced by the previous one, not the original chunk spec. Evolving every codec against the same unthreaded spec is the bug shape that stripped `BytesCodec.endian` at the pipeline level (see `evolve_codecs`) — the inner chain must use the same single source of truth. """ chunk_spec = self._get_chunk_spec(shard_spec) return ChunkTransform(codecs=evolve_codecs(self.codecs, chunk_spec)) def _get_index_chunk_transform(self, chunks_per_shard: tuple[int, ...]) -> Any: """The synchronous transform for the index codec chain. Memoized via instance-local `lru_cache`. """ index_spec = self._get_index_chunk_spec(chunks_per_shard) return ChunkTransform(codecs=evolve_codecs(self.index_codecs, index_spec)) def _decode_shard_index_sync( self, index_bytes: Buffer, chunks_per_shard: tuple[int, ...] ) -> _ShardIndex: """Decode shard index synchronously using ChunkTransform.""" self._check_shard_index_size(index_bytes, chunks_per_shard) index_transform = self._get_index_chunk_transform(chunks_per_shard) index_spec = self._get_index_chunk_spec(chunks_per_shard) index_array = index_transform.decode_chunk(index_bytes, index_spec) return _ShardIndex(chunks_per_shard, index_array.as_numpy_array()) def _encode_shard_index_sync(self, index: _ShardIndex) -> Buffer: """Encode shard index synchronously using ChunkTransform.""" index_transform = self._get_index_chunk_transform(index.chunks_per_shard) index_spec = self._get_index_chunk_spec(index.chunks_per_shard) index_nd = get_ndbuffer_class().from_numpy_array(index.offsets_and_lengths) result: Buffer | None = index_transform.encode_chunk(index_nd, index_spec) if result is None: raise RuntimeError("Encoding the shard index produced no bytes.") return result def _shard_reader_from_bytes_sync( self, buf: Buffer, chunks_per_shard: tuple[int, ...] ) -> _ShardReader: """Sync version of _ShardReader.from_bytes.""" shard_index_size = self._shard_index_size(chunks_per_shard) if self.index_location == "start": shard_index_bytes = buf[:shard_index_size] else: shard_index_bytes = buf[-shard_index_size:] index = self._decode_shard_index_sync(shard_index_bytes, chunks_per_shard) reader = _ShardReader() reader.buf = buf reader.index = index return reader def _decode_sync( self, shard_bytes: Buffer, shard_spec: ArraySpec, ) -> NDBuffer: """Decode a full shard synchronously. Sync counterpart to `_decode_single`. Same semantics (decode every inner chunk and assemble the full shard array) but routes through `ChunkTransform` instead of the async codec pipeline, so it can run on the sync codec-pipeline fast path without an event loop. For a partial read where the caller only needs a slice of the shard, use `_decode_partial_sync` instead — it fetches only the byte ranges that overlap the selection. This method does not parallelize decompression, but should. See TODO: make issue for handling subchunk parallelism """ shard_shape = shard_spec.shape chunk_shape = self.chunk_shape chunks_per_shard = self._get_chunks_per_shard(shard_spec) chunk_spec = self._get_chunk_spec(shard_spec) inner_transform = self._get_inner_chunk_transform(shard_spec) indexer = BasicIndexer( tuple(slice(0, s) for s in shard_shape), shape=shard_shape, chunk_grid=ChunkGrid.from_sizes(shard_shape, chunk_shape), ) out = chunk_spec.prototype.nd_buffer.empty( shape=shard_shape, dtype=shard_spec.dtype.to_native_dtype(), order=shard_spec.order, ) shard_dict = self._shard_reader_from_bytes_sync(shard_bytes, chunks_per_shard) if shard_dict.index.is_all_empty(): out.fill(shard_spec.fill_value) return out for chunk_coords, chunk_selection, out_selection, _ in indexer: # the GetResult status is discarded: missing INNER chunks of a # present shard always fill (read_missing_chunks is a store-key # level promise, applied to top-level statuses at the array layer) decode_and_scatter_chunk( shard_dict.get(chunk_coords), out, chunk_spec=chunk_spec, chunk_selection=chunk_selection, out_selection=out_selection, drop_axes=(), decode=inner_transform.decode_chunk, ) return out def _encode_sync( self, shard_array: NDBuffer, shard_spec: ArraySpec, ) -> Buffer | None: """Encode a full shard synchronously. Sync counterpart to `_encode_single`. This is reached when a `ShardingCodec` is an *inner* codec of another sharding codec (nested sharding): the outer codec encodes each inner chunk through its `ChunkTransform`, which calls this method on the inner `ShardingCodec`. Each inner chunk is encoded through the inner `ChunkTransform` and collected into an intermediate `dict`. The dict's key order is immaterial — the physical on-disk layout is decided downstream by the `subchunk_write_order` loop in `_encode_shard_dict_sync` (this method does NOT impose a layout). Empty inner chunks become `None` entries when `write_empty_chunks` is False, signalling `_encode_shard_dict_sync` to elide them from the data section and mark them empty in the shard index. Returns `None` if every inner chunk was elided (an all-empty shard) — callers treat that as "delete the shard key". This method does not parallelize compression, but should. See TODO: make issue for handling subchunk parallelism For a partial write that only touches some inner chunks, use `_encode_partial_sync` instead. """ shard_shape = shard_spec.shape chunks_per_shard = self._get_chunks_per_shard(shard_spec) chunk_spec = self._get_chunk_spec(shard_spec) inner_transform = self._get_inner_chunk_transform(shard_spec) indexer = BasicIndexer( tuple(slice(0, s) for s in shard_shape), shape=shard_shape, chunk_grid=ChunkGrid.from_sizes(shard_shape, self.chunk_shape), ) # Key order here is immaterial; _encode_shard_dict_sync lays the present # chunks out in subchunk_write_order. shard_builder: dict[tuple[int, ...], Buffer | None] = dict.fromkeys( lexicographic_order_coords(chunks_per_shard) ) for chunk_coords, _chunk_selection, out_selection, _ in indexer: # None = chunk normalized to missing (see encode_or_elide_chunk) shard_builder[chunk_coords] = encode_or_elide_chunk( shard_array[out_selection], chunk_spec, inner_transform.encode_chunk ) return self._encode_shard_dict_sync( shard_builder, chunks_per_shard=chunks_per_shard, buffer_prototype=default_buffer_prototype(), ) def _encode_partial_sync( self, byte_setter: Any, value: NDBuffer, selection: SelectorTuple, shard_spec: ArraySpec, ) -> None: """Sync equivalent of `_encode_partial_single`. Receives the source data for the written region (not a pre-merged shard array) and the selection within the shard, matching the calling convention of the async partial-encode path used by `BatchedCodecPipeline`. This method does not parallelize compression, but should. See TODO: make issue for handling subchunk parallelism Loads the existing shard, merges the written region into the affected inner chunks, and rewrites the whole shard. """ chunks_per_shard = self._get_chunks_per_shard(shard_spec) chunk_spec = self._get_chunk_spec(shard_spec) inner_transform = self._get_inner_chunk_transform(shard_spec) indexer, value = self._get_shard_indexer_and_value(selection, shard_spec, value) is_complete = self._is_complete_shard_write(indexer, chunks_per_shard) is_scalar = len(value.shape) == 0 # Load existing inner-chunk bytes into a dict (same structure as # the async path's shard_dict). if is_complete: shard_dict: dict[tuple[int, ...], Buffer | None] = dict.fromkeys( lexicographic_order_coords(chunks_per_shard) ) else: existing_bytes = byte_setter.get_sync(prototype=shard_spec.prototype) if existing_bytes is not None: shard_reader_fb = self._shard_reader_from_bytes_sync( existing_bytes, chunks_per_shard ) # Build the dict with one vectorized index lookup over all chunks, # matching the async _encode_partial_single path. A per-coordinate # __getitem__ loop here is O(n_chunks) Python overhead that dominates # partial writes into shards with many inner chunks. The coordinate # array and keys are cached on the reader, so neither is rebuilt here. shard_dict = shard_reader_fb.to_dict_vectorized() else: shard_dict = dict.fromkeys(lexicographic_order_coords(chunks_per_shard)) # Merge, encode, and store each affected inner chunk into shard_dict via # the canonical merge_and_encode_chunk (None = normalized to missing). # # Scalar fast path: when the written value is a scalar broadcast, every # *complete* inner chunk is byte-for-byte identical — same fill, same # empty-check, same encoded bytes. Compute that outcome once and reuse it # for all complete chunks instead of re-merging, re-checking, and # re-encoding tens of thousands of identical chunks. Incomplete (edge) # chunks still merge against their own existing data individually. # `_sentinel` distinguishes "not computed yet" from a memoized `None` # (an empty chunk). _sentinel = object() scalar_complete_result: Buffer | object | None = _sentinel for chunk_coords, chunk_sel, out_sel, is_complete_chunk in indexer: if is_scalar and is_complete_chunk: if scalar_complete_result is _sentinel: scalar_complete_result = merge_and_encode_chunk( None, value, chunk_spec=chunk_spec, chunk_selection=chunk_sel, out_selection=out_sel, is_complete=is_complete_chunk, drop_axes=(), decode=inner_transform.decode_chunk, encode=inner_transform.encode_chunk, ) shard_dict[chunk_coords] = scalar_complete_result # type: ignore[assignment] continue # A complete chunk fully overwrites: skip decoding what it replaces. existing_raw = None if is_complete_chunk else shard_dict.get(chunk_coords) shard_dict[chunk_coords] = merge_and_encode_chunk( existing_raw, value, chunk_spec=chunk_spec, chunk_selection=chunk_sel, out_selection=out_sel, is_complete=is_complete_chunk, drop_axes=(), decode=inner_transform.decode_chunk, encode=inner_transform.encode_chunk, ) blob = self._encode_shard_dict_sync( shard_dict, chunks_per_shard=chunks_per_shard, buffer_prototype=default_buffer_prototype(), ) if blob is None: byte_setter.delete_sync() else: byte_setter.set_sync(blob) def _build_shard_layout( self, shard_dict: ShardMapping, chunks_per_shard: tuple[int, ...], ) -> tuple[_ShardIndex, list[Buffer]] | None: """Lay out the present inner chunks of a shard. Pure compute, no IO. Packs the encoded inner chunks (in the codec's `subchunk_write_order`) into a contiguous data section and builds a shard index pointing each present chunk at its ABSOLUTE byte offset within the final blob: when the index is stored at the start, offsets are pre-shifted by the index size (known without encoding — the index codecs are fixed-size, which every index read path already relies on), so the index can be encoded exactly once by the caller. Returns `(index, data_buffers)`, or `None` for an all-empty shard (no chunks present). Shared by the sync and async `_encode_shard_dict*` so the layout/offset logic cannot drift between them. """ index = _ShardIndex.create_empty(chunks_per_shard) buffers: list[Buffer] = [] chunk_start = ( self._shard_index_size(chunks_per_shard) if self.index_location == "start" else 0 ) for chunk_coords in self._subchunk_order_iter(chunks_per_shard, self.subchunk_write_order): value = shard_dict.get(chunk_coords) if value is None or len(value) == 0: continue chunk_length = len(value) buffers.append(value) index.set_chunk_slice(chunk_coords, slice(chunk_start, chunk_start + chunk_length)) chunk_start += chunk_length if len(buffers) == 0: return None return index, buffers def _assemble_shard( self, index_bytes: Buffer, buffers: list[Buffer], buffer_prototype: BufferPrototype, *, chunks_per_shard: tuple[int, ...], ) -> Buffer: """Concatenate the encoded index and data buffers into the shard blob. The layout from `_build_shard_layout` already assumes the index size, so the encoded index length must match `_shard_index_size` exactly — guard that assumption rather than silently corrupt offsets. The guard lives here, once, for both the sync and async encode paths. """ if len(index_bytes) != self._shard_index_size(chunks_per_shard): raise RuntimeError( "encoded shard index size does not match _shard_index_size; " "variable-size index codecs are not supported" ) if self.index_location == "start": buffers.insert(0, index_bytes) else: buffers.append(index_bytes) template = buffer_prototype.buffer.create_zero_length() return template.combine(buffers) def _encode_shard_dict_sync( self, shard_dict: ShardMapping, chunks_per_shard: tuple[int, ...], buffer_prototype: BufferPrototype, ) -> Buffer | None: """Sync version of _encode_shard_dict. Layout via the shared `_build_shard_layout` (offsets already absolute), then a single index encode and concatenation. Returns `None` for an all-empty shard (no chunks present). """ layout = self._build_shard_layout(shard_dict, chunks_per_shard) if layout is None: return None index, buffers = layout index_bytes = self._encode_shard_index_sync(index) return self._assemble_shard( index_bytes, buffers, buffer_prototype, chunks_per_shard=chunks_per_shard ) async def _decode_single( self, shard_bytes: Buffer, shard_spec: ArraySpec, ) -> NDBuffer: shard_shape = shard_spec.shape chunk_shape = self.chunk_shape chunks_per_shard = self._get_chunks_per_shard(shard_spec) chunk_spec = self._get_chunk_spec(shard_spec) indexer = BasicIndexer( tuple(slice(0, s) for s in shard_shape), shape=shard_shape, chunk_grid=ChunkGrid.from_sizes(shard_shape, chunk_shape), ) # setup output array out = chunk_spec.prototype.nd_buffer.empty( shape=shard_shape, dtype=shard_spec.dtype.to_native_dtype(), order=shard_spec.order, ) shard_dict = await _ShardReader.from_bytes(shard_bytes, self, chunks_per_shard) if shard_dict.index.is_all_empty(): out.fill(shard_spec.fill_value) return out # decoding chunks and writing them into the output buffer await self._get_inner_pipeline(shard_spec).read( [ ( _ShardingByteGetter(shard_dict, chunk_coords), chunk_spec, chunk_selection, out_selection, is_complete_shard, ) for chunk_coords, chunk_selection, out_selection, is_complete_shard in indexer ], out, ) return out async def _decode_partial_single( self, byte_getter: ByteGetter, selection: SelectorTuple, shard_spec: ArraySpec, ) -> NDBuffer | None: shard_shape = shard_spec.shape chunk_shape = self.chunk_shape chunks_per_shard = self._get_chunks_per_shard(shard_spec) chunk_spec = self._get_chunk_spec(shard_spec) indexer = get_indexer( selection, shape=shard_shape, chunk_grid=ChunkGrid.from_sizes(shard_shape, chunk_shape), ) # setup output array out = shard_spec.prototype.nd_buffer.empty( shape=indexer.shape, dtype=shard_spec.dtype.to_native_dtype(), order=shard_spec.order, ) indexed_chunks = list(indexer) all_chunk_coords = {chunk_coords for chunk_coords, *_ in indexed_chunks} # reading bytes of all requested chunks shard_dict_maybe: ShardMapping | None if self._is_total_shard(all_chunk_coords, chunks_per_shard): # read entire shard shard_dict_maybe = await self._load_full_shard_maybe( byte_getter=byte_getter, prototype=chunk_spec.prototype, chunks_per_shard=chunks_per_shard, ) else: # read some chunks within the shard shard_dict_maybe = await self._load_partial_shard_maybe( byte_getter, chunk_spec.prototype, chunks_per_shard, all_chunk_coords, max_gap_bytes=shard_spec.config.sharding_coalesce_max_gap_bytes, max_coalesced_bytes=shard_spec.config.sharding_coalesce_max_bytes, ) if shard_dict_maybe is None: return None shard_dict = shard_dict_maybe # decoding chunks and writing them into the output buffer await self._get_inner_pipeline(shard_spec).read( [ ( _ShardingByteGetter(shard_dict, chunk_coords), chunk_spec, chunk_selection, out_selection, is_complete_shard, ) for chunk_coords, chunk_selection, out_selection, is_complete_shard in indexer ], out, ) if hasattr(indexer, "sel_shape"): return out.reshape(indexer.sel_shape) else: return out def _subchunk_order_iter( self, chunks_per_shard: tuple[int, ...], subchunk_write_order: SubchunkWriteOrder ) -> Iterable[tuple[int, ...]]: subchunk_iter: Iterable[tuple[int, ...]] match subchunk_write_order: case "morton": subchunk_iter = morton_order_coords(chunks_per_shard) case "lexicographic": subchunk_iter = lexicographic_order_coords(chunks_per_shard) case "colexicographic": subchunk_iter = colexicographic_order_coords(chunks_per_shard) case "unordered": # "unordered" promises no particular layout; today it happens to be # lexicographic, but callers must not rely on that. subchunk_iter = np.ndindex(chunks_per_shard) case _: raise ValueError(f"Unrecognized subchunk write order: {subchunk_write_order!r}.") return subchunk_iter def _decode_full_shard_bulk_if_uncompressed( self, shard_bytes: Buffer, shard_spec: ArraySpec, indexer: Any, ) -> NDBuffer | None: """Vectorized whole-shard decode for dense, fixed-size, uncompressed shards. Returns the assembled shard array, or None if the fast path does not apply (so the caller falls back to the per-chunk loop). Conditions: - inner codec chain is fixed-size (no compression / variable-length); - the inner codec chain is exactly a single BytesCodec — decode is a dtype/endian view with no reordering. A trailing crc32c is NOT accepted (the bulk path can't verify per-chunk checksums, so crc shards keep the per-chunk path's corruption detection); - the data type is not structured (the byte-order handling below has no `Struct` branch); - `indexer` is an identity full-shard read (`_is_identity_full_read`); - the stored index is dense (every chunk present, equal fixed length, exactly tiling the data section) so the data section is a regular grid of chunk payloads. Chunk positions are read from the stored index, so this is correct for any `subchunk_write_order` (morton / lexicographic / colexicographic / unordered). The on-disk byte order is taken from the BytesCodec's `endian`, so big- and little-endian shards both decode correctly. """ # --- gate on a trivial, fixed-size inner codec chain --- if not self._inner_codecs_fixed_size: return None # The inner chain must be exactly a single BytesCodec (a dtype/endian # view, no reordering). A trailing Crc32cCodec is excluded on purpose: # the bulk path would have to strip-and-discard the per-chunk checksum # bytes, silently dropping the corruption detection the per-chunk path # enforces (Crc32cCodec._decode_sync raises on mismatch). crc-protected # shards therefore fall through to the per-chunk path. if len(self.codecs) != 1 or not isinstance(self.codecs[0], BytesCodec): return None ab_codec = self.codecs[0] # The byte-order handling below lacks the structured-dtype branch of # `BytesCodec._decode_sync` (which applies `newbyteorder` to multi-byte # struct fields), so structured dtypes must take the per-chunk path. if isinstance(shard_spec.dtype, Struct): return None chunks_per_shard = self._get_chunks_per_shard(shard_spec) chunk_spec = self._get_chunk_spec(shard_spec) n_chunks = product(chunks_per_shard) if n_chunks == 0: return None # Only valid for an identity full-shard read, where each chunk lands at # its natural grid position. The per-dimension check is load-bearing: # a gather selection (an `OrthogonalIndexer` with an integer-array or # boolean dimension, from `arr[perm, :]` / `arr.oindex[...]`, or a # `CoordinateIndexer` from vindex) can have an output `.shape` equal to # the shard shape while reordering or duplicating points — serving it # from the bulk path would return the shard in natural order, silently # dropping the reordering. Anything that is not a full-slice-per- # dimension read falls through to the per-chunk path so # chunk_selection / out_selection are honored. if not _is_identity_full_read(indexer, shard_spec.shape): return None chunk_byte_length = self._inner_chunk_byte_length(chunk_spec) shard_index_size = self._shard_index_size(chunks_per_shard) if len(shard_bytes) != n_chunks * chunk_byte_length + shard_index_size: return None # not a dense fixed-size shard # --- decode the index; require dense layout --- if self.index_location == "start": index_bytes = shard_bytes[:shard_index_size] else: index_bytes = shard_bytes[-shard_index_size:] index = self._decode_shard_index_sync(index_bytes, chunks_per_shard) data_section_start = shard_index_size if self.index_location == "start" else 0 if not index.is_dense(chunk_byte_length, data_section_start=data_section_start): return None # --- bulk reconstruct --- # The index gives each chunk's absolute byte offset within the blob; with # a dense, crc-free, fixed-size layout the payload length is exactly the # encoded item-bytes of one chunk. native_dtype = shard_spec.dtype.to_native_dtype() raw = shard_bytes.as_numpy_array().view(np.uint8) payload = chunk_byte_length cs = self.chunk_shape # On-disk byte order is carried by the BytesCodec's `endian`, NOT by the # data type (zarr v3). Build the read-view dtype from the codec's endian # exactly as BytesCodec._decode_sync does, so a big-endian shard read on a # little-endian host (or vice versa) is interpreted correctly. Assigning # into the native-dtype `out` then performs any needed byteswap. `endian` # is now a plain Literal['little', 'big'] | None string (no longer an enum). endian_str = ab_codec.endian if isinstance(chunk_spec.dtype, HasEndianness): stored_dtype = replace(chunk_spec.dtype, endianness=endian_str).to_native_dtype() # type: ignore[call-arg] else: stored_dtype = chunk_spec.dtype.to_native_dtype() offsets = index.offsets_and_lengths[..., 0].reshape(-1) # localized coords, C-order coords_c = list(np.ndindex(chunks_per_shard)) out = shard_spec.prototype.nd_buffer.empty( shape=indexer.shape, dtype=native_dtype, order=shard_spec.order ) for flat, coord in enumerate(coords_c): start = int(offsets[flat]) chunk = raw[start : start + payload].view(stored_dtype).reshape(cs) sel = tuple(slice(c * s, c * s + s) for c, s in zip(coord, cs, strict=True)) out[sel] = chunk return out def _decode_partial_sync( self, byte_getter: Any, selection: SelectorTuple, shard_spec: ArraySpec, ) -> NDBuffer | None: """Sync equivalent of `_decode_partial_single`. Reads only the inner-chunk byte ranges that overlap `selection` (plus the shard index) and decodes them through the inner codec chain. The store must support `get_sync` with byte ranges. This method does not parallelize decompression, but should. See TODO: make issue for handling subchunk parallelism Two sub-paths: - If `selection` covers the entire shard, just fetch the whole blob — that's strictly cheaper than two round trips (index, then data) plus the per-chunk overhead of partial fetches. - Otherwise fetch the index alone, look up only the byte slices of the inner chunks the selection touches, fetch those, and decode. """ shard_shape = shard_spec.shape chunk_shape = self.chunk_shape chunks_per_shard = self._get_chunks_per_shard(shard_spec) chunk_spec = self._get_chunk_spec(shard_spec) inner_transform = self._get_inner_chunk_transform(shard_spec) indexer = get_indexer( selection, shape=shard_shape, chunk_grid=ChunkGrid.from_sizes(shard_shape, chunk_shape), ) out = shard_spec.prototype.nd_buffer.empty( shape=indexer.shape, dtype=shard_spec.dtype.to_native_dtype(), order=shard_spec.order, ) indexed_chunks = list(indexer) all_chunk_coords = {chunk_coords for chunk_coords, *_ in indexed_chunks} # Read just the inner chunks we need. if self._is_total_shard(all_chunk_coords, chunks_per_shard): shard_bytes = byte_getter.get_sync(prototype=chunk_spec.prototype) if shard_bytes is None: return None bulk = self._decode_full_shard_bulk_if_uncompressed(shard_bytes, shard_spec, indexer) if bulk is not None: # The bulk path only fires for an identity full-shard read # (`_is_identity_full_read`), so the result is already # shard-shaped and in natural order — no reshape needed. return bulk shard_reader = self._shard_reader_from_bytes_sync(shard_bytes, chunks_per_shard) shard_dict: ShardMapping = shard_reader else: # Partial read: fetch only the touched inner chunks, coalescing # adjacent byte ranges (mirrors the async _load_partial_shard_maybe # / #3004). Returns None if the shard is absent. partial = self._load_partial_shard_maybe_sync( byte_getter, chunk_spec.prototype, chunks_per_shard, all_chunk_coords, max_gap_bytes=shard_spec.config.sharding_coalesce_max_gap_bytes, max_coalesced_bytes=shard_spec.config.sharding_coalesce_max_bytes, ) if partial is None: return None shard_dict = partial # Decode each needed inner chunk and scatter into out (statuses # discarded: missing inner chunks fill, see _decode_sync). for chunk_coords, chunk_selection, out_selection, _ in indexed_chunks: decode_and_scatter_chunk( shard_dict.get(chunk_coords), out, chunk_spec=chunk_spec, chunk_selection=chunk_selection, out_selection=out_selection, drop_axes=(), decode=inner_transform.decode_chunk, ) if hasattr(indexer, "sel_shape"): return out.reshape(indexer.sel_shape) return out async def _encode_single( self, shard_array: NDBuffer, shard_spec: ArraySpec, ) -> Buffer | None: shard_shape = shard_spec.shape chunk_shape = self.chunk_shape chunks_per_shard = self._get_chunks_per_shard(shard_spec) chunk_spec = self._get_chunk_spec(shard_spec) indexer = list( BasicIndexer( tuple(slice(0, s) for s in shard_shape), shape=shard_shape, chunk_grid=ChunkGrid.from_sizes(shard_shape, chunk_shape), ) ) shard_builder = dict.fromkeys(lexicographic_order_coords(chunks_per_shard)) await self._get_inner_pipeline(shard_spec).write( [ ( _ShardingByteSetter(shard_builder, chunk_coords), chunk_spec, chunk_selection, out_selection, is_complete_shard, ) for chunk_coords, chunk_selection, out_selection, is_complete_shard in indexer ], shard_array, ) return await self._encode_shard_dict( shard_builder, chunks_per_shard=chunks_per_shard, buffer_prototype=default_buffer_prototype(), ) async def _encode_partial_single( self, byte_setter: ByteSetter, shard_array: NDBuffer, selection: SelectorTuple, shard_spec: ArraySpec, ) -> None: chunks_per_shard = self._get_chunks_per_shard(shard_spec) chunk_spec = self._get_chunk_spec(shard_spec) indexer, shard_array = self._get_shard_indexer_and_value(selection, shard_spec, shard_array) if self._is_complete_shard_write(indexer, chunks_per_shard): shard_dict = dict.fromkeys(lexicographic_order_coords(chunks_per_shard)) else: shard_reader = await self._load_full_shard_maybe( byte_getter=byte_setter, prototype=chunk_spec.prototype, chunks_per_shard=chunks_per_shard, ) if shard_reader is None: shard_reader = _ShardReader.create_empty(chunks_per_shard) # Use vectorized lookup for better performance. The lexicographic # coordinate array and keys are cached, so neither is rebuilt on # every write. shard_dict = shard_reader.to_dict_vectorized() await self._get_inner_pipeline(shard_spec).write( [ ( _ShardingByteSetter(shard_dict, chunk_coords), chunk_spec, chunk_selection, out_selection, is_complete_shard, ) for chunk_coords, chunk_selection, out_selection, is_complete_shard in indexer ], shard_array, ) buf = await self._encode_shard_dict( shard_dict, chunks_per_shard=chunks_per_shard, buffer_prototype=default_buffer_prototype(), ) if buf is None: await byte_setter.delete() else: await byte_setter.set(buf) async def _encode_shard_dict( self, map: ShardMapping, chunks_per_shard: tuple[int, ...], buffer_prototype: BufferPrototype, ) -> Buffer | None: """Layout via the shared `_build_shard_layout` (offsets already absolute), then a single index encode and concatenation. Async twin of `_encode_shard_dict_sync`.""" layout = self._build_shard_layout(map, chunks_per_shard) if layout is None: return None index, buffers = layout index_bytes = await self._encode_shard_index(index) return self._assemble_shard( index_bytes, buffers, buffer_prototype, chunks_per_shard=chunks_per_shard ) def _get_shard_indexer_and_value( self, selection: SelectorTuple, shard_spec: ArraySpec, value: NDBuffer ) -> tuple[list[ChunkProjection], NDBuffer]: """Index ``selection`` over the inner chunk grid, flattening ``value`` to match. ``get_indexer`` classifies a tuple of integer arrays as a coordinate selection, and a ``CoordinateIndexer`` addresses the value buffer as 1-D. An ``OrthogonalIndexer`` with two or more array-indexed axes hands down an ``np.ix_`` tuple, so ``sel_shape`` is N-D, while the caller shaped ``value`` like the orthogonal result: the broadcast shape minus the integer-indexed axes, which ``np.ix_`` keeps as length-1 axes. That value has the element count and C order of the flattened projections, so ravel it. Only that value shape is ravelled. A mask or coordinate selection arrives with a 1-D ``sel_shape`` and a value that must already be flat, and an orthogonal value with an axis the selection does not have is invalid. Both are left alone so the write fails the same way it does on an unsharded array; the shard-level selection alone cannot tell orthogonal from mask indexing, the value shape can. Scalars pass through and are broadcast downstream. The partial-decode paths apply the inverse reshape, to ``indexer.sel_shape``, on the way out. """ shard_shape = shard_spec.shape indexer = get_indexer( selection, shape=shard_shape, chunk_grid=ChunkGrid.from_sizes(shard_shape, self.chunk_shape), ) if ( isinstance(indexer, CoordinateIndexer) and len(value.shape) > 1 and _drops_only_unit_axes(value.shape, indexer.sel_shape) ): value = value.reshape(indexer.shape) return list(indexer), value def _is_total_shard( self, all_chunk_coords: set[tuple[int, ...]], chunks_per_shard: tuple[int, ...] ) -> bool: # `all_chunk_coords` comes from an indexer over this shard's chunk grid, so # it is always a subset of that grid (`validate` requires the shard shape to # be divisible by the inner chunk shape, so the indexer cannot produce an # out-of-grid coordinate). A subset whose size equals the grid's is the # whole grid, so the count check alone proves totality — no need to build # and membership-test the full coordinate set on this hot path. return len(all_chunk_coords) == product(chunks_per_shard) def _is_complete_shard_write( self, indexed_chunks: Sequence[ChunkProjection], chunks_per_shard: tuple[int, ...], ) -> bool: all_chunk_coords = {chunk_coords for chunk_coords, *_ in indexed_chunks} return self._is_total_shard(all_chunk_coords, chunks_per_shard) and all( is_complete_chunk for *_, is_complete_chunk in indexed_chunks ) @property def _sync_capable(self) -> bool: """Dynamic opt-out consulted by `_codec_supports_sync` / `ChunkTransform`. This codec structurally satisfies `SupportsSyncCodec`, but every sync method (`_decode_sync`, `_encode_sync`, `_decode_partial_sync`, `_encode_partial_sync`) delegates to the inner and index codec chains through `ChunkTransform`, so it can only run synchronously when every codec in BOTH chains is itself sync-capable. Reporting False here makes `ChunkTransform` construction raise, which in turn makes `FusedCodecPipeline.evolve_from_array_spec` set `sync_transform=None` — the whole pipeline then declines the sync fast path and routes through the async paths (partial shard decode / async fallback write), exactly as it does for an async-only TOP-level codec or a non-sync store. """ return self._inner_codecs_sync_capable() and self._index_codecs_sync_capable() def _inner_codecs_sync_capable(self) -> bool: # _codec_supports_sync (not bare isinstance) so a nested sharding codec # with an async-only inner chain propagates its opt-out outward. return all(_codec_supports_sync(c) for c in self.codecs) def _index_codecs_sync_capable(self) -> bool: return all(_codec_supports_sync(c) for c in self.index_codecs) async def _decode_shard_index( self, index_bytes: Buffer, chunks_per_shard: tuple[int, ...] ) -> _ShardIndex: # Pure compute (the bytes are already in hand): delegate to the sync # implementation instead of spinning up a pipeline + per-call # AsyncChunkTransform for a tiny fixed-size decode. The default # (bytes + crc32c) index chain is sync-capable; an async-only # third-party index codec falls back to the full async pipeline, which # the synchronous read paths cannot use but this async path still can. if self._index_codecs_sync_capable(): return self._decode_shard_index_sync(index_bytes, chunks_per_shard) self._check_shard_index_size(index_bytes, chunks_per_shard) index_array = next( iter( await get_pipeline_class() .from_codecs(self.index_codecs) .decode([(index_bytes, self._get_index_chunk_spec(chunks_per_shard))]) ) ) if index_array is None: raise RuntimeError("Decoding the shard index produced no array.") return _ShardIndex(chunks_per_shard, index_array.as_numpy_array()) async def _encode_shard_index(self, index: _ShardIndex) -> Buffer: # Pure compute: delegate to the sync implementation, with the same # async-pipeline fallback as _decode_shard_index. if self._index_codecs_sync_capable(): return self._encode_shard_index_sync(index) index_bytes = next( iter( await get_pipeline_class() .from_codecs(self.index_codecs) .encode( [ ( get_ndbuffer_class().from_numpy_array(index.offsets_and_lengths), self._get_index_chunk_spec(index.chunks_per_shard), ) ] ) ) ) if index_bytes is None: raise RuntimeError("Encoding the shard index produced no bytes.") return index_bytes def _check_shard_index_size( self, index_bytes: Buffer, chunks_per_shard: tuple[int, ...] ) -> None: """Raise if `index_bytes` is shorter than the encoded shard index. A stored shard always holds its index, so a shorter read means the stored value is truncated (a zero-length value included). Every read and write path decodes the index through here, so they all reject it alike. zarrs and tensorstore also reject such a shard. """ expected = self._shard_index_size(chunks_per_shard) if len(index_bytes) < expected: raise ValueError( f"The stored shard is too short to hold its index: read {len(index_bytes)} " f"bytes of index, expected {expected}. The shard is truncated or corrupt." ) def _shard_index_size(self, chunks_per_shard: tuple[int, ...]) -> int: return ( get_pipeline_class() .from_codecs(self.index_codecs) .compute_encoded_size( 16 * product(chunks_per_shard), self._get_index_chunk_spec(chunks_per_shard) ) ) def _get_index_chunk_spec(self, chunks_per_shard: tuple[int, ...]) -> ArraySpec: return ArraySpec( shape=chunks_per_shard + (2,), dtype=UInt64(endianness="little"), fill_value=MAX_UINT_64, config=ArrayConfig( order="C", write_empty_chunks=False ), # Note: this is hard-coded for simplicity -- it is not surfaced into user code, prototype=default_buffer_prototype(), ) def _get_chunk_spec(self, shard_spec: ArraySpec) -> ArraySpec: return ArraySpec( shape=self.chunk_shape, dtype=shard_spec.dtype, fill_value=shard_spec.fill_value, config=shard_spec.config, prototype=shard_spec.prototype, ) def _get_chunks_per_shard(self, shard_spec: ArraySpec) -> tuple[int, ...]: return tuple( s // c for s, c in zip( shard_spec.shape, self.chunk_shape, strict=False, ) ) def _shard_index_byte_range( self, chunks_per_shard: tuple[int, ...] ) -> RangeByteRequest | SuffixByteRequest: """Byte range of the shard index within the shard blob. Single source of truth for the index-location arithmetic, shared by the sync and async index loaders so they cannot drift. """ shard_index_size = self._shard_index_size(chunks_per_shard) if self.index_location == "start": return RangeByteRequest(0, shard_index_size) return SuffixByteRequest(shard_index_size) @staticmethod def _pair_chunks_with_byte_ranges( shard_index: _ShardIndex, all_chunk_coords: set[tuple[int, ...]] ) -> list[tuple[tuple[int, ...], RangeByteRequest]]: """Pair each requested chunk coord with its byte range in the shard. Coords whose chunk is absent from the index are omitted. Shared by the sync and async partial-shard loaders. """ chunk_coord_byte_ranges: list[tuple[tuple[int, ...], RangeByteRequest]] = [] for chunk_coord in all_chunk_coords: chunk_byte_slice = shard_index.get_chunk_slice(chunk_coord) if chunk_byte_slice is not None: chunk_coord_byte_ranges.append( (chunk_coord, RangeByteRequest(chunk_byte_slice[0], chunk_byte_slice[1])) ) return chunk_coord_byte_ranges async def _load_shard_index_maybe( self, byte_getter: ByteGetter, chunks_per_shard: tuple[int, ...] ) -> _ShardIndex | None: index_bytes = await byte_getter.get( prototype=numpy_buffer_prototype(), byte_range=self._shard_index_byte_range(chunks_per_shard), ) if index_bytes is not None: return await self._decode_shard_index(index_bytes, chunks_per_shard) return None async def _load_full_shard_maybe( self, byte_getter: ByteGetter, prototype: BufferPrototype, chunks_per_shard: tuple[int, ...] ) -> _ShardReader | None: shard_bytes = await byte_getter.get(prototype=prototype) return ( await _ShardReader.from_bytes(shard_bytes, self, chunks_per_shard) if shard_bytes is not None else None ) @property def _inner_codecs_fixed_size(self) -> bool: """True when all inner codecs produce fixed-size output (no compression).""" return all(c.is_fixed_size for c in self.codecs) def _inner_chunk_byte_length(self, chunk_spec: ArraySpec) -> int: """Encoded byte length of a single inner chunk. Only valid when _inner_codecs_fixed_size.""" raw_byte_length = 1 for s in self.chunk_shape: raw_byte_length *= s raw_byte_length *= chunk_spec.dtype.item_size # type: ignore[attr-defined] return int(self.codec_pipeline.compute_encoded_size(raw_byte_length, chunk_spec)) async def _load_partial_shard_maybe( self, byte_getter: ByteGetter, prototype: BufferPrototype, chunks_per_shard: tuple[int, ...], all_chunk_coords: set[tuple[int, ...]], max_gap_bytes: int, max_coalesced_bytes: int, ) -> ShardMapping | None: """ Read chunks from `byte_getter` for the case where the read is less than a full shard. Returns a mapping of chunk coordinates to bytes or None. `max_gap_bytes` and `max_coalesced_bytes` are forwarded to `Store.get_ranges` to control byte-range coalescing across the requested chunks. """ shard_index = await self._load_shard_index_maybe(byte_getter, chunks_per_shard) if shard_index is None: return None chunk_coord_byte_ranges = self._pair_chunks_with_byte_ranges(shard_index, all_chunk_coords) if not chunk_coord_byte_ranges: return {} shard_dict: ShardMutableMapping = {} if isinstance(byte_getter, StorePath): # External store: use Store.get_ranges for coalescing + concurrency. byte_ranges = [byte_range for _, byte_range in chunk_coord_byte_ranges] try: async for group in byte_getter.store.get_ranges( byte_getter.path, byte_ranges, prototype=prototype, max_gap_bytes=max_gap_bytes, max_coalesced_bytes=max_coalesced_bytes, ): for idx, buf in group: if buf is not None: chunk_coord, _ = chunk_coord_byte_ranges[idx] shard_dict[chunk_coord] = buf except BaseExceptionGroup as eg: # `Store.get_ranges` raises FileNotFoundError (wrapped in a # BaseExceptionGroup) if any underlying fetch indicates the key is # absent. The shard index loaded above, so this typically means a # race where the shard was deleted mid-read; treat it as "shard # gone" to match the index-missing branch (return None). Anything # else in the group (e.g. IO errors) is re-raised. _, rest = eg.split(FileNotFoundError) if rest is not None: raise rest from None return None else: # Any other ByteGetter. In practice only `_ShardingByteGetter` for # nested sharding, which slices an in-memory buffer (no I/O to coalesce). for chunk_coord, byte_range in chunk_coord_byte_ranges: buf = await byte_getter.get(prototype, byte_range) if buf is not None: shard_dict[chunk_coord] = buf return shard_dict def _load_shard_index_maybe_sync( self, byte_getter: Any, chunks_per_shard: tuple[int, ...] ) -> _ShardIndex | None: """Sync counterpart of `_load_shard_index_maybe`.""" index_bytes = byte_getter.get_sync( prototype=numpy_buffer_prototype(), byte_range=self._shard_index_byte_range(chunks_per_shard), ) if index_bytes is not None: return self._decode_shard_index_sync(index_bytes, chunks_per_shard) return None def _load_partial_shard_maybe_sync( self, byte_getter: Any, prototype: BufferPrototype, chunks_per_shard: tuple[int, ...], all_chunk_coords: set[tuple[int, ...]], *, max_gap_bytes: int, max_coalesced_bytes: int, ) -> ShardMapping | None: """Sync counterpart of `_load_partial_shard_maybe` (the #3004 read path). Reads the shard index, then fetches only the touched inner chunks via the store's coalescing `get_ranges_sync` (merging adjacent ranges into fewer reads), matching the async path's IO shape without an event loop. `max_gap_bytes` and `max_coalesced_bytes` control the coalescing, forwarded from the array's `sharding_coalesce_*` config exactly as the async path. """ shard_index = self._load_shard_index_maybe_sync(byte_getter, chunks_per_shard) if shard_index is None: return None chunk_coord_byte_ranges = self._pair_chunks_with_byte_ranges(shard_index, all_chunk_coords) if not chunk_coord_byte_ranges: return {} shard_dict: ShardMutableMapping = {} store = byte_getter.store if hasattr(byte_getter, "store") else None if isinstance(store, Store) and _store_supports_sync_io(store): # External store: coalesce via get_ranges_sync (mirrors get_ranges). byte_ranges = [byte_range for _, byte_range in chunk_coord_byte_ranges] try: for idx, buf in store.get_ranges_sync( byte_getter.path, byte_ranges, prototype=prototype, max_gap_bytes=max_gap_bytes, max_coalesced_bytes=max_coalesced_bytes, ): if buf is not None: chunk_coord, _ = chunk_coord_byte_ranges[idx] shard_dict[chunk_coord] = buf except BaseExceptionGroup as eg: # Mirror the async path: a FileNotFoundError means the shard was # deleted mid-read -> treat as "gone" (None). Re-raise anything else. _, rest = eg.split(FileNotFoundError) if rest is not None: raise rest from None return None else: # Nested sharding: an in-memory _ShardingByteGetter, no IO to coalesce. for chunk_coord, byte_range in chunk_coord_byte_ranges: buf = byte_getter.get_sync(prototype=prototype, byte_range=byte_range) if buf is not None: shard_dict[chunk_coord] = buf return shard_dict def compute_encoded_size(self, input_byte_length: int, shard_spec: ArraySpec) -> int: chunks_per_shard = self._get_chunks_per_shard(shard_spec) return input_byte_length + self._shard_index_size(chunks_per_shard) zarr-developers-zarr-python-a2f2415/src/zarr/codecs/transpose.py000066400000000000000000000105771527014137600250740ustar00rootroot00000000000000from __future__ import annotations from collections.abc import Iterable from dataclasses import dataclass, replace from typing import TYPE_CHECKING, cast import numpy as np from zarr.abc.codec import ArrayArrayCodec from zarr.core.array_spec import ArraySpec from zarr.core.common import JSON, parse_named_configuration if TYPE_CHECKING: from typing import Self from zarr.core.buffer import NDBuffer from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType from zarr.core.metadata.v3 import ChunkGridMetadata def parse_transpose_order(data: JSON | Iterable[int]) -> tuple[int, ...]: if not isinstance(data, Iterable): raise TypeError(f"Expected an iterable. Got {data} instead.") if not all(isinstance(a, int) for a in data): raise TypeError(f"Expected an iterable of integers. Got {data} instead.") return tuple(cast("Iterable[int]", data)) @dataclass(frozen=True) class TransposeCodec(ArrayArrayCodec): """Transpose codec""" is_fixed_size = True order: tuple[int, ...] def __init__(self, *, order: Iterable[int]) -> None: order_parsed = parse_transpose_order(order) object.__setattr__(self, "order", order_parsed) @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: _, configuration_parsed = parse_named_configuration(data, "transpose") return cls(**configuration_parsed) # type: ignore[arg-type] def to_dict(self) -> dict[str, JSON]: return {"name": "transpose", "configuration": {"order": tuple(self.order)}} def validate( self, shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGridMetadata, ) -> None: if len(self.order) != len(shape): raise ValueError( f"The `order` tuple must have as many entries as there are dimensions in the array. Got {self.order}." ) if len(self.order) != len(set(self.order)): raise ValueError( f"There must not be duplicates in the `order` tuple. Got {self.order}." ) if not all(0 <= x < len(shape) for x in self.order): raise ValueError( f"All entries in the `order` tuple must be between 0 and the number of dimensions in the array. Got {self.order}." ) def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: ndim = array_spec.ndim if len(self.order) != ndim: raise ValueError( f"The `order` tuple must have as many entries as there are dimensions in the array. Got {self.order}." ) if len(self.order) != len(set(self.order)): raise ValueError( f"There must not be duplicates in the `order` tuple. Got {self.order}." ) if not all(0 <= x < ndim for x in self.order): raise ValueError( f"All entries in the `order` tuple must be between 0 and the number of dimensions in the array. Got {self.order}." ) order = tuple(self.order) if order != self.order: return replace(self, order=order) return self def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: return ArraySpec( shape=tuple(chunk_spec.shape[self.order[i]] for i in range(chunk_spec.ndim)), dtype=chunk_spec.dtype, fill_value=chunk_spec.fill_value, config=chunk_spec.config, prototype=chunk_spec.prototype, ) def _decode_sync( self, chunk_array: NDBuffer, chunk_spec: ArraySpec, ) -> NDBuffer: inverse_order = tuple(int(i) for i in np.argsort(self.order)) return chunk_array.transpose(inverse_order) async def _decode_single( self, chunk_array: NDBuffer, chunk_spec: ArraySpec, ) -> NDBuffer: return self._decode_sync(chunk_array, chunk_spec) def _encode_sync( self, chunk_array: NDBuffer, _chunk_spec: ArraySpec, ) -> NDBuffer | None: return chunk_array.transpose(self.order) async def _encode_single( self, chunk_array: NDBuffer, _chunk_spec: ArraySpec, ) -> NDBuffer | None: return self._encode_sync(chunk_array, _chunk_spec) def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int: return input_byte_length zarr-developers-zarr-python-a2f2415/src/zarr/codecs/vlen_utf8.py000066400000000000000000000107641527014137600247660ustar00rootroot00000000000000from __future__ import annotations from dataclasses import dataclass from typing import TYPE_CHECKING import numpy as np from numcodecs.vlen import VLenBytes, VLenUTF8 from zarr._compat import _reshape_view from zarr.abc.codec import ArrayBytesCodec from zarr.core.common import JSON, parse_named_configuration if TYPE_CHECKING: from typing import Self from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer, NDBuffer # can use a global because there are no parameters _vlen_utf8_codec = VLenUTF8() _vlen_bytes_codec = VLenBytes() @dataclass(frozen=True) class VLenUTF8Codec(ArrayBytesCodec): """Variable-length UTF8 codec""" @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: _, configuration_parsed = parse_named_configuration( data, "vlen-utf8", require_configuration=False ) configuration_parsed = configuration_parsed or {} return cls(**configuration_parsed) def to_dict(self) -> dict[str, JSON]: return {"name": "vlen-utf8", "configuration": {}} def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: return self def _decode_sync( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> NDBuffer: raw_bytes = chunk_bytes.as_array_like() decoded = _vlen_utf8_codec.decode(raw_bytes) decoded = _reshape_view(decoded, chunk_spec.shape) as_string_dtype = decoded.astype(chunk_spec.dtype.to_native_dtype(), copy=False) return chunk_spec.prototype.nd_buffer.from_numpy_array(as_string_dtype) async def _decode_single( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> NDBuffer: return self._decode_sync(chunk_bytes, chunk_spec) def _encode_sync( self, chunk_array: NDBuffer, chunk_spec: ArraySpec, ) -> Buffer | None: # numcodecs vlen codecs flatten with order="A", so an F-contiguous chunk # would be encoded in transposed element order (gh-3558) return chunk_spec.prototype.buffer.from_bytes( _vlen_utf8_codec.encode(np.ascontiguousarray(chunk_array.as_numpy_array())) ) async def _encode_single( self, chunk_array: NDBuffer, chunk_spec: ArraySpec, ) -> Buffer | None: return self._encode_sync(chunk_array, chunk_spec) def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int: # what is input_byte_length for an object dtype? raise NotImplementedError("compute_encoded_size is not implemented for VLen codecs") @dataclass(frozen=True) class VLenBytesCodec(ArrayBytesCodec): @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: _, configuration_parsed = parse_named_configuration( data, "vlen-bytes", require_configuration=False ) configuration_parsed = configuration_parsed or {} return cls(**configuration_parsed) def to_dict(self) -> dict[str, JSON]: return {"name": "vlen-bytes", "configuration": {}} def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: return self def _decode_sync( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> NDBuffer: raw_bytes = chunk_bytes.as_array_like() decoded = _vlen_bytes_codec.decode(raw_bytes) decoded = _reshape_view(decoded, chunk_spec.shape) return chunk_spec.prototype.nd_buffer.from_numpy_array(decoded) async def _decode_single( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> NDBuffer: return self._decode_sync(chunk_bytes, chunk_spec) def _encode_sync( self, chunk_array: NDBuffer, chunk_spec: ArraySpec, ) -> Buffer | None: # numcodecs vlen codecs flatten with order="A", so an F-contiguous chunk # would be encoded in transposed element order (gh-3558) return chunk_spec.prototype.buffer.from_bytes( _vlen_bytes_codec.encode(np.ascontiguousarray(chunk_array.as_numpy_array())) ) async def _encode_single( self, chunk_array: NDBuffer, chunk_spec: ArraySpec, ) -> Buffer | None: return self._encode_sync(chunk_array, chunk_spec) def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int: # what is input_byte_length for an object dtype? raise NotImplementedError("compute_encoded_size is not implemented for VLen codecs") zarr-developers-zarr-python-a2f2415/src/zarr/codecs/zstd.py000066400000000000000000000063471527014137600240420ustar00rootroot00000000000000from __future__ import annotations import asyncio from dataclasses import dataclass from functools import cached_property from typing import TYPE_CHECKING import numcodecs from numcodecs.zstd import Zstd from packaging.version import Version from zarr.abc.codec import BytesBytesCodec from zarr.core.buffer.cpu import as_numpy_array_wrapper from zarr.core.common import JSON, parse_named_configuration from zarr.core.json_parse import parse_field if TYPE_CHECKING: from typing import Self from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer def parse_zstd_level(data: JSON) -> int: parsed: int = parse_field(data, int, "level", error=TypeError) if parsed >= 23: raise ValueError(f"Value must be less than or equal to 22. Got {parsed} instead.") return parsed def parse_checksum(data: JSON) -> bool: parsed: bool = parse_field(data, bool, "checksum", error=TypeError) return parsed @dataclass(frozen=True) class ZstdCodec(BytesBytesCodec): """zstd codec""" is_fixed_size = False level: int = 0 checksum: bool = False def __init__(self, *, level: int = 0, checksum: bool = False) -> None: # numcodecs 0.13.0 introduces the checksum attribute for the zstd codec _numcodecs_version = Version(numcodecs.__version__) if _numcodecs_version < Version("0.13.0"): raise RuntimeError( "numcodecs version >= 0.13.0 is required to use the zstd codec. " f"Version {_numcodecs_version} is currently installed." ) level_parsed = parse_zstd_level(level) checksum_parsed = parse_checksum(checksum) object.__setattr__(self, "level", level_parsed) object.__setattr__(self, "checksum", checksum_parsed) @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: _, configuration_parsed = parse_named_configuration(data, "zstd") return cls(**configuration_parsed) # type: ignore[arg-type] def to_dict(self) -> dict[str, JSON]: return {"name": "zstd", "configuration": {"level": self.level, "checksum": self.checksum}} @cached_property def _zstd_codec(self) -> Zstd: config_dict = {"level": self.level, "checksum": self.checksum} return Zstd.from_config(config_dict) def _decode_sync( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer: return as_numpy_array_wrapper(self._zstd_codec.decode, chunk_bytes, chunk_spec.prototype) async def _decode_single( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer: return await asyncio.to_thread(self._decode_sync, chunk_bytes, chunk_spec) def _encode_sync( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer | None: return as_numpy_array_wrapper(self._zstd_codec.encode, chunk_bytes, chunk_spec.prototype) async def _encode_single( self, chunk_bytes: Buffer, chunk_spec: ArraySpec, ) -> Buffer | None: return await asyncio.to_thread(self._encode_sync, chunk_bytes, chunk_spec) def compute_encoded_size(self, _input_byte_length: int, _chunk_spec: ArraySpec) -> int: raise NotImplementedError zarr-developers-zarr-python-a2f2415/src/zarr/core/000077500000000000000000000000001527014137600221625ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/core/__init__.py000066400000000000000000000004331527014137600242730ustar00rootroot00000000000000""" The ``zarr.core`` module is considered private API and should not be imported directly by 3rd-party code. """ from __future__ import annotations from zarr.core.buffer import Buffer, NDBuffer # noqa: F401 from zarr.core.codec_pipeline import BatchedCodecPipeline # noqa: F401 zarr-developers-zarr-python-a2f2415/src/zarr/core/_coalesce.py000066400000000000000000000203361527014137600244550ustar00rootroot00000000000000# src/zarr/core/_coalesce.py from __future__ import annotations import asyncio from typing import TYPE_CHECKING, NamedTuple from zarr.abc.store import RangeByteRequest if TYPE_CHECKING: from collections.abc import AsyncGenerator, Awaitable, Callable, Sequence from zarr.abc.store import ByteRequest from zarr.core.buffer import Buffer class _WorkerCtx(NamedTuple): """Shared state passed to the per-task worker coroutines. Bundling these lets the workers declare their dependencies as one parameter instead of capturing them implicitly via closure. """ fetch: Callable[[ByteRequest | None], Awaitable[Buffer | None]] semaphore: asyncio.Semaphore async def _fetch_single( ctx: _WorkerCtx, idx: int, req: ByteRequest | None ) -> Sequence[tuple[int, Buffer | None]]: """Fetch one byte range. Raises FileNotFoundError if the key is absent.""" async with ctx.semaphore: buf = await ctx.fetch(req) if buf is None: raise FileNotFoundError return ((idx, buf),) async def _fetch_group( ctx: _WorkerCtx, members: list[tuple[int, RangeByteRequest]] ) -> Sequence[tuple[int, Buffer | None]]: """Fetch one merged byte range and slice it back into per-input buffers. `members` must already be sorted by `start`; callers in this module build it from the sorted mergeable list. Raises `FileNotFoundError` if the key is absent. """ if len(members) == 1: solo_idx, solo_req = members[0] return await _fetch_single(ctx, solo_idx, solo_req) start = members[0][1].start end = max(r.end for _, r in members) async with ctx.semaphore: big = await ctx.fetch(RangeByteRequest(start, end)) if big is None: raise FileNotFoundError sliced = [(idx, big[r.start - start : r.end - start]) for idx, r in members] return tuple(sliced) def coalesce_ranges( byte_ranges: Sequence[ByteRequest | None], *, max_gap_bytes: int, max_coalesced_bytes: int, ) -> tuple[ list[list[tuple[int, RangeByteRequest]]], list[tuple[int, ByteRequest | None]], ]: """Plan a set of byte-range fetches: which inputs merge, which stand alone. Pure (no I/O). The result is the I/O plan a caller would execute: each group corresponds to one fetch of a coalesced byte range, and each uncoalescable item corresponds to one fetch of the original request. All tuning knobs are required keyword arguments. `Store.get_ranges` is the public entry point and owns the canonical default values; this function takes them explicitly to avoid duplicating policy. Parameters ---------- byte_ranges Input ranges. `None` means "the whole value". max_gap_bytes Two `RangeByteRequest`s separated by at most this many bytes may be merged into one fetch. max_coalesced_bytes Upper bound on the size of a single merged fetch. Returns ------- groups List of merged groups. Each group is a list of `(input_index, RangeByteRequest)` pairs sorted by `start`. A single-element group represents a `RangeByteRequest` that did not merge with any neighbor. uncoalescable List of `(input_index, request)` pairs for inputs that are not `RangeByteRequest` (`OffsetByteRequest`, `SuffixByteRequest`, `None`). Indices are preserved from the input order. Notes ----- Only `RangeByteRequest` inputs participate in coalescing. Two ranges merge when both: their gap (next `start` minus current group's running `end`) is `<= max_gap_bytes`, and the resulting merged span is `<= max_coalesced_bytes`. """ indexed = list(enumerate(byte_ranges)) mergeable = [(i, r) for i, r in indexed if isinstance(r, RangeByteRequest)] uncoalescable: list[tuple[int, ByteRequest | None]] = [ (i, r) for i, r in indexed if not isinstance(r, RangeByteRequest) ] # Sort mergeables by start offset, then merge. Track running start/end of the # current group so each merge step is O(1) instead of O(group size). mergeable.sort(key=lambda pair: pair[1].start) groups: list[list[tuple[int, RangeByteRequest]]] = [] group_start = 0 group_end = 0 for pair in mergeable: _i, r = pair if groups and r.start - group_end <= max_gap_bytes: prospective_end = max(group_end, r.end) if prospective_end - group_start <= max_coalesced_bytes: groups[-1].append(pair) group_end = prospective_end continue groups.append([pair]) group_start = r.start group_end = r.end return groups, uncoalescable async def coalesced_get( fetch: Callable[[ByteRequest | None], Awaitable[Buffer | None]], byte_ranges: Sequence[ByteRequest | None], *, max_concurrency: int, max_gap_bytes: int, max_coalesced_bytes: int, ) -> AsyncGenerator[Sequence[tuple[int, Buffer | None]]]: """Read many byte ranges through `fetch` with coalescing and concurrency. Nearby ranges are merged into a single underlying I/O, and merged fetches are run concurrently. Each yield corresponds to exactly one underlying I/O operation: a sequence of `(input_index, result)` tuples for all input ranges served by that I/O. Tuples within a yielded sequence are ordered by start offset. Yields across groups are in completion order, not input order. All tuning knobs are required keyword arguments. `Store.get_ranges` is the public entry point and owns the canonical default values; this function takes them explicitly to avoid duplicating policy. Parameters ---------- fetch Callable that reads one byte range and returns a `Buffer` (or `None` if the underlying key does not exist). Typically constructed via `functools.partial(store.get, key, prototype)`. byte_ranges Input ranges. `None` means "the whole value". max_concurrency Maximum number of merged fetches in flight at once. max_gap_bytes Forwarded to `coalesce_ranges`. max_coalesced_bytes Forwarded to `coalesce_ranges`. Yields ------ Sequence[tuple[int, Buffer | None]] Per-I/O batch of `(input_index, result)` tuples. Notes ----- - Only `RangeByteRequest` inputs are coalesced. `OffsetByteRequest`, `SuffixByteRequest`, and `None` are each treated as uncoalescable (one fetch, one single-tuple yield per input). - Failures from underlying fetches surface as a `BaseExceptionGroup` (PEP 654). Inner exceptions include `FileNotFoundError` if a fetch returns `None`, plus any exception `fetch` raises. Pending fetches are cancelled as soon as one task fails, so the group typically contains a single non-`CancelledError` exception even under high concurrency. - Groups completed before the failure remain observable on the yields preceding the raise. - `GeneratorExit` raised by `aclose()` is filtered out so the iterator closes cleanly; callers don't see a group containing only it. """ if not byte_ranges: return groups, singles = coalesce_ranges( byte_ranges, max_gap_bytes=max_gap_bytes, max_coalesced_bytes=max_coalesced_bytes, ) ctx = _WorkerCtx(fetch=fetch, semaphore=asyncio.Semaphore(max_concurrency)) # Launch all work as tasks. The semaphore bounds actual I/O concurrency. # TaskGroup wraps task exceptions in BaseExceptionGroup; we propagate the # group unchanged as part of the public contract (callers handle batch # failures via `except*` / PEP 654). GeneratorExit (raised when the # consumer calls aclose()) is filtered out so close completes cleanly. try: async with asyncio.TaskGroup() as tg: tasks = [ *(tg.create_task(_fetch_group(ctx, group)) for group in groups), *(tg.create_task(_fetch_single(ctx, i, single)) for i, single in singles), ] for fut in asyncio.as_completed(tasks): yield await fut except BaseExceptionGroup as eg: # Strip GeneratorExits (consumer aclose()) and propagate whatever remains. _, other_errors = eg.split(GeneratorExit) if other_errors is not None: raise other_errors from None zarr-developers-zarr-python-a2f2415/src/zarr/core/_info.py000066400000000000000000000121111527014137600236220ustar00rootroot00000000000000from __future__ import annotations import dataclasses import textwrap from typing import TYPE_CHECKING, Literal if TYPE_CHECKING: from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec from zarr.abc.numcodec import Numcodec from zarr.core.common import ZarrFormat from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType @dataclasses.dataclass(kw_only=True) class GroupInfo: """ Visual summary for a Group. Note that this method and its properties is not part of Zarr's public API. """ _name: str _type: Literal["Group"] = "Group" _zarr_format: ZarrFormat _read_only: bool _store_type: str _count_members: int | None = None _count_arrays: int | None = None _count_groups: int | None = None def __repr__(self) -> str: template = textwrap.dedent("""\ Name : {_name} Type : {_type} Zarr format : {_zarr_format} Read-only : {_read_only} Store type : {_store_type}""") if self._count_members is not None: template += "\nNo. members : {_count_members}" if self._count_arrays is not None: template += "\nNo. arrays : {_count_arrays}" if self._count_groups is not None: template += "\nNo. groups : {_count_groups}" return template.format(**dataclasses.asdict(self)) def human_readable_size(size: int) -> str: if size < 2**10: return f"{size}" elif size < 2**20: return f"{size / float(2**10):.1f}K" elif size < 2**30: return f"{size / float(2**20):.1f}M" elif size < 2**40: return f"{size / float(2**30):.1f}G" elif size < 2**50: return f"{size / float(2**40):.1f}T" else: return f"{size / float(2**50):.1f}P" def byte_info(size: int) -> str: if size < 2**10: return str(size) else: return f"{size} ({human_readable_size(size)})" @dataclasses.dataclass(kw_only=True, frozen=True, slots=True) class ArrayInfo: """ Visual summary for an Array. Note that this method and its properties is not part of Zarr's public API. """ _type: Literal["Array"] = "Array" _zarr_format: ZarrFormat _data_type: ZDType[TBaseDType, TBaseScalar] _fill_value: object _shape: tuple[int, ...] # "" marks a sharded array whose shard grid is rectilinear, so # there is no uniform shard shape; None means the array is not sharded. _shard_shape: tuple[int, ...] | Literal[""] | None = None _chunk_shape: tuple[int, ...] | None = None _order: Literal["C", "F"] _read_only: bool _store_type: str _filters: tuple[Numcodec, ...] | tuple[ArrayArrayCodec, ...] = () _serializer: ArrayBytesCodec | None = None _compressors: tuple[Numcodec, ...] | tuple[BytesBytesCodec, ...] = () _count_bytes: int | None = None _count_bytes_stored: int | None = None _count_chunks_initialized: int | None = None def __repr__(self) -> str: template = textwrap.dedent("""\ Type : {_type} Zarr format : {_zarr_format} Data type : {_data_type} Fill value : {_fill_value} Shape : {_shape}""") if self._shard_shape is not None: template += textwrap.dedent(""" Shard shape : {_shard_shape}""") template += textwrap.dedent(""" Chunk shape : {_chunk_shape} Order : {_order} Read-only : {_read_only} Store type : {_store_type}""") # We can't use dataclasses.asdict, because we only want a shallow dict kwargs = {field.name: getattr(self, field.name) for field in dataclasses.fields(self)} if self._chunk_shape is None: # for non-regular chunk grids kwargs["_chunk_shape"] = "" template += "\nFilters : {_filters}" if self._serializer is not None: template += "\nSerializer : {_serializer}" template += "\nCompressors : {_compressors}" if self._count_bytes is not None: template += "\nNo. bytes : {_count_bytes}" kwargs["_count_bytes"] = byte_info(self._count_bytes) if self._count_bytes_stored is not None: template += "\nNo. bytes stored : {_count_bytes_stored}" kwargs["_count_bytes_stored"] = byte_info(self._count_bytes_stored) if ( self._count_bytes is not None and self._count_bytes_stored is not None and self._count_bytes_stored > 0 ): template += "\nStorage ratio : {_storage_ratio}" kwargs["_storage_ratio"] = f"{self._count_bytes / self._count_bytes_stored:.1f}" if self._count_chunks_initialized is not None: if self._shard_shape is not None: template += "\nShards Initialized : {_count_chunks_initialized}" else: template += "\nChunks Initialized : {_count_chunks_initialized}" return template.format(**kwargs) zarr-developers-zarr-python-a2f2415/src/zarr/core/_json.py000066400000000000000000000110501527014137600236410ustar00rootroot00000000000000"""Helpers for moving JSON documents in and out of zarr stores. These are free functions, deliberately not methods on the ``Store`` ABC: reading and writing JSON is a composition of the store's ``get``/``set`` primitives with a buffer/JSON conversion, not part of the store contract. Keeping them as functions means stores cannot (and need not) override them, and the ``Store`` definition stays free of any dependency on the buffer prototype. These functions are pure: the JSON encoding parameters (``indent``, ``allow_nan``) are explicit arguments rather than read from the global config. Callers that want zarr's configured indentation pass ``indent=config.get("json_indent")``. Two layers: - ``buffer_to_json`` / ``json_to_buffer`` convert between a ``Buffer`` and a parsed JSON value. The buffer prototype lives here, at buffer construction, where it is meaningful. - ``get_json`` / ``set_json`` compose those with ``Store.get`` / ``Store.set``. ``get_json`` returns ``None`` for a missing key (the contract most callers want); callers that require presence check for ``None`` themselves. """ from __future__ import annotations import json from typing import TYPE_CHECKING, cast from zarr.core.buffer import default_buffer_prototype if TYPE_CHECKING: from zarr.abc.store import ByteRequest, Store from zarr.core.buffer import Buffer, BufferPrototype from zarr.core.common import JSON def buffer_to_json(buffer: Buffer) -> JSON: """Parse the contents of a `Buffer` as a JSON value.""" # json.loads is typed as returning Any; the result is by definition JSON. return cast("JSON", json.loads(buffer.to_bytes())) def json_equal(a: JSON, b: JSON) -> bool: """Whether two JSON values have the same JSON encoding. Python compares `True` and `1`, or `1.0` and `1`, as equal; JSON does not.""" return json.dumps(a) == json.dumps(b) def buffer_to_json_object(buffer: Buffer) -> dict[str, JSON]: """Parse the contents of a `Buffer` as a JSON object (a `dict`). Every metadata document zarr reads is a JSON object, so this narrows the `JSON` union to `dict[str, JSON]` once, here, instead of at each call site. Parameters ---------- buffer The buffer whose contents are parsed as a JSON object. Raises ------ TypeError If the parsed value is not a JSON object. """ obj = buffer_to_json(buffer) if not isinstance(obj, dict): raise TypeError(f"Expected a JSON object, got {type(obj).__name__}.") return obj def json_to_buffer( obj: JSON, *, prototype: BufferPrototype | None = None, indent: int | None = None, allow_nan: bool = True, ) -> Buffer: """Serialize a JSON value into a `Buffer`. Parameters ---------- obj The JSON-serializable value to encode. prototype The buffer prototype to construct the result with. Defaults to `default_buffer_prototype()`. indent Indentation passed to `json.dumps`. `None` (the default) writes without newline indentation, using json's default separators. Callers that want zarr's configured indentation pass `indent=config.get("json_indent")`. allow_nan Whether to permit `NaN`/`Infinity` in the output, passed to `json.dumps`. """ if prototype is None: prototype = default_buffer_prototype() return prototype.buffer.from_bytes(json.dumps(obj, indent=indent, allow_nan=allow_nan).encode()) async def get_json(store: Store, key: str, *, byte_range: ByteRequest | None = None) -> JSON | None: """Read and parse the JSON document at `key`, or `None` if it is absent. Parameters ---------- store The store to read from. key The key identifying the JSON document. byte_range If given, read only this portion of the value. Note that a partial read of a JSON document may not be valid JSON. Returns ------- JSON or None The parsed JSON value, or `None` if `key` does not exist. """ buffer = await store.get(key, default_buffer_prototype(), byte_range) return None if buffer is None else buffer_to_json(buffer) async def set_json( store: Store, key: str, obj: JSON, *, prototype: BufferPrototype | None = None, indent: int | None = None, allow_nan: bool = True, ) -> None: """Serialize `obj` as JSON and write it to `key` in `store`. `indent` and `allow_nan` are forwarded to `json_to_buffer`. """ await store.set( key, json_to_buffer(obj, prototype=prototype, indent=indent, allow_nan=allow_nan) ) zarr-developers-zarr-python-a2f2415/src/zarr/core/_tree.py000066400000000000000000000117221527014137600236350ustar00rootroot00000000000000import sys from collections import deque from collections.abc import Sequence from html import escape as html_escape from typing import Any from zarr.core.group import AsyncGroup class TreeRepr: """ A simple object with a tree-like repr for the Zarr Group. Note that this object and it's implementation isn't considered part of Zarr's public API. """ def __init__(self, text: str, html: str, truncated: str = "") -> None: self._text = text self._html = html self._truncated = truncated def __repr__(self) -> str: if self._truncated: return self._truncated + self._text return self._text def _repr_mimebundle_( self, include: Sequence[str] | None = None, exclude: Sequence[str] | None = None, **kwargs: Any, ) -> dict[str, str]: text = self._truncated + self._text if self._truncated else self._text # For jupyter support. html_body = self._truncated + self._html if self._truncated else self._html html = ( '
"
            f"{html_body}
\n" ) return {"text/plain": text, "text/html": html} async def group_tree_async( group: AsyncGroup, max_depth: int | None = None, *, max_nodes: int = 500, plain: bool = False, ) -> TreeRepr: members: list[tuple[str, Any]] = [] truncated = False async for item in group.members(max_depth=max_depth): if len(members) == max_nodes: truncated = True break members.append(item) members.sort(key=lambda key_node: key_node[0]) # Set up styling tokens: ANSI bold for terminals, HTML for Jupyter, # or empty strings when plain=True (useful for LLMs, logging, files). if plain: ansi_open = ansi_close = html_open = html_close = "" else: # Avoid emitting ANSI escape codes when output is piped or in CI. use_ansi = sys.stdout.isatty() ansi_open = "\x1b[1m" if use_ansi else "" ansi_close = "\x1b[0m" if use_ansi else "" html_open = "" html_close = "" # Group members by parent key so we can render the tree level by level. nodes: dict[str, list[tuple[str, Any]]] = {} for key, node in members: if key.count("/") == 0: parent_key = "" else: parent_key = key.rsplit("/", 1)[0] nodes.setdefault(parent_key, []).append((key, node)) # Render the tree iteratively (not recursively) to avoid hitting # Python's recursion limit on deeply nested hierarchies. # Each stack frame is (prefix_string, remaining_children_at_this_level). text_lines = [f"{ansi_open}{group.name}{ansi_close}"] html_lines = [f"{html_open}{html_escape(group.name)}{html_close}"] stack = [("", deque(nodes.get("", [])))] while stack: prefix, remaining = stack[-1] if not remaining: stack.pop() continue key, node = remaining.popleft() name = key.rsplit("/")[-1] escaped_name = html_escape(name) # if we popped the last item then remaining will # now be empty - that's how we got past the if not remaining # above, but this can still be true. is_last = not remaining connector = "└── " if is_last else "├── " if isinstance(node, AsyncGroup): text_lines.append(f"{prefix}{connector}{ansi_open}{name}{ansi_close}") html_lines.append(f"{prefix}{connector}{html_open}{escaped_name}{html_close}") else: text_lines.append( f"{prefix}{connector}{ansi_open}{name}{ansi_close} {node.shape} {node.dtype}" ) html_lines.append( f"{prefix}{connector}{html_open}{escaped_name}{html_close}" f" {html_escape(str(node.shape))} {html_escape(str(node.dtype))}" ) # Descend into children with an accumulated prefix: # Example showing how prefix accumulates: # / # ├── a prefix = "" # │ ├── b prefix = "" + "│ " # │ │ └── x prefix = "" + "│ " + "│ " # │ └── c prefix = "" + "│ " # └── d prefix = "" # └── e prefix = "" + " " if children := nodes.get(key, []): if is_last: child_prefix = prefix + " " else: child_prefix = prefix + "│ " stack.append((child_prefix, deque(children))) text = "\n".join(text_lines) + "\n" html = "\n".join(html_lines) + "\n" note = ( f"Truncated at max_nodes={max_nodes}, some nodes and their children may be missing\n" if truncated else "" ) return TreeRepr(text, html, truncated=note) zarr-developers-zarr-python-a2f2415/src/zarr/core/array.py000066400000000000000000006761411527014137600236710ustar00rootroot00000000000000from __future__ import annotations import contextlib import copy import math import warnings from asyncio import gather from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, field, replace from itertools import starmap from logging import getLogger from typing import ( TYPE_CHECKING, Any, Literal, TypedDict, cast, overload, ) from warnings import warn import numpy as np from typing_extensions import Sentinel, deprecated import zarr from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec from zarr.abc.numcodec import Numcodec, _is_numcodec from zarr.codecs._v2 import V2Codec from zarr.codecs.bytes import BytesCodec from zarr.codecs.vlen_utf8 import VLenBytesCodec, VLenUTF8Codec from zarr.codecs.zstd import ZstdCodec from zarr.core._info import ArrayInfo from zarr.core._json import buffer_to_json_object from zarr.core.array_spec import ArrayConfig, ArrayConfigLike, ArraySpec, parse_array_config from zarr.core.attributes import Attributes from zarr.core.buffer import ( BufferPrototype, NDArrayLike, NDArrayLikeOrScalar, NDBuffer, default_buffer_prototype, ) from zarr.core.buffer.cpu import buffer_prototype as cpu_buffer_prototype from zarr.core.chunk_grids import ( SHARDED_INNER_CHUNK_MAX_BYTES, ChunkGrid, _is_auto, _is_keep, guess_chunks, normalize_chunks_nd, resolve_outer_and_inner_chunks, ) from zarr.core.chunk_key_encodings import ( ChunkKeyEncoding, ChunkKeyEncodingLike, DefaultChunkKeyEncoding, V2ChunkKeyEncoding, parse_chunk_key_encoding, ) from zarr.core.common import ( JSON, ZARR_JSON, ZARRAY_JSON, ZATTRS_JSON, ChunksLike, DimensionNamesLike, MemoryOrder, ShapeLike, ZarrFormat, _default_zarr_format, _warn_order_kwarg, ceildiv_int, concurrent_foreach, concurrent_map, parse_shapelike, product, ) from zarr.core.config import config as zarr_config from zarr.core.dtype import ( Structured, VariableLengthBytes, VariableLengthUTF8, ZDType, ZDTypeLike, parse_dtype, ) from zarr.core.dtype.common import HasEndianness, HasItemSize, HasObjectCodec from zarr.core.indexing import ( AsyncOIndex, AsyncVIndex, BasicIndexer, BasicSelection, BlockIndex, BlockIndexer, CoordinateIndexer, CoordinateSelection, Fields, Indexer, MaskIndexer, MaskSelection, OIndex, OrthogonalIndexer, OrthogonalSelection, Selection, VIndex, _iter_grid, check_fields, check_no_multi_fields, is_pure_fancy_indexing, is_pure_orthogonal_indexing, is_scalar, pop_fields, ) from zarr.core.metadata import ( ArrayMetadata, ArrayMetadataDict, ArrayMetadataJSON_V3, ArrayV2Metadata, ArrayV2MetadataDict, ArrayV3Metadata, ) from zarr.core.metadata.io import ( ARRAY_DOCUMENTS, encode_documents, parse_stored_array, read_documents, save_metadata, save_new_metadata, store_documents, upsert_metadata, ) from zarr.core.metadata.v2 import ( CompressorLikev2, get_object_codec_id, parse_compressor, parse_filters, ) from zarr.core.metadata.v3 import ( ChunkGridMetadata, RectilinearChunkGridMetadata, RegularChunkGridMetadata, create_chunk_grid_metadata, parse_node_type_array, ) from zarr.core.sync import sync from zarr.errors import ( ArrayNotFoundError, ChunkNotFoundError, MetadataValidationError, ZarrDeprecationWarning, ZarrUserWarning, ) from zarr.registry import ( _parse_array_array_codec, _parse_array_bytes_codec, _parse_bytes_bytes_codec, get_pipeline_class, ) from zarr.storage._common import StorePath, make_store_path from zarr.storage._memory import MemoryStore from zarr.storage._utils import _relativize_path if TYPE_CHECKING: from collections.abc import Iterator from typing import Self import numpy.typing as npt from zarr.abc.codec import CodecPipeline from zarr.abc.store import Store from zarr.codecs.sharding import IndexLocation, ShardingCodec from zarr.core.buffer import Buffer from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar from zarr.storage import StoreLike from zarr.types import AnyArray, AnyAsyncArray, ArrayV2, ArrayV3, AsyncArrayV2, AsyncArrayV3 # Array and AsyncArray are defined in the base `zarr` namespace __all__ = [ "DEFAULT_FILL_VALUE", "create_codec_pipeline", "parse_array_metadata", ] logger = getLogger(__name__) DEFAULT_FILL_VALUE = Sentinel("DEFAULT_FILL_VALUE") """ Sentinel indicating that the default fill value should be used. This sentinel exists because conventional values used to convey "defaultness" like `None` or `"auto"` are ambiguous when specifying the fill value parameter of a Zarr array. The value `None` is ambiguous because it is a valid fill value for Zarr V2 (resulting in `"fill_value": null` in array metadata). A string like `"auto"` is ambiguous because such a string is a valid fill value for an array with a string data type. This sentinel lies outside the space of valid fill values, which means it can unambiguously express that the default fill value should be used. """ def _chunk_sizes_from_shape( array_shape: tuple[int, ...], chunk_shape: tuple[int, ...] ) -> tuple[tuple[int, ...], ...]: """Compute dask-style chunk sizes from an array shape and uniform chunk shape.""" result: list[tuple[int, ...]] = [] for s, c in zip(array_shape, chunk_shape, strict=True): nchunks = ceildiv_int(s, c) sizes = tuple(min(c, s - i * c) for i in range(nchunks)) result.append(sizes) return tuple(result) def parse_array_metadata(data: Any, path: str | None = None) -> ArrayMetadata: """Array metadata from a metadata object or a metadata document, naming the array at `path` in warnings about how an invalid document was read. `ArrayV2Metadata` accepts a chunk size of 0, as it always has, though only an invalid document holds one: such metadata is read as the documents it would store are (see `zarr.core.metadata.repair`), so an array can be built from it. No data was read or written under that chunk size, so the reading is silent.""" if isinstance(data, ArrayV2Metadata) and 0 in data.chunks: return parse_stored_array(data.to_buffer_dict(default_buffer_prototype()), 2) if isinstance(data, ArrayMetadata): return data if isinstance(data, dict): zarr_format = data.get("zarr_format") if zarr_format == 3: meta_out = ArrayV3Metadata.from_dict(data, path=path) if len(meta_out.storage_transformers) > 0: msg = ( f"Array metadata contains storage transformers: {meta_out.storage_transformers}." "Arrays with storage transformers are not supported in zarr-python at this time." ) raise ValueError(msg) return meta_out elif zarr_format == 2: return ArrayV2Metadata.from_dict(data, path=path) else: raise ValueError(f"Invalid zarr_format: {zarr_format}. Expected 2 or 3") raise TypeError # pragma: no cover def create_codec_pipeline(metadata: ArrayMetadata, *, store: Store | None = None) -> CodecPipeline: if store is not None: try: return get_pipeline_class().from_array_metadata_and_store( array_metadata=metadata, store=store ) except NotImplementedError: pass if isinstance(metadata, ArrayV3Metadata): # The pipeline built here is a throwaway: `evolve_from_array_spec` below # reconstructs codecs against the evolved spec. `from_codecs` is the # chain's first construction, so its advisory warnings (e.g. sharding's # "disables partial reads" warning) fire here; `evolve_from_array_spec` # re-splits the same already-warned-about chain via # `codecs_from_list_unchecked`, so it does not re-emit them. pipeline = get_pipeline_class().from_codecs(metadata.codecs) # Use the regular chunk shape if available, otherwise use a # placeholder. The ChunkTransform is shape-agnostic — the actual # chunk shape is passed per-call at decode/encode time. if isinstance(metadata.chunk_grid, RegularChunkGridMetadata): chunk_shape = metadata.chunk_grid.chunk_shape else: chunk_shape = (1,) * len(metadata.shape) chunk_spec = ArraySpec( shape=chunk_shape, dtype=metadata.data_type, fill_value=metadata.fill_value, config=ArrayConfig.from_dict({}), prototype=default_buffer_prototype(), ) return pipeline.evolve_from_array_spec(chunk_spec) elif isinstance(metadata, ArrayV2Metadata): v2_codec = V2Codec(filters=metadata.filters, compressor=metadata.compressor) pipeline = get_pipeline_class().from_codecs([v2_codec]) chunk_spec = ArraySpec( shape=metadata.chunks, dtype=metadata.dtype, fill_value=metadata.fill_value, config=ArrayConfig.from_dict({"order": metadata.order}), prototype=default_buffer_prototype(), ) return pipeline.evolve_from_array_spec(chunk_spec) raise TypeError # pragma: no cover async def get_array_metadata( store_path: StorePath, zarr_format: ZarrFormat | None = 3 ) -> dict[str, JSON]: if zarr_format == 2: zarray_bytes, zattrs_bytes = await gather( (store_path / ZARRAY_JSON).get(prototype=cpu_buffer_prototype), (store_path / ZATTRS_JSON).get(prototype=cpu_buffer_prototype), ) if zarray_bytes is None: msg = ( "A Zarr V2 array metadata document was not found in store " f"{store_path.store!r} at path {store_path.path!r}." ) raise ArrayNotFoundError(msg) return _array_metadata_dict_v2(zarray_bytes, zattrs_bytes) elif zarr_format == 3: zarr_json_bytes = await (store_path / ZARR_JSON).get(prototype=cpu_buffer_prototype) if zarr_json_bytes is None: msg = ( "A Zarr V3 array metadata document was not found in store " f"{store_path.store!r} at path {store_path.path!r}." ) raise ArrayNotFoundError(msg) return _array_metadata_dict_v3(zarr_json_bytes) elif zarr_format is None: zarr_json_bytes, zarray_bytes, zattrs_bytes = await gather( (store_path / ZARR_JSON).get(prototype=cpu_buffer_prototype), (store_path / ZARRAY_JSON).get(prototype=cpu_buffer_prototype), (store_path / ZATTRS_JSON).get(prototype=cpu_buffer_prototype), ) if zarr_json_bytes is not None and zarray_bytes is not None: # warn and favor v3 msg = f"Both zarr.json (Zarr format 3) and .zarray (Zarr format 2) metadata objects exist at {store_path}. Zarr v3 will be used." warnings.warn(msg, category=ZarrUserWarning, stacklevel=1) # favor v3 when both are present if zarr_json_bytes is not None: return _array_metadata_dict_v3(zarr_json_bytes) if zarray_bytes is not None: return _array_metadata_dict_v2(zarray_bytes, zattrs_bytes) msg = ( f"Neither Zarr V3 nor Zarr V2 array metadata documents " f"were found in store {store_path.store!r} at path {store_path.path!r}." ) raise ArrayNotFoundError(msg) else: msg = f"Invalid value for 'zarr_format'. Expected 2, 3, or None. Got '{zarr_format}'." # type: ignore[unreachable] raise MetadataValidationError(msg) def _array_metadata_dict_v2(zarray_bytes: Buffer, zattrs_bytes: Buffer | None) -> dict[str, JSON]: """Combine a `.zarray` document and an optional `.zattrs` document into one metadata dict.""" metadata_dict: dict[str, JSON] = buffer_to_json_object(zarray_bytes) metadata_dict["attributes"] = ( buffer_to_json_object(zattrs_bytes) if zattrs_bytes is not None else {} ) return metadata_dict def _array_metadata_dict_v3(zarr_json_bytes: Buffer) -> dict[str, JSON]: """Parse a `zarr.json` document, checking that it describes an array.""" metadata_dict: dict[str, JSON] = buffer_to_json_object(zarr_json_bytes) parse_node_type_array(metadata_dict.get("node_type")) return metadata_dict def _v2_chunks_given(chunks: object) -> bool: """Whether the legacy Zarr format 2 `chunks` argument is given. A falsy `chunks` (such as `None`, 0, `[]` or `False`) is read as not given. A numpy array with more than one element has no truth value and is always given; a shorter one is given when its element is nonzero (an empty array is not given). """ if isinstance(chunks, np.ndarray): return chunks.size > 1 or bool(chunks.any()) return bool(chunks) @dataclass(frozen=True) class AsyncArray[T_ArrayMetadata: (ArrayV2Metadata, ArrayV3Metadata)]: """ An asynchronous array class representing a chunked array stored in a Zarr store. Parameters ---------- metadata : ArrayMetadata The metadata of the array. store_path : StorePath The path to the Zarr store. config : ArrayConfigLike, optional The runtime configuration of the array, by default None. Attributes ---------- metadata : ArrayMetadata The metadata of the array. store_path : StorePath The path to the Zarr store. codec_pipeline : CodecPipeline The codec pipeline used for encoding and decoding chunks. config : ArrayConfig The runtime configuration of the array. """ metadata: T_ArrayMetadata store_path: StorePath codec_pipeline: CodecPipeline = field(init=False) _chunk_grid: ChunkGrid = field(init=False) config: ArrayConfig @overload def __init__( self: AsyncArrayV2, metadata: ArrayV2Metadata | ArrayV2MetadataDict, store_path: StorePath, config: ArrayConfigLike | None = None, ) -> None: ... @overload def __init__( self: AsyncArrayV3, metadata: ArrayV3Metadata | ArrayMetadataJSON_V3, store_path: StorePath, config: ArrayConfigLike | None = None, ) -> None: ... def __init__( self, metadata: ArrayMetadata | ArrayMetadataDict, store_path: StorePath, config: ArrayConfigLike | None = None, ) -> None: metadata_parsed = parse_array_metadata(metadata, str(store_path)) config_parsed = parse_array_config(config) object.__setattr__(self, "metadata", metadata_parsed) object.__setattr__(self, "store_path", store_path) object.__setattr__(self, "config", config_parsed) object.__setattr__(self, "_chunk_grid", ChunkGrid.from_metadata(metadata_parsed)) object.__setattr__( self, "codec_pipeline", create_codec_pipeline(metadata=metadata_parsed, store=store_path.store), ) @classmethod async def _create( cls, store: StoreLike, *, # v2 and v3 shape: ShapeLike, dtype: ZDTypeLike | ZDType[TBaseDType, TBaseScalar], zarr_format: ZarrFormat = 3, fill_value: Any | None = DEFAULT_FILL_VALUE, attributes: dict[str, JSON] | None = None, # v3 only chunk_shape: ChunksLike | None = None, chunk_key_encoding: ( ChunkKeyEncodingLike | tuple[Literal["default"], Literal[".", "/"]] | tuple[Literal["v2"], Literal[".", "/"]] | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: DimensionNamesLike = None, # v2 only chunks: ChunksLike | None = None, dimension_separator: Literal[".", "/"] | None = None, order: MemoryOrder | None = None, filters: Iterable[dict[str, JSON] | Numcodec] | None = None, compressor: CompressorLike = "auto", # runtime overwrite: bool = False, data: npt.ArrayLike | None = None, config: ArrayConfigLike | None = None, ) -> AnyAsyncArray: """Method to create a new asynchronous array instance. Deprecated in favor of [`zarr.api.asynchronous.create_array`][]. """ dtype_parsed = parse_dtype(dtype, zarr_format=zarr_format) store_path = await make_store_path(store) shape = parse_shapelike(shape) if chunks is not None and chunk_shape is not None: raise ValueError("Only one of chunk_shape or chunks can be provided.") # Unify the v2 (chunks) and v3 (chunk_shape) parameter names _raw_chunks = chunks if chunks is not None else chunk_shape config_parsed = parse_array_config(config) result: AnyAsyncArray if zarr_format == 3: if dimension_separator is not None: raise ValueError( "dimension_separator cannot be used for arrays with zarr_format 3. Use chunk_key_encoding instead." ) if filters is not None: raise ValueError( "filters cannot be used for arrays with zarr_format 3. Use array-to-array codecs instead." ) if compressor != "auto": raise ValueError( "compressor cannot be used for arrays with zarr_format 3. Use bytes-to-bytes codecs instead." ) if order is not None: _warn_order_kwarg() item_size = 1 if isinstance(dtype_parsed, HasItemSize): item_size = dtype_parsed.item_size if _raw_chunks is None: outer_chunks = guess_chunks(shape, item_size) else: outer_chunks = normalize_chunks_nd(_raw_chunks, shape) chunk_grid = create_chunk_grid_metadata(outer_chunks) result = await cls._create_v3( store_path, shape=shape, dtype=dtype_parsed, fill_value=fill_value, chunk_key_encoding=chunk_key_encoding, codecs=codecs, dimension_names=dimension_names, attributes=attributes, overwrite=overwrite, config=config_parsed, chunk_grid=chunk_grid, ) elif zarr_format == 2: if codecs is not None: raise ValueError( "codecs cannot be used for arrays with zarr_format 2. Use filters and compressor instead." ) if chunk_key_encoding is not None: raise ValueError( "chunk_key_encoding cannot be used for arrays with zarr_format 2. Use dimension_separator instead." ) if dimension_names is not None: raise ValueError("dimension_names cannot be used for arrays with zarr_format 2.") item_size = 1 if isinstance(dtype_parsed, HasItemSize): item_size = dtype_parsed.item_size _raw_v2 = chunks if _v2_chunks_given(chunks) else chunk_shape if _raw_v2 is None: outer_chunks = guess_chunks(shape, item_size) else: outer_chunks = normalize_chunks_nd(_raw_v2, shape) if not outer_chunks.is_regular: raise ValueError("Zarr format 2 does not support rectilinear chunk grids.") _chunks = outer_chunks.chunk_shape if order is None: order_parsed = config_parsed.order else: order_parsed = order config_parsed = replace(config_parsed, order=order) result = await cls._create_v2( store_path, shape=shape, dtype=dtype_parsed, chunks=_chunks, dimension_separator=dimension_separator, fill_value=fill_value, order=order_parsed, config=config_parsed, filters=filters, compressor=compressor, attributes=attributes, overwrite=overwrite, ) else: raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover if data is not None: # insert user-provided data await result.setitem(..., data) return result @staticmethod def _create_metadata_v3( shape: ShapeLike, dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGridMetadata, fill_value: Any | None = DEFAULT_FILL_VALUE, chunk_key_encoding: ChunkKeyEncodingLike | None = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: DimensionNamesLike = None, attributes: dict[str, JSON] | None = None, ) -> ArrayV3Metadata: """Create an instance of ArrayV3Metadata.""" filters: tuple[ArrayArrayCodec, ...] compressors: tuple[BytesBytesCodec, ...] shape = parse_shapelike(shape) if codecs is None: # no data types have default filters filters = () serializer = default_serializer_v3(dtype) compressors = default_compressors_v3(dtype) codecs_parsed = (*filters, serializer, *compressors) else: codecs_parsed = tuple(codecs) chunk_key_encoding_parsed: ChunkKeyEncodingLike if chunk_key_encoding is None: chunk_key_encoding_parsed = {"name": "default", "separator": "/"} else: chunk_key_encoding_parsed = chunk_key_encoding if fill_value is DEFAULT_FILL_VALUE or fill_value is None: # Use dtype's default scalar for the DEFAULT_FILL_VALUE sentinel # For v3, None is converted to DEFAULT_FILL_VALUE behavior fill_value_parsed = dtype.default_scalar() else: fill_value_parsed = fill_value return ArrayV3Metadata( shape=shape, data_type=dtype, chunk_grid=chunk_grid, chunk_key_encoding=chunk_key_encoding_parsed, fill_value=fill_value_parsed, codecs=codecs_parsed, # type: ignore[arg-type] dimension_names=tuple(dimension_names) if dimension_names else None, attributes=attributes or {}, ) @classmethod async def _create_v3( cls, store_path: StorePath, *, shape: ShapeLike, dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGridMetadata, config: ArrayConfig, fill_value: Any | None = DEFAULT_FILL_VALUE, chunk_key_encoding: ( ChunkKeyEncodingLike | tuple[Literal["default"], Literal[".", "/"]] | tuple[Literal["v2"], Literal[".", "/"]] | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: DimensionNamesLike = None, attributes: dict[str, JSON] | None = None, overwrite: bool = False, ) -> AsyncArrayV3: if isinstance(chunk_key_encoding, tuple): chunk_key_encoding = ( V2ChunkKeyEncoding(separator=chunk_key_encoding[1]) if chunk_key_encoding[0] == "v2" else DefaultChunkKeyEncoding(separator=chunk_key_encoding[1]) ) metadata = cls._create_metadata_v3( shape=shape, dtype=dtype, chunk_grid=chunk_grid, fill_value=fill_value, chunk_key_encoding=chunk_key_encoding, codecs=codecs, dimension_names=dimension_names, attributes=attributes, ) array = cls(metadata=metadata, store_path=store_path, config=config) await save_new_metadata(store_path, metadata, overwrite=overwrite, ensure_parents=True) return array @staticmethod def _create_metadata_v2( shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunks: tuple[int, ...], order: MemoryOrder, dimension_separator: Literal[".", "/"] | None = None, fill_value: Any | None = DEFAULT_FILL_VALUE, filters: Iterable[dict[str, JSON] | Numcodec] | None = None, compressor: CompressorLikev2 = None, attributes: dict[str, JSON] | None = None, ) -> ArrayV2Metadata: if dimension_separator is None: dimension_separator = "." # Handle the DEFAULT_FILL_VALUE sentinel if fill_value is DEFAULT_FILL_VALUE: fill_value_parsed: Any = dtype.default_scalar() else: # For v2, preserve None as-is (backward compatibility) fill_value_parsed = fill_value return ArrayV2Metadata( shape=shape, dtype=dtype, chunks=chunks, order=order, dimension_separator=dimension_separator, fill_value=fill_value_parsed, compressor=compressor, filters=filters, attributes=attributes, ) @classmethod async def _create_v2( cls, store_path: StorePath, *, shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunks: tuple[int, ...], order: MemoryOrder, config: ArrayConfig, dimension_separator: Literal[".", "/"] | None = None, fill_value: Any | None = DEFAULT_FILL_VALUE, filters: Iterable[dict[str, JSON] | Numcodec] | None = None, compressor: CompressorLike = "auto", attributes: dict[str, JSON] | None = None, overwrite: bool = False, ) -> AsyncArrayV2: compressor_parsed: CompressorLikev2 if compressor == "auto": compressor_parsed = default_compressor_v2(dtype) elif isinstance(compressor, BytesBytesCodec): raise ValueError( "Cannot use a BytesBytesCodec as a compressor for zarr v2 arrays. " "Use a numcodecs codec directly instead." ) else: compressor_parsed = compressor if filters is None: filters = default_filters_v2(dtype) metadata = cls._create_metadata_v2( shape=shape, dtype=dtype, chunks=chunks, order=order, dimension_separator=dimension_separator, fill_value=fill_value, filters=filters, compressor=compressor_parsed, attributes=attributes, ) array = cls(metadata=metadata, store_path=store_path, config=config) await save_new_metadata(store_path, metadata, overwrite=overwrite, ensure_parents=True) return array @classmethod def from_dict( cls, store_path: StorePath, data: dict[str, JSON], ) -> AnyAsyncArray: """ Create a Zarr array from a dictionary, with support for both Zarr format 2 and 3 metadata. Parameters ---------- store_path : StorePath The path within the store where the array should be created. data : dict A dictionary representing the array data. This dictionary should include necessary metadata for the array, such as shape, dtype, and other attributes. The format of the metadata will determine whether a Zarr format 2 or 3 array is created. Returns ------- AsyncArrayV3 or AsyncArrayV2 The created Zarr array, either using Zarr format 2 or 3 metadata based on the provided data. Raises ------ ValueError If the dictionary data is invalid or incompatible with either Zarr format 2 or 3 array creation. """ metadata = parse_array_metadata(data, str(store_path)) return cls(metadata=metadata, store_path=store_path) @classmethod async def open( cls, store: StoreLike, zarr_format: ZarrFormat | None = 3, *, config: ArrayConfigLike | None = None, ) -> AnyAsyncArray: """ Async method to open an existing Zarr array from a given store. Parameters ---------- store : StoreLike The store containing the Zarr array. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. zarr_format : ZarrFormat | None, optional The Zarr format version (default is 3). config : ArrayConfigLike or None, default=None Runtime configuration for the opened array. Keys not specified are taken from the global configuration. Returns ------- AsyncArray The opened Zarr array. Examples -------- ```python import asyncio import zarr from zarr.core.array import AsyncArray async def example(): store = zarr.storage.MemoryStore() # First create an array to open await zarr.api.asynchronous.create_array( store=store, shape=(100, 100), dtype="int32" ) # Now open it async_arr = await AsyncArray.open(store) return async_arr async_arr = asyncio.run(example()) # ``` """ store_path = await make_store_path(store) metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format) # TODO: remove this cast when we have better type hints _metadata_dict = cast("ArrayMetadataJSON_V3", metadata_dict) return cls(store_path=store_path, metadata=_metadata_dict, config=config) @property def store(self) -> Store: return self.store_path.store @property @deprecated("Use AsyncArray.config instead.", category=ZarrDeprecationWarning) def _config(self) -> ArrayConfig: return self.config @property def ndim(self) -> int: """Returns the number of dimensions in the Array. Returns ------- int The number of dimensions in the Array. """ return len(self.metadata.shape) @property def shape(self) -> tuple[int, ...]: """Returns the shape of the Array. Returns ------- tuple The shape of the Array. """ return self.metadata.shape @property def chunks(self) -> tuple[int, ...]: """Returns the chunk shape of the Array. If sharding is used the inner chunk shape is returned, which is defined for any chunk grid (inner chunks are always regular). Otherwise, only defined for arrays using a regular chunk grid: for a non-sharded array with a rectilinear chunk grid, `NotImplementedError` is raised. Use `read_chunk_sizes` for general access. Returns ------- tuple[int, ...]: The chunk shape of the Array. """ # TODO: move sharding awareness out of metadata return self.metadata.chunks @property def read_chunk_sizes(self) -> tuple[tuple[int, ...], ...]: """Per-dimension data sizes of chunks used for reading, clipped to the array extent. Boundary chunks that extend past the array shape are clipped, so the last size along a dimension may be smaller than the declared chunk size. This matches the dask `Array.chunks` convention. When sharding is used, returns the inner chunk sizes. Otherwise, returns the outer chunk sizes (same as `write_chunk_sizes`). Returns ------- tuple[tuple[int, ...], ...] One inner tuple per dimension containing the data size of each chunk (not the encoded buffer size). Examples -------- Without sharding, `read_chunk_sizes` and `write_chunk_sizes` are the same: >>> arr = zarr.create_array({}, dtype="i1", shape=(100, 80), chunks=(30, 40)) >>> arr.read_chunk_sizes ((30, 30, 30, 10), (40, 40)) For a sharded array the two differ: reads are efficient at inner-chunk granularity, while writes go to storage one shard at a time: >>> sharded = zarr.create_array({}, dtype="i1", shape=(40,), chunks=(10,), shards=(20,)) >>> sharded.read_chunk_sizes ((10, 10, 10, 10),) >>> sharded.write_chunk_sizes ((20, 20),) """ if (sharding_codec := _sharding_codec(self.metadata)) is not None: return _chunk_sizes_from_shape(self.shape, sharding_codec.chunk_shape) return self._chunk_grid.chunk_sizes @property def write_chunk_sizes(self) -> tuple[tuple[int, ...], ...]: """Per-dimension data sizes of storage chunks, clipped to the array extent. Always returns the outer chunk sizes, regardless of sharding. Boundary chunks that extend past the array shape are clipped, so the last size along a dimension may be smaller than the declared chunk size. This matches the dask `Array.chunks` convention. Returns ------- tuple[tuple[int, ...], ...] One inner tuple per dimension containing the data size of each chunk (not the encoded buffer size). Examples -------- Without sharding, `write_chunk_sizes` and `read_chunk_sizes` are the same: >>> arr = zarr.create_array({}, dtype="i1", shape=(100, 80), chunks=(30, 40)) >>> arr.write_chunk_sizes ((30, 30, 30, 10), (40, 40)) For a sharded array the two differ: writes go to storage one shard at a time, while reads are efficient at inner-chunk granularity: >>> sharded = zarr.create_array({}, dtype="i1", shape=(40,), chunks=(10,), shards=(20,)) >>> sharded.write_chunk_sizes ((20, 20),) >>> sharded.read_chunk_sizes ((10, 10, 10, 10),) """ return self._chunk_grid.chunk_sizes @property def shards(self) -> tuple[int, ...] | None: """Returns the shard shape of the Array. Returns None if sharding is not used. Only defined when the shard grid is regular: for a sharded array with a rectilinear chunk grid, `NotImplementedError` is raised. Use `write_chunk_sizes` for general access. Returns ------- tuple[int, ...]: The shard shape of the Array. """ return self.metadata.shards @property def size(self) -> int: """Returns the total number of elements in the array Returns ------- int Total number of elements in the array """ return math.prod(self.metadata.shape) @property def filters(self) -> tuple[Numcodec, ...] | tuple[ArrayArrayCodec, ...]: """ Filters that are applied to each chunk of the array, in order, before serializing that chunk to bytes. """ if self.metadata.zarr_format == 2: filters = self.metadata.filters if filters is None: return () return filters return tuple( codec for codec in self.metadata.inner_codecs if isinstance(codec, ArrayArrayCodec) ) @property def serializer(self) -> ArrayBytesCodec | None: """ Array-to-bytes codec to use for serializing the chunks into bytes. """ if self.metadata.zarr_format == 2: return None return next( codec for codec in self.metadata.inner_codecs if isinstance(codec, ArrayBytesCodec) ) @property @deprecated("Use AsyncArray.compressors instead.", category=ZarrDeprecationWarning) def compressor(self) -> Numcodec | None: """ Compressor that is applied to each chunk of the array. !!! warning "Deprecated" `Array.compressor` is deprecated since v3.0.0 and will be removed in a future release. Use [`Array.compressors`][zarr.AsyncArray.compressors] instead. """ if self.metadata.zarr_format == 2: return self.metadata.compressor raise TypeError("`compressor` is not available for Zarr format 3 arrays.") @property def compressors(self) -> tuple[Numcodec, ...] | tuple[BytesBytesCodec, ...]: """ Compressors that are applied to each chunk of the array. Compressors are applied in order, and after any filters are applied (if any are specified) and the data is serialized into bytes. """ if self.metadata.zarr_format == 2: if self.metadata.compressor is not None: return (self.metadata.compressor,) return () return tuple( codec for codec in self.metadata.inner_codecs if isinstance(codec, BytesBytesCodec) ) @property def _zdtype(self) -> ZDType[TBaseDType, TBaseScalar]: """ The zarr-specific representation of the array data type """ # `dtype` returns the zarr dtype object for both v2 and v3 metadata # (on v3 it is an alias for `data_type`). return self.metadata.dtype @property def dtype(self) -> TBaseDType: """Returns the data type of the array. Returns ------- np.dtype Data type of the array """ return self._zdtype.to_native_dtype() @property def order(self) -> MemoryOrder: """Returns the memory order of the array. Returns ------- bool Memory order of the array """ if self.metadata.zarr_format == 2: return self.metadata.order else: return self.config.order @property def attrs(self) -> dict[str, JSON]: """Returns the attributes of the array. Returns ------- dict Attributes of the array """ return self.metadata.attributes @property def read_only(self) -> bool: """Returns True if the array is read-only. Returns ------- bool True if the array is read-only """ # Backwards compatibility for 2.x return self.store_path.read_only @property def path(self) -> str: """Storage path. Returns ------- str The path to the array in the Zarr store. """ return self.store_path.path @property def name(self) -> str: """Array name following h5py convention. Returns ------- str The name of the array. """ # follow h5py convention: add leading slash name = self.path if not name.startswith("/"): name = "/" + name return name @property def basename(self) -> str: """Final component of name. Returns ------- str The basename or final component of the array name. """ return self.name.split("/")[-1] @property def cdata_shape(self) -> tuple[int, ...]: """ The number of chunks along each dimension. When sharding is used, this counts inner chunks (not shards) per dimension. Returns ------- tuple[int, ...] The number of chunks along each dimension. """ return self._chunk_grid_shape @property def _chunk_grid_shape(self) -> tuple[int, ...]: """ The number of chunks along each dimension. When sharding is used, this counts inner chunks (not shards) per dimension. Returns ------- tuple[int, ...] The number of chunks along each dimension. """ if (sharding_codec := _sharding_codec(self.metadata)) is not None: # When sharding, count inner chunks across the whole array chunk_shape = sharding_codec.chunk_shape return tuple(starmap(ceildiv_int, zip(self.shape, chunk_shape, strict=True))) return self._chunk_grid.grid_shape @property def _shard_grid_shape(self) -> tuple[int, ...]: """ The shape of the shard grid for this array. When no shards are present this will automatically fall back to the chunk grid. Returns ------- tuple[int, ...] The shape of the shard grid for this array. """ # The stored chunk grid is the shard grid when sharding is used, the # chunk grid otherwise. Works for regular and rectilinear grids alike. return self._chunk_grid.grid_shape @property def nchunks(self) -> int: """ The number of chunks in this array. Note that if a sharding codec is used, then the number of chunks may exceed the number of stored objects supporting this array. Returns ------- int The total number of chunks in the array. """ return product(self._chunk_grid_shape) @property def _nshards(self) -> int: """ The number of shards in this array. If no shards are present this will fall back to giving the number of chunks Returns ------- int The total number of shards or if absent, chunks in the array. """ return product(self._shard_grid_shape) @overload def with_config(self: AsyncArrayV2, config: ArrayConfigLike) -> AsyncArrayV2: ... @overload def with_config(self: AsyncArrayV3, config: ArrayConfigLike) -> AsyncArrayV3: ... def with_config(self, config: ArrayConfigLike) -> Self: """ Return a copy of this Array with a new runtime configuration. Parameters ---------- config : ArrayConfigLike The runtime config for the new Array. Any keys not specified will be inherited from the current array's config. Returns ------- A new Array """ if isinstance(config, ArrayConfig): new_config = config else: # Merge new config with existing config, so missing keys are inherited # from the current array rather than from global defaults new_config = ArrayConfig(**{**self.config.to_dict(), **config}) # type: ignore[arg-type] return type(self)(metadata=self.metadata, store_path=self.store_path, config=new_config) async def nchunks_initialized(self) -> int: """ Calculate the number of chunks that have been initialized in storage. This value is calculated as the sum of the number of chunks in every initialized shard (shard sizes can vary when the shard grid is rectilinear). Each initialized shard is counted at its full declared size, so for an edge shard that extends beyond the array extent this includes chunks that lie entirely outside the array, and the result can exceed `nchunks`. For arrays that do not use sharding, each stored object holds one chunk, so the number of chunks initialized is the same as the number of stored objects associated with an array. Returns ------- nchunks_initialized : int The number of chunks that have been initialized. Notes ----- On [`AsyncArray`][zarr.AsyncArray] this is an asynchronous method, unlike the (synchronous) property [`Array.nchunks_initialized`][zarr.Array.nchunks_initialized]. Examples -------- ```python import asyncio import zarr.api.asynchronous async def example(): arr = await zarr.api.asynchronous.create(shape=(10,), chunks=(1,)) count = await arr.nchunks_initialized() print(f"Initial: {count}") #> Initial: 0 await arr.setitem(slice(5), 1) count = await arr.nchunks_initialized() print(f"After write: {count}") #> After write: 5 return count result = asyncio.run(example()) ``` """ return await _nchunks_initialized(self) async def _nshards_initialized(self) -> int: """ Calculate the number of shards that have been initialized in storage. This is the number of shards that have been persisted to the storage backend. Returns ------- nshards_initialized : int The number of shards that have been initialized. Notes ----- On [`AsyncArray`][zarr.AsyncArray] this is an asynchronous method, unlike the (synchronous) property [`Array._nshards_initialized`][zarr.Array._nshards_initialized]. Examples -------- ```python import asyncio import zarr.api.asynchronous async def example(): arr = await zarr.api.asynchronous.create(shape=(10,), chunks=(2,)) count = await arr._nshards_initialized() print(f"Initial: {count}") #> Initial: 0 await arr.setitem(slice(5), 1) count = await arr._nshards_initialized() print(f"After write: {count}") #> After write: 3 return count result = asyncio.run(example()) ``` """ return await _nshards_initialized(self) async def nbytes_stored(self) -> int: return await _nbytes_stored(self.store_path) def _iter_chunk_coords( self, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None ) -> Iterator[tuple[int, ...]]: """ Create an iterator over the coordinates of chunks in chunk grid space. If the `origin` keyword is used, iteration will start at the chunk index specified by `origin`. The default behavior is to start at the origin of the grid coordinate space. If the `selection_shape` keyword is used, iteration will be bounded over a contiguous region ranging from `[origin, origin selection_shape]`, where the upper bound is exclusive as per python indexing conventions. Parameters ---------- origin : Sequence[int] | None, default=None The origin of the selection relative to the array's chunk grid. selection_shape : Sequence[int] | None, default=None The shape of the selection in chunk grid coordinates. Yields ------ chunk_coords: tuple[int, ...] The coordinates of each chunk in the selection. """ return _iter_chunk_coords( array=self, origin=origin, selection_shape=selection_shape, ) def _iter_shard_coords( self, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None ) -> Iterator[tuple[int, ...]]: """ Create an iterator over the coordinates of shards in shard grid space. This will fall back to chunk grid space in case no shards are present. Note that If the `origin` keyword is used, iteration will start at the shard index specified by `origin`. The default behavior is to start at the origin of the grid coordinate space. If the `selection_shape` keyword is used, iteration will be bounded over a contiguous region ranging from `[origin, origin selection_shape]`, where the upper bound is exclusive as per python indexing conventions. Parameters ---------- origin : Sequence[int] | None, default=None The origin of the selection relative to the array's shard grid. selection_shape : Sequence[int] | None, default=None The shape of the selection in shard grid coordinates. Yields ------ chunk_coords: tuple[int, ...] The coordinates of each shard in the selection or chunk in case of no shard being present. """ return _iter_shard_coords( array=self, origin=origin, selection_shape=selection_shape, ) def _iter_shard_keys( self, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None ) -> Iterator[str]: """ Iterate over the keys of the stored objects supporting this array. Although only stored objects, e.g. shards should have keys, in case no shards are present this automatically falls back to chunks. Parameters ---------- origin : Sequence[int] | None, default=None The origin of the selection relative to the array's chunk grid. selection_shape : Sequence[int] | None, default=None The shape of the selection in shard grid coordinates. Yields ------ key: str The storage key of each shard in the selection or in case of no shard present of each chunk although the latter case is technically incorrect. """ # Iterate over the coordinates of chunks in chunk grid space. return _iter_shard_keys( array=self, origin=origin, selection_shape=selection_shape, ) def _iter_chunk_regions( self, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None ) -> Iterator[tuple[slice, ...]]: """ Iterate over the regions spanned by each chunk. Parameters ---------- origin : Sequence[int] | None, default=None The origin of the selection relative to the array's chunk grid. selection_shape : Sequence[int] | None, default=None The shape of the selection in chunk grid coordinates. Yields ------ region: tuple[slice, ...] A tuple of slice objects representing the region spanned by each chunk in the selection. """ return _iter_chunk_regions( array=self, origin=origin, selection_shape=selection_shape, ) def _iter_shard_regions( self, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None ) -> Iterator[tuple[slice, ...]]: """ Iterate over the regions spanned by each shard. This will automatically fall back to chunks if no shards are present. Parameters ---------- origin : Sequence[int] | None, default=None The origin of the selection relative to the array's shard grid. selection_shape : Sequence[int] | None, default=None The shape of the selection in shard grid coordinates. Yields ------ region: tuple[slice, ...] A tuple of slice objects representing the region spanned by each shard in the selection or chunk in the absence of shards. """ return _iter_shard_regions(array=self, origin=origin, selection_shape=selection_shape) @property def nbytes(self) -> int: """ The total number of bytes that can be stored in the chunks of this array. Notes ----- This value is calculated by multiplying the number of elements in the array and the size of each element, the latter of which is determined by the dtype of the array. For this reason, `nbytes` will likely be inaccurate for arrays with variable-length dtypes. It is not possible to determine the size of an array with variable-length elements from the shape and dtype alone. """ return self.size * self.dtype.itemsize async def _get_selection( self, indexer: Indexer, *, prototype: BufferPrototype, out: NDBuffer | None = None, fields: Fields | None = None, ) -> NDArrayLikeOrScalar: return await _get_selection( self.store_path, self.metadata, self.codec_pipeline, self.config, self._chunk_grid, indexer, prototype=prototype, out=out, fields=fields, ) async def getitem( self, selection: BasicSelection, *, prototype: BufferPrototype | None = None, ) -> NDArrayLikeOrScalar: """ Asynchronous function that retrieves a subset of the array's data based on the provided selection. Parameters ---------- selection : BasicSelection A selection object specifying the subset of data to retrieve. prototype : BufferPrototype, optional A buffer prototype to use for the retrieved data (default is None). Returns ------- NDArrayLikeOrScalar The retrieved subset of the array's data. Examples -------- >>> async def example(): ... import zarr.api.asynchronous ... import zarr.storage ... ... async_arr = await zarr.api.asynchronous.create_array( ... store={}, ... shape=(100,100), ... chunks=(10,10), ... dtype="i4", ... fill_value=0, ... ) ... ... return await async_arr.getitem((0, 1)) >>> import asyncio >>> asyncio.run(example()) np.int32(0) """ return await _getitem( self.store_path, self.metadata, self.codec_pipeline, self.config, self._chunk_grid, selection, prototype=prototype, ) async def get_orthogonal_selection( self, selection: OrthogonalSelection, *, out: NDBuffer | None = None, fields: Fields | None = None, prototype: BufferPrototype | None = None, ) -> NDArrayLikeOrScalar: if prototype is None: prototype = default_buffer_prototype() indexer = OrthogonalIndexer(selection, self.metadata.shape, self._chunk_grid) return await _get_selection( self.store_path, self.metadata, self.codec_pipeline, self.config, self._chunk_grid, indexer=indexer, out=out, fields=fields, prototype=prototype, ) async def get_mask_selection( self, mask: MaskSelection, *, out: NDBuffer | None = None, fields: Fields | None = None, prototype: BufferPrototype | None = None, ) -> NDArrayLikeOrScalar: if prototype is None: prototype = default_buffer_prototype() indexer = MaskIndexer(mask, self.metadata.shape, self._chunk_grid) return await _get_selection( self.store_path, self.metadata, self.codec_pipeline, self.config, self._chunk_grid, indexer=indexer, out=out, fields=fields, prototype=prototype, ) async def get_coordinate_selection( self, selection: CoordinateSelection, *, out: NDBuffer | None = None, fields: Fields | None = None, prototype: BufferPrototype | None = None, ) -> NDArrayLikeOrScalar: if prototype is None: prototype = default_buffer_prototype() indexer = CoordinateIndexer(selection, self.metadata.shape, self._chunk_grid) out_array = await _get_selection( self.store_path, self.metadata, self.codec_pipeline, self.config, self._chunk_grid, indexer=indexer, out=out, fields=fields, prototype=prototype, ) if hasattr(out_array, "shape"): # restore shape out_array = cast("NDArrayLikeOrScalar", np.array(out_array).reshape(indexer.sel_shape)) return out_array async def _save_metadata(self, metadata: ArrayMetadata, ensure_parents: bool = False) -> None: """Store `metadata` as this array's own documents, then clear the `_stored_document` mark (see `_stored_document_replaced`). `_resize` stores the documents it encoded before deleting chunks directly, then clears the mark the same way.""" await save_metadata(self.store_path, metadata, ensure_parents=ensure_parents) self._stored_document_replaced() def _stored_document_replaced(self) -> None: """Record that the store no longer holds a document of this array that needs an repair: it holds the repair, a valid document, or none. The metadata this handle holds, which a consolidated group handle may share, then stops standing for the document it was read from (see `mark_repaired`), so no later write through either handle stores that document again.""" object.__setattr__(self.metadata, "_stored_document", None) async def _store_repaired_document(self) -> None: """Store the repair of this array's current stored document, if it needs one, before chunks are written under this handle's metadata. Only for metadata read from a document whose repair moves chunks (see `zarr.core.metadata.repair`). The document is read again, because the store may hold a newer one than this handle's metadata. If that one lays out chunks differently (another writer stored a different chunk size since), this handle would write chunks no reader finds, so it raises and stores nothing. If it needs no repair (the array was re-saved since, possibly by another implementation), it is left as written; if there is none, there is nothing to repair. Storing the same repair twice is harmless, so handles that write chunks concurrently need no coordination. The read and the store are not one atomic step, though: a metadata write by another handle between them (a resize, an attribute update) is replaced by the repair of the document read before it, as with any two metadata writes that race. """ if self.metadata._stored_document is None: return zarr_format = self.metadata.zarr_format documents = await read_documents(self.store_path, ARRAY_DOCUMENTS[zarr_format]) try: current = parse_stored_array(documents, zarr_format, str(self.store_path)) except ArrayNotFoundError: pass else: if _chunk_layout(current) != _chunk_layout(self.metadata): raise ValueError( f"Array {str(self.store_path)!r}: the metadata stored has changed since " "this array was opened; reopen the array to write to it. Nothing was stored." ) if current._stored_document is not None: await upsert_metadata(self.store_path, current, documents) self._stored_document_replaced() async def _set_selection( self, indexer: Indexer, value: npt.ArrayLike, *, prototype: BufferPrototype, fields: Fields | None = None, ) -> None: if product(indexer.shape) > 0: # Chunks are about to be stored under the repaired metadata, so store it # first: every reader of the store then agrees with them. await self._store_repaired_document() return await _set_selection( self.store_path, self.metadata, self.codec_pipeline, self.config, self._chunk_grid, indexer, value, prototype=prototype, fields=fields, ) async def setitem( self, selection: BasicSelection, value: npt.ArrayLike, prototype: BufferPrototype | None = None, ) -> None: """ Asynchronously set values in the array using basic indexing. Parameters ---------- selection : BasicSelection The selection defining the region of the array to set. value : numpy.typing.ArrayLike The values to be written into the selected region of the array. prototype : BufferPrototype or None, optional A prototype buffer that defines the structure and properties of the array chunks being modified. If None, the default buffer prototype is used. Default is None. Returns ------- None This method does not return any value. Raises ------ IndexError If the selection is out of bounds for the array. ValueError If the values are not compatible with the array's dtype or shape. Notes ----- - This method is asynchronous and should be awaited. - Supports basic indexing, where the selection is contiguous and does not involve advanced indexing. """ if prototype is None: prototype = default_buffer_prototype() indexer = BasicIndexer(selection, shape=self.metadata.shape, chunk_grid=self._chunk_grid) return await self._set_selection(indexer, value, prototype=prototype) @property def oindex(self) -> AsyncOIndex[T_ArrayMetadata]: """Shortcut for orthogonal (outer) indexing, see [get_orthogonal_selection][zarr.Array.get_orthogonal_selection] and [set_orthogonal_selection][zarr.Array.set_orthogonal_selection] for documentation and examples.""" return AsyncOIndex(self) @property def vindex(self) -> AsyncVIndex[T_ArrayMetadata]: """Shortcut for vectorized (inner) indexing, see [get_coordinate_selection][zarr.Array.get_coordinate_selection], [set_coordinate_selection][zarr.Array.set_coordinate_selection], [get_mask_selection][zarr.Array.get_mask_selection] and [set_mask_selection][zarr.Array.set_mask_selection] for documentation and examples.""" return AsyncVIndex(self) async def resize(self, new_shape: ShapeLike, delete_outside_chunks: bool = True) -> None: """ Asynchronously resize the array to a new shape. Parameters ---------- new_shape : tuple[int, ...] The desired new shape of the array. delete_outside_chunks : bool, optional If True (default), chunks that fall outside the new shape will be deleted. If False, the data in those chunks will be preserved. Returns ------- AsyncArray The resized array. Raises ------ ValueError If the new shape is incompatible with the current array's chunking configuration. Notes ----- - This method is asynchronous and should be awaited. """ return await _resize(self, new_shape, delete_outside_chunks) async def append(self, data: npt.ArrayLike, axis: int = 0) -> tuple[int, ...]: """Append `data` to `axis`. Parameters ---------- data : array-like Data to be appended. axis : int Axis along which to append. Returns ------- new_shape : tuple Notes ----- The size of all dimensions other than `axis` must match between this array and `data`. """ return await _append(self, data, axis) async def update_attributes(self, new_attributes: dict[str, JSON]) -> Self: """ Asynchronously update the array's attributes. Parameters ---------- new_attributes : dict of str to JSON A dictionary of new attributes to update or add to the array. The keys represent attribute names, and the values must be JSON-compatible. Returns ------- AsyncArray The array with the updated attributes. Raises ------ ValueError If the attributes are invalid or incompatible with the array's metadata. Notes ----- - This method is asynchronous and should be awaited. - The updated attributes will be merged with existing attributes, and any conflicts will be overwritten by the new values. """ await _update_attributes(self, new_attributes) return self def __repr__(self) -> str: return f"" @property def info(self) -> Any: """ Return the statically known information for an array. Returns ------- ArrayInfo Related ------- [zarr.AsyncArray.info_complete][] - All information about a group, including dynamic information like the number of bytes and chunks written. Examples -------- >>> import asyncio >>> arr = asyncio.run( ... zarr.api.asynchronous.create( ... path="array", shape=(3, 4, 5), chunks=(2, 2, 2) ... ) ... ) >>> arr.info Type : Array Zarr format : 3 Data type : Float64(endianness=...) Fill value : 0.0 Shape : (3, 4, 5) Chunk shape : (2, 2, 2) Order : C Read-only : False Store type : MemoryStore Filters : () Serializer : BytesCodec(endian='little') Compressors : (ZstdCodec(level=0, checksum=False),) No. bytes : 480 """ return self._info() async def info_complete(self) -> Any: """ Return all the information for an array, including dynamic information like a storage size. In addition to the static information, this provides - The count of chunks initialized - The sum of the bytes written Returns ------- ArrayInfo Related ------- [zarr.AsyncArray.info][] - A property giving just the statically known information about an array. """ return await _info_complete(self) def _info( self, count_chunks_initialized: int | None = None, count_bytes_stored: int | None = None ) -> Any: rectilinear_grid = _stored_rectilinear_grid_or_none(self.metadata) sharded = _sharding_codec(self.metadata) is not None # `.chunks` (the inner chunk shape when sharded) is undefined only for a # non-sharded rectilinear grid, which ArrayInfo renders as ""; # `.shards` is undefined for a rectilinear shard grid, where the # "" sentinel keeps the array rendered as sharded. chunk_shape = self.chunks if (rectilinear_grid is None or sharded) else None shard_shape: tuple[int, ...] | Literal[""] | None if rectilinear_grid is None: shard_shape = self.shards elif sharded: shard_shape = "" else: shard_shape = None return ArrayInfo( _zarr_format=self.metadata.zarr_format, _data_type=self._zdtype, _fill_value=self.metadata.fill_value, _shape=self.shape, _order=self.order, _shard_shape=shard_shape, _chunk_shape=chunk_shape, _read_only=self.read_only, _compressors=self.compressors, _filters=self.filters, _serializer=self.serializer, _store_type=type(self.store_path.store).__name__, _count_bytes=self.nbytes, _count_bytes_stored=count_bytes_stored, _count_chunks_initialized=count_chunks_initialized, ) # TODO: Array can be a frozen data class again once property setters (e.g. shape) are removed @dataclass(frozen=False) class Array[T_ArrayMetadata: (ArrayV2Metadata, ArrayV3Metadata)]: """ A Zarr array. """ _async_array: AsyncArray[T_ArrayMetadata] @property def async_array(self) -> AsyncArray[T_ArrayMetadata]: """An asynchronous version of the current array. Useful for batching requests. Returns ------- An asynchronous array whose metadata + store matches that of this synchronous array. """ return self._async_array @property def config(self) -> ArrayConfig: """ The runtime configuration for this array. This is a read-only property. To modify the runtime configuration, use `Array.with_config` to create a new `Array` with the modified configuration. Returns ------- An `ArrayConfig` object that defines the runtime configuration for the array. """ return self.async_array.config @property def _chunk_grid(self) -> ChunkGrid: """The chunk grid for this array, bound to the array's shape.""" return self.async_array._chunk_grid @classmethod def _create( cls, store: StoreLike, *, # v2 and v3 shape: tuple[int, ...], dtype: ZDTypeLike, zarr_format: ZarrFormat = 3, fill_value: Any | None = DEFAULT_FILL_VALUE, attributes: dict[str, JSON] | None = None, # v3 only chunk_shape: tuple[int, ...] | None = None, chunk_key_encoding: ( ChunkKeyEncoding | tuple[Literal["default"], Literal[".", "/"]] | tuple[Literal["v2"], Literal[".", "/"]] | None ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: DimensionNamesLike = None, # v2 only chunks: tuple[int, ...] | None = None, dimension_separator: Literal[".", "/"] | None = None, order: MemoryOrder | None = None, filters: list[dict[str, JSON]] | None = None, compressor: CompressorLike = "auto", # runtime overwrite: bool = False, config: ArrayConfigLike | None = None, ) -> Self: """Creates a new Array instance from an initialized store. Deprecated in favor of [`zarr.create_array`][]. """ async_array = sync( AsyncArray._create( store=store, shape=shape, dtype=dtype, zarr_format=zarr_format, attributes=attributes, fill_value=fill_value, chunk_shape=chunk_shape, chunk_key_encoding=chunk_key_encoding, codecs=codecs, dimension_names=dimension_names, chunks=chunks, dimension_separator=dimension_separator, order=order, filters=filters, compressor=compressor, overwrite=overwrite, config=config, ), ) return cls(async_array) @classmethod def from_dict( cls, store_path: StorePath, data: dict[str, JSON], ) -> Self: """ Create a Zarr array from a dictionary. Parameters ---------- store_path : StorePath The path within the store where the array should be created. data : dict A dictionary representing the array data. This dictionary should include necessary metadata for the array, such as shape, dtype, fill value, and attributes. Returns ------- Array The created Zarr array. Raises ------ ValueError If the dictionary data is invalid or missing required fields for array creation. """ async_array = AsyncArray.from_dict(store_path=store_path, data=data) return cls(async_array) @classmethod def open( cls, store: StoreLike, *, config: ArrayConfigLike | None = None, ) -> Self: """Opens an existing Array from a store. Parameters ---------- store : StoreLike Store containing the Array. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. config : ArrayConfigLike or None, default=None Runtime configuration for the opened array. Keys not specified are taken from the global configuration. Returns ------- Array Array opened from the store. """ async_array = sync(AsyncArray.open(store, config=config)) return cls(async_array) @property def store(self) -> Store: return self.async_array.store @property def ndim(self) -> int: """Returns the number of dimensions in the array. Returns ------- int The number of dimensions in the array. """ return self.async_array.ndim @property def shape(self) -> tuple[int, ...]: """Returns the shape of the array. Returns ------- tuple[int, ...] The shape of the array. """ return self.async_array.shape @shape.setter def shape(self, value: tuple[int, ...]) -> None: """Sets the shape of the array by calling resize.""" self.resize(value) @property def chunks(self) -> tuple[int, ...]: """Returns a tuple of integers describing the length of each dimension of a chunk of the array. If sharding is used the inner chunk shape is returned, which is defined for any chunk grid (inner chunks are always regular). Otherwise, only defined for arrays using a regular chunk grid: for a non-sharded array with a rectilinear chunk grid, `NotImplementedError` is raised. Use `read_chunk_sizes` for general access. Returns ------- tuple A tuple of integers representing the length of each dimension of a chunk. """ return self.async_array.chunks @property def read_chunk_sizes(self) -> tuple[tuple[int, ...], ...]: """Per-dimension data sizes of chunks used for reading, clipped to the array extent. Boundary chunks that extend past the array shape are clipped, so the last size along a dimension may be smaller than the declared chunk size. This matches the dask `Array.chunks` convention. When sharding is used, returns the inner chunk sizes. Otherwise, returns the outer chunk sizes (same as `write_chunk_sizes`). Returns ------- tuple[tuple[int, ...], ...] One inner tuple per dimension containing the data size of each chunk (not the encoded buffer size). Examples -------- Without sharding, `read_chunk_sizes` and `write_chunk_sizes` are the same: >>> import zarr >>> arr = zarr.create_array({}, dtype="i1", shape=(100, 80), chunks=(30, 40)) >>> arr.read_chunk_sizes ((30, 30, 30, 10), (40, 40)) For a sharded array the two differ: reads are efficient at inner-chunk granularity, while writes go to storage one shard at a time: >>> sharded = zarr.create_array({}, dtype="i1", shape=(40,), chunks=(10,), shards=(20,)) >>> sharded.read_chunk_sizes ((10, 10, 10, 10),) >>> sharded.write_chunk_sizes ((20, 20),) """ return self.async_array.read_chunk_sizes @property def write_chunk_sizes(self) -> tuple[tuple[int, ...], ...]: """Per-dimension data sizes of storage chunks, clipped to the array extent. Always returns the outer chunk sizes, regardless of sharding. Boundary chunks that extend past the array shape are clipped, so the last size along a dimension may be smaller than the declared chunk size. This matches the dask `Array.chunks` convention. Returns ------- tuple[tuple[int, ...], ...] One inner tuple per dimension containing the data size of each chunk (not the encoded buffer size). Examples -------- Without sharding, `write_chunk_sizes` and `read_chunk_sizes` are the same: >>> import zarr >>> arr = zarr.create_array({}, dtype="i1", shape=(100, 80), chunks=(30, 40)) >>> arr.write_chunk_sizes ((30, 30, 30, 10), (40, 40)) For a sharded array the two differ: writes go to storage one shard at a time, while reads are efficient at inner-chunk granularity: >>> sharded = zarr.create_array({}, dtype="i1", shape=(40,), chunks=(10,), shards=(20,)) >>> sharded.write_chunk_sizes ((20, 20),) >>> sharded.read_chunk_sizes ((10, 10, 10, 10),) """ return self.async_array.write_chunk_sizes @property def shards(self) -> tuple[int, ...] | None: """Returns a tuple of integers describing the length of each dimension of a shard of the array. Returns None if sharding is not used. Only defined when the shard grid is regular: for a sharded array with a rectilinear chunk grid, `NotImplementedError` is raised. Use `write_chunk_sizes` for general access. Returns ------- tuple | None A tuple of integers representing the length of each dimension of a shard or None if sharding is not used. """ return self.async_array.shards @property def size(self) -> int: """Returns the total number of elements in the array. Returns ------- int Total number of elements in the array. """ return self.async_array.size @property def dtype(self) -> np.dtype[Any]: """Returns the NumPy data type. Returns ------- np.dtype The NumPy data type. """ return self.async_array.dtype @property def attrs(self) -> Attributes: """Returns a [MutableMapping][collections.abc.MutableMapping] containing user-defined attributes. Returns ------- attrs A [MutableMapping][collections.abc.MutableMapping] object containing user-defined attributes. Notes ----- Note that attribute values must be JSON serializable. """ return Attributes(self) @property def path(self) -> str: """Storage path.""" return self.async_array.path @property def name(self) -> str: """Array name following h5py convention.""" return self.async_array.name @property def basename(self) -> str: """Final component of name.""" return self.async_array.basename @property def metadata(self) -> ArrayMetadata: return self.async_array.metadata @property def store_path(self) -> StorePath: return self.async_array.store_path @property def order(self) -> MemoryOrder: return self.async_array.order @property def read_only(self) -> bool: return self.async_array.read_only @property def fill_value(self) -> Any: return self.metadata.fill_value @property def filters(self) -> tuple[Numcodec, ...] | tuple[ArrayArrayCodec, ...]: """ Filters that are applied to each chunk of the array, in order, before serializing that chunk to bytes. """ return self.async_array.filters @property def serializer(self) -> ArrayBytesCodec | None: """ Array-to-bytes codec to use for serializing the chunks into bytes. """ return self.async_array.serializer @property @deprecated("Use Array.compressors instead.", category=ZarrDeprecationWarning) def compressor(self) -> Numcodec | None: """ Compressor that is applied to each chunk of the array. !!! warning "Deprecated" `array.compressor` is deprecated since v3.0.0 and will be removed in a future release. Use [`array.compressors`][zarr.Array.compressors] instead. """ return self.async_array.compressor @property def compressors(self) -> tuple[Numcodec, ...] | tuple[BytesBytesCodec, ...]: """ Compressors that are applied to each chunk of the array. Compressors are applied in order, and after any filters are applied (if any are specified) and the data is serialized into bytes. """ return self.async_array.compressors @property def cdata_shape(self) -> tuple[int, ...]: """ The number of chunks along each dimension. When sharding is used, this counts inner chunks (not shards) per dimension. """ return self._chunk_grid_shape @property def _chunk_grid_shape(self) -> tuple[int, ...]: """ The number of chunks along each dimension. When sharding is used, this counts inner chunks (not shards) per dimension. Returns ------- tuple[int, ...] The number of chunks along each dimension. """ return self.async_array._chunk_grid_shape @property def _shard_grid_shape(self) -> tuple[int, ...]: """ The shape of the shard grid for this array. """ return self.async_array._shard_grid_shape @property def nchunks(self) -> int: """ The number of chunks in this array. Note that if a sharding codec is used, then the number of chunks may exceed the number of stored objects supporting this array. """ return self.async_array.nchunks @property def _nshards(self) -> int: """ The number of shards in the stored representation of this array. """ return self.async_array._nshards @overload def with_config(self: ArrayV2, config: ArrayConfigLike) -> ArrayV2: ... @overload def with_config(self: ArrayV3, config: ArrayConfigLike) -> ArrayV3: ... def with_config(self, config: ArrayConfigLike) -> Self: """ Return a copy of this Array with a new runtime configuration. Parameters ---------- config : ArrayConfigLike The runtime config for the new Array. Any keys not specified will be inherited from the current array's config. Returns ------- A new Array """ return type(self)(self._async_array.with_config(config)) @property def nbytes(self) -> int: """ The total number of bytes that can be stored in the chunks of this array. Notes ----- This value is calculated by multiplying the number of elements in the array and the size of each element, the latter of which is determined by the dtype of the array. For this reason, `nbytes` will likely be inaccurate for arrays with variable-length dtypes. It is not possible to determine the size of an array with variable-length elements from the shape and dtype alone. """ return self.async_array.nbytes @property def nchunks_initialized(self) -> int: """ Calculate the number of chunks that have been initialized in storage. This value is calculated as the sum of the number of chunks in every initialized shard (shard sizes can vary when the shard grid is rectilinear). Each initialized shard is counted at its full declared size, so for an edge shard that extends beyond the array extent this includes chunks that lie entirely outside the array, and the result can exceed `nchunks`. For arrays that do not use sharding, each stored object holds one chunk, so the number of chunks initialized is the same as the number of stored objects associated with an array. For a direct count of the number of initialized stored objects, see `nshards_initialized`. Returns ------- nchunks_initialized : int The number of chunks that have been initialized. Examples -------- >>> arr = zarr.create_array(store={}, dtype="i1", shape=(10,), chunks=(1,), shards=(2,)) >>> arr.nchunks_initialized 0 >>> arr[:5] = 1 >>> arr.nchunks_initialized 6 """ return sync(self.async_array.nchunks_initialized()) @property def _nshards_initialized(self) -> int: """ Calculate the number of shards that have been initialized, i.e. the number of shards that have been persisted to the storage backend. Returns ------- nshards_initialized : int The number of shards that have been initialized. Examples -------- >>> arr = zarr.create(shape=(10,), chunks=(2,)) >>> arr._nshards_initialized 0 >>> arr[:5] = 1 >>> arr._nshards_initialized 3 """ return sync(self.async_array._nshards_initialized()) def nbytes_stored(self) -> int: """ Determine the size, in bytes, of the array actually written to the store. Returns ------- size : int """ return sync(self.async_array.nbytes_stored()) def _iter_shard_keys( self, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None ) -> Iterator[str]: """ Iterate over the storage keys of each shard, relative to an optional origin, and optionally limited to a contiguous region in chunk grid coordinates. If no shards are present this falls back to chunks, though in this case these are then actually not storage keys. Parameters ---------- origin : Sequence[int] | None, default=None The origin of the selection relative to the array's shard grid. selection_shape : Sequence[int] | None, default=None The shape of the selection in shard grid coordinates. Yields ------ str The storage key of each shard in the selection or chunk though chunks technically do not have storage keys. """ return self.async_array._iter_shard_keys(origin=origin, selection_shape=selection_shape) def _iter_chunk_coords( self, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None ) -> Iterator[tuple[int, ...]]: """ Create an iterator over the coordinates of chunks in chunk grid space. If the `origin` keyword is used, iteration will start at the chunk index specified by `origin`. The default behavior is to start at the origin of the grid coordinate space. If the `selection_shape` keyword is used, iteration will be bounded over a contiguous region ranging from `[origin, origin + selection_shape]`, where the upper bound is exclusive as per python indexing conventions. Parameters ---------- origin : Sequence[int] | None, default=None The origin of the selection relative to the array's chunk grid. selection_shape : Sequence[int] | None, default=None The shape of the selection in chunk grid coordinates. Yields ------ tuple[int, ...] The coordinates of each chunk in the selection. """ return self.async_array._iter_chunk_coords(origin=origin, selection_shape=selection_shape) def _iter_shard_coords( self, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None ) -> Iterator[tuple[int, ...]]: """ Create an iterator over the coordinates of shards in shard grid space. If the `origin` keyword is used, iteration will start at the shard index specified by `origin`. The default behavior is to start at the origin of the grid coordinate space. If the `selection_shape` keyword is used, iteration will be bounded over a contiguous region ranging from `[origin, origin selection_shape]`, where the upper bound is exclusive as per python indexing conventions. Parameters ---------- origin : Sequence[int] | None, default=None The origin of the selection relative to the array's shard grid. selection_shape : Sequence[int] | None, default=None The shape of the selection in shard grid coordinates. Yields ------ tuple[int, ...] The coordinates of each shard in the selection. """ return self.async_array._iter_shard_coords(origin=origin, selection_shape=selection_shape) def _iter_chunk_regions( self, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None ) -> Iterator[tuple[slice, ...]]: """ Iterate over the regions spanned by each chunk. Parameters ---------- origin : Sequence[int] | None, default=None The origin of the selection relative to the array's chunk grid. selection_shape : Sequence[int] | None, default=None The shape of the selection in chunk grid coordinates. Yields ------ tuple[slice, ...] A tuple of slice objects representing the region spanned by each chunk in the selection. """ return self.async_array._iter_chunk_regions(origin=origin, selection_shape=selection_shape) def _iter_shard_regions( self, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None ) -> Iterator[tuple[slice, ...]]: """ Iterate over the regions spanned by each shard or chunk if no shard is present. Parameters ---------- origin : Sequence[int] | None, default=None The origin of the selection relative to the array's chunk grid. selection_shape : Sequence[int] | None, default=None The shape of the selection in chunk grid coordinates. Yields ------ tuple[slice, ...] A tuple of slice objects representing the region spanned by each shard or if no shard is present, chunk in the selection. """ return self.async_array._iter_shard_regions(origin=origin, selection_shape=selection_shape) def __array__( self, dtype: npt.DTypeLike | None = None, copy: bool | None = None ) -> NDArrayLike: """ This method is used by numpy when converting zarr.Array into a numpy array. For more information, see https://numpy.org/devdocs/user/basics.interoperability.html#the-array-method """ if copy is False: msg = "`copy=False` is not supported. This method always creates a copy." raise ValueError(msg) arr = self[...] arr_np = np.array(arr, dtype=dtype) if dtype is not None: arr_np = arr_np.astype(dtype) return arr_np def __getitem__(self, selection: Selection) -> NDArrayLikeOrScalar: """Retrieve data for an item or region of the array. Parameters ---------- selection : tuple An integer index or slice or tuple of int/slice objects specifying the requested item or region for each dimension of the array. Returns ------- NDArrayLikeOrScalar An array-like or scalar containing the data for the requested region. Examples -------- Setup a 1-dimensional array: >>> import zarr >>> import numpy as np >>> data = np.arange(100, dtype="uint16") >>> z = zarr.create_array( ... {}, ... shape=data.shape, ... chunks=(10,), ... dtype=data.dtype, ... ) >>> z[:] = data Retrieve a single item: >>> z[5] array(5, dtype=uint16) Retrieve a region via slicing: >>> z[:5] array([0, 1, 2, 3, 4], dtype=uint16) >>> z[-5:] array([95, 96, 97, 98, 99], dtype=uint16) >>> z[5:10] array([5, 6, 7, 8, 9], dtype=uint16) >>> z[5:10:2] array([5, 7, 9], dtype=uint16) >>> z[::2] array([ 0, 2, 4, ..., 94, 96, 98], dtype=uint16) Load the entire array into memory: >>> z[...] array([ 0, 1, 2, ..., 97, 98, 99], dtype=uint16) Setup a 2-dimensional array: >>> data = np.arange(100, dtype="uint16").reshape(10, 10) >>> z = zarr.create_array( ... {}, ... shape=data.shape, ... chunks=(10, 10), ... dtype=data.dtype, ... ) >>> z[:] = data Retrieve an item: >>> z[2, 2] array(22, dtype=uint16) Retrieve a region via slicing: >>> z[1:3, 1:3] array([[11, 12], [21, 22]], dtype=uint16) >>> z[1:3, :] array([[10, 11, 12, 13, 14, 15, 16, 17, 18, 19], [20, 21, 22, 23, 24, 25, 26, 27, 28, 29]], dtype=uint16) >>> z[:, 1:3] array([[ 1, 2], [11, 12], [21, 22], [31, 32], [41, 42], [51, 52], [61, 62], [71, 72], [81, 82], [91, 92]], dtype=uint16) >>> z[0:5:2, 0:5:2] array([[ 0, 2, 4], [20, 22, 24], [40, 42, 44]], dtype=uint16) >>> z[::2, ::2] array([[ 0, 2, 4, 6, 8], [20, 22, 24, 26, 28], [40, 42, 44, 46, 48], [60, 62, 64, 66, 68], [80, 82, 84, 86, 88]], dtype=uint16) Load the entire array into memory: >>> z[...] array([[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], [20, 21, 22, 23, 24, 25, 26, 27, 28, 29], [30, 31, 32, 33, 34, 35, 36, 37, 38, 39], [40, 41, 42, 43, 44, 45, 46, 47, 48, 49], [50, 51, 52, 53, 54, 55, 56, 57, 58, 59], [60, 61, 62, 63, 64, 65, 66, 67, 68, 69], [70, 71, 72, 73, 74, 75, 76, 77, 78, 79], [80, 81, 82, 83, 84, 85, 86, 87, 88, 89], [90, 91, 92, 93, 94, 95, 96, 97, 98, 99]], dtype=uint16) Notes ----- Slices with step > 1 are supported, but slices with negative step are not. For arrays with a structured dtype, see Zarr format 2 for examples of how to use fields Currently the implementation for __getitem__ is provided by [`vindex`][zarr.Array.vindex] if the indexing is pure fancy indexing (ie a broadcast-compatible tuple of integer array indices), or by [`set_basic_selection`][zarr.Array.set_basic_selection] otherwise. Effectively, this means that the following indexing modes are supported: - integer indexing - slice indexing - mixed slice and integer indexing - boolean indexing - fancy indexing (vectorized list of integers) For specific indexing options including outer indexing, see the methods listed under Related. Related ------- [get_basic_selection][zarr.Array.get_basic_selection], [set_basic_selection][zarr.Array.set_basic_selection] [get_mask_selection][zarr.Array.get_mask_selection], [set_mask_selection][zarr.Array.set_mask_selection], [get_coordinate_selection][zarr.Array.get_coordinate_selection], [set_coordinate_selection][zarr.Array.set_coordinate_selection], [get_orthogonal_selection][zarr.Array.get_orthogonal_selection], [set_orthogonal_selection][zarr.Array.set_orthogonal_selection], [get_block_selection][zarr.Array.get_block_selection], [set_block_selection][zarr.Array.set_block_selection], [vindex][zarr.Array.vindex], [oindex][zarr.Array.oindex], [blocks][zarr.Array.blocks], [__setitem__][zarr.Array.__setitem__] """ fields, pure_selection = pop_fields(selection) if is_pure_fancy_indexing(pure_selection, self.ndim): return self.vindex[cast("CoordinateSelection | MaskSelection", selection)] elif is_pure_orthogonal_indexing(pure_selection, self.ndim): return self.get_orthogonal_selection(pure_selection, fields=fields) else: return self.get_basic_selection(cast("BasicSelection", pure_selection), fields=fields) def __setitem__(self, selection: Selection, value: npt.ArrayLike) -> None: """Modify data for an item or region of the array. Parameters ---------- selection : tuple An integer index or slice or tuple of int/slice specifying the requested region for each dimension of the array. value : npt.ArrayLike An array-like containing the data to be stored in the selection. Examples -------- Setup a 1-dimensional array: >>> import zarr >>> z = zarr.zeros( ... shape=(100,), ... store={}, ... chunk_shape=(5,), ... dtype="i4", ... ) Set all array elements to the same scalar value: >>> z[...] = 42 >>> z[...] array([42, 42, 42, ..., 42, 42, 42], dtype=int32) Set a portion of the array: >>> z[:10] = np.arange(10) >>> z[-10:] = np.arange(10)[::-1] >>> z[...] array([ 0, 1, 2, ..., 2, 1, 0], dtype=int32) Setup a 2-dimensional array: >>> z = zarr.zeros( ... shape=(5, 5), ... store={}, ... chunk_shape=(5, 5), ... dtype="i4", ... ) Set all array elements to the same scalar value: >>> z[...] = 42 Set a portion of the array: >>> z[0, :] = np.arange(z.shape[1]) >>> z[:, 0] = np.arange(z.shape[0]) >>> z[...] array([[ 0, 1, 2, 3, 4], [ 1, 42, 42, 42, 42], [ 2, 42, 42, 42, 42], [ 3, 42, 42, 42, 42], [ 4, 42, 42, 42, 42]], dtype=int32) Notes ----- Slices with step > 1 are supported, but slices with negative step are not. For arrays with a structured dtype, see Zarr format 2 for examples of how to use fields Currently the implementation for __setitem__ is provided by [`vindex`][zarr.Array.vindex] if the indexing is pure fancy indexing (ie a broadcast-compatible tuple of integer array indices), or by [`set_basic_selection`][zarr.Array.set_basic_selection] otherwise. Effectively, this means that the following indexing modes are supported: - integer indexing - slice indexing - mixed slice and integer indexing - boolean indexing - fancy indexing (vectorized list of integers) For specific indexing options including outer indexing, see the methods listed under Related. Related ------- [get_basic_selection][zarr.Array.get_basic_selection], [set_basic_selection][zarr.Array.set_basic_selection], [get_mask_selection][zarr.Array.get_mask_selection], [set_mask_selection][zarr.Array.set_mask_selection], [get_coordinate_selection][zarr.Array.get_coordinate_selection], [set_coordinate_selection][zarr.Array.set_coordinate_selection], [get_orthogonal_selection][zarr.Array.get_orthogonal_selection], [set_orthogonal_selection][zarr.Array.set_orthogonal_selection], [get_block_selection][zarr.Array.get_block_selection], [set_block_selection][zarr.Array.set_block_selection], [vindex][zarr.Array.vindex], [oindex][zarr.Array.oindex], [blocks][zarr.Array.blocks], [__getitem__][zarr.Array.__getitem__] """ # Converting a zarr Array to numpy here avoids a SyncError that occurs when # value.__getitem__ is called inside the async codec pipeline (which already # runs within a running event loop). np.asarray triggers Array.__array__, # which reads the data synchronously before we enter the async context. if isinstance(value, Array): value = np.asarray(value) fields, pure_selection = pop_fields(selection) if is_pure_fancy_indexing(pure_selection, self.ndim): self.vindex[cast("CoordinateSelection | MaskSelection", selection)] = value elif is_pure_orthogonal_indexing(pure_selection, self.ndim): self.set_orthogonal_selection(pure_selection, value, fields=fields) else: self.set_basic_selection(cast("BasicSelection", pure_selection), value, fields=fields) def get_basic_selection( self, selection: BasicSelection = Ellipsis, *, out: NDBuffer | None = None, prototype: BufferPrototype | None = None, fields: Fields | None = None, ) -> NDArrayLikeOrScalar: """Retrieve data for an item or region of the array. Parameters ---------- selection : BasicSelection A selection specifying the requested item or region for each dimension of the array. May be any combination of int and/or slice or ellipsis for multidimensional arrays. out : NDBuffer, optional If given, load the selected data directly into this buffer. prototype : BufferPrototype, optional The prototype of the buffer to use for the output data. If not provided, the default buffer prototype is used. fields : str or sequence of str, optional For arrays with a structured dtype, one or more fields can be specified to extract data for. Returns ------- NDArrayLikeOrScalar An array-like or scalar containing the data for the requested region. Examples -------- Setup a 1-dimensional array: >>> import zarr >>> import numpy as np >>> data = np.arange(100, dtype="uint16") >>> z = zarr.create_array( ... {}, ... shape=data.shape, ... chunks=(3,), ... dtype=data.dtype, ... ) >>> z[:] = data Retrieve a single item: >>> z.get_basic_selection(5) np.uint16(5) Retrieve a region via slicing: >>> z.get_basic_selection(slice(5)) array([0, 1, 2, 3, 4], dtype=uint16) >>> z.get_basic_selection(slice(-5, None)) array([95, 96, 97, 98, 99], dtype=uint16) >>> z.get_basic_selection(slice(5, 10)) array([5, 6, 7, 8, 9], dtype=uint16) >>> z.get_basic_selection(slice(5, 10, 2)) array([5, 7, 9], dtype=uint16) >>> z.get_basic_selection(slice(None, None, 2)) array([ 0, 2, 4, ..., 94, 96, 98], dtype=uint16) Setup a 3-dimensional array: >>> data = np.arange(1000).reshape(10, 10, 10) >>> z = zarr.create_array( ... {}, ... shape=data.shape, ... chunks=(5, 5, 5), ... dtype=data.dtype, ... ) >>> z[:] = data Retrieve an item: >>> z.get_basic_selection((1, 2, 3)) np.int64(123) Retrieve a region via slicing and Ellipsis: >>> z.get_basic_selection((slice(1, 3), slice(1, 3), 0)) array([[110, 120], [210, 220]]) >>> z.get_basic_selection((0, slice(1, 3), slice(None))) array([[10, 11, 12, 13, 14, 15, 16, 17, 18, 19], [20, 21, 22, 23, 24, 25, 26, 27, 28, 29]]) >>> z.get_basic_selection((..., 2)) array([[ 2, 12, 22, 32, 42, 52, 62, 72, 82, 92], [102, 112, 122, 132, 142, 152, 162, 172, 182, 192], ... [802, 812, 822, 832, 842, 852, 862, 872, 882, 892], [902, 912, 922, 932, 942, 952, 962, 972, 982, 992]]) Notes ----- Slices with step > 1 are supported, but slices with negative step are not. For arrays with a structured dtype, see Zarr format 2 for examples of how to use the `fields` parameter. This method provides the implementation for accessing data via the square bracket notation (__getitem__). See [`__getitem__`][zarr.Array.__getitem__] for examples using the alternative notation. Related ------- [set_basic_selection][zarr.Array.set_basic_selection], [get_mask_selection][zarr.Array.get_mask_selection], [set_mask_selection][zarr.Array.set_mask_selection], [get_coordinate_selection][zarr.Array.get_coordinate_selection], [set_coordinate_selection][zarr.Array.set_coordinate_selection], [get_orthogonal_selection][zarr.Array.get_orthogonal_selection], [set_orthogonal_selection][zarr.Array.set_orthogonal_selection], [get_block_selection][zarr.Array.get_block_selection], [set_block_selection][zarr.Array.set_block_selection], [vindex][zarr.Array.vindex], [oindex][zarr.Array.oindex], [blocks][zarr.Array.blocks], [__getitem__][zarr.Array.__getitem__], [__setitem__][zarr.Array.__setitem__] """ if prototype is None: prototype = default_buffer_prototype() return sync( self.async_array._get_selection( BasicIndexer(selection, self.shape, self._chunk_grid), out=out, fields=fields, prototype=prototype, ) ) def set_basic_selection( self, selection: BasicSelection, value: npt.ArrayLike, *, fields: Fields | None = None, prototype: BufferPrototype | None = None, ) -> None: """Modify data for an item or region of the array. Parameters ---------- selection : tuple A tuple specifying the requested item or region for each dimension of the array. May be any combination of int and/or slice or ellipsis for multidimensional arrays. value : npt.ArrayLike An array-like containing values to be stored into the array. fields : str or sequence of str, optional For arrays with a structured dtype, one or more fields can be specified to set data for. prototype : BufferPrototype, optional The prototype of the buffer used for setting the data. If not provided, the default buffer prototype is used. Examples -------- Setup a 1-dimensional array: >>> import zarr >>> z = zarr.zeros( ... shape=(100,), ... store={}, ... chunk_shape=(100,), ... dtype="i4", ... ) Set all array elements to the same scalar value: >>> z.set_basic_selection(..., 42) >>> z[...] array([42, 42, 42, ..., 42, 42, 42], dtype=int32) Set a portion of the array: >>> z.set_basic_selection(slice(10), np.arange(10)) >>> z.set_basic_selection(slice(-10, None), np.arange(10)[::-1]) >>> z[...] array([ 0, 1, 2, ..., 2, 1, 0], dtype=int32) Setup a 2-dimensional array: >>> z = zarr.zeros( ... shape=(5, 5), ... store={}, ... chunk_shape=(5, 5), ... dtype="i4", ... ) Set all array elements to the same scalar value: >>> z.set_basic_selection(..., 42) Set a portion of the array: >>> z.set_basic_selection((0, slice(None)), np.arange(z.shape[1])) >>> z.set_basic_selection((slice(None), 0), np.arange(z.shape[0])) >>> z[...] array([[ 0, 1, 2, 3, 4], [ 1, 42, 42, 42, 42], [ 2, 42, 42, 42, 42], [ 3, 42, 42, 42, 42], [ 4, 42, 42, 42, 42]], dtype=int32) Notes ----- For arrays with a structured dtype, see Zarr format 2 for examples of how to use the `fields` parameter. This method provides the underlying implementation for modifying data via square bracket notation, see [`__setitem__`][zarr.Array.__setitem__] for equivalent examples using the alternative notation. Related ------- [get_basic_selection][zarr.Array.get_basic_selection], [get_mask_selection][zarr.Array.get_mask_selection], [set_mask_selection][zarr.Array.set_mask_selection], [get_coordinate_selection][zarr.Array.get_coordinate_selection], [set_coordinate_selection][zarr.Array.set_coordinate_selection], [get_orthogonal_selection][zarr.Array.get_orthogonal_selection], [set_orthogonal_selection][zarr.Array.set_orthogonal_selection], [get_block_selection][zarr.Array.get_block_selection], [set_block_selection][zarr.Array.set_block_selection], [vindex][zarr.Array.vindex], [oindex][zarr.Array.oindex], [blocks][zarr.Array.blocks], [__getitem__][zarr.Array.__getitem__], [__setitem__][zarr.Array.__setitem__] """ if prototype is None: prototype = default_buffer_prototype() indexer = BasicIndexer(selection, self.shape, self._chunk_grid) sync(self.async_array._set_selection(indexer, value, fields=fields, prototype=prototype)) def get_orthogonal_selection( self, selection: OrthogonalSelection, *, out: NDBuffer | None = None, fields: Fields | None = None, prototype: BufferPrototype | None = None, ) -> NDArrayLikeOrScalar: """Retrieve data by making a selection for each dimension of the array. For example, if an array has 2 dimensions, allows selecting specific rows and/or columns. The selection for each dimension can be either an integer (indexing a single item), a slice, an array of integers, or a Boolean array where True values indicate a selection. Parameters ---------- selection : tuple A selection for each dimension of the array. May be any combination of int, slice, integer array or Boolean array. out : NDBuffer, optional If given, load the selected data directly into this buffer. fields : str or sequence of str, optional For arrays with a structured dtype, one or more fields can be specified to extract data for. prototype : BufferPrototype, optional The prototype of the buffer to use for the output data. If not provided, the default buffer prototype is used. Returns ------- NDArrayLikeOrScalar An array-like or scalar containing the data for the requested selection. Examples -------- Setup a 2-dimensional array: >>> import zarr >>> import numpy as np >>> data = np.arange(100).reshape(10, 10) >>> z = zarr.create_array( ... {}, ... shape=data.shape, ... chunks=data.shape, ... dtype=data.dtype, ... ) >>> z[:] = data Retrieve rows and columns via any combination of int, slice, integer array and/or Boolean array: >>> z.get_orthogonal_selection(([1, 4], slice(None))) array([[10, 11, 12, 13, 14, 15, 16, 17, 18, 19], [40, 41, 42, 43, 44, 45, 46, 47, 48, 49]]) >>> z.get_orthogonal_selection((slice(None), [1, 4])) array([[ 1, 4], [11, 14], [21, 24], [31, 34], [41, 44], [51, 54], [61, 64], [71, 74], [81, 84], [91, 94]]) >>> z.get_orthogonal_selection(([1, 4], [1, 4])) array([[11, 14], [41, 44]]) >>> sel = np.zeros(z.shape[0], dtype=bool) >>> sel[1] = True >>> sel[4] = True >>> z.get_orthogonal_selection((sel, sel)) array([[11, 14], [41, 44]]) For convenience, the orthogonal selection functionality is also available via the `oindex` property, e.g.: >>> z.oindex[[1, 4], :] array([[10, 11, 12, 13, 14, 15, 16, 17, 18, 19], [40, 41, 42, 43, 44, 45, 46, 47, 48, 49]]) >>> z.oindex[:, [1, 4]] array([[ 1, 4], [11, 14], [21, 24], [31, 34], [41, 44], [51, 54], [61, 64], [71, 74], [81, 84], [91, 94]]) >>> z.oindex[[1, 4], [1, 4]] array([[11, 14], [41, 44]]) >>> sel = np.zeros(z.shape[0], dtype=bool) >>> sel[1] = True >>> sel[4] = True >>> z.oindex[sel, sel] array([[11, 14], [41, 44]]) Notes ----- Orthogonal indexing is also known as outer indexing. Slices with step > 1 are supported, but slices with negative step are not. Related ------- [get_basic_selection][zarr.Array.get_basic_selection], [set_basic_selection][zarr.Array.set_basic_selection], [get_mask_selection][zarr.Array.get_mask_selection], [set_mask_selection][zarr.Array.set_mask_selection], [get_coordinate_selection][zarr.Array.get_coordinate_selection], [set_coordinate_selection][zarr.Array.set_coordinate_selection], [set_orthogonal_selection][zarr.Array.set_orthogonal_selection], [get_block_selection][zarr.Array.get_block_selection], [set_block_selection][zarr.Array.set_block_selection], [vindex][zarr.Array.vindex], [oindex][zarr.Array.oindex], [blocks][zarr.Array.blocks], [__getitem__][zarr.Array.__getitem__], [__setitem__][zarr.Array.__setitem__] """ if prototype is None: prototype = default_buffer_prototype() indexer = OrthogonalIndexer(selection, self.shape, self._chunk_grid) return sync( self.async_array._get_selection( indexer=indexer, out=out, fields=fields, prototype=prototype ) ) def set_orthogonal_selection( self, selection: OrthogonalSelection, value: npt.ArrayLike, *, fields: Fields | None = None, prototype: BufferPrototype | None = None, ) -> None: """Modify data via a selection for each dimension of the array. Parameters ---------- selection : tuple A selection for each dimension of the array. May be any combination of int, slice, integer array or Boolean array. value : npt.ArrayLike An array-like array containing the data to be stored in the array. fields : str or sequence of str, optional For arrays with a structured dtype, one or more fields can be specified to set data for. prototype : BufferPrototype, optional The prototype of the buffer used for setting the data. If not provided, the default buffer prototype is used. Examples -------- Setup a 2-dimensional array: >>> import zarr >>> z = zarr.zeros( ... shape=(5, 5), ... store={}, ... chunk_shape=(5, 5), ... dtype="i4", ... ) Set data for a selection of rows: >>> z.set_orthogonal_selection(([1, 4], slice(None)), 1) >>> z[...] array([[0, 0, 0, 0, 0], [1, 1, 1, 1, 1], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [1, 1, 1, 1, 1]], dtype=int32) Set data for a selection of columns: >>> z.set_orthogonal_selection((slice(None), [1, 4]), 2) >>> z[...] array([[0, 2, 0, 0, 2], [1, 2, 1, 1, 2], [0, 2, 0, 0, 2], [0, 2, 0, 0, 2], [1, 2, 1, 1, 2]], dtype=int32) Set data for a selection of rows and columns: >>> z.set_orthogonal_selection(([1, 4], [1, 4]), 3) >>> z[...] array([[0, 2, 0, 0, 2], [1, 3, 1, 1, 3], [0, 2, 0, 0, 2], [0, 2, 0, 0, 2], [1, 3, 1, 1, 3]], dtype=int32) Set data from a 2D array: >>> values = np.arange(10).reshape(2, 5) >>> z.set_orthogonal_selection(([0, 3], ...), values) >>> z[...] array([[0, 1, 2, 3, 4], [1, 3, 1, 1, 3], [0, 2, 0, 0, 2], [5, 6, 7, 8, 9], [1, 3, 1, 1, 3]], dtype=int32) For convenience, this functionality is also available via the `oindex` property: >>> z.oindex[[1, 4], [1, 4]] = 4 >>> z[...] array([[0, 1, 2, 3, 4], [1, 4, 1, 1, 4], [0, 2, 0, 0, 2], [5, 6, 7, 8, 9], [1, 4, 1, 1, 4]], dtype=int32) Notes ----- Orthogonal indexing is also known as outer indexing. Slices with step > 1 are supported, but slices with negative step are not. Related ------- [get_basic_selection][zarr.Array.get_basic_selection], [set_basic_selection][zarr.Array.set_basic_selection], [get_mask_selection][zarr.Array.get_mask_selection], [set_mask_selection][zarr.Array.set_mask_selection], [get_coordinate_selection][zarr.Array.get_coordinate_selection], [set_coordinate_selection][zarr.Array.set_coordinate_selection], [get_orthogonal_selection][zarr.Array.get_orthogonal_selection], [get_block_selection][zarr.Array.get_block_selection], [set_block_selection][zarr.Array.set_block_selection], [vindex][zarr.Array.vindex], [oindex][zarr.Array.oindex], [blocks][zarr.Array.blocks], [__getitem__][zarr.Array.__getitem__], [__setitem__][zarr.Array.__setitem__] """ if prototype is None: prototype = default_buffer_prototype() indexer = OrthogonalIndexer(selection, self.shape, self._chunk_grid) return sync( self.async_array._set_selection(indexer, value, fields=fields, prototype=prototype) ) def get_mask_selection( self, mask: MaskSelection, *, out: NDBuffer | None = None, fields: Fields | None = None, prototype: BufferPrototype | None = None, ) -> NDArrayLikeOrScalar: """Retrieve a selection of individual items, by providing a Boolean array of the same shape as the array against which the selection is being made, where True values indicate a selected item. Parameters ---------- mask : ndarray, bool A Boolean array of the same shape as the array against which the selection is being made. out : NDBuffer, optional If given, load the selected data directly into this buffer. fields : str or sequence of str, optional For arrays with a structured dtype, one or more fields can be specified to extract data for. prototype : BufferPrototype, optional The prototype of the buffer to use for the output data. If not provided, the default buffer prototype is used. Returns ------- NDArrayLikeOrScalar An array-like or scalar containing the data for the requested selection. Examples -------- Setup a 2-dimensional array: >>> import zarr >>> import numpy as np >>> data = np.arange(100).reshape(10, 10) >>> z = zarr.create_array( ... {}, ... shape=data.shape, ... chunks=data.shape, ... dtype=data.dtype, ... ) >>> z[:] = data Retrieve items by specifying a mask: >>> sel = np.zeros_like(z, dtype=bool) >>> sel[1, 1] = True >>> sel[4, 4] = True >>> z.get_mask_selection(sel) array([11, 44]) For convenience, the mask selection functionality is also available via the `vindex` property: >>> z.vindex[sel] array([11, 44]) Notes ----- Mask indexing is a form of vectorized or inner indexing, and is equivalent to coordinate indexing. Internally the mask array is converted to coordinate arrays by calling `np.nonzero`. Related ------- [get_basic_selection][zarr.Array.get_basic_selection], [set_basic_selection][zarr.Array.set_basic_selection], [set_mask_selection][zarr.Array.set_mask_selection], [get_orthogonal_selection][zarr.Array.get_orthogonal_selection], [set_orthogonal_selection][zarr.Array.set_orthogonal_selection], [get_coordinate_selection][zarr.Array.get_coordinate_selection], [set_coordinate_selection][zarr.Array.set_coordinate_selection], [get_block_selection][zarr.Array.get_block_selection], [set_block_selection][zarr.Array.set_block_selection], [vindex][zarr.Array.vindex], [oindex][zarr.Array.oindex], [blocks][zarr.Array.blocks], [__getitem__][zarr.Array.__getitem__], [__setitem__][zarr.Array.__setitem__] """ if prototype is None: prototype = default_buffer_prototype() indexer = MaskIndexer(mask, self.shape, self._chunk_grid) return sync( self.async_array._get_selection( indexer=indexer, out=out, fields=fields, prototype=prototype ) ) def set_mask_selection( self, mask: MaskSelection, value: npt.ArrayLike, *, fields: Fields | None = None, prototype: BufferPrototype | None = None, ) -> None: """Modify a selection of individual items, by providing a Boolean array of the same shape as the array against which the selection is being made, where True values indicate a selected item. Parameters ---------- mask : ndarray, bool A Boolean array of the same shape as the array against which the selection is being made. value : npt.ArrayLike An array-like containing values to be stored into the array. fields : str or sequence of str, optional For arrays with a structured dtype, one or more fields can be specified to set data for. Examples -------- Setup a 2-dimensional array: >>> import zarr >>> z = zarr.zeros( ... shape=(5, 5), ... store={}, ... chunk_shape=(5, 5), ... dtype="i4", ... ) Set data for a selection of items: >>> sel = np.zeros_like(z, dtype=bool) >>> sel[1, 1] = True >>> sel[4, 4] = True >>> z.set_mask_selection(sel, 1) >>> z[...] array([[0, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 1]], dtype=int32) For convenience, this functionality is also available via the `vindex` property: >>> z.vindex[sel] = 2 >>> z[...] array([[0, 0, 0, 0, 0], [0, 2, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 2]], dtype=int32) Notes ----- Mask indexing is a form of vectorized or inner indexing, and is equivalent to coordinate indexing. Internally the mask array is converted to coordinate arrays by calling `np.nonzero`. Related ------- [get_basic_selection][zarr.Array.get_basic_selection], [set_basic_selection][zarr.Array.set_basic_selection], [get_mask_selection][zarr.Array.get_mask_selection], [get_orthogonal_selection][zarr.Array.get_orthogonal_selection], [set_orthogonal_selection][zarr.Array.set_orthogonal_selection], [get_coordinate_selection][zarr.Array.get_coordinate_selection], [set_coordinate_selection][zarr.Array.set_coordinate_selection], [get_block_selection][zarr.Array.get_block_selection], [set_block_selection][zarr.Array.set_block_selection], [vindex][zarr.Array.vindex], [oindex][zarr.Array.oindex], [blocks][zarr.Array.blocks], [__getitem__][zarr.Array.__getitem__], [__setitem__][zarr.Array.__setitem__] """ if prototype is None: prototype = default_buffer_prototype() indexer = MaskIndexer(mask, self.shape, self._chunk_grid) sync(self.async_array._set_selection(indexer, value, fields=fields, prototype=prototype)) def get_coordinate_selection( self, selection: CoordinateSelection, *, out: NDBuffer | None = None, fields: Fields | None = None, prototype: BufferPrototype | None = None, ) -> NDArrayLikeOrScalar: """Retrieve a selection of individual items, by providing the indices (coordinates) for each selected item. Parameters ---------- selection : tuple An integer (coordinate) array for each dimension of the array. out : NDBuffer, optional If given, load the selected data directly into this buffer. fields : str or sequence of str, optional For arrays with a structured dtype, one or more fields can be specified to extract data for. prototype : BufferPrototype, optional The prototype of the buffer to use for the output data. If not provided, the default buffer prototype is used. Returns ------- NDArrayLikeOrScalar An array-like or scalar containing the data for the requested coordinate selection. Examples -------- Setup a 2-dimensional array: >>> import zarr >>> import numpy as np >>> data = np.arange(0, 100, dtype="uint16").reshape((10, 10)) >>> z = zarr.create_array( ... {}, ... shape=data.shape, ... chunks=(3, 3), ... dtype=data.dtype, ... ) >>> z[:] = data Retrieve items by specifying their coordinates: >>> z.get_coordinate_selection(([1, 4], [1, 4])) array([11, 44], dtype=uint16) For convenience, the coordinate selection functionality is also available via the `vindex` property: >>> z.vindex[[1, 4], [1, 4]] array([11, 44], dtype=uint16) Notes ----- Coordinate indexing is also known as point selection, and is a form of vectorized or inner indexing. Slices are not supported. Coordinate arrays must be provided for all dimensions of the array. Coordinate arrays may be multidimensional, in which case the output array will also be multidimensional. Coordinate arrays are broadcast against each other before being applied. The shape of the output will be the same as the shape of each coordinate array after broadcasting. Related ------- [get_basic_selection][zarr.Array.get_basic_selection], [set_basic_selection][zarr.Array.set_basic_selection], [get_mask_selection][zarr.Array.get_mask_selection], [set_mask_selection][zarr.Array.set_mask_selection], [get_orthogonal_selection][zarr.Array.get_orthogonal_selection], [set_orthogonal_selection][zarr.Array.set_orthogonal_selection], [set_coordinate_selection][zarr.Array.set_coordinate_selection], [get_block_selection][zarr.Array.get_block_selection], [set_block_selection][zarr.Array.set_block_selection], [vindex][zarr.Array.vindex], [oindex][zarr.Array.oindex], [blocks][zarr.Array.blocks], [__getitem__][zarr.Array.__getitem__], [__setitem__][zarr.Array.__setitem__] """ if prototype is None: prototype = default_buffer_prototype() indexer = CoordinateIndexer(selection, self.shape, self._chunk_grid) out_array = sync( self.async_array._get_selection( indexer=indexer, out=out, fields=fields, prototype=prototype ) ) if hasattr(out_array, "shape"): # restore shape out_array = np.array(out_array).reshape(indexer.sel_shape) return out_array def set_coordinate_selection( self, selection: CoordinateSelection, value: npt.ArrayLike, *, fields: Fields | None = None, prototype: BufferPrototype | None = None, ) -> None: """Modify a selection of individual items, by providing the indices (coordinates) for each item to be modified. Parameters ---------- selection : tuple An integer (coordinate) array for each dimension of the array. value : npt.ArrayLike An array-like containing values to be stored into the array. fields : str or sequence of str, optional For arrays with a structured dtype, one or more fields can be specified to set data for. Examples -------- Setup a 2-dimensional array: >>> import zarr >>> z = zarr.zeros( ... shape=(5, 5), ... store={}, ... chunk_shape=(5, 5), ... dtype="i4", ... ) Set data for a selection of items: >>> z.set_coordinate_selection(([1, 4], [1, 4]), 1) >>> z[...] array([[0, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 1]], dtype=int32) For convenience, this functionality is also available via the `vindex` property: >>> z.vindex[[1, 4], [1, 4]] = 2 >>> z[...] array([[0, 0, 0, 0, 0], [0, 2, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 2]], dtype=int32) Notes ----- Coordinate indexing is also known as point selection, and is a form of vectorized or inner indexing. Slices are not supported. Coordinate arrays must be provided for all dimensions of the array. Related ------- [get_basic_selection][zarr.Array.get_basic_selection], [set_basic_selection][zarr.Array.set_basic_selection], [get_mask_selection][zarr.Array.get_mask_selection], [set_mask_selection][zarr.Array.set_mask_selection], [get_orthogonal_selection][zarr.Array.get_orthogonal_selection], [set_orthogonal_selection][zarr.Array.set_orthogonal_selection], [get_coordinate_selection][zarr.Array.get_coordinate_selection], [get_block_selection][zarr.Array.get_block_selection], [set_block_selection][zarr.Array.set_block_selection], [vindex][zarr.Array.vindex], [oindex][zarr.Array.oindex], [blocks][zarr.Array.blocks], [__getitem__][zarr.Array.__getitem__], [__setitem__][zarr.Array.__setitem__] """ if prototype is None: prototype = default_buffer_prototype() # setup indexer indexer = CoordinateIndexer(selection, self.shape, self._chunk_grid) # handle value - need ndarray-like flatten value if not is_scalar(value, self.dtype): try: from numcodecs.compat import ensure_ndarray_like value = ensure_ndarray_like(value) # TODO replace with agnostic except TypeError: # Handle types like `list` or `tuple` value = np.array(value) # TODO replace with agnostic if hasattr(value, "shape") and len(value.shape) > 1: value = np.array(value).reshape(-1) if not is_scalar(value, self.dtype) and ( isinstance(value, NDArrayLike) and indexer.shape != value.shape ): raise ValueError( f"Attempting to set a selection of {indexer.sel_shape[0]} " f"elements with an array of {value.shape[0]} elements." ) sync(self.async_array._set_selection(indexer, value, fields=fields, prototype=prototype)) def get_block_selection( self, selection: BasicSelection, *, out: NDBuffer | None = None, fields: Fields | None = None, prototype: BufferPrototype | None = None, ) -> NDArrayLikeOrScalar: """Retrieve a selection of individual items, by providing the indices (coordinates) for each selected item. Parameters ---------- selection : int or slice or tuple of int or slice An integer (coordinate) or slice for each dimension of the array. out : NDBuffer, optional If given, load the selected data directly into this buffer. fields : str or sequence of str, optional For arrays with a structured dtype, one or more fields can be specified to extract data for. prototype : BufferPrototype, optional The prototype of the buffer to use for the output data. If not provided, the default buffer prototype is used. Returns ------- NDArrayLikeOrScalar An array-like or scalar containing the data for the requested block selection. Examples -------- Setup a 2-dimensional array: >>> import zarr >>> import numpy as np >>> data = np.arange(0, 100, dtype="uint16").reshape((10, 10)) >>> z = zarr.create_array( ... {}, ... shape=data.shape, ... chunks=(3, 3), ... dtype=data.dtype, ... ) >>> z[:] = data Retrieve items by specifying their block coordinates: >>> z.get_block_selection((1, slice(None))) array([[30, 31, 32, 33, 34, 35, 36, 37, 38, 39], [40, 41, 42, 43, 44, 45, 46, 47, 48, 49], [50, 51, 52, 53, 54, 55, 56, 57, 58, 59]], dtype=uint16) Which is equivalent to: >>> z[3:6, :] array([[30, 31, 32, 33, 34, 35, 36, 37, 38, 39], [40, 41, 42, 43, 44, 45, 46, 47, 48, 49], [50, 51, 52, 53, 54, 55, 56, 57, 58, 59]], dtype=uint16) For convenience, the block selection functionality is also available via the `blocks` property: >>> z.blocks[1] array([[30, 31, 32, 33, 34, 35, 36, 37, 38, 39], [40, 41, 42, 43, 44, 45, 46, 47, 48, 49], [50, 51, 52, 53, 54, 55, 56, 57, 58, 59]], dtype=uint16) Notes ----- Block indexing is a convenience indexing method to work on individual chunks with chunk index slicing. It has the same concept as Dask's `Array.blocks` indexing. Slices are supported. However, only with a step size of one. Block index arrays may be multidimensional to index multidimensional arrays: >>> z.blocks[0, 1:3] array([[ 3, 4, 5, 6, 7, 8], [13, 14, 15, 16, 17, 18], [23, 24, 25, 26, 27, 28]], dtype=uint16) Related ------- [get_basic_selection][zarr.Array.get_basic_selection], [set_basic_selection][zarr.Array.set_basic_selection], [get_mask_selection][zarr.Array.get_mask_selection], [set_mask_selection][zarr.Array.set_mask_selection], [get_orthogonal_selection][zarr.Array.get_orthogonal_selection], [set_orthogonal_selection][zarr.Array.set_orthogonal_selection], [get_coordinate_selection][zarr.Array.get_coordinate_selection], [set_coordinate_selection][zarr.Array.set_coordinate_selection], [set_block_selection][zarr.Array.set_block_selection], [vindex][zarr.Array.vindex], [oindex][zarr.Array.oindex], [blocks][zarr.Array.blocks], [__getitem__][zarr.Array.__getitem__], [__setitem__][zarr.Array.__setitem__] """ if prototype is None: prototype = default_buffer_prototype() indexer = BlockIndexer(selection, self.shape, self._chunk_grid) return sync( self.async_array._get_selection( indexer=indexer, out=out, fields=fields, prototype=prototype ) ) def set_block_selection( self, selection: BasicSelection, value: npt.ArrayLike, *, fields: Fields | None = None, prototype: BufferPrototype | None = None, ) -> None: """Modify a selection of individual blocks, by providing the chunk indices (coordinates) for each block to be modified. Parameters ---------- selection : tuple An integer (coordinate) or slice for each dimension of the array. value : npt.ArrayLike An array-like containing the data to be stored in the block selection. fields : str or sequence of str, optional For arrays with a structured dtype, one or more fields can be specified to set data for. prototype : BufferPrototype, optional The prototype of the buffer used for setting the data. If not provided, the default buffer prototype is used. Examples -------- Set up a 2-dimensional array: >>> import zarr >>> z = zarr.zeros( ... shape=(6, 6), ... store={}, ... chunk_shape=(2, 2), ... dtype="i4", ... ) Set data for a selection of items: >>> z.set_block_selection((1, 0), 1) >>> z[...] array([[0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [1, 1, 0, 0, 0, 0], [1, 1, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0]], dtype=int32) For convenience, this functionality is also available via the `blocks` property: >>> z.blocks[2, 1] = 4 >>> z[...] array([[0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [1, 1, 0, 0, 0, 0], [1, 1, 0, 0, 0, 0], [0, 0, 4, 4, 0, 0], [0, 0, 4, 4, 0, 0]], dtype=int32) >>> z.blocks[:, 2] = 7 >>> z[...] array([[0, 0, 0, 0, 7, 7], [0, 0, 0, 0, 7, 7], [1, 1, 0, 0, 7, 7], [1, 1, 0, 0, 7, 7], [0, 0, 4, 4, 7, 7], [0, 0, 4, 4, 7, 7]], dtype=int32) Notes ----- Block indexing is a convenience indexing method to work on individual chunks with chunk index slicing. It has the same concept as Dask's `Array.blocks` indexing. Slices are supported. However, only with a step size of one. Related ------- [get_basic_selection][zarr.Array.get_basic_selection], [set_basic_selection][zarr.Array.set_basic_selection], [get_mask_selection][zarr.Array.get_mask_selection], [set_mask_selection][zarr.Array.set_mask_selection], [get_orthogonal_selection][zarr.Array.get_orthogonal_selection], [set_orthogonal_selection][zarr.Array.set_orthogonal_selection], [get_coordinate_selection][zarr.Array.get_coordinate_selection], [get_block_selection][zarr.Array.get_block_selection], [set_block_selection][zarr.Array.set_block_selection], [vindex][zarr.Array.vindex], [oindex][zarr.Array.oindex], [blocks][zarr.Array.blocks], [__getitem__][zarr.Array.__getitem__], [__setitem__][zarr.Array.__setitem__] """ if prototype is None: prototype = default_buffer_prototype() indexer = BlockIndexer(selection, self.shape, self._chunk_grid) sync(self.async_array._set_selection(indexer, value, fields=fields, prototype=prototype)) @property def vindex(self) -> VIndex: """Shortcut for vectorized (inner) indexing, see [get_coordinate_selection][zarr.Array.get_coordinate_selection], [set_coordinate_selection][zarr.Array.set_coordinate_selection], [get_mask_selection][zarr.Array.get_mask_selection] and [set_mask_selection][zarr.Array.set_mask_selection] for documentation and examples.""" return VIndex(self) @property def oindex(self) -> OIndex: """Shortcut for orthogonal (outer) indexing, see [get_orthogonal_selection][zarr.Array.get_orthogonal_selection] and [set_orthogonal_selection][zarr.Array.set_orthogonal_selection] for documentation and examples.""" return OIndex(self) @property def blocks(self) -> BlockIndex: """Shortcut for blocked chunked indexing, see [get_block_selection][zarr.Array.get_block_selection] and [set_block_selection][zarr.Array.set_block_selection] for documentation and examples.""" return BlockIndex(self) def resize(self, new_shape: ShapeLike) -> None: """ Change the shape of the array by growing or shrinking one or more dimensions. This is an in-place operation that modifies the array. Parameters ---------- new_shape : tuple New shape of the array. Notes ----- If one or more dimensions are shrunk, any chunks falling outside the new array shape will be deleted from the underlying store. However, it is noteworthy that the chunks partially falling inside the new array (i.e. boundary chunks) will remain intact, and therefore, the data falling outside the new array but inside the boundary chunks would be restored by a subsequent resize operation that grows the array size. Examples -------- ```python import zarr z = zarr.zeros(shape=(10000, 10000), chunk_shape=(1000, 1000), dtype="int32",) z.shape #> (10000, 10000) z.resize((20000, 1000)) z.shape #> (20000, 1000) z.resize((50, 50)) z.shape #>(50, 50) ``` """ sync(self.async_array.resize(new_shape)) def append(self, data: npt.ArrayLike, axis: int = 0) -> tuple[int, ...]: """Append `data` to `axis`. Parameters ---------- data : array-like Data to be appended. axis : int Axis along which to append. Returns ------- new_shape : tuple Notes ----- The size of all dimensions other than `axis` must match between this array and `data`. Examples -------- >>> import numpy as np >>> import zarr >>> a = np.arange(10000000, dtype="i4").reshape(10000, 1000) >>> z = zarr.array(a, chunks=(1000, 100)) >>> z.shape (10000, 1000) >>> z.append(a) (20000, 1000) >>> z.append(np.vstack([a, a]), axis=1) (20000, 2000) >>> z.shape (20000, 2000) """ return sync(self.async_array.append(data, axis=axis)) def update_attributes(self, new_attributes: dict[str, JSON]) -> Self: """ Update the array's attributes. Parameters ---------- new_attributes : dict A dictionary of new attributes to update or add to the array. The keys represent attribute names, and the values must be JSON-compatible. Returns ------- Array The array with the updated attributes. Raises ------ ValueError If the attributes are invalid or incompatible with the array's metadata. Notes ----- - The updated attributes will be merged with existing attributes, and any conflicts will be overwritten by the new values. """ new_array = sync(self.async_array.update_attributes(new_attributes)) return type(self)(new_array) def __repr__(self) -> str: return f"" @property def info(self) -> Any: """ Return the statically known information for an array. Returns ------- ArrayInfo Related ------- [zarr.Array.info_complete][] - All information about a group, including dynamic information like the number of bytes and chunks written. Examples -------- >>> arr = zarr.create(shape=(10,), chunks=(2,), dtype="float32") >>> arr.info Type : Array Zarr format : 3 Data type : Float32(endianness=...) Fill value : 0.0 Shape : (10,) Chunk shape : (2,) Order : C Read-only : False Store type : MemoryStore Filters : () Serializer : BytesCodec(endian='little') Compressors : (ZstdCodec(level=0, checksum=False),) No. bytes : 40 """ return self.async_array.info def info_complete(self) -> Any: """ Returns all the information about an array, including information from the Store. In addition to the statically known information like `name` and `zarr_format`, this includes additional information like the size of the array in bytes and the number of chunks written. Note that this method will need to read metadata from the store. Returns ------- ArrayInfo Related ------- [zarr.Array.info][] - The statically known subset of metadata about an array. """ return sync(self.async_array.info_complete()) async def _shards_initialized( array: AnyAsyncArray, ) -> tuple[str, ...]: """ Return the keys of the shards that have been persisted to the storage backend. This will fall back to chunks in case no shards are present. Parameters ---------- array : AsyncArray The array to inspect. Returns ------- chunks_initialized : tuple[str, ...] The keys of the shards or if these are not present, chunks that have been initialized. Related ------- [nchunks_initialized][zarr.Array.nchunks_initialized] """ store_contents = [ x async for x in array.store_path.store.list_prefix(prefix=array.store_path.path) ] store_contents_relative = [ _relativize_path(path=key, prefix=array.store_path.path) for key in store_contents # obstore can include a directory marker whose key matches the listed prefix; # it is not an initialized shard and must be excluded before relativizing. if array.store_path.path == "" or key != array.store_path.path ] return tuple( chunk_key for chunk_key in array._iter_shard_keys() if chunk_key in store_contents_relative ) type FiltersLike = ( Iterable[dict[str, JSON] | ArrayArrayCodec | Numcodec] | ArrayArrayCodec | Iterable[Numcodec] | Numcodec | Literal["auto"] | None ) # Union of acceptable types for users to pass in for both v2 and v3 compressors type CompressorLike = dict[str, JSON] | BytesBytesCodec | Numcodec | Literal["auto"] | None type CompressorsLike = ( Iterable[dict[str, JSON] | BytesBytesCodec | Numcodec] | Mapping[str, JSON] | BytesBytesCodec | Numcodec | Literal["auto"] | None ) type SerializerLike = dict[str, JSON] | ArrayBytesCodec | Literal["auto"] class ShardsConfigParam(TypedDict): shape: tuple[int, ...] index_location: IndexLocation | None # A shard shape is given in any form a chunk shape is, or as a sharding configuration. type ShardsLike = ChunksLike | ShardsConfigParam | Literal["auto"] def _stores_share_data(a: Store, b: Store) -> bool: """Whether two stores are views onto the same underlying data. ``Store.__eq__`` can't answer this directly. Several stores include ``read_only`` in their equality check, so a read-only and a writable view of the same data are unequal. ``MemoryStore`` compares its dicts by value, so two distinct stores with equal contents are equal. """ if type(a) is MemoryStore and type(b) is MemoryStore: return a._store_dict is b._store_dict with contextlib.suppress(NotImplementedError): a = a.with_read_only(b.read_only) return a == b async def from_array( store: StoreLike, *, data: AnyArray | npt.ArrayLike, write_data: bool = True, name: str | None = None, chunks: ChunksLike | Literal["auto", "keep"] = "keep", shards: ShardsLike | Literal["keep"] | None = "keep", filters: FiltersLike | Literal["keep"] = "keep", compressors: CompressorsLike | Literal["keep"] = "keep", serializer: SerializerLike | Literal["keep"] = "keep", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, zarr_format: ZarrFormat | None = None, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfigLike | None = None, ) -> AnyAsyncArray: """Create an array from an existing array or array-like. Parameters ---------- store : StoreLike StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. data : Array | array-like The array to copy. write_data : bool, default True Whether to copy the data from the input array to the new array. If `write_data` is `False`, the new array will be created with the same metadata as the input array, but without any data. name : str or None, optional The name of the array within the store. If `name` is `None`, the array will be located at the root of the store. chunks : tuple[int, ...] or Sequence[Sequence[int]] or "auto" or "keep", optional Chunk shape of the array. Following values are supported: - "auto": Automatically determine the chunk shape based on the array's shape and dtype. - "keep": Retain the chunk grid of the data array if it is a zarr Array. - tuple[int, ...]: A tuple of integers representing the chunk shape (regular grid). - Sequence[Sequence[int]]: Per-dimension chunk edge lists (rectilinear grid). Rectilinear chunk grids are experimental and must be explicitly enabled with `zarr.config.set({'array.rectilinear_chunks': True})` while the feature is stabilizing. If not specified, defaults to "keep" if data is a zarr Array, otherwise "auto". shards : tuple[int, ...], optional Shard shape of the array. Following values are supported: - "auto": Automatically determine the shard shape based on the array's shape and chunk shape. - "keep": Retain the shard shape of the data array if it is a zarr Array. - tuple[int, ...]: A tuple of integers representing the shard shape. - None: No sharding. If not specified, defaults to "keep" if data is a zarr Array, otherwise None. filters : Iterable[Codec] | Literal["auto", "keep"], optional Iterable of filters to apply to each chunk of the array, in order, before serializing that chunk to bytes. For Zarr format 3, a "filter" is a codec that takes an array and returns an array, and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the order of your filters is consistent with the behavior of each filter. The default value of `"keep"` instructs Zarr to infer `filters` from `data`. If that inference is not possible, Zarr will fall back to the behavior specified by `"auto"`, which is to choose default filters based on the data type of the array and the Zarr format specified. For all data types in Zarr V3, and most data types in Zarr V2, the default filters are the empty tuple `()`. The only cases where default filters are not empty is when the Zarr format is 2, and the data type is a variable-length data type like [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthUTF8`][]. In these cases, the default filters is a tuple with a single element which is a codec specific to that particular data type. To create an array with no filters, provide an empty iterable or the value `None`. compressors : Iterable[Codec] or "auto" or "keep", optional List of compressors to apply to the array. Compressors are applied in order, and after any filters are applied (if any are specified) and the data is serialized into bytes. For Zarr format 3, a "compressor" is a codec that takes a bytestream, and returns another bytestream. Multiple compressors may be provided for Zarr format 3. For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may be provided for Zarr format 2. Following values are supported: - Iterable[Codec]: List of compressors to apply to the array. - "auto": Automatically determine the compressors based on the array's dtype. - "keep": Retain the compressors of the input array if it is a zarr Array. If no `compressors` are provided, defaults to "keep" if data is a zarr Array, otherwise "auto". serializer : dict[str, JSON] | ArrayBytesCodec or "auto" or "keep", optional Array-to-bytes codec to use for encoding the array data. Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. Following values are supported: - dict[str, JSON]: A dict representation of an `ArrayBytesCodec`. - ArrayBytesCodec: An instance of `ArrayBytesCodec`. - "auto": a default serializer will be used. These defaults can be changed by modifying the value of `array.v3_default_serializer` in [`zarr.config`][zarr.config]. - "keep": Retain the serializer of the input array if it is a zarr Array. fill_value : Any, optional Fill value for the array. If not specified, defaults to the fill value of the data array. Pass `None` explicitly to use the default scalar of the data type (Zarr format 3) or a null fill value (Zarr format 2) instead. order : {"C", "F"}, optional The memory order of the array (default is "C"). For Zarr format 2, this parameter sets the memory order of the array. For Zarr format 3, this parameter is deprecated, because memory order is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. If not specified, defaults to the memory order of the data array. zarr_format : {2, 3}, optional The zarr format to use when saving. If not specified, defaults to the zarr format of the data array. attributes : dict, optional Attributes for the array. If not specified, the source Zarr array's attributes are deep-copied so nested containers are independent. Deeply nested attributes can raise `RecursionError` during copying even if they can be stored and reopened; the threshold depends on Python's recursion limit and the current call stack. Pass an empty dict to create the array with no attributes. chunk_key_encoding : ChunkKeyEncoding, optional A specification of how the chunk keys are represented in storage. For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. If not specified and the data array has the same zarr format as the target array, the chunk key encoding of the data array is used. dimension_names : Iterable[str | None] | None The names of the dimensions (default is None). Zarr format 3 only. Zarr format 2 arrays should not use this parameter. If not specified, defaults to the dimension names of the data array. storage_options : dict, optional If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. overwrite : bool, default False Whether to overwrite an array with the same name in the store, if one exists. config : ArrayConfig or ArrayConfigLike, optional Runtime configuration for the array. Returns ------- AsyncArray The array. Examples -------- Create an array from an existing Array: >>> import asyncio >>> import zarr >>> store = zarr.storage.LocalStore("example.zarr") >>> arr = zarr.create_array( ... store={}, ... shape=(100,100), ... chunks=(10,10), ... dtype="int32", ... fill_value=0, ... ) >>> arr2 = asyncio.run(from_array(store, data=arr, overwrite=True)) >>> arr2 >>> asyncio.run(store.clear()) # Remove files generated by test Create an array from an existing NumPy array: >>> arr3 = asyncio.run( ... from_array({}, data=np.arange(10000, dtype="i4").reshape(100, 100)) ... ) >>> arr3 Create an array from any array-like object: >>> arr4 = asyncio.run(from_array({}, data=[[1, 2], [3, 4]])) >>> arr4 >>> asyncio.run(arr4.getitem(...)) array([[1, 2], [3, 4]]) Create an array from an existing Array without copying the data: >>> arr5 = asyncio.run(from_array({}, data=Array(arr4), write_data=False)) >>> arr5 >>> asyncio.run(arr5.getitem(...)) array([[0, 0], [0, 0]]) """ mode: Literal["a"] = "a" config_parsed = parse_array_config(config) store_path = await make_store_path(store, path=name, mode=mode, storage_options=storage_options) ( chunks, shards, filters, compressors, serializer, fill_value, order, zarr_format, chunk_key_encoding, dimension_names, attributes, ) = _parse_keep_array_attr( data=data, chunks=chunks, shards=shards, filters=filters, compressors=compressors, serializer=serializer, fill_value=fill_value, order=order, zarr_format=zarr_format, chunk_key_encoding=chunk_key_encoding, dimension_names=dimension_names, attributes=attributes, ) if not hasattr(data, "dtype") or not hasattr(data, "shape"): data = np.array(data) # init_array deletes everything under the destination before the copy streams from the # source, so a destination that overlaps the source would copy only fill values. if ( write_data and overwrite and isinstance(data, Array) and _stores_share_data(data.store_path.store, store_path.store) ): src_prefix = f"{data.store_path.path}/" if data.store_path.path else "" dest_prefix = f"{store_path.path}/" if store_path.path else "" if src_prefix.startswith(dest_prefix) or dest_prefix.startswith(src_prefix): raise ValueError( f"Cannot overwrite {store_path.path!r} with the array at " f"{data.store_path.path!r} because the paths overlap. Write to a different " "path, or pass data=array[...]." ) result = await init_array( store_path=store_path, shape=data.shape, # A native object dtype cannot identify the source's Zarr data type. dtype=data._async_array._zdtype if isinstance(data, Array) else data.dtype, chunks=chunks, shards=shards, filters=filters, compressors=compressors, serializer=serializer, fill_value=fill_value, order=order, zarr_format=zarr_format, attributes=attributes, chunk_key_encoding=chunk_key_encoding, dimension_names=dimension_names, overwrite=overwrite, config=config_parsed, ) if write_data: if isinstance(data, Array): async def _copy_array_region( chunk_coords: tuple[int, ...] | slice, _data: AnyArray ) -> None: arr = await _data.async_array.getitem(chunk_coords) await result.setitem(chunk_coords, arr) # Stream data from the source array to the new array await concurrent_map( [(region, data) for region in result._iter_shard_regions()], _copy_array_region, zarr.core.config.config.get("async.concurrency"), ) else: async def _copy_arraylike_region(chunk_coords: slice, _data: NDArrayLike) -> None: await result.setitem(chunk_coords, _data[chunk_coords]) # Stream data from the source array to the new array await concurrent_map( [(region, data) for region in result._iter_shard_regions()], _copy_arraylike_region, zarr.core.config.config.get("async.concurrency"), ) return result async def init_array( *, store_path: StorePath, shape: ShapeLike, dtype: ZDTypeLike, chunks: ChunksLike | Literal["auto"] = "auto", shards: ShardsLike | None = None, filters: FiltersLike = "auto", compressors: CompressorsLike = "auto", serializer: SerializerLike = "auto", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, zarr_format: ZarrFormat | None = 3, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, overwrite: bool = False, config: ArrayConfigLike | None = None, ) -> AnyAsyncArray: """Create and persist an array metadata document. Parameters ---------- store_path : StorePath StorePath instance. The path attribute is the name of the array to initialize. shape : tuple[int, ...] Shape of the array. dtype : ZDTypeLike Data type of the array. chunks : tuple[int, ...], optional Chunk shape of the array. If not specified, default are guessed based on the shape and dtype. shards : tuple[int, ...], optional Shard shape of the array. The default value of `None` results in no sharding at all. filters : Iterable[Codec] | Literal["auto"], optional Iterable of filters to apply to each chunk of the array, in order, before serializing that chunk to bytes. For Zarr format 3, a "filter" is a codec that takes an array and returns an array, and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the order of your filters is consistent with the behavior of each filter. The default value of `"auto"` instructs Zarr to use a default based on the data type of the array and the Zarr format specified. For all data types in Zarr V3, and most data types in Zarr V2, the default filters are empty. The only cases where default filters are not empty is when the Zarr format is 2, and the data type is a variable-length data type like [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, the default filters contains a single element which is a codec specific to that particular data type. To create an array with no filters, provide an empty iterable or the value `None`. compressors : Iterable[Codec] | Literal["auto"], optional List of compressors to apply to the array. Compressors are applied in order, and after any filters are applied (if any are specified) and the data is serialized into bytes. The default value of `"auto"` instructs Zarr to use a default of [`zarr.codecs.ZstdCodec`][]. To create an array with no compressors, provide an empty iterable or the value `None`. serializer : dict[str, JSON] | ArrayBytesCodec | Literal["auto"], optional Array-to-bytes codec to use for encoding the array data. Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. The default value of `"auto"` instructs Zarr to use a default codec based on the data type of the array. For most data types this default codec is [`zarr.codecs.BytesCodec`][]. For [`zarr.dtype.VariableLengthUTF8`][], the default codec is [`zarr.codecs.VlenUTF8Codec`][]. For [`zarr.dtype.VariableLengthBytes`][], the default codec is [`zarr.codecs.VlenBytesCodec`][]. fill_value : Any, optional Fill value for the array. order : {"C", "F"}, optional The memory order of the array (default is "C"). For Zarr format 2, this parameter sets the memory order of the array. For Zarr format 3, this parameter is deprecated, because memory order is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. If no `order` is provided, a default order will be used. This default can be changed by modifying the value of `array.order` in [`zarr.config`][zarr.config]. zarr_format : {2, 3}, optional The zarr format to use when saving. attributes : dict, optional Attributes for the array. chunk_key_encoding : ChunkKeyEncodingLike, optional A specification of how the chunk keys are represented in storage. For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. dimension_names : Iterable[str], optional The names of the dimensions (default is None). Zarr format 3 only. Zarr format 2 arrays should not use this parameter. overwrite : bool, default False Whether to overwrite an array with the same name in the store, if one exists. config : ArrayConfigLike or None, default=None Configuration for this array. If `None`, the default array runtime configuration will be used. This default is stored in the global configuration object. Returns ------- AsyncArray The AsyncArray. """ if zarr_format is None: zarr_format = _default_zarr_format() from zarr.codecs.sharding import ShardingCodec zdtype = parse_dtype(dtype, zarr_format=zarr_format) shape_parsed = parse_shapelike(shape) item_size = zdtype.item_size if isinstance(zdtype, HasItemSize) else 1 chunk_key_encoding_parsed = _parse_chunk_key_encoding( chunk_key_encoding, zarr_format=zarr_format ) # Normalize the user's chunks into a canonical ChunkGrid if _is_auto(chunks): max_bytes = None if shards is None else SHARDED_INNER_CHUNK_MAX_BYTES chunks_normalized = guess_chunks(shape_parsed, item_size, max_bytes=max_bytes) else: chunks_normalized = normalize_chunks_nd(chunks, shape_parsed) # Validate rectilinear chunks constraints. The normalized grid is the one # judge of what the user declared. if not chunks_normalized.is_regular: if zarr_format == 2: raise ValueError("Zarr format 2 does not support rectilinear chunk grids.") if shards is not None: raise ValueError( "Rectilinear chunks with sharding is not supported. " "Use rectilinear shards instead: " "chunks=(inner_size, ...), shards=[[shard_sizes], ...]" ) # Resolve chunks + shards into outer_chunks (grid metadata) and # inner (sub-chunk structure for ShardingCodec, None if no sharding) outer_chunks, inner = resolve_outer_and_inner_chunks( array_shape=shape_parsed, chunks=chunks_normalized, shard_shape=shards, item_size=item_size, ) meta: ArrayV2Metadata | ArrayV3Metadata if zarr_format == 2: if inner is not None: msg = ( "Zarr format 2 arrays can only be created with `shard_shape` set to `None`. " f"Got `shard_shape={shards}` instead." ) raise ValueError(msg) if serializer != "auto": raise ValueError("Zarr format 2 arrays do not support `serializer`.") filters_parsed, compressor_parsed = _parse_chunk_encoding_v2( compressor=compressors, filters=filters, dtype=zdtype ) if dimension_names is not None: raise ValueError("Zarr format 2 arrays do not support dimension names.") if order is None: order_parsed = zarr_config.get("array.order") else: order_parsed = order chunk_key_encoding_parsed = cast("V2ChunkKeyEncoding", chunk_key_encoding_parsed) meta = AsyncArray._create_metadata_v2( shape=shape_parsed, dtype=zdtype, chunks=outer_chunks.chunk_shape, dimension_separator=chunk_key_encoding_parsed.separator, fill_value=fill_value, order=order_parsed, filters=filters_parsed, compressor=compressor_parsed, attributes=attributes, ) else: array_array, array_bytes, bytes_bytes = _parse_chunk_encoding_v3( compressors=compressors, filters=filters, serializer=serializer, dtype=zdtype, ) sub_codecs = cast("tuple[Codec, ...]", (*array_array, array_bytes, *bytes_bytes)) # A stored chunk grid passed as chunks= / shards= (e.g. by the "keep" # path of from_array) becomes the stored grid verbatim, so the new # array preserves the source grid exactly — including trailing edges # beyond the extent left behind by a shrinking resize, and bare-int # shorthand dimensions. if isinstance(shards, ChunkGridMetadata): grid = shards elif shards is None and isinstance(chunks, ChunkGridMetadata): grid = chunks else: grid = create_chunk_grid_metadata(outer_chunks) codecs_out: tuple[Codec, ...] if inner is not None: inner_chunks_flat = inner.outer_chunks.chunk_shape index_location: IndexLocation = "end" if isinstance(shards, dict): index_location = cast("IndexLocation", shards.get("index_location", "end")) sharding_codec = ShardingCodec( chunk_shape=inner_chunks_flat, codecs=sub_codecs, index_location=index_location ) sharding_codec.validate( shape=inner_chunks_flat, dtype=zdtype, chunk_grid=grid, ) codecs_out = (sharding_codec,) else: codecs_out = sub_codecs if order is not None: _warn_order_kwarg() meta = AsyncArray._create_metadata_v3( shape=shape_parsed, dtype=zdtype, fill_value=fill_value, chunk_grid=grid, chunk_key_encoding=chunk_key_encoding_parsed, codecs=codecs_out, dimension_names=dimension_names, attributes=attributes, ) arr = AsyncArray(metadata=meta, store_path=store_path, config=config) await save_new_metadata(store_path, meta, overwrite=overwrite, ensure_parents=True) return arr async def create_array( store: StoreLike, *, name: str | None = None, shape: ShapeLike | None = None, dtype: ZDTypeLike | None = None, data: np.ndarray[Any, np.dtype[Any]] | None = None, chunks: ChunksLike | Literal["auto"] = "auto", shards: ShardsLike | None = None, filters: FiltersLike = "auto", compressors: CompressorsLike = "auto", serializer: SerializerLike = "auto", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, zarr_format: ZarrFormat | None = 3, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfigLike | None = None, write_data: bool = True, ) -> AnyAsyncArray: """Create an array. Parameters ---------- store : StoreLike StoreLike object to open. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. name : str or None, optional The name of the array within the store. If `name` is `None`, the array will be located at the root of the store. shape : ShapeLike, optional Shape of the array. Must be `None` if `data` is provided. dtype : ZDTypeLike | None Data type of the array. Must be `None` if `data` is provided. data : np.ndarray, optional Array-like data to use for initializing the array. If this parameter is provided, the `shape` and `dtype` parameters must be `None`. chunks : tuple[int, ...] | Sequence[Sequence[int]] | Literal["auto"], default="auto" Chunk shape of the array. If chunks is "auto", a chunk shape is guessed based on the shape of the array and the dtype. A nested list of per-dimension edge sizes creates a rectilinear grid. Rectilinear chunk grids are experimental and must be explicitly enabled with `zarr.config.set({'array.rectilinear_chunks': True})` while the feature is stabilizing. shards : tuple[int, ...], optional Shard shape of the array. The default value of `None` results in no sharding at all. filters : Iterable[Codec] | Literal["auto"], optional Iterable of filters to apply to each chunk of the array, in order, before serializing that chunk to bytes. For Zarr format 3, a "filter" is a codec that takes an array and returns an array, and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the order of your filters is consistent with the behavior of each filter. The default value of `"auto"` instructs Zarr to use a default based on the data type of the array and the Zarr format specified. For all data types in Zarr V3, and most data types in Zarr V2, the default filters are empty. The only cases where default filters are not empty is when the Zarr format is 2, and the data type is a variable-length data type like [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, the default filters contains a single element which is a codec specific to that particular data type. To create an array with no filters, provide an empty iterable or the value `None`. compressors : Iterable[Codec], optional List of compressors to apply to the array. Compressors are applied in order, and after any filters are applied (if any are specified) and the data is serialized into bytes. For Zarr format 3, a "compressor" is a codec that takes a bytestream, and returns another bytestream. These values must be instances of [`zarr.abc.codec.BytesBytesCodec`][], or dict representations of [`zarr.abc.codec.BytesBytesCodec`][]. Multiple compressors may be provided for Zarr format 3. Codecs that take an array and return bytes are serializers and must be supplied with `serializer` instead. If no `compressors` are provided, the default for Zarr format 3 is `(ZstdCodec(),)`. Use `None` to omit default compressors. For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may be provided for Zarr format 2. If no `compressor` is provided, a default compressor will be used. This default can be changed by modifying the value of `array.v2_default_compressor` in [`zarr.config`][zarr.config]. Use `None` to omit the default compressor. serializer : dict[str, JSON] | ArrayBytesCodec, optional Array-to-bytes codec to use for encoding the array data. Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. Codecs that are instances of [`zarr.abc.codec.ArrayBytesCodec`][] must be supplied here, not with `compressors`. If no `serializer` is provided, a default serializer is chosen based on the data type: `VLenUTF8Codec` for variable-length strings, `VLenBytesCodec` for variable-length bytes, and `BytesCodec` otherwise. fill_value : Any, optional Fill value for the array. order : {"C", "F"}, optional The memory order of the array (default is "C"). For Zarr format 2, this parameter sets the memory order of the array. For Zarr format 3, this parameter is deprecated, because memory order is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. If no `order` is provided, a default order will be used. This default can be changed by modifying the value of `array.order` in [`zarr.config`][zarr.config]. zarr_format : {2, 3}, optional The zarr format to use when saving. attributes : dict, optional Attributes for the array. chunk_key_encoding : ChunkKeyEncodingLike, optional A specification of how the chunk keys are represented in storage. For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. dimension_names : Iterable[str], optional The names of the dimensions (default is None). Zarr format 3 only. Zarr format 2 arrays should not use this parameter. storage_options : dict, optional If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. overwrite : bool, default False Whether to overwrite an array with the same name in the store, if one exists. If `True`, any existing keys under that path are deleted first. config : ArrayConfigLike, optional Runtime configuration for the array. write_data : bool If a pre-existing array-like object was provided to this function via the `data` parameter then `write_data` determines whether the values in that array-like object should be written to the Zarr array created by this function. If `write_data` is `False`, then the array will be left empty. Returns ------- AsyncArray The array. Examples -------- >>> import asyncio >>> import zarr >>> asyncio.run( ... zarr.api.asynchronous.create_array( ... store={}, ... shape=(100,100), ... chunks=(10,10), ... dtype="i4", ... fill_value=0 ... ) ... ) """ data_parsed, shape_parsed, dtype_parsed = _parse_data_params( data=data, shape=shape, dtype=dtype ) if data_parsed is not None: return await from_array( store, data=data_parsed, write_data=write_data, name=name, chunks=chunks, shards=shards, filters=filters, compressors=compressors, serializer=serializer, fill_value=fill_value, order=order, zarr_format=zarr_format, attributes=attributes, chunk_key_encoding=chunk_key_encoding, dimension_names=dimension_names, storage_options=storage_options, overwrite=overwrite, config=config, ) else: mode: Literal["a"] = "a" store_path = await make_store_path( store, path=name, mode=mode, storage_options=storage_options ) return await init_array( store_path=store_path, shape=shape_parsed, dtype=dtype_parsed, chunks=chunks, shards=shards, filters=filters, compressors=compressors, serializer=serializer, fill_value=fill_value, order=order, zarr_format=zarr_format, attributes=attributes, chunk_key_encoding=chunk_key_encoding, dimension_names=dimension_names, overwrite=overwrite, config=config, ) def _chunk_layout( metadata: ArrayMetadata, ) -> tuple[tuple[int, ...] | ChunkGridMetadata, tuple[int, ...] | None]: """How an array's chunks are laid out: its chunk grid and, if it is sharded, the inner chunk shape.""" grid = metadata.chunks if isinstance(metadata, ArrayV2Metadata) else metadata.chunk_grid sharding = _sharding_codec(metadata) return grid, None if sharding is None else sharding.chunk_shape def _sharding_codec(metadata: ArrayMetadata) -> ShardingCodec | None: """The array's sharding codec, or None if the array is not sharded. Zarr format 2 arrays are never sharded. """ if isinstance(metadata, ArrayV3Metadata): return metadata.sharding_codec return None def _stored_rectilinear_grid_or_none( metadata: ArrayMetadata, ) -> RectilinearChunkGridMetadata | None: """The *stored* rectilinear chunk grid, or None if the stored grid is regular (in which case `.chunks` and `.shards` are defined). Dispatches on the stored metadata, not the runtime `ChunkGrid`: a rectilinear grid whose dimensions are all bare-int step sizes is regular at runtime, while its stored metadata — and therefore `.chunks` — is rectilinear. Zarr format 2 grids are always regular. """ if isinstance(metadata, ArrayV3Metadata) and isinstance( metadata.chunk_grid, RectilinearChunkGridMetadata ): return metadata.chunk_grid return None def _parse_keep_array_attr( data: AnyArray | npt.ArrayLike, chunks: ChunksLike | Literal["auto", "keep"], shards: ShardsLike | Literal["keep"] | None, filters: FiltersLike | Literal["keep"], compressors: CompressorsLike | Literal["keep"], serializer: SerializerLike | Literal["keep"], fill_value: Any | None, order: MemoryOrder | None, zarr_format: ZarrFormat | None, chunk_key_encoding: ChunkKeyEncodingLike | None, dimension_names: DimensionNamesLike, attributes: dict[str, JSON] | None, ) -> tuple[ ChunksLike | Literal["auto"], ShardsLike | None, FiltersLike, CompressorsLike, SerializerLike, Any | None, MemoryOrder | None, ZarrFormat, ChunkKeyEncodingLike | None, DimensionNamesLike, dict[str, JSON] | None, ]: # chunks / shards may be numpy arrays, so their sentinel checks go through the # isinstance-guarded helpers rather than a bare ``==`` against the string. if isinstance(data, Array): rectilinear_grid = _stored_rectilinear_grid_or_none(data.metadata) sharded = _sharding_codec(data.metadata) is not None if _is_keep(chunks): if rectilinear_grid is None or sharded: # `.chunks` is the inner chunk shape when sharding is used, and # inner chunks are regular whatever the shape of the shard grid. chunks = data.chunks else: # Pass the stored grid through as-is: it is O(ndim), not # O(nchunks), and init_array stores it verbatim, so the copy # preserves the grid exactly — bare-int shorthand dimensions, # and trailing edges beyond the extent left behind by a # shrinking resize. chunks = rectilinear_grid if _is_keep(shards): if rectilinear_grid is None: shards = data.shards elif sharded: # The stored grid is the shard grid; init_array accepts it as # the `shards=` parameter and stores it verbatim. shards = rectilinear_grid else: shards = None if zarr_format is None: zarr_format = data.metadata.zarr_format if filters == "keep": if zarr_format == data.metadata.zarr_format: filters = data.filters or None else: filters = "auto" if compressors == "keep": if zarr_format == data.metadata.zarr_format: compressors = data.compressors or None else: compressors = "auto" if serializer == "keep": if zarr_format == 3 and data.metadata.zarr_format == 3: serializer = cast("SerializerLike", data.serializer) else: serializer = "auto" if fill_value is DEFAULT_FILL_VALUE: fill_value = data.fill_value if data.metadata.zarr_format == 2 and zarr_format == 3 and data.order == "F": # Can't set order="F" for v3 arrays warnings.warn( "The 'order' attribute of a Zarr format 2 array does not have a direct analogue in Zarr format 3. " "The existing order='F' of the source Zarr format 2 array will be ignored.", ZarrUserWarning, stacklevel=2, ) elif order is None and zarr_format == 2: order = data.order if chunk_key_encoding is None and zarr_format == data.metadata.zarr_format: if isinstance(data.metadata, ArrayV2Metadata): chunk_key_encoding = {"name": "v2", "separator": data.metadata.dimension_separator} elif isinstance(data.metadata, ArrayV3Metadata): chunk_key_encoding = data.metadata.chunk_key_encoding if dimension_names is None and data.metadata.zarr_format == 3: dimension_names = data.metadata.dimension_names if attributes is None: # Deep copy so nested containers are not shared between the source # array's in-memory metadata and the new array's. attributes = copy.deepcopy(dict(data.attrs)) else: if _is_keep(chunks): chunks = "auto" if _is_keep(shards): shards = None if zarr_format is None: zarr_format = 3 if filters == "keep": filters = "auto" if compressors == "keep": compressors = "auto" if serializer == "keep": serializer = "auto" # After resolving "keep" above, chunks is never "keep" at this point. chunks_out: ChunksLike | Literal["auto"] = chunks return ( chunks_out, shards, filters, compressors, serializer, fill_value, order, zarr_format, chunk_key_encoding, dimension_names, attributes, ) def _parse_chunk_key_encoding( data: ChunkKeyEncodingLike | None, zarr_format: ZarrFormat ) -> ChunkKeyEncoding: """ Take an implicit specification of a chunk key encoding and parse it into a ChunkKeyEncoding object. """ if data is None: if zarr_format == 2: data = {"name": "v2", "configuration": {"separator": "."}} else: data = {"name": "default", "configuration": {"separator": "/"}} result = parse_chunk_key_encoding(data) if zarr_format == 2 and result.name != "v2": msg = ( "Invalid chunk key encoding. For Zarr format 2 arrays, the `name` field of the " f"chunk key encoding must be 'v2'. Got `name` = {result.name} instead." ) raise ValueError(msg) return result def default_compressors_v3(dtype: ZDType[Any, Any]) -> tuple[BytesBytesCodec, ...]: """ Given a data type, return the default compressors for that data type. This is just a tuple containing `ZstdCodec` """ return (ZstdCodec(),) def default_serializer_v3(dtype: ZDType[Any, Any]) -> ArrayBytesCodec: """ Given a data type, return the default serializer for that data type. The default serializer for most data types is the `BytesCodec`, which may or may not be parameterized with an endianness, depending on whether the data type has endianness. Variable length strings and variable length bytes have hard-coded serializers -- `VLenUTF8Codec` and `VLenBytesCodec`, respectively. Structured data types with multi-byte fields use `BytesCodec` with little-endian encoding. """ serializer: ArrayBytesCodec = BytesCodec(endian=None) if isinstance(dtype, HasEndianness) or ( isinstance(dtype, Structured) and dtype.has_multi_byte_fields() ): serializer = BytesCodec(endian="little") elif isinstance(dtype, HasObjectCodec): if dtype.object_codec_id == "vlen-bytes": serializer = VLenBytesCodec() elif dtype.object_codec_id == "vlen-utf8": serializer = VLenUTF8Codec() else: msg = f"Data type {dtype} requires an unknown object codec: {dtype.object_codec_id!r}." raise ValueError(msg) return serializer def default_filters_v2(dtype: ZDType[Any, Any]) -> tuple[Numcodec] | None: """ Given a data type, return the default filters for that data type. For data types that require an object codec, namely variable length data types, this is a tuple containing the object codec. Otherwise it's `None`. """ if isinstance(dtype, HasObjectCodec): if dtype.object_codec_id == "vlen-bytes": from numcodecs import VLenBytes return (VLenBytes(),) elif dtype.object_codec_id == "vlen-utf8": from numcodecs import VLenUTF8 return (VLenUTF8(),) else: msg = f"Data type {dtype} requires an unknown object codec: {dtype.object_codec_id!r}." raise ValueError(msg) return None def default_compressor_v2(dtype: ZDType[Any, Any]) -> Numcodec: """ Given a data type, return the default compressors for that data type. This is just the numcodecs `Zstd` codec. """ from numcodecs import Zstd return Zstd(level=0, checksum=False) # type: ignore[no-any-return] def _parse_chunk_encoding_v2( *, compressor: CompressorsLike, filters: FiltersLike, dtype: ZDType[TBaseDType, TBaseScalar], ) -> tuple[tuple[Numcodec, ...] | None, Numcodec | None]: """ Generate chunk encoding classes for Zarr format 2 arrays with optional defaults. """ _filters: tuple[Numcodec, ...] | None _compressor: Numcodec | None if compressor is None or compressor == (): _compressor = None elif compressor == "auto": _compressor = default_compressor_v2(dtype) elif isinstance(compressor, tuple | list) and len(compressor) == 1: _compressor = parse_compressor(compressor[0]) else: if isinstance(compressor, Iterable) and not isinstance(compressor, dict): msg = f"For Zarr format 2 arrays, the `compressor` must be a single codec. Got an iterable with type {type(compressor)} instead." raise TypeError(msg) _compressor = parse_compressor(compressor) if filters is None: _filters = None elif filters == "auto": _filters = default_filters_v2(dtype) else: if isinstance(filters, Iterable): for idx, f in enumerate(filters): if not _is_numcodec(f): msg = ( "For Zarr format 2 arrays, all elements of `filters` must be numcodecs codecs. " f"Element at index {idx} has type {type(f)}, which is not a numcodecs codec." ) raise TypeError(msg) _filters = parse_filters(filters) if isinstance(dtype, HasObjectCodec): # check the filters and the compressor for the object codec required for this data type if _filters is None: if _compressor is None: object_codec_id = None else: object_codec_id = get_object_codec_id((_compressor.get_config(),)) else: object_codec_id = get_object_codec_id( ( *[f.get_config() for f in _filters], _compressor.get_config() if _compressor is not None else None, ) ) if object_codec_id is None: if isinstance(dtype, VariableLengthUTF8): # type: ignore[unreachable] codec_name = "the numcodecs.VLenUTF8 codec" # type: ignore[unreachable] elif isinstance(dtype, VariableLengthBytes): # type: ignore[unreachable] codec_name = "the numcodecs.VLenBytes codec" # type: ignore[unreachable] else: codec_name = f"an unknown object codec with id {dtype.object_codec_id!r}" msg = ( f"Data type {dtype} requires {codec_name}, " "but no such codec was specified in the filters or compressor parameters for " "this array. " ) raise ValueError(msg) return _filters, _compressor def _parse_chunk_encoding_v3( *, compressors: CompressorsLike, filters: FiltersLike, serializer: SerializerLike, dtype: ZDType[TBaseDType, TBaseScalar], ) -> tuple[tuple[ArrayArrayCodec, ...], ArrayBytesCodec, tuple[BytesBytesCodec, ...]]: """ Generate chunk encoding classes for v3 arrays with optional defaults. """ if filters is None: out_array_array: tuple[ArrayArrayCodec, ...] = () elif filters == "auto": # no data types have default filters out_array_array = () else: maybe_array_array: Iterable[Codec | dict[str, JSON]] if isinstance(filters, dict | Codec): maybe_array_array = (filters,) else: maybe_array_array = cast("Iterable[Codec | dict[str, JSON]]", filters) out_array_array = tuple(_parse_array_array_codec(c) for c in maybe_array_array) if serializer == "auto": out_array_bytes = default_serializer_v3(dtype) else: # TODO: ensure that the serializer is compatible with the ndarray produced by the # array-array codecs. For example, if a sequence of array-array codecs produces an # array with a single-byte data type, then the serializer should not specify endiannesss. out_array_bytes = _parse_array_bytes_codec(serializer) if compressors is None: out_bytes_bytes: tuple[BytesBytesCodec, ...] = () elif compressors == "auto": out_bytes_bytes = default_compressors_v3(dtype) else: maybe_bytes_bytes: Iterable[Codec | dict[str, JSON]] if isinstance(compressors, dict | Codec): maybe_bytes_bytes = (compressors,) else: maybe_bytes_bytes = cast("Iterable[Codec | dict[str, JSON]]", compressors) out_bytes_bytes = tuple(_parse_bytes_bytes_codec(c) for c in maybe_bytes_bytes) # TODO: ensure that the serializer is compatible with the ndarray produced by the # array-array codecs. For example, if a sequence of array-array codecs produces an # array with a single-byte data type, then the serializer should not specify endiannesss. # TODO: add checks to ensure that the right serializer is used for vlen data types return out_array_array, out_array_bytes, out_bytes_bytes def _parse_deprecated_compressor( compressor: CompressorLike | None, compressors: CompressorsLike, zarr_format: int = 3 ) -> CompressorsLike | None: if compressor != "auto": if compressors != "auto": raise ValueError("Cannot specify both `compressor` and `compressors`.") if zarr_format == 3: warn( "The `compressor` argument is deprecated. Use `compressors` instead.", category=ZarrUserWarning, stacklevel=2, ) if compressor is None: # "no compression" compressors = () else: compressors = (compressor,) elif zarr_format == 2 and compressor == compressors == "auto": compressors = ({"id": "blosc"},) return compressors def _parse_data_params( *, data: np.ndarray[Any, np.dtype[Any]] | None, shape: ShapeLike | None, dtype: ZDTypeLike | None, ) -> tuple[np.ndarray[Any, np.dtype[Any]] | None, ShapeLike, ZDTypeLike]: """ Ensure an array-like `data` parameter is consistent with the `dtype` and `shape` parameters. """ if data is None: if shape is None: msg = ( "The data parameter was set to None, but shape was not specified. " "Either provide a value for data, or specify shape." ) raise ValueError(msg) shape_out = shape if dtype is None: msg = ( "The data parameter was set to None, but dtype was not specified. " "Either provide an array-like value for data, or specify dtype." ) raise ValueError(msg) dtype_out = dtype else: if shape is not None: msg = ( "The data parameter was used, but the shape parameter was also " "used. This is an error. Either use the data parameter, or the shape parameter, " "but not both." ) raise ValueError(msg) shape_out = data.shape if dtype is not None: msg = ( "The data parameter was used, but the dtype parameter was also " "used. This is an error. Either use the data parameter, or the dtype parameter, " "but not both." ) raise ValueError(msg) dtype_out = data.dtype return data, shape_out, dtype_out def _iter_chunk_coords( array: AnyArray | AnyAsyncArray, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None, ) -> Iterator[tuple[int, ...]]: """ Create an iterator over the coordinates of chunks in chunk grid space. If the `origin` keyword is used, iteration will start at the chunk index specified by `origin`. The default behavior is to start at the origin of the grid coordinate space. If the `selection_shape` keyword is used, iteration will be bounded over a contiguous region ranging from `[origin, origin selection_shape]`, where the upper bound is exclusive as per python indexing conventions. Parameters ---------- array : Array | AsyncArray The array to iterate over. origin : Sequence[int] | None, default=None The origin of the selection in grid coordinates. selection_shape : Sequence[int] | None, default=None The shape of the selection in grid coordinates. Yields ------ chunk_coords: tuple[int, ...] The coordinates of each chunk in the selection. """ return _iter_grid(array._chunk_grid_shape, origin=origin, selection_shape=selection_shape) def _iter_shard_coords( array: AnyArray | AnyAsyncArray, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None, ) -> Iterator[tuple[int, ...]]: """ Create an iterator over the coordinates of shards in shard grid space. If the `origin` keyword is used, iteration will start at the shard index specified by `origin`. The default behavior is to start at the origin of the grid coordinate space. If the `selection_shape` keyword is used, iteration will be bounded over a contiguous region ranging from `[origin, origin selection_shape]`, where the upper bound is exclusive as per python indexing conventions. If no shards are present this will iterate over the coordinates of chunks in chunk grid space instead. Parameters ---------- array : Array | AsyncArray The array to iterate over. origin : Sequence[int] | None, default=None The origin of the selection in grid coordinates. selection_shape : Sequence[int] | None, default=None The shape of the selection in grid coordinates. Yields ------ chunk_coords: tuple[int, ...] The coordinates of each shard in the selection or chunks if no shards are present. """ return _iter_grid(array._shard_grid_shape, origin=origin, selection_shape=selection_shape) def _iter_shard_keys( array: AnyArray | AnyAsyncArray, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None, ) -> Iterator[str]: """ Iterate over the storage keys of each shard, relative to an optional origin, and optionally limited to a contiguous region in shard grid coordinates. This automatically falls back to chunks when no shards are present. Parameters ---------- array : Array | AsyncArray The array to iterate over. origin : Sequence[int] | None, default=None The origin of the selection in grid coordinates. selection_shape : Sequence[int] | None, default=None The shape of the selection in grid coordinates. Yields ------ key: str The storage key of each shard in the selection or chunk when no shards are present. """ # Iterate over the coordinates of chunks in chunk grid space. _iter = _iter_grid(array._shard_grid_shape, origin=origin, selection_shape=selection_shape) return (array.metadata.encode_chunk_key(k) for k in _iter) def _iter_shard_regions( array: AnyArray | AnyAsyncArray, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None, ) -> Iterator[tuple[slice, ...]]: """ Iterate over the regions spanned by each shard. These are the smallest regions of the array that are safe to write concurrently. When no shards are present this will fall back to chunks. Parameters ---------- array : Array | AsyncArray The array to iterate over. origin : Sequence[int] | None, default=None The origin of the selection relative to the array's shard grid. selection_shape : Sequence[int] | None, default=None The shape of the selection in shard grid coordinates. Yields ------ region: tuple[slice, ...] A tuple of slice objects representing the region spanned by each shard in the selection or chunk when no shards are present. """ # The stored chunk grid always describes the write regions: the shard grid # when sharding is used, the chunk grid otherwise. Iterating it directly # works for regular and rectilinear grids alike. return array._chunk_grid.iter_chunk_regions(origin=origin, selection_shape=selection_shape) def _iter_chunk_regions( array: AnyArray | AnyAsyncArray, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None, ) -> Iterator[tuple[slice, ...]]: """ Iterate over the regions spanned by each shard. These are the smallest regions of the array that are efficient to read concurrently. Parameters ---------- array : Array | AsyncArray The array to iterate over. origin : Sequence[int] | None, default=None The origin of the selection in grid coordinates. selection_shape : Sequence[int] | None, default=None The shape of the selection in grid coordinates. Returns ------- region: tuple[slice, ...] A tuple of slice objects representing the region spanned by each shard in the selection. """ return array._chunk_grid.iter_chunk_regions(origin=origin, selection_shape=selection_shape) async def _nchunks_initialized( array: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata], ) -> int: """ Calculate the number of chunks that have been initialized in storage. This value is calculated as the sum of the number of chunks in every initialized shard (shard sizes can vary when the shard grid is rectilinear). Each initialized shard is counted at its full declared size, so for an edge shard that extends beyond the array extent this includes chunks that lie entirely outside the array, and the result can exceed `nchunks`. For arrays that do not use sharding, each stored object holds one chunk, so the number of chunks initialized is the same as the number of stored objects associated with an array. Parameters ---------- array : AsyncArray The array to inspect. Returns ------- nchunks_initialized : int The number of chunks that have been initialized. """ meta = array.metadata if not isinstance(meta, ArrayV3Metadata) or meta.sharding_codec is None: return await _nshards_initialized(array) # `.chunks` is the inner chunk shape, which is defined for any sharded array. inner_chunks = array.chunks if isinstance(meta.chunk_grid, RegularChunkGridMetadata): # Uniform shard shape: the exact chunk count per shard is a single # multiply, O(1) after the storage listing. chunks_per_shard = product( tuple(a // b for a, b in zip(meta.chunk_grid.chunk_shape, inner_chunks, strict=True)) ) return (await _nshards_initialized(array)) * chunks_per_shard # Rectilinear shard grid: the chunk count varies per shard, so decode the # initialized keys and sum each shard's own count. grid = array._chunk_grid total = 0 for key in await _shards_initialized(array): spec = grid[meta.chunk_key_encoding.decode_chunk_key(key)] if spec is not None: total += product( tuple(s // c for s, c in zip(spec.codec_shape, inner_chunks, strict=True)) ) return total async def _nshards_initialized( array: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata], ) -> int: """ Calculate the number of shards that have been initialized in storage. This is the number of shards that have been persisted to the storage backend. Parameters ---------- array : AsyncArray The array to inspect. Returns ------- nshards_initialized : int The number of shards that have been initialized. """ return len(await _shards_initialized(array)) async def _nbytes_stored( store_path: StorePath, ) -> int: """ Calculate the number of bytes stored for an array. Parameters ---------- store_path : StorePath The store path of the array. Returns ------- nbytes_stored : int The number of bytes stored. """ return await store_path.store.getsize_prefix(store_path.path) def _get_chunk_spec( metadata: ArrayMetadata, chunk_grid: ChunkGrid, chunk_coords: tuple[int, ...], array_config: ArrayConfig, prototype: BufferPrototype, ) -> ArraySpec: """Build an ArraySpec for a single chunk using the ChunkGrid.""" spec = chunk_grid[chunk_coords] if spec is None: raise IndexError(f"Chunk coordinates {chunk_coords} are out of bounds.") return ArraySpec( shape=spec.codec_shape, dtype=metadata.dtype, fill_value=metadata.fill_value, config=array_config, prototype=prototype, ) async def _get_selection( store_path: StorePath, metadata: ArrayMetadata, codec_pipeline: CodecPipeline, config: ArrayConfig, chunk_grid: ChunkGrid, indexer: Indexer, *, prototype: BufferPrototype, out: NDBuffer | None = None, fields: Fields | None = None, ) -> NDArrayLikeOrScalar: """ Get a selection from an array. Parameters ---------- store_path : StorePath The store path of the array. metadata : ArrayMetadata The array metadata. codec_pipeline : CodecPipeline The codec pipeline for encoding/decoding. config : ArrayConfig The array configuration. indexer : Indexer The indexer specifying the selection. prototype : BufferPrototype A buffer prototype to use for the retrieved data. out : NDBuffer | None, optional An output buffer to write the data to. fields : Fields | None, optional Fields to select from structured arrays. Returns ------- NDArrayLikeOrScalar The selected data. """ # `dtype` returns the zarr dtype object for both v2 and v3 metadata. zdtype = metadata.dtype dtype = zdtype.to_native_dtype() # Determine memory order if metadata.zarr_format == 2: order = metadata.order else: order = config.order # check fields are sensible out_dtype = check_fields(fields, dtype) # setup output buffer if out is not None: if isinstance(out, NDBuffer): out_buffer = out else: raise TypeError(f"out argument needs to be an NDBuffer. Got {type(out)!r}") if out_buffer.shape != indexer.shape: raise ValueError( f"shape of out argument doesn't match. Expected {indexer.shape}, got {out.shape}" ) else: out_buffer = prototype.nd_buffer.empty( shape=indexer.shape, dtype=out_dtype, order=order, ) if product(indexer.shape) > 0: # need to use the order from the metadata for v2 _config = config if metadata.zarr_format == 2: _config = replace(_config, order=order) # reading chunks and decoding them indexed_chunks = list(indexer) # For regular grids, all chunks share the same ArraySpec, so build it once # and reuse it to avoid per-chunk ChunkGrid lookups and ArraySpec construction. regular_grid = chunk_grid.is_regular if regular_grid: regular_chunk_spec = ArraySpec( shape=chunk_grid.chunk_shape, dtype=metadata.dtype, fill_value=metadata.fill_value, config=_config, prototype=prototype, ) results = await codec_pipeline.read( [ ( store_path / metadata.encode_chunk_key(chunk_coords), regular_chunk_spec if regular_grid else _get_chunk_spec(metadata, chunk_grid, chunk_coords, _config, prototype), chunk_selection, out_selection, is_complete_chunk, ) for chunk_coords, chunk_selection, out_selection, is_complete_chunk in indexed_chunks ], out_buffer, drop_axes=indexer.drop_axes, ) if _config.read_missing_chunks is False: missing_info = [] for i, result in enumerate(results): if result["status"] == "missing": coords = indexed_chunks[i][0] key = metadata.encode_chunk_key(coords) missing_info.append(f" chunk '{key}' (grid position {coords})") if missing_info: chunks_str = "\n".join(missing_info) raise ChunkNotFoundError( f"{len(missing_info)} chunk(s) not found in store '{store_path}'.\n" f"Set the 'array.read_missing_chunks' config to True to fill " f"missing chunks with the fill value.\n" f"Missing chunks:\n{chunks_str}" ) if isinstance(indexer, BasicIndexer) and indexer.shape == (): return out_buffer.as_scalar() return out_buffer.as_ndarray_like() async def _getitem( store_path: StorePath, metadata: ArrayMetadata, codec_pipeline: CodecPipeline, config: ArrayConfig, chunk_grid: ChunkGrid, selection: BasicSelection, *, prototype: BufferPrototype | None = None, ) -> NDArrayLikeOrScalar: """ Retrieve a subset of the array's data based on the provided selection. Parameters ---------- store_path : StorePath The store path of the array. metadata : ArrayMetadata The array metadata. codec_pipeline : CodecPipeline The codec pipeline for encoding/decoding. config : ArrayConfig The array configuration. chunk_grid : ChunkGrid The chunk grid. selection : BasicSelection A selection object specifying the subset of data to retrieve. prototype : BufferPrototype, optional A buffer prototype to use for the retrieved data (default is None). Returns ------- NDArrayLikeOrScalar The retrieved subset of the array's data. """ if prototype is None: prototype = default_buffer_prototype() indexer = BasicIndexer( selection, shape=metadata.shape, chunk_grid=chunk_grid, ) return await _get_selection( store_path, metadata, codec_pipeline, config, chunk_grid, indexer, prototype=prototype ) async def _set_selection( store_path: StorePath, metadata: ArrayMetadata, codec_pipeline: CodecPipeline, config: ArrayConfig, chunk_grid: ChunkGrid, indexer: Indexer, value: npt.ArrayLike, *, prototype: BufferPrototype, fields: Fields | None = None, ) -> None: """ Set a selection in an array. Parameters ---------- store_path : StorePath The store path of the array. metadata : ArrayMetadata The array metadata. codec_pipeline : CodecPipeline The codec pipeline for encoding/decoding. config : ArrayConfig The array configuration. chunk_grid : ChunkGrid The chunk grid. indexer : Indexer The indexer specifying the selection. value : npt.ArrayLike The values to write. prototype : BufferPrototype A buffer prototype to use. fields : Fields | None, optional Fields to select from structured arrays. """ # `dtype` returns the zarr dtype object for both v2 and v3 metadata. zdtype = metadata.dtype dtype = zdtype.to_native_dtype() # check fields are sensible check_fields(fields, dtype) fields = check_no_multi_fields(fields) # check value shape if np.isscalar(value): array_like = prototype.buffer.create_zero_length().as_array_like() if isinstance(array_like, np._typing._SupportsArrayFunc): # TODO: need to handle array types that don't support __array_function__ # like PyTorch and JAX array_like_ = cast("np._typing._SupportsArrayFunc", array_like) value = np.asanyarray(value, dtype=dtype, like=array_like_) else: if not hasattr(value, "shape"): value = np.asarray(value, dtype) # assert ( # value.shape == indexer.shape # ), f"shape of value doesn't match indexer shape. Expected {indexer.shape}, got {value.shape}" if not hasattr(value, "dtype") or value.dtype.name != dtype.name: if hasattr(value, "astype"): # Handle things that are already NDArrayLike more efficiently value = value.astype(dtype=dtype, order="A") else: value = np.array(value, dtype=dtype, order="A") value = cast("NDArrayLike", value) # We accept any ndarray like object from the user and convert it # to an NDBuffer (or subclass). From this point onwards, we only pass # Buffer and NDBuffer between components. value_buffer = prototype.nd_buffer.from_ndarray_like(value) # Determine memory order if metadata.zarr_format == 2: order = metadata.order else: order = config.order # need to use the order from the metadata for v2 _config = config if metadata.zarr_format == 2: _config = replace(_config, order=order) # merging with existing data and encoding chunks # For regular grids, all chunks share the same ArraySpec, so build it once # and reuse it to avoid per-chunk ChunkGrid lookups and ArraySpec construction. regular_grid = chunk_grid.is_regular if regular_grid: regular_chunk_spec = ArraySpec( shape=chunk_grid.chunk_shape, dtype=metadata.dtype, fill_value=metadata.fill_value, config=_config, prototype=prototype, ) await codec_pipeline.write( [ ( store_path / metadata.encode_chunk_key(chunk_coords), regular_chunk_spec if regular_grid else _get_chunk_spec(metadata, chunk_grid, chunk_coords, _config, prototype), chunk_selection, out_selection, is_complete_chunk, ) for chunk_coords, chunk_selection, out_selection, is_complete_chunk in indexer ], value_buffer, drop_axes=indexer.drop_axes, ) def _iter_chunk_coords_to_delete( old_grid_shape: tuple[int, ...], new_grid_shape: tuple[int, ...] ) -> Iterator[tuple[int, ...]]: """Yield the old grid minus the new grid using only O(ndim) auxiliary space.""" remaining_shape = list(old_grid_shape) for axis, (old, new) in enumerate(zip(old_grid_shape, new_grid_shape, strict=True)): if new >= old: continue # Assign each coordinate to its first axis outside the new grid. Earlier # axes have already been clipped, so these slabs never overlap. slab_shape = remaining_shape.copy() slab_shape[axis] = old - new for index in range(math.prod(slab_shape)): coords = [0] * len(slab_shape) # itertools.product caches its inputs, even when they are ranges. for dim in range(len(slab_shape) - 1, -1, -1): index, coords[dim] = divmod(index, slab_shape[dim]) coords[axis] += new yield tuple(coords) remaining_shape[axis] = new async def _resize( array: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata], new_shape: ShapeLike, delete_outside_chunks: bool = True, ) -> None: """ Resize an array to a new shape. Parameters ---------- array : AsyncArray The array to resize. new_shape : ShapeLike The desired new shape of the array. delete_outside_chunks : bool, optional If True (default), chunks that fall outside the new shape will be deleted. If False, the data in those chunks will be preserved. """ new_shape = parse_shapelike(new_shape) if len(new_shape) != len(array.metadata.shape): raise ValueError( f"The new shape must have the same number of dimensions as the array. " f"Got {len(new_shape)} dimension(s), expected {len(array.metadata.shape)}." ) new_metadata = array.metadata.update_shape(new_shape) new_chunk_grid = ChunkGrid.from_metadata(new_metadata) # Encode the new metadata before deleting any chunk: metadata that cannot be stored # then fails with the store untouched. documents = encode_documents(array.store_path, new_metadata) if delete_outside_chunks: old_grid_shape = array._chunk_grid.grid_shape new_grid_shape = new_chunk_grid.grid_shape # Chunks of the old grid outside the box it shares with the new grid. # Zero whenever no axis loses a whole chunk, including pure growth. n_delete = math.prod(old_grid_shape) - math.prod( min(old, new) for old, new in zip(old_grid_shape, new_grid_shape, strict=True) ) if n_delete > 0: async def _delete_chunk(coords: tuple[int, ...]) -> None: await (array.store_path / array.metadata.encode_chunk_key(coords)).delete() # `None` lifts the I/O concurrency limit, but the worker pool still # needs a bound; 1000 keeps the pool far smaller than the grid. limit = zarr_config.get("async.concurrency") or 1000 await concurrent_foreach( ( (coords,) for coords in _iter_chunk_coords_to_delete(old_grid_shape, new_grid_shape) ), _delete_chunk, min(limit, n_delete), ) # Write new metadata await store_documents(array.store_path, documents) array._stored_document_replaced() # Update metadata and chunk_grid (in place) object.__setattr__(array, "metadata", new_metadata) object.__setattr__(array, "_chunk_grid", new_chunk_grid) async def _append( array: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata], data: npt.ArrayLike, axis: int = 0, ) -> tuple[int, ...]: """ Append data to an array along the specified axis. Parameters ---------- array : AsyncArray The array to append to. data : npt.ArrayLike Data to be appended. axis : int Axis along which to append. Returns ------- new_shape : tuple[int, ...] The new shape of the array after appending. Notes ----- The size of all dimensions other than `axis` must match between the array and `data`. """ # ensure data is array-like if not hasattr(data, "shape"): data = np.asanyarray(data) self_shape_preserved = tuple(s for i, s in enumerate(array.shape) if i != axis) data_shape_preserved = tuple(s for i, s in enumerate(data.shape) if i != axis) if self_shape_preserved != data_shape_preserved: raise ValueError( f"shape of data to append is not compatible with the array. " f"The shape of the data is ({data_shape_preserved})" f"and the shape of the array is ({self_shape_preserved})." "All dimensions must match except for the dimension being " "appended." ) # remember old shape old_shape = array.shape # determine new shape new_shape = tuple( array.shape[i] if i != axis else array.shape[i] + data.shape[i] for i in range(len(array.shape)) ) # resize await _resize(array, new_shape) # store data append_selection = tuple( slice(None) if i != axis else slice(old_shape[i], new_shape[i]) for i in range(len(array.shape)) ) await array.setitem(append_selection, data) return new_shape async def _update_attributes( array: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata], new_attributes: dict[str, JSON], ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: """ Update the array's attributes. Parameters ---------- array : AsyncArray The array whose attributes to update. new_attributes : dict[str, JSON] A dictionary of new attributes to update or add to the array. Returns ------- AsyncArray The array with the updated attributes. """ array.metadata.attributes.update(new_attributes) # Write new metadata await array._save_metadata(array.metadata) return array async def _info_complete( array: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata], ) -> Any: """ Return all the information for an array, including dynamic information like storage size. Parameters ---------- array : AsyncArray The array to get info for. Returns ------- ArrayInfo Complete information about the array including: - The count of chunks initialized - The sum of the bytes written """ return array._info( await _nshards_initialized(array), await array.store_path.store.getsize_prefix(array.store_path.path), ) zarr-developers-zarr-python-a2f2415/src/zarr/core/array_spec.py000066400000000000000000000156051527014137600246730ustar00rootroot00000000000000from __future__ import annotations from dataclasses import dataclass, fields from typing import TYPE_CHECKING, Any, Literal, Self, TypedDict, cast import numpy as np from zarr.core.common import ( MemoryOrder, parse_bool, parse_fill_value, parse_int, parse_order, parse_shapelike, ) from zarr.core.config import config as zarr_config if TYPE_CHECKING: from typing import NotRequired from zarr.core.buffer import BufferPrototype from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType class ArrayConfigParams(TypedDict): """ A TypedDict model of the attributes of an ArrayConfig class, but with no required fields. This allows for partial construction of an ArrayConfig, with the assumption that the unset keys will be taken from a global configuration. """ order: NotRequired[MemoryOrder] write_empty_chunks: NotRequired[bool] read_missing_chunks: NotRequired[bool] sharding_coalesce_max_gap_bytes: NotRequired[int] sharding_coalesce_max_bytes: NotRequired[int] @dataclass(frozen=True) class ArrayConfig: """ A model of the runtime configuration of an array. Parameters ---------- order : MemoryOrder The memory layout of the arrays returned when reading data from the store. write_empty_chunks : bool If True, empty chunks will be written to the store. read_missing_chunks : bool If True, missing chunks will be filled with the array's fill value on read. If False, reading missing chunks will raise a ``ChunkNotFoundError``. sharding_coalesce_max_gap_bytes : int When reading multiple chunks from the same shard, nearby byte ranges separated by no more than this many bytes are coalesced into a single request to the store. sharding_coalesce_max_bytes : int Requests will not be coalesced if doing so would exceed this byte size. """ order: MemoryOrder write_empty_chunks: bool read_missing_chunks: bool sharding_coalesce_max_gap_bytes: int sharding_coalesce_max_bytes: int def __init__( self, order: MemoryOrder, write_empty_chunks: bool, *, read_missing_chunks: bool = True, sharding_coalesce_max_gap_bytes: int = 1 << 20, # 1 MiB sharding_coalesce_max_bytes: int = 16 << 20, # 16 MiB ) -> None: order_parsed = parse_order(order) write_empty_chunks_parsed = parse_bool(write_empty_chunks) read_missing_chunks_parsed = parse_bool(read_missing_chunks) sharding_coalesce_max_gap_bytes_parsed = parse_int(sharding_coalesce_max_gap_bytes) sharding_coalesce_max_bytes_parsed = parse_int(sharding_coalesce_max_bytes) object.__setattr__(self, "order", order_parsed) object.__setattr__(self, "write_empty_chunks", write_empty_chunks_parsed) object.__setattr__(self, "read_missing_chunks", read_missing_chunks_parsed) object.__setattr__( self, "sharding_coalesce_max_gap_bytes", sharding_coalesce_max_gap_bytes_parsed ) object.__setattr__(self, "sharding_coalesce_max_bytes", sharding_coalesce_max_bytes_parsed) @classmethod def from_dict(cls, data: ArrayConfigParams) -> Self: """ Create an ArrayConfig from a dict. The keys of that dict are a subset of the attributes of the ArrayConfig class. Any keys missing from that dict will be set to the the values in the ``array`` namespace of ``zarr.config``. Keys that are not attributes of the ArrayConfig class raise a `TypeError`. """ unknown = set(data) - {f.name for f in fields(ArrayConfig)} if unknown: raise TypeError( f"Unknown array config keys: {sorted(unknown)}. " f"Valid keys are {[f.name for f in fields(ArrayConfig)]}." ) kwargs_out: ArrayConfigParams = {} for f in fields(ArrayConfig): field_name = cast( "Literal['order', 'write_empty_chunks', 'read_missing_chunks', 'sharding_coalesce_max_gap_bytes', 'sharding_coalesce_max_bytes']", f.name, ) if field_name not in data: kwargs_out[field_name] = zarr_config.get(f"array.{field_name}") else: kwargs_out[field_name] = data[field_name] return cls(**kwargs_out) def to_dict(self) -> ArrayConfigParams: """ Serialize an instance of this class to a dict. """ return { "order": self.order, "write_empty_chunks": self.write_empty_chunks, "read_missing_chunks": self.read_missing_chunks, "sharding_coalesce_max_gap_bytes": self.sharding_coalesce_max_gap_bytes, "sharding_coalesce_max_bytes": self.sharding_coalesce_max_bytes, } ArrayConfigLike = ArrayConfig | ArrayConfigParams def parse_array_config(data: ArrayConfigLike | None) -> ArrayConfig: """ Convert various types of data to an ArrayConfig. """ if data is None: return ArrayConfig.from_dict({}) elif isinstance(data, ArrayConfig): return data else: return ArrayConfig.from_dict(data) @dataclass(frozen=True, eq=False) class ArraySpec: shape: tuple[int, ...] dtype: ZDType[TBaseDType, TBaseScalar] fill_value: Any config: ArrayConfig prototype: BufferPrototype def __init__( self, shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], fill_value: Any, config: ArrayConfig, prototype: BufferPrototype, ) -> None: shape_parsed = parse_shapelike(shape) fill_value_parsed = parse_fill_value(fill_value) object.__setattr__(self, "shape", shape_parsed) object.__setattr__(self, "dtype", dtype) object.__setattr__(self, "fill_value", fill_value_parsed) object.__setattr__(self, "config", config) object.__setattr__(self, "prototype", prototype) def _key(self) -> tuple[object, ...]: """Returns the tuple used for equality/hash identity.""" fill_value = self.fill_value if isinstance(fill_value, np.generic): # fill_values should be byte-identical, otherwise they correspond to different values in memory / on disk. # Importantly, this ensures np.nan == np.nan, NaT == NaT, and -0.0 != 0.0. # It also fixes np.void fill_values being unhashable (#3054). fill_value = fill_value.tobytes() return (self.shape, self.dtype, fill_value, self.config, self.prototype) def __eq__(self, other: object) -> bool: if not isinstance(other, ArraySpec): return NotImplemented return self._key() == other._key() def __hash__(self) -> int: return hash(self._key()) @property def ndim(self) -> int: return len(self.shape) @property def order(self) -> MemoryOrder: return self.config.order zarr-developers-zarr-python-a2f2415/src/zarr/core/attributes.py000066400000000000000000000032271527014137600247260ustar00rootroot00000000000000from __future__ import annotations from collections.abc import MutableMapping from typing import TYPE_CHECKING from zarr.core.common import JSON if TYPE_CHECKING: from collections.abc import Iterator from zarr.core.group import Group from zarr.types import AnyArray class Attributes(MutableMapping[str, JSON]): def __init__(self, obj: AnyArray | Group) -> None: # key=".zattrs", read_only=False, cache=True, synchronizer=None self._obj = obj def __getitem__(self, key: str) -> JSON: return self._obj.metadata.attributes[key] def __setitem__(self, key: str, value: JSON) -> None: new_attrs = dict(self._obj.metadata.attributes) new_attrs[key] = value self._obj = self._obj.update_attributes(new_attrs) def __delitem__(self, key: str) -> None: new_attrs = dict(self._obj.metadata.attributes) del new_attrs[key] self.put(new_attrs) def __iter__(self) -> Iterator[str]: return iter(self._obj.metadata.attributes) def __len__(self) -> int: return len(self._obj.metadata.attributes) def put(self, d: dict[str, JSON]) -> None: """ Overwrite all attributes with the values from `d`. Equivalent to the following pseudo-code, but performed atomically. ```python attrs = {"a": 1, "b": 2} attrs.clear() attrs.update({"a": "3", "c": 4}) print(attrs) #> {'a': '3', 'c': 4} ``` """ self._obj.metadata.attributes.clear() self._obj = self._obj.update_attributes(d) def asdict(self) -> dict[str, JSON]: return dict(self._obj.metadata.attributes) zarr-developers-zarr-python-a2f2415/src/zarr/core/buffer/000077500000000000000000000000001527014137600234335ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/core/buffer/__init__.py000066400000000000000000000006451527014137600255510ustar00rootroot00000000000000from zarr.core.buffer.core import ( ArrayLike, Buffer, BufferPrototype, NDArrayLike, NDArrayLikeOrScalar, NDBuffer, default_buffer_prototype, ) from zarr.core.buffer.cpu import numpy_buffer_prototype __all__ = [ "ArrayLike", "Buffer", "BufferPrototype", "NDArrayLike", "NDArrayLikeOrScalar", "NDBuffer", "default_buffer_prototype", "numpy_buffer_prototype", ] zarr-developers-zarr-python-a2f2415/src/zarr/core/buffer/core.py000066400000000000000000000450301527014137600247370ustar00rootroot00000000000000from __future__ import annotations import sys from abc import ABC, abstractmethod from collections.abc import Iterable from typing import ( TYPE_CHECKING, Any, Literal, NamedTuple, Protocol, SupportsIndex, cast, runtime_checkable, ) import numpy as np import numpy.typing as npt if TYPE_CHECKING: from collections.abc import Iterable, Sequence from typing import Self from zarr.codecs.bytes import EndianLiteral from zarr.core.common import BytesLike # Everything here is imported into ``zarr.core.buffer`` namespace. __all__: list[str] = [] # Integer dtypes used to compare raw bytes when `fill_value` is zero, keyed by # itemsize. A view to a dtype of the same width is always permitted by numpy. _BITWISE_DTYPE = {1: "u1", 2: "u2", 4: "u4", 8: "u8"} @runtime_checkable class ArrayLike(Protocol): """Protocol for the array-like type that underlie Buffer""" @property def dtype(self) -> np.dtype[Any]: ... @property def ndim(self) -> int: ... @property def size(self) -> int: ... def __getitem__(self, key: slice) -> Self: ... def __setitem__(self, key: slice, value: Any) -> None: ... def copy(self) -> Self: ... @runtime_checkable class NDArrayLike(Protocol): """Protocol for the nd-array-like type that underlie NDBuffer""" @property def dtype(self) -> np.dtype[Any]: ... @property def ndim(self) -> int: ... @property def size(self) -> int: ... @property def shape(self) -> tuple[int, ...]: ... def __len__(self) -> int: ... def __getitem__(self, key: slice) -> Self: ... def __setitem__(self, key: slice, value: Any) -> None: ... def __array__(self) -> npt.NDArray[Any]: ... def reshape( self, shape: tuple[int, ...], /, *, order: Literal["A", "C", "F"] | None = ..., copy: bool | None = ..., ) -> NDArrayLike: ... def view(self, dtype: npt.DTypeLike) -> Self: ... def astype( self, dtype: npt.DTypeLike, order: Literal["K", "A", "C", "F"] = ..., *, copy: bool = ..., ) -> Self: ... def fill(self, value: Any) -> None: ... def copy(self) -> Self: ... def transpose(self, axes: SupportsIndex | Sequence[SupportsIndex] | None) -> Self: ... def ravel(self, order: Literal["K", "A", "C", "F"] = ...) -> Self: ... def all(self) -> np.bool_: ... def __eq__(self, other: object) -> Self: # type: ignore[override] """Element-wise equal Notes ----- Type checkers such as mypy complains because the return type isn't a bool like its supertype "object", which violates the Liskov substitution principle. This is true, but since NumPy's ndarray is defined as an element-wise equal, our hands are tied. """ ScalarType = int | float | complex | bytes | str | bool | np.generic NDArrayLikeOrScalar = ScalarType | NDArrayLike def check_item_key_is_1d_contiguous(key: Any) -> None: """Raises error if `key` isn't a 1d contiguous slice""" if not isinstance(key, slice): raise TypeError( f"Item key has incorrect type (expected slice, got {key.__class__.__name__})" ) if not (key.step is None or key.step == 1): raise ValueError("slice must be contiguous") class Buffer(ABC): """A flat contiguous memory block We use Buffer throughout Zarr to represent a contiguous block of memory. A Buffer is backed by an underlying array-like instance that represents the memory. The memory type is unspecified; can be regular host memory, CUDA device memory, or something else. The only requirement is that the array-like instance can be copied/converted to a regular Numpy array (host memory). Notes ----- This buffer is untyped, so all indexing and sizes are in bytes. Parameters ---------- array_like array-like object that must be 1-dim, contiguous, and byte dtype. """ def __init__(self, array_like: ArrayLike) -> None: if array_like.ndim != 1: raise ValueError("array_like: only 1-dim allowed") if array_like.dtype != np.dtype("B"): raise ValueError("array_like: only byte dtype allowed") self._data = array_like @classmethod @abstractmethod def create_zero_length(cls) -> Self: """Create an empty buffer with length zero Returns ------- New empty 0-length buffer """ if cls is Buffer: raise NotImplementedError("Cannot call abstract method on the abstract class 'Buffer'") return cls( cast("ArrayLike", None) ) # This line will never be reached, but it satisfies the type checker @classmethod def from_array_like(cls, array_like: ArrayLike) -> Self: """Create a new buffer of an array-like object Parameters ---------- array_like array-like object that must be 1-dim, contiguous, and byte dtype. Returns ------- New buffer representing `array_like` """ return cls(array_like) @classmethod @abstractmethod def from_buffer(cls, buffer: Buffer) -> Self: """Create a new buffer of an existing Buffer This is useful if you want to ensure that an existing buffer is of the correct subclass of Buffer. E.g., MemoryStore uses this to return a buffer instance of the subclass specified by its BufferPrototype argument. Typically, this only copies data if the data has to be moved between memory types, such as from host to device memory. Parameters ---------- buffer buffer object. Returns ------- A new buffer representing the content of the input buffer Notes ----- Subclasses of `Buffer` must override this method to implement more optimal conversions that avoid copies where possible """ if cls is Buffer: raise NotImplementedError("Cannot call abstract method on the abstract class 'Buffer'") return cls( cast("ArrayLike", None) ) # This line will never be reached, but it satisfies the type checker @classmethod @abstractmethod def from_bytes(cls, bytes_like: BytesLike) -> Self: """Create a new buffer of a bytes-like object (host memory) Parameters ---------- bytes_like bytes-like object Returns ------- New buffer representing `bytes_like` """ if cls is Buffer: raise NotImplementedError("Cannot call abstract method on the abstract class 'Buffer'") return cls( cast("ArrayLike", None) ) # This line will never be reached, but it satisfies the type checker def as_array_like(self) -> ArrayLike: """Returns the underlying array (host or device memory) of this buffer This will never copy data. Returns ------- The underlying 1d array such as a NumPy or CuPy array. """ return self._data @abstractmethod def as_numpy_array(self) -> npt.NDArray[Any]: """Returns the buffer as a NumPy array (host memory). Notes ----- Might have to copy data, consider using `.as_array_like()` instead. Returns ------- NumPy array of this buffer (might be a data copy) """ ... def as_buffer_like(self) -> BytesLike: """Returns the buffer as an object that implements the Python buffer protocol. Notes ----- Might have to copy data, since the implementation uses `.as_numpy_array()`. Returns ------- An object that implements the Python buffer protocol """ return memoryview(self.as_numpy_array()) def to_bytes(self) -> bytes: """Returns the buffer as `bytes` (host memory). Warnings -------- Will always copy data, only use this method for small buffers such as metadata buffers. If possible, use `.as_numpy_array()` or `.as_array_like()` instead. Returns ------- `bytes` of this buffer (data copy) """ return bytes(self.as_numpy_array()) def __getitem__(self, key: slice) -> Self: check_item_key_is_1d_contiguous(key) return self.__class__(self._data.__getitem__(key)) def __setitem__(self, key: slice, value: Any) -> None: check_item_key_is_1d_contiguous(key) self._data.__setitem__(key, value) def __len__(self) -> int: return self._data.size @abstractmethod def combine(self, others: Iterable[Buffer]) -> Self: """Concatenate many buffers""" ... def __add__(self, other: Buffer) -> Self: """Concatenate two buffers""" return self.combine([other]) def __eq__(self, other: object) -> bool: # Another Buffer class can override this to choose a more efficient path return isinstance(other, Buffer) and np.array_equal( self.as_numpy_array(), other.as_numpy_array() ) class NDBuffer: """An n-dimensional memory block We use NDBuffer throughout Zarr to represent a n-dimensional memory block. An NDBuffer is backed by an underlying ndarray-like instance that represents the memory. The memory type is unspecified; can be regular host memory, CUDA device memory, or something else. The only requirement is that the ndarray-like instance can be copied/converted to a regular Numpy array (host memory). Notes ----- The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer is a special case of NDBuffer where dim=1, stride=1, and dtype="B". However, in order to use Python's type system to differentiate between the contiguous Buffer and the n-dim (non-contiguous) NDBuffer, we keep the definition of the two classes separate. Parameters ---------- array : ndarray_like ndarray-like object that is convertible to a regular Numpy array. """ def __init__(self, array: NDArrayLike) -> None: self._data = array @classmethod @abstractmethod def create( cls, *, shape: Iterable[int], dtype: npt.DTypeLike, order: Literal["C", "F"] = "C", fill_value: Any | None = None, ) -> Self: """Create a new buffer and its underlying ndarray-like object Parameters ---------- shape The shape of the buffer and its underlying ndarray-like object dtype The datatype of the buffer and its underlying ndarray-like object order Whether to store multi-dimensional data in row-major (C-style) or column-major (Fortran-style) order in memory. fill_value If not None, fill the new buffer with a scalar value. Returns ------- New buffer representing a new ndarray_like object Notes ----- A subclass can overwrite this method to create an ndarray-like object other then the default Numpy array. """ if cls is NDBuffer: raise NotImplementedError( "Cannot call abstract method on the abstract class 'NDBuffer'" ) return cls( cast("NDArrayLike", None) ) # This line will never be reached, but it satisfies the type checker @classmethod def empty( cls, shape: tuple[int, ...], dtype: npt.DTypeLike, order: Literal["C", "F"] = "C" ) -> Self: """ Create an empty buffer with the given shape, dtype, and order. This method can be faster than ``NDBuffer.create`` because it doesn't have to initialize the memory used by the underlying ndarray-like object. Parameters ---------- shape The shape of the buffer and its underlying ndarray-like object dtype The datatype of the buffer and its underlying ndarray-like object order Whether to store multi-dimensional data in row-major (C-style) or column-major (Fortran-style) order in memory. Returns ------- buffer New buffer representing a new ndarray_like object with empty data. See Also -------- NDBuffer.create Create a new buffer with some initial fill value. """ # Implementations should override this method if they have a faster way # to allocate an empty buffer. return cls.create(shape=shape, dtype=dtype, order=order) @classmethod def from_ndarray_like(cls, ndarray_like: NDArrayLike) -> Self: """Create a new buffer of an ndarray-like object Parameters ---------- ndarray_like ndarray-like object Returns ------- New buffer representing `ndarray_like` """ return cls(ndarray_like) @classmethod @abstractmethod def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self: """Create a new buffer of Numpy array-like object Parameters ---------- array_like Object that can be coerced into a Numpy array Returns ------- New buffer representing `array_like` """ if cls is NDBuffer: raise NotImplementedError( "Cannot call abstract method on the abstract class 'NDBuffer'" ) return cls( cast("NDArrayLike", None) ) # This line will never be reached, but it satisfies the type checker def as_ndarray_like(self) -> NDArrayLike: """Returns the underlying array (host or device memory) of this buffer This will never copy data. Returns ------- The underlying array such as a NumPy or CuPy array. """ return self._data @abstractmethod def as_numpy_array(self) -> npt.NDArray[Any]: """Returns the buffer as a NumPy array (host memory). Warnings -------- Might have to copy data, consider using `.as_ndarray_like()` instead. Returns ------- NumPy array of this buffer (might be a data copy) """ ... def as_scalar(self) -> ScalarType: """Returns the buffer as a scalar value""" if self._data.size != 1: raise ValueError("Buffer does not contain a single scalar value") return cast("ScalarType", self.as_numpy_array()[()]) @property def dtype(self) -> np.dtype[Any]: return self._data.dtype @property def shape(self) -> tuple[int, ...]: return self._data.shape @property def byteorder(self) -> EndianLiteral: if self.dtype.byteorder == "<": return "little" elif self.dtype.byteorder == ">": return "big" else: return sys.byteorder def reshape(self, newshape: tuple[int, ...] | Literal[-1]) -> Self: # numpy accepts a bare -1, but the NDArrayLike protocol only types the # tuple form; normalize so the forwarded value matches the protocol. shape = (newshape,) if newshape == -1 else newshape return self.__class__(self._data.reshape(shape)) def squeeze(self, axis: tuple[int, ...]) -> Self: newshape = tuple(a for i, a in enumerate(self.shape) if i not in axis) return self.__class__(self._data.reshape(newshape)) def astype(self, dtype: npt.DTypeLike, order: Literal["K", "A", "C", "F"] = "K") -> Self: return self.__class__(self._data.astype(dtype=dtype, order=order)) @abstractmethod def __getitem__(self, key: Any) -> Self: ... @abstractmethod def __setitem__(self, key: Any, value: Any) -> None: ... def __len__(self) -> int: return self._data.__len__() def __repr__(self) -> str: return f"" def all_equal(self, other: Any, equal_nan: bool = True) -> bool: """Whether every element of this buffer equals `other`.""" if other is None: # Handle None fill_value for Zarr V2 return False return _array_all_equal(self._data, other, equal_nan) def fill(self, value: Any) -> None: self._data.fill(value) def copy(self) -> Self: return self.__class__(self._data.copy()) def transpose(self, axes: SupportsIndex | Sequence[SupportsIndex] | None) -> Self: return self.__class__(self._data.transpose(axes)) def _array_all_equal(data: NDArrayLike, other: Any, equal_nan: bool) -> bool: """Whether every element of `data` equals `other`, inspecting all of it.""" # Handle positive and negative zero by comparing bit patterns: if ( np.asarray(other).dtype.kind == "f" and other == 0.0 and data.dtype.kind not in ("U", "S", "T", "O", "V") ): _data, other = np.broadcast_arrays(data, np.asarray(other, data.dtype)) # Read the bytes as unsigned integers rather than as a void dtype. # Both compare bit patterns, so both keep -0.0 distinct from 0.0, # but numpy has vectorised integer comparison loops and no void # one: the void form falls back to a generic elementwise path that # is more than an order of magnitude slower. Widths with no integer # of the same size (longdouble, complex128) keep the void form. bitwise_dtype = _BITWISE_DTYPE.get(_data.dtype.itemsize, f"V{_data.dtype.itemsize}") return bool(np.array_equal(_data.view(bitwise_dtype), other.view(bitwise_dtype))) # use array_equal to obtain equal_nan=True functionality _data, other = np.broadcast_arrays(data, other) return bool( np.array_equal( _data, other, equal_nan=equal_nan if data.dtype.kind not in ("U", "S", "T", "O", "V") else False, ) ) class BufferPrototype(NamedTuple): """Prototype of the Buffer and NDBuffer class The protocol must be pickable. Attributes ---------- buffer The Buffer class to use when Zarr needs to create new Buffer. nd_buffer The NDBuffer class to use when Zarr needs to create new NDBuffer. """ buffer: type[Buffer] nd_buffer: type[NDBuffer] # The default buffer prototype used throughout the Zarr codebase. def default_buffer_prototype() -> BufferPrototype: from zarr.registry import ( get_buffer_class, get_ndbuffer_class, ) return BufferPrototype(buffer=get_buffer_class(), nd_buffer=get_ndbuffer_class()) zarr-developers-zarr-python-a2f2415/src/zarr/core/buffer/cpu.py000066400000000000000000000265271527014137600246100ustar00rootroot00000000000000from __future__ import annotations from typing import ( TYPE_CHECKING, Any, Literal, cast, ) import numpy as np import numpy.typing as npt from zarr.core.buffer import core from zarr.registry import ( register_buffer, register_ndbuffer, ) if TYPE_CHECKING: from collections.abc import Callable, Iterable from typing import Self from zarr.core.buffer.core import ArrayLike, NDArrayLike from zarr.core.common import BytesLike # Elements in the first slice examined by `NDBuffer.all_equal`. Large enough to # amortise one numpy call, small enough that a mismatch in it is nearly free. _ALL_EQUAL_BLOCK = 1 << 14 class Buffer(core.Buffer): """A flat contiguous memory block We use Buffer throughout Zarr to represent a contiguous block of memory. A Buffer is backed by an underlying array-like instance that represents the memory. The memory type is unspecified; can be regular host memory, CUDA device memory, or something else. The only requirement is that the array-like instance can be copied/converted to a regular Numpy array (host memory). Notes ----- This buffer is untyped, so all indexing and sizes are in bytes. Parameters ---------- array_like array-like object that must be 1-dim, contiguous, and byte dtype. """ def __init__(self, array_like: ArrayLike) -> None: super().__init__(array_like) @classmethod def create_zero_length(cls) -> Self: return cls(np.array([], dtype="B")) @classmethod def from_buffer(cls, buffer: core.Buffer) -> Self: """Create a new buffer of an existing Buffer This is useful if you want to ensure that an existing buffer is of the correct subclass of Buffer. E.g., MemoryStore uses this to return a buffer instance of the subclass specified by its BufferPrototype argument. Typically, this only copies data if the data has to be moved between memory types, such as from host to device memory. Parameters ---------- buffer buffer object. Returns ------- A new buffer representing the content of the input buffer Notes ----- Subclasses of `Buffer` must override this method to implement more optimal conversions that avoid copies where possible """ return cls.from_array_like(buffer.as_numpy_array()) @classmethod def from_bytes(cls, bytes_like: BytesLike) -> Self: """Create a new buffer of a bytes-like object (host memory) Parameters ---------- bytes_like bytes-like object Returns ------- New buffer representing `bytes_like` """ return cls.from_array_like(np.frombuffer(bytes_like, dtype="B")) def as_numpy_array(self) -> npt.NDArray[Any]: """Returns the buffer as a NumPy array (host memory). Notes ----- Might have to copy data, consider using `.as_array_like()` instead. Returns ------- NumPy array of this buffer (might be a data copy) """ return np.asanyarray(self._data) def combine(self, others: Iterable[core.Buffer]) -> Self: data = [np.asanyarray(self._data)] for buf in others: other_array = buf.as_array_like() data.append(np.asanyarray(other_array)) return self.__class__(np.concatenate(data)) class NDBuffer(core.NDBuffer): """An n-dimensional memory block We use NDBuffer throughout Zarr to represent a n-dimensional memory block. An NDBuffer is backed by an underlying ndarray-like instance that represents the memory. The memory type is unspecified; can be regular host memory, CUDA device memory, or something else. The only requirement is that the ndarray-like instance can be copied/converted to a regular Numpy array (host memory). Notes ----- The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer is a special case of NDBuffer where dim=1, stride=1, and dtype="B". However, in order to use Python's type system to differentiate between the contiguous Buffer and the n-dim (non-contiguous) NDBuffer, we keep the definition of the two classes separate. Parameters ---------- array ndarray-like object that is convertible to a regular Numpy array. """ def __init__(self, array: NDArrayLike) -> None: super().__init__(array) @classmethod def create( cls, *, shape: Iterable[int], dtype: npt.DTypeLike, order: Literal["C", "F"] = "C", fill_value: Any | None = None, ) -> Self: parsed_dtype = np.dtype(dtype) zero_fill = fill_value is None or (isinstance(fill_value, int) and fill_value == 0) if parsed_dtype.kind in "mM": # NumPy allocation can drop a generic time unit's scale factor and byte order # (`np.zeros(1, ">M8[2generic]").dtype` is ` Self: data = np.empty(shape=shape, dtype=dtype, order=order) if data.dtype.kind in "mM": # See `create`: a view restores what allocation drops from generic time dtypes. data = data.view(dtype=dtype) return cls(data) @classmethod def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self: return cls.from_ndarray_like(np.asanyarray(array_like)) def astype(self, dtype: npt.DTypeLike, order: Literal["K", "A", "C", "F"] = "K") -> Self: target = np.dtype(dtype) if ( self.dtype.kind in "mM" and isinstance(target, np.dtypes.DateTime64DType | np.dtypes.TimeDelta64DType) and target.kind == self.dtype.kind and np.datetime_data(self.dtype)[0] == "generic" and np.datetime_data(self.dtype) == np.datetime_data( cast("np.dtypes.DateTime64DType | np.dtypes.TimeDelta64DType", target) ) ): # NumPy's astype between generic time dtypes ignores the target and keeps the # source byte order. Convert the underlying counts for an endian-only cast. counts = self.as_numpy_array().view(self.dtype.byteorder + "i8") converted = counts.astype(target.byteorder + "i8", order=order) return self.__class__(converted.view(target)) return super().astype(dtype, order=order) def as_numpy_array(self) -> npt.NDArray[Any]: """Returns the buffer as a NumPy array (host memory). Warnings -------- Might have to copy data, consider using `.as_ndarray_like()` instead. Returns ------- NumPy array of this buffer (might be a data copy) """ return np.asanyarray(self._data) def __getitem__(self, key: Any) -> Self: # a trailing Ellipsis makes numpy return a 0-d view instead of a scalar, which # np.asanyarray would copy (slow for large fixed-length string dtypes) if isinstance(key, tuple) and not any(k is Ellipsis for k in key): key = (*key, Ellipsis) return self.__class__(np.asanyarray(self._data.__getitem__(key))) def __setitem__(self, key: Any, value: Any) -> None: if isinstance(value, NDBuffer): value = value._data self._data.__setitem__(key, value) def all_equal(self, other: Any, equal_nan: bool = True) -> bool: """Whether every element of this buffer equals `other`. Returns as soon as part of the buffer is found to differ, rather than reading all of it. """ if other is None: # Handle None fill_value for Zarr V2 return False data = self._data if data.size == 0 or data.ndim == 0 or np.ndim(other) > 0: # nothing to slice, or `other` has to broadcast against all of `data` return core._array_all_equal(data, other, equal_nan) # A buffer that is not uniformly `other` still has to be read in full # before a whole-buffer comparison can report the first mismatch, and # on the write path that is the common case. Walking it in slices of # the leading axis returns as soon as one slice differs. Slices double # in length, so a buffer that really is uniform is covered in O(log n) # comparisons and costs the same as a single scan. A buffer no larger # than one block is covered by the first slice. # # The leading axis is used rather than a flattened view because a chunk # is usually a strided view into a larger array, which cannot be # flattened without copying it. Slicing axis 0 is a view whatever the # layout. leading = data.shape[0] step = max(1, _ALL_EQUAL_BLOCK // (data.size // leading)) start = 0 while start < leading: stop = min(start + step, leading) if not core._array_all_equal(data[start:stop], other, equal_nan): return False start = stop step *= 2 return True def as_numpy_array_wrapper( func: Callable[[npt.NDArray[Any]], bytes], buf: core.Buffer, prototype: core.BufferPrototype ) -> core.Buffer: """Converts the input of `func` to a numpy array and the output back to `Buffer`. This function is useful when calling a `func` that only support host memory such as `GZip.decode` and `Blosc.decode`. In this case, use this wrapper to convert the input `buf` to a Numpy array and convert the result back into a `Buffer`. Parameters ---------- func The callable that will be called with the converted `buf` as input. `func` must return bytes, which will be converted into a `Buffer` before returned. buf The buffer that will be converted to a Numpy array before given as input to `func`. prototype The prototype of the output buffer. Returns ------- The result of `func` converted to a `Buffer` """ return prototype.buffer.from_bytes(func(buf.as_numpy_array())) # CPU buffer prototype using numpy arrays buffer_prototype = core.BufferPrototype(buffer=Buffer, nd_buffer=NDBuffer) # default_buffer_prototype = buffer_prototype # The numpy prototype used for E.g. when reading the shard index def numpy_buffer_prototype() -> core.BufferPrototype: return core.BufferPrototype(buffer=Buffer, nd_buffer=NDBuffer) register_buffer(Buffer, qualname="zarr.buffer.cpu.Buffer") register_ndbuffer(NDBuffer, qualname="zarr.buffer.cpu.NDBuffer") # backwards compatibility register_buffer(Buffer, qualname="zarr.core.buffer.cpu.Buffer") register_ndbuffer(NDBuffer, qualname="zarr.core.buffer.cpu.NDBuffer") zarr-developers-zarr-python-a2f2415/src/zarr/core/buffer/gpu.py000066400000000000000000000171531527014137600246070ustar00rootroot00000000000000from __future__ import annotations import warnings from typing import ( TYPE_CHECKING, Any, Literal, cast, ) import numpy as np import numpy.typing as npt from zarr.core.buffer import core from zarr.core.buffer.core import ArrayLike, BufferPrototype, NDArrayLike from zarr.errors import ZarrUserWarning from zarr.registry import ( register_buffer, register_ndbuffer, ) if TYPE_CHECKING: from collections.abc import Iterable from typing import Self from zarr.core.common import BytesLike try: import cupy as cp except ImportError: cp = None class Buffer(core.Buffer): """A flat contiguous memory block on the GPU We use Buffer throughout Zarr to represent a contiguous block of memory. A Buffer is backed by an underlying array-like instance that represents the memory. The memory type is unspecified; can be regular host memory, CUDA device memory, or something else. The only requirement is that the array-like instance can be copied/converted to a regular Numpy array (host memory). Notes ----- This buffer is untyped, so all indexing and sizes are in bytes. Parameters ---------- array_like array-like object that must be 1-dim, contiguous, and byte dtype. """ def __init__(self, array_like: ArrayLike) -> None: if cp is None: raise ImportError( "Cannot use zarr.buffer.gpu.Buffer without cupy. Please install cupy." ) if array_like.ndim != 1: raise ValueError("array_like: only 1-dim allowed") if array_like.dtype != np.dtype("B"): raise ValueError("array_like: only byte dtype allowed") if not hasattr(array_like, "__cuda_array_interface__"): # Slow copy based path for arrays that don't support the __cuda_array_interface__ # TODO: Add a fast zero-copy path for arrays that support the dlpack protocol msg = ( "Creating a zarr.buffer.gpu.Buffer with an array that does not support the " "__cuda_array_interface__ for zero-copy transfers, " "falling back to slow copy based path" ) warnings.warn( msg, category=ZarrUserWarning, stacklevel=2, ) self._data = cp.asarray(array_like) @classmethod def create_zero_length(cls) -> Self: """Create an empty buffer with length zero Returns ------- New empty 0-length buffer """ return cls(cp.array([], dtype="B")) @classmethod def from_buffer(cls, buffer: core.Buffer) -> Self: """Create a GPU Buffer given an arbitrary Buffer This will try to be zero-copy if `buffer` is already on the GPU and will trigger a copy if not. Returns ------- New GPU Buffer constructed from `buffer` """ return cls(buffer.as_array_like()) @classmethod def from_bytes(cls, bytes_like: BytesLike) -> Self: return cls.from_array_like(cp.frombuffer(bytes_like, dtype="B")) def as_numpy_array(self) -> npt.NDArray[Any]: return cast("npt.NDArray[Any]", cp.asnumpy(self._data)) def combine(self, others: Iterable[core.Buffer]) -> Self: data = [cp.asanyarray(self._data)] for other in others: other_array = other.as_array_like() gpu_other = Buffer(other_array) gpu_other_array = gpu_other.as_array_like() data.append(cp.asanyarray(gpu_other_array)) return self.__class__(cp.concatenate(data)) class NDBuffer(core.NDBuffer): """A n-dimensional memory block on the GPU We use NDBuffer throughout Zarr to represent a n-dimensional memory block. An NDBuffer is backed by an underlying ndarray-like instance that represents the memory. The memory type is unspecified; can be regular host memory, CUDA device memory, or something else. The only requirement is that the ndarray-like instance can be copied/converted to a regular Numpy array (host memory). Notes ----- The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer is a special case of NDBuffer where dim=1, stride=1, and dtype="B". However, in order to use Python's type system to differentiate between the contiguous Buffer and the n-dim (non-contiguous) NDBuffer, we keep the definition of the two classes separate. Parameters ---------- array ndarray-like object that is convertible to a regular Numpy array. """ def __init__(self, array: NDArrayLike) -> None: if cp is None: raise ImportError( "Cannot use zarr.buffer.gpu.NDBuffer without cupy. Please install cupy." ) if array.dtype == object: raise TypeError("zarr.buffer.gpu.NDBuffer does not support arrays with object dtype.") self._data = array if not hasattr(array, "__cuda_array_interface__"): # Slow copy based path for arrays that don't support the __cuda_array_interface__ # TODO: Add a fast zero-copy path for arrays that support the dlpack protocol msg = ( "Creating a zarr.buffer.gpu.NDBuffer with an array that does not support the " "__cuda_array_interface__ for zero-copy transfers, " "falling back to slow copy based path" ) warnings.warn( msg, stacklevel=2, ) self._data = cp.asarray(array) @classmethod def create( cls, *, shape: Iterable[int], dtype: npt.DTypeLike, order: Literal["C", "F"] = "C", fill_value: Any | None = None, ) -> Self: ret = cls(cp.empty(shape=tuple(shape), dtype=dtype, order=order)) if fill_value is not None: ret.fill(fill_value) return ret @classmethod def empty( cls, shape: tuple[int, ...], dtype: npt.DTypeLike, order: Literal["C", "F"] = "C" ) -> Self: return cls(cp.empty(shape=shape, dtype=dtype, order=order)) @classmethod def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self: """Create a new buffer of Numpy array-like object Parameters ---------- array_like Object that can be coerced into a Numpy array Returns ------- New buffer representing `array_like` """ return cls(cp.asarray(array_like)) def as_numpy_array(self) -> npt.NDArray[Any]: """Returns the buffer as a NumPy array (host memory). Warnings -------- Might have to copy data, consider using `.as_ndarray_like()` instead. Returns ------- NumPy array of this buffer (might be a data copy) """ return cast("npt.NDArray[Any]", cp.asnumpy(self._data)) def __getitem__(self, key: Any) -> Self: return self.__class__(self._data.__getitem__(key)) def __setitem__(self, key: Any, value: Any) -> None: if isinstance(value, NDBuffer): value = value._data elif isinstance(value, core.NDBuffer): gpu_value = NDBuffer(value.as_ndarray_like()) value = gpu_value._data self._data.__setitem__(key, value) buffer_prototype = BufferPrototype(buffer=Buffer, nd_buffer=NDBuffer) register_buffer(Buffer, qualname="zarr.buffer.gpu.Buffer") register_ndbuffer(NDBuffer, qualname="zarr.buffer.gpu.NDBuffer") # backwards compatibility register_buffer(Buffer, qualname="zarr.core.buffer.gpu.Buffer") register_ndbuffer(NDBuffer, qualname="zarr.core.buffer.gpu.NDBuffer") zarr-developers-zarr-python-a2f2415/src/zarr/core/chunk_grids.py000066400000000000000000001133301527014137600250350ustar00rootroot00000000000000from __future__ import annotations import bisect import itertools import math import operator import warnings from dataclasses import dataclass, field from functools import reduce from typing import ( TYPE_CHECKING, Any, Literal, NamedTuple, Protocol, SupportsIndex, cast, runtime_checkable, ) import numpy as np import numpy.typing as npt from typing_extensions import TypeIs import zarr from zarr.core.common import ( ShapeLike, ceildiv_int, parse_shapelike, ) from zarr.errors import ZarrUserWarning if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Sequence from zarr.core.array import ShardsLike from zarr.core.common import ChunksLike from zarr.core.metadata import ArrayMetadata SHARDED_INNER_CHUNK_MAX_BYTES: int = 1048576 """Target ceiling in bytes for the auto-chunking heuristic when sharding is active (1 MiB). Applied when `chunks` is left to auto-chunking (`None` or `"auto"`) and `shards` is not `None`. Explicit chunk sizes are not affected by this value. """ @dataclass(frozen=True) class FixedDimension: """Uniform chunk size. Boundary chunks contain less data but are encoded at full size by the codec pipeline.""" size: int # chunk edge length (>= 1) extent: int # array dimension length (>= 0) nchunks: int = field(init=False, repr=False) ngridcells: int = field(init=False, repr=False) def __post_init__(self) -> None: if self.size < 1: raise ValueError(f"FixedDimension size must be >= 1, got {self.size}") if self.extent < 0: raise ValueError(f"FixedDimension extent must be >= 0, got {self.extent}") n = ceildiv_int(self.extent, self.size) object.__setattr__(self, "nchunks", n) object.__setattr__(self, "ngridcells", n) def index_to_chunk(self, idx: int) -> int: if idx < 0: raise IndexError(f"Negative index {idx} is not allowed") if idx >= self.extent: raise IndexError(f"Index {idx} is out of bounds for extent {self.extent}") return idx // self.size def chunk_offset(self, chunk_ix: int) -> int: """Byte-aligned start position of chunk *chunk_ix* in array coordinates. Does not validate *chunk_ix* — callers must ensure it is in ``[0, nchunks)``. Use ``ChunkGrid.__getitem__`` for safe access. """ return chunk_ix * self.size def chunk_size(self, chunk_ix: int) -> int: """Buffer size for codec processing — always uniform. Does not validate *chunk_ix* — callers must ensure it is in ``[0, nchunks)``. Use ``ChunkGrid.__getitem__`` for safe access. """ return self.size def data_size(self, chunk_ix: int) -> int: """Valid data region within the buffer — clipped at extent. Does not validate *chunk_ix* — callers must ensure it is in ``[0, nchunks)``. Use ``ChunkGrid.__getitem__`` for safe access. """ return max(0, min(self.size, self.extent - chunk_ix * self.size)) @property def _unique_edge_lengths(self) -> Iterable[int]: """Distinct chunk edge lengths for this dimension. Used by shard validation to check that every unique edge length is divisible by the inner chunk size. O(1) for fixed dimensions since there is only one edge length. """ return (self.size,) def indices_to_chunks(self, indices: npt.NDArray[np.intp]) -> npt.NDArray[np.intp]: return indices // self.size def with_extent(self, new_extent: int) -> FixedDimension: """Re-bind to *new_extent* without modifying edges. Used when constructing a grid from existing metadata where edges are already correct. Raises on ``VaryingDimension`` if edges don't cover the new extent. """ return FixedDimension(size=self.size, extent=new_extent) def resize(self, new_extent: int) -> FixedDimension: """Adapt for a user-initiated array resize, growing edges if needed. For ``FixedDimension`` this is identical to ``with_extent`` since regular grids don't store explicit edges. """ return FixedDimension(size=self.size, extent=new_extent) @property def _size_repr(self) -> str: return str(self.size) @dataclass(frozen=True) class VaryingDimension: """Explicit per-chunk sizes. The last chunk may extend past the array extent (``extent < sum(edges)``), in which case ``data_size`` clips to the valid region while ``chunk_size`` returns the full edge length for codec processing. This underflow is allowed to match how regular grids handle boundary chunks, and to support shrinking an array without rewriting chunk edges (the spec allows trailing edges beyond the extent).""" edges: tuple[int, ...] # per-chunk edge lengths (all > 0) cumulative: tuple[int, ...] # prefix sums for O(log n) lookup extent: int # array dimension length (may be < sum(edges) after resize) nchunks: int = field(init=False, repr=False) # cached at construction ngridcells: int = field(init=False, repr=False) # cached at construction # TODO(perf): for long dimensions (O(million chunks)): # - with_extent/resize recompute cumulative sums and nchunks from scratch; # add a fast path that reuses the existing cumulative tuple. # - Consider storing cumulative as ndarray so bisect calls can use # np.searchsorted. Scalar lookups (chunk_offset, index_to_chunk) # would need benchmarking to confirm no regression. def __init__(self, edges: Sequence[int], extent: int) -> None: edges_tuple = tuple(edges) if not edges_tuple: raise ValueError("VaryingDimension edges must not be empty") if any(e <= 0 for e in edges_tuple): raise ValueError(f"All edge lengths must be > 0, got {edges_tuple}") cumulative = tuple(itertools.accumulate(edges_tuple)) if extent < 0: raise ValueError(f"VaryingDimension extent must be >= 0, got {extent}") if extent > cumulative[-1]: raise ValueError( f"VaryingDimension extent {extent} exceeds sum of edges {cumulative[-1]}" ) object.__setattr__(self, "edges", edges_tuple) object.__setattr__(self, "cumulative", cumulative) object.__setattr__(self, "extent", extent) # Cache nchunks: number of chunks that overlap [0, extent) if extent == 0: n = 0 else: n = bisect.bisect_left(cumulative, extent) + 1 object.__setattr__(self, "nchunks", n) object.__setattr__(self, "ngridcells", len(edges_tuple)) def index_to_chunk(self, idx: int) -> int: if idx < 0 or idx >= self.extent: raise IndexError(f"Index {idx} out of bounds for dimension with extent {self.extent}") return bisect.bisect_right(self.cumulative, idx) def chunk_offset(self, chunk_ix: int) -> int: """Start position of chunk *chunk_ix* in array coordinates. Does not validate *chunk_ix* — callers must ensure it is in ``[0, ngridcells)``. Use ``ChunkGrid.__getitem__`` for safe access. """ return self.cumulative[chunk_ix - 1] if chunk_ix > 0 else 0 def chunk_size(self, chunk_ix: int) -> int: """Buffer size for codec processing. Does not validate *chunk_ix* — callers must ensure it is in ``[0, ngridcells)``. Use ``ChunkGrid.__getitem__`` for safe access. """ return self.edges[chunk_ix] def data_size(self, chunk_ix: int) -> int: """Valid data region within the buffer — clipped at extent. Does not validate *chunk_ix* — callers must ensure it is in ``[0, ngridcells)``. Use ``ChunkGrid.__getitem__`` for safe access. """ offset = self.cumulative[chunk_ix - 1] if chunk_ix > 0 else 0 return max(0, min(self.edges[chunk_ix], self.extent - offset)) @property def _unique_edge_lengths(self) -> Iterable[int]: """Distinct chunk edge lengths for this dimension (lazily deduplicated). Used by shard validation to check that every unique edge length is divisible by the inner chunk size. Lazy deduplication avoids materializing all edges for dimensions with many repeated sizes. """ seen: set[int] = set() for e in self.edges: if e not in seen: seen.add(e) yield e def indices_to_chunks(self, indices: npt.NDArray[np.intp]) -> npt.NDArray[np.intp]: return np.searchsorted(self.cumulative, indices, side="right") def with_extent(self, new_extent: int) -> VaryingDimension: """Re-bind to *new_extent* without modifying edges. Used when constructing a grid from existing metadata where edges are already correct. Raises if the existing edges don't cover *new_extent*. """ edge_sum = self.cumulative[-1] if edge_sum < new_extent: raise ValueError( f"VaryingDimension edge sum {edge_sum} is less than new extent {new_extent}" ) return VaryingDimension(self.edges, extent=new_extent) def resize(self, new_extent: int) -> VaryingDimension: """Adapt for a user-initiated array resize, growing edges if needed. Unlike ``with_extent``, this never fails — if *new_extent* exceeds the current edge sum, a new chunk is appended to cover the gap. Shrinking preserves all edges (the spec allows trailing edges beyond the array extent). """ if new_extent == self.extent: return self elif new_extent > self.cumulative[-1]: expanded_edges = list(self.edges) + [new_extent - self.cumulative[-1]] return VaryingDimension(expanded_edges, extent=new_extent) else: return VaryingDimension(self.edges, extent=new_extent) @property def _size_repr(self) -> str: return repr(tuple(self.edges)) @runtime_checkable class DimensionGrid(Protocol): """Structural interface shared by FixedDimension and VaryingDimension.""" @property def nchunks(self) -> int: ... @property def ngridcells(self) -> int: ... @property def extent(self) -> int: ... def index_to_chunk(self, idx: int) -> int: ... def chunk_offset(self, chunk_ix: int) -> int: ... def chunk_size(self, chunk_ix: int) -> int: ... def data_size(self, chunk_ix: int) -> int: ... def indices_to_chunks(self, indices: npt.NDArray[np.intp]) -> npt.NDArray[np.intp]: ... @property def _unique_edge_lengths(self) -> Iterable[int]: ... def with_extent(self, new_extent: int) -> DimensionGrid: ... def resize(self, new_extent: int) -> DimensionGrid: ... @property def _size_repr(self) -> str: ... @dataclass(frozen=True) class ChunkSpec: """Specification of a single chunk's location and size. ``slices`` gives the valid data region in array coordinates. ``codec_shape`` gives the buffer shape for codec processing. For interior chunks these are equal. For boundary chunks of a regular grid, ``codec_shape`` is the full declared chunk size while ``shape`` is clipped. For rectilinear grids, ``shape == codec_shape`` unless the last chunk extends past the array extent. """ slices: tuple[slice, ...] codec_shape: tuple[int, ...] @property def shape(self) -> tuple[int, ...]: return tuple(s.stop - s.start for s in self.slices) @property def is_boundary(self) -> bool: return self.shape != self.codec_shape # A single dimension's rectilinear chunk spec: bare int (uniform shorthand), # list of ints (explicit edges), or mixed RLE (e.g. [[10, 3], 5]). def _is_auto(spec: object) -> TypeIs[Literal["auto"]]: """Check whether a chunk or shard specification is the ``"auto"`` sentinel. Specifications may be numpy arrays, whose ``==`` against a string is elementwise and cannot be used in a boolean context, so the string check must be guarded by ``isinstance``. """ return isinstance(spec, str) and spec == "auto" def _is_keep(spec: object) -> TypeIs[Literal["keep"]]: """Check whether a chunk or shard specification is the ``"keep"`` sentinel. See `_is_auto` for why this is not a bare ``==`` comparison. """ return isinstance(spec, str) and spec == "keep" def _chunk_int(value: object) -> int | None: """Return `value` as an `int` if it is an integer chunk size, else `None`. This is the chunk normalizer's one integer test. An integer is anything Python's integer protocol (`__index__`) accepts: `int`, numpy integer scalars and 0-d integer arrays. Floats and arrays with dimensions are not integers, nor are numpy booleans; a Python `bool` is an `int`, read as 0 or 1. """ if getattr(value, "dtype", None) == np.bool_: # NumPy before 2.3 still takes a NumPy boolean as an index, with a DeprecationWarning. return None try: return operator.index(cast("SupportsIndex", value)) except TypeError: # Not an integer; numpy arrays define `__index__` but only 0-d integer arrays honour it. return None def _chunk_list(chunks: object) -> list[Any]: """Return the elements of a chunk specification that is not an integer, or raise a `TypeError` naming it (a float, `None`, a 0-d non-integer array).""" try: # `list` also takes a sequence with only `__getitem__` and `__len__`, which # `Iterable` does not recognize. return list(cast("Iterable[Any]", chunks)) except TypeError: # Not iterable, or a 0-d numpy array, whose iteration raises. raise TypeError( f"Chunk specification must be an integer or an iterable of integers; got " f"{chunks!r} of type {type(chunks).__name__}." ) from None def is_regular_1d(dim_chunks: Sequence[int]) -> bool: """Check if a single dimension's chunk sizes represent a regular grid. A regular dimension has either all chunks the same size, or all but the last chunk the same size with the last chunk smaller (boundary chunk). """ if len(dim_chunks) <= 1: return True first = dim_chunks[0] for c in dim_chunks[1:-1]: if c != first: return False # Last chunk must be the same size or a smaller boundary chunk return dim_chunks[-1] <= first def is_regular_nd(chunks: Iterable[Sequence[int]]) -> bool: """Check if an N-dimensional chunk specification represents a regular grid.""" return all(is_regular_1d(d) for d in chunks) @dataclass(frozen=True) class ChunkGrid: """ Unified chunk grid supporting both regular and rectilinear chunking. A chunk grid is a concrete arrangement of chunks for a specific array. It stores the extent (array dimension length) per dimension, enabling ``grid[coords]`` to return a ``ChunkSpec`` without external parameters. Internally represents each dimension as either FixedDimension (uniform chunks) or VaryingDimension (per-chunk edge lengths with prefix sums). """ _dimensions: tuple[DimensionGrid, ...] _is_regular: bool def __init__(self, *, dimensions: tuple[DimensionGrid, ...]) -> None: object.__setattr__(self, "_dimensions", dimensions) object.__setattr__( self, "_is_regular", all(isinstance(d, FixedDimension) for d in dimensions) ) def __repr__(self) -> str: sizes = ", ".join(d._size_repr for d in self._dimensions) shape = tuple(d.extent for d in self._dimensions) return f"ChunkGrid(chunk_sizes=({sizes}), array_shape={shape})" @classmethod def from_metadata(cls, metadata: ArrayMetadata) -> ChunkGrid: """Construct a ChunkGrid from array metadata. For v2 metadata, builds from shape and chunks. For v3 metadata, dispatches on the chunk grid type. """ from zarr.core.metadata import ArrayV2Metadata from zarr.core.metadata.v3 import RectilinearChunkGridMetadata, RegularChunkGridMetadata if isinstance(metadata, ArrayV2Metadata): return cls.from_sizes(metadata.shape, tuple(metadata.chunks)) chunk_grid_meta = metadata.chunk_grid if isinstance(chunk_grid_meta, RegularChunkGridMetadata): return cls.from_sizes(metadata.shape, tuple(chunk_grid_meta.chunk_shape)) elif isinstance(chunk_grid_meta, RectilinearChunkGridMetadata): return cls.from_sizes(metadata.shape, chunk_grid_meta.chunk_shapes) else: raise TypeError(f"Unknown chunk grid metadata type: {type(chunk_grid_meta)}") @classmethod def from_sizes( cls, array_shape: ShapeLike, chunk_sizes: Sequence[int | Sequence[int]], ) -> ChunkGrid: """Create a ChunkGrid from per-dimension chunk size specifications. Parameters ---------- array_shape The array shape (one extent per dimension). chunk_sizes Per-dimension chunk sizes. Each element is either: - An ``int`` — regular (fixed) chunk size for that dimension. - A ``Sequence[int]`` — explicit per-chunk edge lengths, kept as a ``VaryingDimension`` even when the edges are uniform: the spec form declares the grid kind, as in `normalize_chunks_1d`. """ extents = parse_shapelike(array_shape) if len(extents) != len(chunk_sizes): raise ValueError( f"array_shape has {len(extents)} dimensions but chunk_sizes " f"has {len(chunk_sizes)} dimensions" ) dims: list[DimensionGrid] = [] for dim_spec, extent in zip(chunk_sizes, extents, strict=True): if isinstance(dim_spec, int): dims.append(FixedDimension(size=dim_spec, extent=extent)) else: dims.append(VaryingDimension(dim_spec, extent=extent)) return cls(dimensions=tuple(dims)) # -- Properties -- @property def dimensions(self) -> tuple[DimensionGrid, ...]: """The per-dimension grids (`FixedDimension` or `VaryingDimension`).""" return self._dimensions @property def ndim(self) -> int: return len(self._dimensions) @property def is_regular(self) -> bool: return self._is_regular @property def grid_shape(self) -> tuple[int, ...]: """Number of chunks per dimension.""" return tuple(d.nchunks for d in self._dimensions) @property def chunk_shape(self) -> tuple[int, ...]: """Return the uniform chunk shape. Raises if grid is not regular.""" if not self.is_regular: raise ValueError( "chunk_shape is only available for regular chunk grids. " "Use grid[coords] for per-chunk sizes." ) return tuple(d.size for d in self._dimensions if isinstance(d, FixedDimension)) @property def chunk_sizes(self) -> tuple[tuple[int, ...], ...]: """Per-dimension chunk sizes, including the final boundary chunk. Returns the actual data size of each chunk (clipped at the array extent), matching the dask ``Array.chunks`` convention. Works for both regular and rectilinear grids. Returns ------- tuple[tuple[int, ...], ...] One inner tuple per dimension, each containing the data size of every chunk along that dimension. """ return tuple(tuple(d.data_size(i) for i in range(d.nchunks)) for d in self._dimensions) # -- Collection interface -- def __getitem__(self, coords: int | tuple[int, ...]) -> ChunkSpec | None: """Return the ChunkSpec for a chunk at the given grid position, or None if OOB.""" if isinstance(coords, int): coords = (coords,) if len(coords) != self.ndim: raise ValueError( f"Expected {self.ndim} coordinate(s) for a {self.ndim}-d chunk grid, " f"got {len(coords)}." ) slices: list[slice] = [] codec_shape: list[int] = [] for dim, ix in zip(self._dimensions, coords, strict=True): if ix < 0 or ix >= dim.nchunks: return None offset = dim.chunk_offset(ix) slices.append(slice(offset, offset + dim.data_size(ix), 1)) codec_shape.append(dim.chunk_size(ix)) return ChunkSpec(tuple(slices), tuple(codec_shape)) def __iter__(self) -> Iterator[ChunkSpec]: """Iterate all chunks, yielding ChunkSpec for each.""" for coords in itertools.product(*(range(d.nchunks) for d in self._dimensions)): spec = self[coords] if spec is not None: yield spec def all_chunk_coords( self, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None, ) -> Iterator[tuple[int, ...]]: """Iterate over chunk coordinates, optionally restricted to a subregion. Parameters ---------- origin : Sequence[int] | None The first chunk coordinate to return. Defaults to the grid origin. selection_shape : Sequence[int] | None The number of chunks per dimension to iterate. Defaults to the remaining extent from origin. Raises ------ IndexError If the selection extends past the grid shape (matching the bounds check of `zarr.core.indexing._iter_grid`). """ if origin is None: origin_parsed = (0,) * self.ndim else: origin_parsed = tuple(origin) if selection_shape is None: selection_shape_parsed = tuple( g - o for o, g in zip(origin_parsed, self.grid_shape, strict=True) ) else: selection_shape_parsed = tuple(selection_shape) ranges: tuple[range, ...] = () for idx, (o, g, s) in enumerate( zip(origin_parsed, self.grid_shape, selection_shape_parsed, strict=True) ): if o + s > g: raise IndexError( f"Invalid selection shape ({s}) for origin ({o}) and grid shape ({g}) at axis {idx}." ) ranges += (range(o, o + s),) return itertools.product(*ranges) def iter_chunk_regions( self, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None, ) -> Iterator[tuple[slice, ...]]: """Iterate over the data regions (slices) spanned by each chunk. Parameters ---------- origin : Sequence[int] | None The first chunk coordinate to return. Defaults to the grid origin. selection_shape : Sequence[int] | None The number of chunks per dimension to iterate. Defaults to the remaining extent from origin. """ for coords in self.all_chunk_coords(origin=origin, selection_shape=selection_shape): spec = self[coords] if spec is not None: yield spec.slices def get_nchunks(self) -> int: return reduce(operator.mul, (d.nchunks for d in self._dimensions), 1) # -- Resize -- def update_shape(self, new_shape: tuple[int, ...]) -> ChunkGrid: """Return a new ChunkGrid adjusted for *new_shape*. For regular (FixedDimension) axes the extent is simply re-bound. For varying (VaryingDimension) axes: * **grow**: a new chunk whose size equals the growth is appended. * **shrink**: trailing chunks that lie entirely beyond *new_shape* are dropped; the last retained chunk is the one whose cumulative offset first reaches or exceeds the new extent. * **no change**: the dimension is kept as-is. Raises ------ ValueError If *new_shape* has the wrong number of dimensions. """ if len(new_shape) != self.ndim: raise ValueError( f"new_shape has {len(new_shape)} dimensions but " f"chunk grid has {self.ndim} dimensions" ) dims = tuple( dim.resize(new_extent) for dim, new_extent in zip(self._dimensions, new_shape, strict=True) ) return ChunkGrid(dimensions=dims) class ChunkLayout(NamedTuple): """Result of resolving user `chunks`/`shards` into grid metadata inputs. outer_chunks Chunk grid for the chunk grid metadata. When sharding is active this holds the shard sizes; otherwise it holds the user's chunk sizes. inner Recursive sub-structure inside each chunk. `None` means the chunk is opaque (no sharding). When present, `inner.outer_chunks` gives the sub-chunk sizes passed to `ShardingCodec`, and `inner.inner` gives the next level of nesting (for nested sharding), or `None`. """ outer_chunks: ChunkGrid inner: ChunkLayout | None = None def full_span_chunk_size(span: int, unit: int = 1) -> int: """The edge length of one chunk spanning an axis of length `span`. This is the smallest positive multiple of `unit` that covers `span`, so a zero-length axis gets a chunk of size `unit` and zero chunks. `unit` is the size the chunk must be a multiple of: the inner chunk size for a shard, 1 otherwise. """ return unit * max(1, ceildiv_int(span, unit)) def _guess_regular_chunks( shape: tuple[int, ...] | int, typesize: int, *, increment_bytes: int = 256 * 1024, min_bytes: int = 128 * 1024, max_bytes: int = 64 * 1024 * 1024, ) -> tuple[int, ...]: """ Iteratively guess an appropriate chunk layout for an array, given its shape and the size of each element in bytes, and size constraints expressed in bytes. This logic is adapted from h5py. Parameters ---------- shape : tuple[int, ...] The chunk shape. typesize : int The size, in bytes, of each element of the chunk. increment_bytes : int = 256 * 1024 The number of bytes used to increment or decrement the target chunk size in bytes. min_bytes : int = 128 * 1024 The soft lower bound on the final chunk size in bytes. max_bytes : int = 64 * 1024 * 1024 The hard upper bound on the final chunk size in bytes. Returns ------- tuple[int, ...] """ if min_bytes >= max_bytes: raise ValueError(f"Cannot have more min_bytes ({min_bytes}) than max_bytes ({max_bytes})") if isinstance(shape, int): shape = (shape,) # Start from one chunk spanning each axis, then halve axes until the chunk is small enough. chunks = np.array([full_span_chunk_size(s) for s in shape], dtype="=f8") if typesize == 0: return tuple(int(x) for x in chunks) ndims = len(shape) # Determine the optimal chunk size in bytes using a PyTables expression. # This is kept as a float. dset_size = np.prod(chunks) * typesize target_size = increment_bytes * (2 ** np.log10(dset_size / (1024.0 * 1024))) if target_size > max_bytes: target_size = max_bytes elif target_size < min_bytes: target_size = min_bytes idx = 0 while True: # Repeatedly loop over the axes, dividing them by 2. Stop when: # 1a. We're smaller than the target chunk size, OR # 1b. We're within 50% of the target chunk size, AND # 2. The chunk is smaller than the maximum chunk size chunk_bytes = np.prod(chunks) * typesize if ( chunk_bytes < target_size or abs(chunk_bytes - target_size) / target_size < 0.5 ) and chunk_bytes < max_bytes: break if np.prod(chunks) == 1: break # Element size larger than max_bytes chunks[idx % ndims] = math.ceil(chunks[idx % ndims] / 2.0) idx += 1 return tuple(int(x) for x in chunks) def normalize_chunks_1d(chunks: int | Iterable[object], span: int, unit: int = 1) -> DimensionGrid: """ Normalize a one-dimensional chunk specification into a dimension grid: `FixedDimension` for scalar chunk sizes, `VaryingDimension` for explicit per-chunk size lists. Both variants bind the chunk sizes to the span, and the uniform form is O(1) in the number of chunks — a dimension with `2**62` chunks must not materialize one entry per chunk. `-1` means "one chunk covering the entire span", sized by `full_span_chunk_size(span, unit)`. Explicit chunk size lists must sum to the span exactly and always produce `VaryingDimension`, even when the sizes happen to be uniform: the input syntax declares the grid kind, so a per-chunk list is preserved as a rectilinear dimension rather than silently collapsed to a regular one, which would change how the dimension grows on resize. For scalar sizes the last chunk may overhang the span. On a zero-length span any non-empty list of positive sizes is kept: the chunks the axis grows into. """ chunk_size = _chunk_int(chunks) if chunk_size is not None: if chunk_size < -1 or chunk_size == 0: raise ValueError(f"Chunk size must be positive or -1, got {chunk_size}") if chunk_size == -1: return FixedDimension(size=full_span_chunk_size(span, unit), extent=span) return FixedDimension(size=chunk_size, extent=span) else: chunk_list = _chunk_list(chunks) if not chunk_list: raise ValueError("Chunk specification must not be empty") as_ints = [_chunk_int(c) for c in chunk_list] non_int = [ (idx, c) for idx, (c, i) in enumerate(zip(chunk_list, as_ints, strict=True)) if i is None ] if non_int: non_int_idxs, non_int_vals = [*zip(*non_int, strict=False)] raise TypeError( f"Each chunk size must be an integer; got non-integer element(s) {non_int_vals!r} " f"at indices {non_int_idxs!r}. Chunk sizes must be declared as a flat sequence of " f"positive integers (e.g. [3, 3, 1])." ) ints = [i for i in as_ints if i is not None] if any(c <= 0 for c in ints): raise ValueError(f"All chunk sizes must be positive, got {ints}") if span > 0 and sum(ints) != span: raise ValueError(f"Chunk sizes {ints} do not sum to span {span}") return VaryingDimension(ints, extent=span) def normalize_chunks_nd( chunks: ChunksLike | None, shape: tuple[int, ...], unit: tuple[int, ...] | None = None, ) -> ChunkGrid: """ Normalize a chunk specification into a `ChunkGrid`. This is a mechanical transformation — no heuristics, no guessing. Handles `False` ("all data in one chunk"), scalar ints, `-1` sentinels (one chunk per dimension covering the full span), and explicit per-dimension lists of chunk sizes (regular or rectilinear). This is the strict parser for user-supplied chunk specifications. Stored chunk grid metadata (`RegularChunkGridMetadata` / `RectilinearChunkGridMetadata`) is accepted as well and is normalized under the tolerant `ChunkGrid.from_sizes` rules instead (e.g. trailing edges beyond the array extent, as left behind by a shrinking resize). For auto-chunking, use `guess_chunks` which returns a `ChunkGrid` directly. `chunks=None` and `chunks=True` are rejected here — the caller is responsible for choosing between explicit sizes and auto-chunking. `unit` gives, per axis, the size a chunk must be a multiple of (the inner chunk shape, when normalizing a shard shape); it only affects the chunks that `-1` and `False` derive from the span. """ from zarr.core.metadata.v3 import RectilinearChunkGridMetadata, RegularChunkGridMetadata if isinstance(chunks, RegularChunkGridMetadata): return ChunkGrid.from_sizes(shape, tuple(chunks.chunk_shape)) if isinstance(chunks, RectilinearChunkGridMetadata): return ChunkGrid.from_sizes(shape, chunks.chunk_shapes) if chunks is None or chunks is True: raise ValueError( f'{chunks!r} is not a valid chunk input. Use chunks=None or chunks="auto" from the top-level API for auto-chunking, or pass an int / tuple of ints.' ) # handle no chunking: one chunk covering every axis. if chunks is False: chunks = -1 # handle 1D convenience form: one integer applies to every dimension. chunk_size = _chunk_int(chunks) chunks = (chunk_size,) * len(shape) if chunk_size is not None else _chunk_list(chunks) # handle bad dimensionality if len(chunks) != len(shape): raise ValueError( f"chunks has {len(chunks)} dimensions but shape has {len(shape)} dimensions" ) if unit is None: unit = (1,) * len(shape) return ChunkGrid( dimensions=tuple( normalize_chunks_1d(c, span=s, unit=u) for c, s, u in zip(chunks, shape, unit, strict=True) ) ) def guess_chunks( shape: tuple[int, ...], typesize: int, *, max_bytes: int | None = None ) -> ChunkGrid: """ Heuristically determine chunk sizes for an array. This is the policy function — it makes opinionated choices about chunk sizes based on array shape and element size, and returns a normalized `ChunkGrid`. Parameters ---------- shape : tuple[int, ...] Array shape. typesize : int Size of one element in bytes. max_bytes : int or None Target maximum chunk size in bytes. If None, uses the default heuristic from `_guess_regular_chunks`. """ if max_bytes is not None: flat = _guess_regular_chunks(shape, typesize, max_bytes=max_bytes) else: flat = _guess_regular_chunks(shape, typesize) return normalize_chunks_nd(flat, shape) def _guess_num_chunks_per_axis_shard( chunk_shape: tuple[int, ...], item_size: int, max_bytes: int, array_shape: tuple[int, ...] ) -> int: """Generate the number of chunks per axis to hit a target max byte size for a shard. For example, for a (2,2,2) chunk size and item size 4, maximum bytes of 256 would return 2. In other words the shard would be a (2,2,2) grid of (2,2,2) chunks i.e., prod(chunk_shape) * (returned_val ** len(chunk_shape)) * item_size = 256 bytes. Degenerate inputs — a 0-dimensional chunk shape, or a zero-byte chunk — return 1, as the search loop's stopping conditions can never be met. Parameters ---------- chunk_shape The shape of the (inner) chunks. item_size The item size of the data i.e., 2 for uint16. max_bytes The maximum number of bytes per shard to allow. array_shape The shape of the underlying array. Returns ------- The number of chunks per axis. """ bytes_per_chunk = math.prod(chunk_shape) * item_size if max_bytes < bytes_per_chunk: return 1 num_axes = len(chunk_shape) # For a 0-dimensional chunk shape or a zero-byte chunk, both loop conditions # below are constant, so the loop would never terminate. if num_axes == 0 or bytes_per_chunk == 0: return 1 chunks_per_shard = 1 # First check for byte size, second check to make sure we don't go bigger than the array shape while (bytes_per_chunk * ((chunks_per_shard + 1) ** num_axes)) <= max_bytes and all( c * (chunks_per_shard + 1) <= a for c, a in zip(chunk_shape, array_shape, strict=True) ): chunks_per_shard += 1 return chunks_per_shard def resolve_outer_and_inner_chunks( *, array_shape: tuple[int, ...], chunks: ChunkGrid, shard_shape: ShardsLike | None, item_size: int, ) -> ChunkLayout: """Resolve user `chunks`/`shards` into outer and inner chunk specs. Parameters ---------- array_shape The array shape. chunks Normalized chunk grid (the user's `chunks=`). shard_shape Raw shard specification (the user's `shards=`). `None` means no sharding, `"auto"` triggers heuristic inference, a nested sequence is treated as rectilinear shard boundaries, and anything else is used as a regular shard shape. item_size Element size in bytes. Returns ------- ChunkLayout `outer_chunks` is the `ChunkGrid` for chunk grid metadata. `inner` holds the sub-chunk structure for `ShardingCodec`, or is `None` when sharding is not active. """ if shard_shape is None: return ChunkLayout(outer_chunks=chunks) shard_spec: ChunksLike if _is_auto(shard_shape): # Extract the flat chunk shape (uniform size per dimension) for arithmetic. chunk_shape_flat = chunks.chunk_shape warnings.warn( "Automatic shard shape inference is experimental and may change without notice.", ZarrUserWarning, stacklevel=2, ) _shards_out: tuple[int, ...] = () target_shard_size_bytes = zarr.config.get("array.target_shard_size_bytes", None) num_chunks_per_shard_axis = ( _guess_num_chunks_per_axis_shard( chunk_shape=chunk_shape_flat, item_size=item_size, max_bytes=target_shard_size_bytes, array_shape=array_shape, ) if (has_auto_shard := (target_shard_size_bytes is not None)) else 2 ) for a_shape, c_shape in zip(array_shape, chunk_shape_flat, strict=True): can_shard_axis = a_shape // c_shape > 8 if not has_auto_shard else True if can_shard_axis: _shards_out += (c_shape * num_chunks_per_shard_axis,) else: _shards_out += (c_shape,) shard_spec = _shards_out elif isinstance(shard_shape, dict): shard_spec = shard_shape["shape"] else: shard_spec = shard_shape # Regular and rectilinear shard specifications go through the one normalizer. outer = normalize_chunks_nd(shard_spec, array_shape, unit=chunks.chunk_shape) return ChunkLayout(outer_chunks=outer, inner=ChunkLayout(outer_chunks=chunks)) zarr-developers-zarr-python-a2f2415/src/zarr/core/chunk_key_encodings.py000066400000000000000000000107551527014137600265550ustar00rootroot00000000000000from __future__ import annotations from abc import ABC, abstractmethod from dataclasses import dataclass from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypedDict, cast if TYPE_CHECKING: from typing import NotRequired, Self from zarr.abc.metadata import Metadata from zarr.core.common import ( JSON, NamedConfig, parse_named_configuration, ) from zarr.core.json_parse import parse_field from zarr.registry import get_chunk_key_encoding_class, register_chunk_key_encoding SeparatorLiteral = Literal[".", "/"] def parse_separator(data: JSON) -> SeparatorLiteral: return cast("SeparatorLiteral", parse_field(data, Literal[".", "/"], "separator")) class ChunkKeyEncodingParams(TypedDict): name: Literal["v2", "default"] separator: NotRequired[SeparatorLiteral] @dataclass(frozen=True) class ChunkKeyEncoding(ABC, Metadata): """ Defines how chunk coordinates are mapped to store keys. Subclasses must define a class variable `name` and implement `encode_chunk_key`. """ name: ClassVar[str] @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: _, config_parsed = parse_named_configuration(data, require_configuration=False) return cls(**config_parsed or {}) def to_dict(self) -> dict[str, JSON]: return {"name": self.name, "configuration": super().to_dict()} def decode_chunk_key(self, chunk_key: str) -> tuple[int, ...]: """ Optional: decode a chunk key string into chunk coordinates. Not required for normal operation; override if needed for testing or debugging. """ raise NotImplementedError(f"{self.__class__.__name__} does not implement decode_chunk_key.") @abstractmethod def encode_chunk_key(self, chunk_coords: tuple[int, ...]) -> str: """ Encode chunk coordinates into a chunk key string. Must be implemented by subclasses. """ type ChunkKeyEncodingLike = ( dict[str, JSON] | ChunkKeyEncodingParams | ChunkKeyEncoding | NamedConfig[str, Any] ) @dataclass(frozen=True) class DefaultChunkKeyEncoding(ChunkKeyEncoding): name: ClassVar[Literal["default"]] = "default" separator: SeparatorLiteral = "/" def __post_init__(self) -> None: separator_parsed = parse_separator(self.separator) object.__setattr__(self, "separator", separator_parsed) def decode_chunk_key(self, chunk_key: str) -> tuple[int, ...]: if chunk_key == "c": return () # Strip the "c" prefix (e.g. "c/" or "c.") before splitting. prefix = "c" + self.separator if chunk_key.startswith(prefix): return tuple(map(int, chunk_key[len(prefix) :].split(self.separator))) raise ValueError(f"Invalid chunk key for default encoding: {chunk_key!r}") def encode_chunk_key(self, chunk_coords: tuple[int, ...]) -> str: return self.separator.join(map(str, ("c",) + chunk_coords)) @dataclass(frozen=True) class V2ChunkKeyEncoding(ChunkKeyEncoding): name: ClassVar[Literal["v2"]] = "v2" separator: SeparatorLiteral = "." def __post_init__(self) -> None: separator_parsed = parse_separator(self.separator) object.__setattr__(self, "separator", separator_parsed) def decode_chunk_key(self, chunk_key: str) -> tuple[int, ...]: return tuple(map(int, chunk_key.split(self.separator))) def encode_chunk_key(self, chunk_coords: tuple[int, ...]) -> str: chunk_identifier = self.separator.join(map(str, chunk_coords)) return "0" if chunk_identifier == "" else chunk_identifier def parse_chunk_key_encoding(data: ChunkKeyEncodingLike) -> ChunkKeyEncoding: """ Take an implicit specification of a chunk key encoding and parse it into a ChunkKeyEncoding object. """ if isinstance(data, ChunkKeyEncoding): return data # handle ChunkKeyEncodingParams if "name" in data and "separator" in data: data = {"name": data["name"], "configuration": {"separator": data["separator"]}} # type: ignore[typeddict-item] # Now must be a named config data = cast("dict[str, JSON]", data) name_parsed, _ = parse_named_configuration(data, require_configuration=False) try: chunk_key_encoding = get_chunk_key_encoding_class(name_parsed).from_dict(data) except KeyError as e: raise ValueError(f"Unknown chunk key encoding: {e.args[0]!r}") from e return chunk_key_encoding register_chunk_key_encoding("default", DefaultChunkKeyEncoding) register_chunk_key_encoding("v2", V2ChunkKeyEncoding) zarr-developers-zarr-python-a2f2415/src/zarr/core/chunk_utils.py000066400000000000000000000362631527014137600250760ustar00rootroot00000000000000from __future__ import annotations from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, cast from zarr.abc.codec import GetResult, SupportsSyncCodec, _codec_supports_sync from zarr.core.indexing import is_scalar if TYPE_CHECKING: from collections.abc import Callable, Iterable from zarr.abc.codec import Codec from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer, NDBuffer from zarr.core.indexing import SelectorTuple def evolve_codecs(codecs: Iterable[Codec], array_spec: ArraySpec) -> tuple[Codec, ...]: """Evolve a codec chain against ``array_spec``, threading the spec forward. Each codec is evolved against the spec produced by the previous one — NOT the original ``array_spec`` — because earlier array->array codecs may transform the chunk spec (e.g. ``cast_value`` widening int8 -> int16). A later codec (notably the array->bytes serializer) must be evolved against the spec it will actually see at run time; evolving every codec against the unthreaded original spec would, for example, strip a ``BytesCodec``'s ``endian`` (it sees the single-byte source dtype) and then fail at decode time on the multi-byte target. This is the single source of truth for pipeline-construction-time codec evolution, shared by every ``CodecPipeline.evolve_from_array_spec``. (The per-chunk decode/encode counterpart is ``resolve_aa_specs``.) """ evolved: list[Codec] = [] spec = array_spec for codec in codecs: evolved_codec = codec.evolve_from_array_spec(array_spec=spec) evolved.append(evolved_codec) spec = evolved_codec.resolve_metadata(spec) return tuple(evolved) def encode_or_elide_chunk( chunk_array: NDBuffer, chunk_spec: ArraySpec, encode: Callable[[NDBuffer, ArraySpec], Buffer | None], ) -> Buffer | None: """Encode a merged chunk, normalizing empties to missing. Returns the bytes to store, or ``None`` meaning the chunk must NOT be stored (either it normalized to empty per `chunk_is_empty`, or the codec chain elided it). ``None`` is the single "missing" convention shared by the chunk write paths and the shard dicts. """ if chunk_is_empty(chunk_array, chunk_spec): return None return encode(chunk_array, chunk_spec) def fill_value_or_default(chunk_spec: ArraySpec) -> Any: fill_value = chunk_spec.fill_value if fill_value is None: # Zarr V2 allowed `fill_value` to be null in the metadata. # Zarr V3 requires it to be set. This has already been # validated when decoding the metadata, but we support reading # Zarr V2 data and need to support the case where fill_value # is None. return chunk_spec.dtype.default_scalar() else: return fill_value def chunk_is_empty(chunk_array: NDBuffer, chunk_spec: ArraySpec) -> bool: """THE empty-chunk normalization rule, in one place. With ``write_empty_chunks=False`` (the default), a chunk whose decoded content equals the fill value normalizes to *missing*: it must not be stored, and readers reconstruct it from the fill value. Every write path (fused, async fallback, shard inner chunks) must apply this same rule — scattering inline ``all_equal`` checks is how the rule drifts. """ return not chunk_spec.config.write_empty_chunks and chunk_array.all_equal( fill_value_or_default(chunk_spec) ) def scatter_chunk( selected: NDBuffer | None, out: NDBuffer, *, chunk_spec: ArraySpec, out_selection: SelectorTuple, drop_axes: tuple[int, ...], ) -> GetResult: """Scatter one chunk's (already-selected) decoded region into ``out``. ``None`` = the chunk is missing: the fill value is scattered instead and a ``missing`` status is returned. POLICY-FREE by design: whether a missing chunk is an error (``read_missing_chunks=False``) is decided by the array layer from the returned statuses — which is also what makes missing INNER chunks of a present shard fill rather than raise (the sharding codec discards the nested read's statuses; only top-level statuses reach the array layer). """ if selected is None: out[out_selection] = fill_value_or_default(chunk_spec) return GetResult(status="missing") if drop_axes: selected = selected.squeeze(axis=drop_axes) out[out_selection] = selected return GetResult(status="present") def _merge_chunk_array( existing_chunk_array: NDBuffer | None, value: NDBuffer, out_selection: SelectorTuple, chunk_spec: ArraySpec, chunk_selection: SelectorTuple, is_complete_chunk: bool, drop_axes: tuple[int, ...], ) -> NDBuffer: """Merge `value` into a full-chunk-shaped NDBuffer at `chunk_selection`. If `is_complete_chunk` and `value[out_selection]` is exactly chunk-shaped, that VIEW of the caller's `value` is returned without copying — callers (and the codecs they pass it to) must treat it as read-only, since mutating it would corrupt the user's source array. Otherwise, a writable buffer is materialized — either from `existing_chunk_array.copy()` if one was read from the store, or freshly allocated and filled with the chunk's fill value — and the relevant slice of `value` is written into it. """ if is_complete_chunk and value.shape != (): selected = value[out_selection] # The shape check guards against a partial edge chunk arriving with # is_complete_chunk=True, and against dropped axes (size-1 integer # dims), where the selection is not exactly chunk-shaped. if selected.shape == chunk_spec.shape: return selected if existing_chunk_array is None: chunk_array = chunk_spec.prototype.nd_buffer.create( shape=chunk_spec.shape, dtype=chunk_spec.dtype.to_native_dtype(), order=chunk_spec.order, fill_value=fill_value_or_default(chunk_spec), ) else: chunk_array = existing_chunk_array.copy() if chunk_selection == () or is_scalar( value.as_ndarray_like(), chunk_spec.dtype.to_native_dtype() ): chunk_value = value else: chunk_value = value[out_selection] if drop_axes: item = tuple( None if idx in drop_axes else slice(None) for idx in range(chunk_spec.ndim) ) chunk_value = chunk_value[item] chunk_array[chunk_selection] = chunk_value return chunk_array def merge_and_encode_chunk( existing_bytes: Buffer | None, value: NDBuffer, *, chunk_spec: ArraySpec, chunk_selection: SelectorTuple, out_selection: SelectorTuple, is_complete: bool, drop_axes: tuple[int, ...], decode: Callable[[Buffer, ArraySpec], NDBuffer], encode: Callable[[NDBuffer, ArraySpec], Buffer | None], ) -> Buffer | None: """The canonical single-chunk write: merge, normalize, encode. decode existing (``None`` = chunk currently missing) -> merge ``value`` at ``chunk_selection`` -> normalize empties to missing -> encode. Returns the bytes to store or ``None`` = do not store / delete. This is the one state-transition every per-chunk write path expresses; only the IO around it (where ``existing_bytes`` comes from, where the result goes) differs. """ existing_array = decode(existing_bytes, chunk_spec) if existing_bytes is not None else None merged = _merge_chunk_array( existing_array, value, out_selection, chunk_spec, chunk_selection, is_complete, drop_axes ) return encode_or_elide_chunk(merged, chunk_spec, encode) def decode_and_scatter_chunk( chunk_bytes: Buffer | None, out: NDBuffer, *, chunk_spec: ArraySpec, chunk_selection: SelectorTuple, out_selection: SelectorTuple, drop_axes: tuple[int, ...], decode: Callable[[Buffer, ArraySpec], NDBuffer], ) -> GetResult: """The canonical single-chunk read: decode stored bytes (``None`` = missing), select, and scatter into ``out`` via `scatter_chunk`. The read twin of `merge_and_encode_chunk`. """ if chunk_bytes is None: return scatter_chunk( None, out, chunk_spec=chunk_spec, out_selection=out_selection, drop_axes=drop_axes ) selected = decode(chunk_bytes, chunk_spec)[chunk_selection] return scatter_chunk( selected, out, chunk_spec=chunk_spec, out_selection=out_selection, drop_axes=drop_axes ) @dataclass(slots=True, kw_only=True) class ChunkTransform: """A synchronous codec chain. Provides `encode_chunk` and `decode_chunk` for pure-compute codec operations (no IO, no threading, no batching). The `chunk_spec` is supplied per call so the same transform can be reused across chunks with different shapes, prototypes, etc. All codecs must implement `SupportsSyncCodec`. Construction will raise `TypeError` if any codec does not. """ codecs: tuple[Codec, ...] _aa_codecs: tuple[SupportsSyncCodec[NDBuffer, NDBuffer], ...] = field( init=False, repr=False, compare=False ) _ab_codec: SupportsSyncCodec[NDBuffer, Buffer] = field(init=False, repr=False, compare=False) _bb_codecs: tuple[SupportsSyncCodec[Buffer, Buffer], ...] = field( init=False, repr=False, compare=False ) def __post_init__(self) -> None: from zarr.core.codec_pipeline import codecs_from_list_unchecked # _codec_supports_sync, not a bare isinstance check: a codec can satisfy # the SupportsSyncCodec protocol structurally yet be unable to run # synchronously (ShardingCodec whose inner/index chain contains an # async-only codec). Such codecs opt out via `_sync_capable`, and the # TypeError here is what makes FusedCodecPipeline.evolve_from_array_spec # decline the sync fast path and fall back to the async pipeline. non_sync = [c for c in self.codecs if not _codec_supports_sync(c)] if non_sync: names = ", ".join(type(c).__name__ for c in non_sync) raise TypeError( f"All codecs must implement SupportsSyncCodec. The following do not: {names}" ) # `ChunkTransform` is built from a codec chain that already went # through `codecs_from_list` when the owning pipeline was constructed # (see `FusedCodecPipeline.evolve_from_array_spec`), so re-splitting it # here must not re-emit that chain's advisory warnings. aa, ab, bb = codecs_from_list_unchecked(list(self.codecs)) # SupportsSyncCodec was verified above; the cast is purely for mypy. self._aa_codecs = cast("tuple[SupportsSyncCodec[NDBuffer, NDBuffer], ...]", tuple(aa)) self._ab_codec = cast("SupportsSyncCodec[NDBuffer, Buffer]", ab) self._bb_codecs = cast("tuple[SupportsSyncCodec[Buffer, Buffer], ...]", tuple(bb)) # The whole cache entry — (key, aa_specs, ab_spec) — is stored as ONE field # and replaced with a single attribute write. A `ChunkTransform` is shared # across thread-pool workers (read_sync/write_sync with max_workers > 1), and # storing the key separately from the specs would race: a worker could read a # freshly-set key while the matching specs were still the previous (or None) # value. A single tuple assignment is atomic under the GIL, so a reader sees # either the complete old entry or the complete new one — never a torn mix. _cache: tuple[ArraySpec, tuple[ArraySpec, ...], ArraySpec] | None = field( init=False, repr=False, compare=False, default=None ) def _resolve_specs(self, chunk_spec: ArraySpec) -> tuple[tuple[ArraySpec, ...], ArraySpec]: """Return per-AA-codec input specs and the AB spec for `chunk_spec`. The resolved chain depends only on the value of `chunk_spec`, so we cache it keyed on `chunk_spec` itself (ArraySpec is a frozen, hashable dataclass — value identity). Keying on `id(chunk_spec)` would be unsafe: ids are recycled after garbage collection, so a freed spec's id reused by a different spec (same shape, different prototype/dtype/config) could yield a stale hit. Value identity avoids that entirely. Thread-safety: a benign construction race is possible (two workers with different specs may each compute and overwrite the single-entry cache — last writer wins), but a torn read is not, because the entry is written atomically as one tuple. Worst case is a recompute, never a wrong result. """ from zarr.core.codec_pipeline import resolve_aa_specs if not self._aa_codecs: return (), chunk_spec cache = self._cache if cache is not None and cache[0] == chunk_spec: return cache[1], cache[2] aa_specs_t, spec = resolve_aa_specs(cast("tuple[Codec, ...]", self._aa_codecs), chunk_spec) self._cache = (chunk_spec, aa_specs_t, spec) return aa_specs_t, spec def decode_chunk(self, chunk_bytes: Buffer, chunk_spec: ArraySpec) -> NDBuffer: """Decode a single chunk through the full codec chain, synchronously. Pure compute -- no IO. Parameters ---------- chunk_bytes : Buffer The encoded chunk bytes. chunk_spec : ArraySpec The array spec describing shape, dtype, fill value, and codec configuration for this chunk. """ aa_specs, ab_spec = self._resolve_specs(chunk_spec) data: Buffer = chunk_bytes for bb_codec in reversed(self._bb_codecs): data = bb_codec._decode_sync(data, ab_spec) chunk_array: NDBuffer = self._ab_codec._decode_sync(data, ab_spec) for aa_codec, aa_spec in zip(reversed(self._aa_codecs), reversed(aa_specs), strict=True): chunk_array = aa_codec._decode_sync(chunk_array, aa_spec) return chunk_array def encode_chunk(self, chunk_array: NDBuffer, chunk_spec: ArraySpec) -> Buffer | None: """Encode a single chunk through the full codec chain, synchronously. Pure compute -- no IO. Parameters ---------- chunk_array : NDBuffer The chunk data to encode. chunk_spec : ArraySpec The array spec describing shape, dtype, fill value, and codec configuration for this chunk. """ aa_specs, ab_spec = self._resolve_specs(chunk_spec) aa_data: NDBuffer = chunk_array for aa_codec, aa_spec in zip(self._aa_codecs, aa_specs, strict=True): aa_result = aa_codec._encode_sync(aa_data, aa_spec) if aa_result is None: return None aa_data = aa_result ab_result = self._ab_codec._encode_sync(aa_data, ab_spec) if ab_result is None: return None bb_data: Buffer = ab_result for bb_codec in self._bb_codecs: bb_result = bb_codec._encode_sync(bb_data, ab_spec) if bb_result is None: return None bb_data = bb_result return bb_data def compute_encoded_size(self, byte_length: int, array_spec: ArraySpec) -> int: for codec in self.codecs: byte_length = codec.compute_encoded_size(byte_length, array_spec) array_spec = codec.resolve_metadata(array_spec) return byte_length zarr-developers-zarr-python-a2f2415/src/zarr/core/codec_pipeline.py000066400000000000000000001604411527014137600255040ustar00rootroot00000000000000from __future__ import annotations import asyncio import os import threading from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from itertools import batched, chain, pairwise from typing import TYPE_CHECKING, Any from warnings import warn from zarr.abc.codec import ( ArrayArrayCodec, ArrayBytesCodec, ArrayBytesCodecPartialDecodeMixin, ArrayBytesCodecPartialEncodeMixin, BytesBytesCodec, Codec, CodecPipeline, GetResult, ) from zarr.core.chunk_utils import ( ChunkTransform, _merge_chunk_array, chunk_is_empty, decode_and_scatter_chunk, evolve_codecs, fill_value_or_default, merge_and_encode_chunk, scatter_chunk, ) from zarr.core.common import concurrent_iter, concurrent_map from zarr.core.config import config from zarr.errors import ZarrUserWarning from zarr.registry import register_pipeline if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Sequence from typing import Self from zarr.abc.store import ByteGetter, ByteSetter from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer, BufferPrototype, NDBuffer from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType from zarr.core.indexing import SelectorTuple from zarr.core.metadata.v3 import ChunkGridMetadata _pool: ThreadPoolExecutor | None = None _pool_size: int = 0 _pool_lock = threading.Lock() def _resolve_max_workers() -> int: """Helper for getting the maximum number of workers available to the `FusedCodecPipeline`""" import os as _os default = _os.cpu_count() or 1 cfg = config.get("codec_pipeline.max_workers", default=None) if cfg is None: return default try: return max(1, int(cfg)) except (TypeError, ValueError): # This value arrives via the config/env layer (e.g. # `ZARR_CODEC_PIPELINE__MAX_WORKERS`), so tolerate bad input here # instead of raising mid-read. warn( f"Ignoring invalid `codec_pipeline.max_workers` config value {cfg!r}; " f"falling back to {default}.", category=ZarrUserWarning, stacklevel=2, ) return default def _get_pool(max_workers: int) -> ThreadPoolExecutor: """Get or create the module-level thread pool, sized to `max_workers`. The pool grows on demand — if a request arrives for more workers than the current pool has, it is replaced with a larger one. The previous pool is NOT shut down here: another thread may be holding a reference to it and about to submit (`shutdown` would make its `pool.map` raise "cannot schedule new futures after shutdown"). The orphaned pool finishes its in-flight tasks and is garbage-collected once no caller references it. The pool only grows, never shrinks (a shrink request reuses the larger pool, leaving workers idle). Callers that want sequential execution should not call this — they should run the task list inline. `max_workers` must be >= 1. """ global _pool, _pool_size if max_workers < 1: raise ValueError(f"max_workers must be >= 1, got {max_workers}") if _pool is None or _pool_size < max_workers: with _pool_lock: if _pool is None or _pool_size < max_workers: # Replace without shutting down the old pool (see docstring): # avoids a race with a concurrent in-flight pool.map on it. _pool = ThreadPoolExecutor(max_workers=max_workers) _pool_size = max_workers return _pool def _reset_pool_after_fork() -> None: """Discard the pool a forked child inherits from its parent. Executor threads do not survive `fork()`, so the inherited pool accepts submissions that no thread ever runs. The lock is recreated too: it may have been held by a thread that no longer exists in the child. """ global _pool, _pool_size, _pool_lock _pool = None # pragma: no cover _pool_size = 0 # pragma: no cover _pool_lock = threading.Lock() # pragma: no cover # this is only available on certain operating systems if hasattr(os, "register_at_fork"): os.register_at_fork(after_in_child=_reset_pool_after_fork) def _unzip2[T, U](iterable: Iterable[tuple[T, U]]) -> tuple[list[T], list[U]]: out0: list[T] = [] out1: list[U] = [] for item0, item1 in iterable: out0.append(item0) out1.append(item1) return (out0, out1) def resolve_batched(codec: Codec, chunk_specs: Iterable[ArraySpec]) -> Iterable[ArraySpec]: return [codec.resolve_metadata(chunk_spec) for chunk_spec in chunk_specs] def resolve_aa_specs( aa_codecs: tuple[Codec, ...], chunk_spec: ArraySpec ) -> tuple[tuple[ArraySpec, ...], ArraySpec]: """Resolve the per-stage chunk specs for a single chunk's codec chain. Threads `chunk_spec` forward through the array->array codecs via `resolve_metadata` (each codec sees the spec produced by the previous one), returning `(aa_specs, ab_spec)`: * `aa_specs[i]` is the spec the i-th AA codec operates on (its *input* on encode / *output* on decode); * `ab_spec` is the spec after all AA codecs — what the array->bytes codec and the bytes->bytes codecs operate on. This is the single source of truth for per-stage spec evolution, shared by the synchronous `ChunkTransform` and the asynchronous `AsyncChunkTransform`. It is pure metadata (only `resolve_metadata`), so it places no synchronous-codec requirement on the codecs. """ aa_specs: list[ArraySpec] = [] spec = chunk_spec for aa_codec in aa_codecs: aa_specs.append(spec) spec = aa_codec.resolve_metadata(spec) return tuple(aa_specs), spec def pipeline_supports_partial_decode( array_bytes_codec: ArrayBytesCodec, *, array_array_codecs: tuple[ArrayArrayCodec, ...], bytes_bytes_codecs: tuple[BytesBytesCodec, ...], require_no_aa_bb: bool, ) -> bool: """Whether a codec pipeline can decode a partial selection without a full read. Requires the array->bytes codec to implement ``ArrayBytesCodecPartialDecodeMixin``. When ``require_no_aa_bb`` is True it additionally requires no array->array / bytes->bytes codecs, because those can change the slice<->byte-range correspondence (an AA codec can make the selection non-contiguous, a BB codec can rewrite the bytes), making partial decode infeasible. Both pipelines pass `require_no_aa_bb=True`: an outer AA/BB codec (e.g. a compressor wrapping a sharding serializer) must see every byte of the chunk, so a partial branch that only re-decodes/re-encodes the inner sharding codec would silently bypass it. """ if require_no_aa_bb and (len(array_array_codecs) + len(bytes_bytes_codecs)) != 0: return False return isinstance(array_bytes_codec, ArrayBytesCodecPartialDecodeMixin) def pipeline_supports_partial_encode( array_bytes_codec: ArrayBytesCodec, *, array_array_codecs: tuple[ArrayArrayCodec, ...], bytes_bytes_codecs: tuple[BytesBytesCodec, ...], require_no_aa_bb: bool, ) -> bool: """Whether a codec pipeline can encode a partial selection without a full rewrite. Mirror of `pipeline_supports_partial_decode` for encoding. """ if require_no_aa_bb and (len(array_array_codecs) + len(bytes_bytes_codecs)) != 0: return False return isinstance(array_bytes_codec, ArrayBytesCodecPartialEncodeMixin) async def _cancel_and_drain(futures: Iterable[asyncio.Future[Any]]) -> None: """Cancel every not-yet-done future/task and await its outcome. Used to clean up work spawned by a drain loop (`asyncio.as_completed` + `await`) when the loop exits early via exception. Without this, tasks already spawned keep running unattended after the caller has moved on, and an eventual failure surfaces as an unraisable "exception was never retrieved" warning instead of being observed here. """ pending = [f for f in futures if not f.done()] if len(pending) == 0: return for f in pending: f.cancel() await asyncio.gather(*pending, return_exceptions=True) async def _fetch_and_decode_as_completed( batch: Sequence[tuple[ByteGetter | None, ArraySpec]], transform: ChunkTransform, ) -> list[NDBuffer | None]: """Concurrently fetch each chunk's bytes and decode it as fetches complete. Decoding overlaps with in-flight fetches: each chunk is decoded the moment its bytes arrive (in a thread pool when one is available, otherwise inline) rather than waiting for the whole batch to land. A `None` byte getter fetches nothing and decodes to `None`. Results are returned in input order. """ max_workers = _resolve_max_workers() pool = _get_pool(max_workers) if max_workers > 1 else None loop = asyncio.get_running_loop() decode_futures: list[asyncio.Future[NDBuffer | None]] = [loop.create_future() for _ in batch] async def _fetch( idx: int, byte_getter: ByteGetter | None, prototype: BufferPrototype ) -> tuple[int, Buffer | None]: return idx, None if byte_getter is None else await byte_getter.get(prototype) def _decode(buffer: Buffer | None, chunk_spec: ArraySpec) -> NDBuffer | None: return None if buffer is None else transform.decode_chunk(buffer, chunk_spec) fetch_tasks = concurrent_iter( [ (idx, byte_getter, chunk_spec.prototype) for idx, (byte_getter, chunk_spec) in enumerate(batch) ], _fetch, config.get("async.concurrency"), ) try: for fetch_coro in asyncio.as_completed(fetch_tasks): idx, buffer = await fetch_coro chunk_spec = batch[idx][1] # Bridge both paths to asyncio.Future so the final collection loop # can `await` uniformly without blocking the event loop. For the # pool path that means `wrap_future` (not `pool.submit(...).result()`, # which would block the loop thread for the duration of every decode # — freezing any unrelated coroutines sharing this loop). if pool is None: decode_futures[idx].set_result(_decode(buffer, chunk_spec)) else: decode_futures[idx] = asyncio.wrap_future(pool.submit(_decode, buffer, chunk_spec)) return await asyncio.gather(*decode_futures) finally: # On the happy path every future here is already done, so this is a # no-op; on failure it stops abandoned fetches/decodes from # continuing to run unattended after this function has raised. A # single call over both iterables (not two sequential calls) so that # outer-task cancellation during the first drain can't skip the # second, leaving its futures/tasks unobserved. await _cancel_and_drain(chain(fetch_tasks, decode_futures)) async def _encode_and_write_as_completed( batch: Sequence[tuple[ByteSetter, NDBuffer | None, ArraySpec]], transform: ChunkTransform, ) -> None: """Encode each chunk and write it out as encodes complete. The reverse of `_fetch_and_decode_as_completed`: each chunk is encoded (in a thread pool when one is available, otherwise inline) and its write is launched the moment that chunk's encode finishes — overlapping IO writes with still-running encodes rather than waiting for the whole batch to encode. A `None` chunk array encodes to `None` and is `delete`d. Writes are bounded by `async.concurrency`. """ max_workers = _resolve_max_workers() pool = _get_pool(max_workers) if max_workers > 1 else None loop = asyncio.get_running_loop() semaphore = asyncio.Semaphore(config.get("async.concurrency")) def _encode( idx: int, chunk_array: NDBuffer | None, chunk_spec: ArraySpec ) -> tuple[int, Buffer | None]: return idx, None if chunk_array is None else transform.encode_chunk(chunk_array, chunk_spec) async def _write(idx: int, chunk_bytes: Buffer | None) -> None: byte_setter = batch[idx][0] async with semaphore: if chunk_bytes is None: await byte_setter.delete() else: await byte_setter.set(chunk_bytes) # Submit every encode up front. The pool path bridges to asyncio.Future via # `wrap_future` (not `pool.submit(...).result()`, which would block the loop # thread); the inline path resolves immediately. encode_futures: list[asyncio.Future[tuple[int, Buffer | None]]] = [] for idx, (_, chunk_array, chunk_spec) in enumerate(batch): if pool is None: fut: asyncio.Future[tuple[int, Buffer | None]] = loop.create_future() fut.set_result(_encode(idx, chunk_array, chunk_spec)) else: fut = asyncio.wrap_future(pool.submit(_encode, idx, chunk_array, chunk_spec)) encode_futures.append(fut) # Kick off each chunk's write the instant its encode lands, so writes of # already-compressed chunks proceed while the rest are still encoding. write_tasks: list[asyncio.Task[None]] = [] try: for encode_coro in asyncio.as_completed(encode_futures): idx, chunk_bytes = await encode_coro write_tasks.append(asyncio.ensure_future(_write(idx, chunk_bytes))) await asyncio.gather(*write_tasks) finally: # On the happy path every future here is already done, so this is a # no-op; on failure (an encode or a write raising) it stops # already-spawned writes from continuing in the background after # this function has raised. A single call over both iterables (not # two sequential calls) so that outer-task cancellation during the # first drain can't skip the second, leaving its futures/tasks # unobserved. await _cancel_and_drain(chain(write_tasks, encode_futures)) async def _async_read_fallback( pipeline: CodecPipeline, batch: list[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], out: NDBuffer, drop_axes: tuple[int, ...], ) -> tuple[GetResult, ...]: """Async fallback read used when no fast-path is available. Fetches every chunk's bytes via `concurrent_map` (sized by `async.concurrency`), decodes the batch through `pipeline.decode`, then scatters each decoded chunk into `out` at its `out_selection`. Used by both `BatchedCodecPipeline.read_batch` (non-partial-decode branch) and `FusedCodecPipeline.read` (when the store does not advertise sync IO / sync transform is unavailable). """ chunk_array_batch: list[NDBuffer | None] if isinstance(pipeline, FusedCodecPipeline) and pipeline.sync_transform is not None: chunk_array_batch = await _fetch_and_decode_as_completed( [(byte_getter, chunk_spec) for byte_getter, chunk_spec, *_ in batch], pipeline.sync_transform, ) else: chunk_bytes_batch = await concurrent_map( [(byte_getter, array_spec.prototype) for byte_getter, array_spec, *_ in batch], lambda byte_getter, prototype: byte_getter.get(prototype), config.get("async.concurrency"), ) chunk_array_batch = list( await pipeline.decode( [ (chunk_bytes, chunk_spec) for chunk_bytes, (_, chunk_spec, *_) in zip( chunk_bytes_batch, batch, strict=False ) ], ) ) results: list[GetResult] = [] for chunk_array, (_, chunk_spec, chunk_selection, out_selection, _) in zip( chunk_array_batch, batch, strict=True ): selected = None if chunk_array is None else chunk_array[chunk_selection] results.append( scatter_chunk( selected, out, chunk_spec=chunk_spec, out_selection=out_selection, drop_axes=drop_axes, ) ) return tuple(results) async def _async_write_fallback( pipeline: CodecPipeline, batch: list[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], value: NDBuffer, drop_axes: tuple[int, ...], ) -> None: """Async fallback write used when no fast-path is available. For each chunk in `batch`: read its existing bytes from the store (skipping the read for complete chunks), decode the batch via `pipeline.decode`, merge `value` into each decoded chunk via `_merge_chunk_array`, drop chunks that are all-fill when `write_empty_chunks` is False, encode the surviving chunks via `pipeline.encode`, then `set` the encoded bytes (or `delete` if encoding produced `None` or the chunk dropped). Used by both `BatchedCodecPipeline.write_batch` (non-partial-encode branch) and `FusedCodecPipeline.write` (when the store does not advertise sync IO / sync transform is unavailable). """ sync_transform = pipeline.sync_transform if isinstance(pipeline, FusedCodecPipeline) else None if sync_transform is not None: # Read each chunk's existing bytes (skipping complete chunks) and decode # as fetches complete, overlapping the sync decode with in-flight reads. chunk_array_decoded: Iterable[NDBuffer | None] = await _fetch_and_decode_as_completed( [ (None if is_complete_chunk else byte_setter, chunk_spec) for byte_setter, chunk_spec, _, _, is_complete_chunk in batch ], sync_transform, ) else: async def _read_key( byte_setter: ByteSetter | None, prototype: BufferPrototype ) -> Buffer | None: if byte_setter is None: return None return await byte_setter.get(prototype=prototype) chunk_bytes_batch: Iterable[Buffer | None] = await concurrent_map( [ ( None if is_complete_chunk else byte_setter, chunk_spec.prototype, ) for byte_setter, chunk_spec, chunk_selection, _, is_complete_chunk in batch ], _read_key, config.get("async.concurrency"), ) chunk_array_decoded = await pipeline.decode( [ (chunk_bytes, chunk_spec) for chunk_bytes, (_, chunk_spec, *_) in zip(chunk_bytes_batch, batch, strict=False) ], ) chunk_array_merged = [ _merge_chunk_array( chunk_array, value, out_selection, chunk_spec, chunk_selection, is_complete_chunk, drop_axes, ) for chunk_array, ( _, chunk_spec, chunk_selection, out_selection, is_complete_chunk, ) in zip(chunk_array_decoded, batch, strict=False) ] # _merge_chunk_array always returns a real NDBuffer (never None), so the only # way a chunk drops to None here is the empty-chunk normalization. chunk_array_batch: list[NDBuffer | None] = [ None if chunk_is_empty(chunk_array, chunk_spec) else chunk_array for chunk_array, (_, chunk_spec, *_) in zip(chunk_array_merged, batch, strict=False) ] if sync_transform is not None: await _encode_and_write_as_completed( [ (byte_setter, chunk_array, chunk_spec) for chunk_array, (byte_setter, chunk_spec, *_) in zip( chunk_array_batch, batch, strict=False ) ], sync_transform, ) else: chunk_bytes_batch = await pipeline.encode( [ (chunk_array, chunk_spec) for chunk_array, (_, chunk_spec, *_) in zip(chunk_array_batch, batch, strict=False) ], ) async def _write_key(byte_setter: ByteSetter, chunk_bytes: Buffer | None) -> None: if chunk_bytes is None: await byte_setter.delete() else: await byte_setter.set(chunk_bytes) await concurrent_map( [ (byte_setter, chunk_bytes) for chunk_bytes, (byte_setter, *_) in zip(chunk_bytes_batch, batch, strict=False) ], _write_key, config.get("async.concurrency"), ) @dataclass(slots=True, kw_only=True) class AsyncChunkTransform: """A per-chunk asynchronous codec chain — the async mirror of ChunkTransform. Decodes/encodes a SINGLE chunk through the full codec chain, awaiting each codec's per-chunk async method (`_decode_single`/`_encode_single`) with the correctly-evolved per-stage spec (via the shared `resolve_aa_specs`). Unlike ChunkTransform it places no `SupportsSyncCodec` requirement on the codecs, so it works for async-only codecs. Unlike the batched codec API it operates on one chunk at a time — the mini-batch fan-out is a BatchedCodecPipeline concern and deliberately not reintroduced here. """ codecs: tuple[Codec, ...] _aa_codecs: tuple[ArrayArrayCodec, ...] = field(init=False, repr=False, compare=False) _ab_codec: ArrayBytesCodec = field(init=False, repr=False, compare=False) _bb_codecs: tuple[BytesBytesCodec, ...] = field(init=False, repr=False, compare=False) def __post_init__(self) -> None: # `AsyncChunkTransform` is (re)constructed per decode/encode call from a # codec chain that already went through `codecs_from_list` when the # pipeline itself was built, so re-splitting it here must not re-emit # that chain's advisory warnings on every call. aa, ab, bb = codecs_from_list_unchecked(list(self.codecs)) self._aa_codecs = aa self._ab_codec = ab self._bb_codecs = bb async def decode_chunk(self, chunk_bytes: Buffer, chunk_spec: ArraySpec) -> NDBuffer: """Decode one chunk through the chain (bb -> ab -> aa), async.""" aa_specs, ab_spec = resolve_aa_specs(self._aa_codecs, chunk_spec) data: Buffer = chunk_bytes for bb_codec in reversed(self._bb_codecs): data = await bb_codec._decode_single(data, ab_spec) chunk_array: NDBuffer = await self._ab_codec._decode_single(data, ab_spec) for aa_codec, aa_spec in zip(reversed(self._aa_codecs), reversed(aa_specs), strict=True): chunk_array = await aa_codec._decode_single(chunk_array, aa_spec) return chunk_array async def encode_chunk(self, chunk_array: NDBuffer, chunk_spec: ArraySpec) -> Buffer | None: """Encode one chunk through the chain (aa -> ab -> bb), async. Returns None if any stage drops the chunk (e.g. an all-fill chunk under write_empty_chunks=False), matching ChunkTransform.encode_chunk. """ aa_specs, ab_spec = resolve_aa_specs(self._aa_codecs, chunk_spec) aa_data: NDBuffer = chunk_array for aa_codec, aa_spec in zip(self._aa_codecs, aa_specs, strict=True): aa_result = await aa_codec._encode_single(aa_data, aa_spec) if aa_result is None: return None aa_data = aa_result ab_result = await self._ab_codec._encode_single(aa_data, ab_spec) if ab_result is None: return None bb_data: Buffer = ab_result for bb_codec in self._bb_codecs: bb_result = await bb_codec._encode_single(bb_data, ab_spec) if bb_result is None: return None bb_data = bb_result return bb_data @dataclass(frozen=True) class BatchedCodecPipeline(CodecPipeline): """Default codec pipeline. This batched codec pipeline divides the chunk batches into batches of a configurable batch size ("mini-batch"). Fetching, decoding, encoding and storing are performed in lock step for each mini-batch. Multiple mini-batches are processing concurrently. """ array_array_codecs: tuple[ArrayArrayCodec, ...] array_bytes_codec: ArrayBytesCodec bytes_bytes_codecs: tuple[BytesBytesCodec, ...] batch_size: int def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: # Re-splits an already-`codecs_from_list`-validated (and warned-about) # chain against the evolved spec, so this uses the quiet variant rather # than routing through `from_codecs` (which would re-warn). evolved_codecs = evolve_codecs(self, array_spec) array_array_codecs, array_bytes_codec, bytes_bytes_codecs = codecs_from_list_unchecked( evolved_codecs ) return type(self)( array_array_codecs=array_array_codecs, array_bytes_codec=array_bytes_codec, bytes_bytes_codecs=bytes_bytes_codecs, batch_size=self.batch_size, ) @classmethod def from_codecs(cls, codecs: Iterable[Codec], *, batch_size: int | None = None) -> Self: array_array_codecs, array_bytes_codec, bytes_bytes_codecs = codecs_from_list(codecs) return cls( array_array_codecs=array_array_codecs, array_bytes_codec=array_bytes_codec, bytes_bytes_codecs=bytes_bytes_codecs, batch_size=batch_size or config.get("codec_pipeline.batch_size"), ) @property def supports_partial_decode(self) -> bool: # Only a single ArrayBytesCodec that supports partial decoding, and no # AA/BB codecs (they break the slice<->byte-range correspondence). return pipeline_supports_partial_decode( self.array_bytes_codec, array_array_codecs=self.array_array_codecs, bytes_bytes_codecs=self.bytes_bytes_codecs, require_no_aa_bb=True, ) @property def supports_partial_encode(self) -> bool: return pipeline_supports_partial_encode( self.array_bytes_codec, array_array_codecs=self.array_array_codecs, bytes_bytes_codecs=self.bytes_bytes_codecs, require_no_aa_bb=True, ) def __iter__(self) -> Iterator[Codec]: yield from self.array_array_codecs yield self.array_bytes_codec yield from self.bytes_bytes_codecs def validate( self, *, shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGridMetadata, ) -> None: for codec in self: codec.validate(shape=shape, dtype=dtype, chunk_grid=chunk_grid) def compute_encoded_size(self, byte_length: int, array_spec: ArraySpec) -> int: for codec in self: byte_length = codec.compute_encoded_size(byte_length, array_spec) array_spec = codec.resolve_metadata(array_spec) return byte_length def _codecs_with_resolved_metadata_batched( self, chunk_specs: Iterable[ArraySpec] ) -> tuple[ list[tuple[ArrayArrayCodec, list[ArraySpec]]], tuple[ArrayBytesCodec, list[ArraySpec]], list[tuple[BytesBytesCodec, list[ArraySpec]]], ]: aa_codecs_with_spec: list[tuple[ArrayArrayCodec, list[ArraySpec]]] = [] chunk_specs = list(chunk_specs) for aa_codec in self.array_array_codecs: aa_codecs_with_spec.append((aa_codec, chunk_specs)) chunk_specs = [aa_codec.resolve_metadata(chunk_spec) for chunk_spec in chunk_specs] ab_codec_with_spec = (self.array_bytes_codec, chunk_specs) chunk_specs = [ self.array_bytes_codec.resolve_metadata(chunk_spec) for chunk_spec in chunk_specs ] bb_codecs_with_spec: list[tuple[BytesBytesCodec, list[ArraySpec]]] = [] for bb_codec in self.bytes_bytes_codecs: bb_codecs_with_spec.append((bb_codec, chunk_specs)) chunk_specs = [bb_codec.resolve_metadata(chunk_spec) for chunk_spec in chunk_specs] return (aa_codecs_with_spec, ab_codec_with_spec, bb_codecs_with_spec) async def decode_batch( self, chunk_bytes_and_specs: Iterable[tuple[Buffer | None, ArraySpec]], ) -> Iterable[NDBuffer | None]: chunk_bytes_batch: Iterable[Buffer | None] chunk_bytes_batch, chunk_specs = _unzip2(chunk_bytes_and_specs) ( aa_codecs_with_spec, ab_codec_with_spec, bb_codecs_with_spec, ) = self._codecs_with_resolved_metadata_batched(chunk_specs) for bb_codec, chunk_spec_batch in bb_codecs_with_spec[::-1]: chunk_bytes_batch = await bb_codec.decode( zip(chunk_bytes_batch, chunk_spec_batch, strict=False) ) ab_codec, chunk_spec_batch = ab_codec_with_spec chunk_array_batch = await ab_codec.decode( zip(chunk_bytes_batch, chunk_spec_batch, strict=False) ) for aa_codec, chunk_spec_batch in aa_codecs_with_spec[::-1]: chunk_array_batch = await aa_codec.decode( zip(chunk_array_batch, chunk_spec_batch, strict=False) ) return chunk_array_batch async def decode_partial_batch( self, batch_info: Iterable[tuple[ByteGetter, SelectorTuple, ArraySpec]], ) -> Iterable[NDBuffer | None]: codec = self.array_bytes_codec if not self.supports_partial_decode or not isinstance( codec, ArrayBytesCodecPartialDecodeMixin ): raise ValueError("This codec pipeline does not support partial decoding.") return await codec.decode_partial(batch_info) async def encode_batch( self, chunk_arrays_and_specs: Iterable[tuple[NDBuffer | None, ArraySpec]], ) -> Iterable[Buffer | None]: chunk_array_batch: Iterable[NDBuffer | None] chunk_specs: Iterable[ArraySpec] chunk_array_batch, chunk_specs = _unzip2(chunk_arrays_and_specs) for aa_codec in self.array_array_codecs: chunk_array_batch = await aa_codec.encode( zip(chunk_array_batch, chunk_specs, strict=False) ) chunk_specs = resolve_batched(aa_codec, chunk_specs) chunk_bytes_batch = await self.array_bytes_codec.encode( zip(chunk_array_batch, chunk_specs, strict=False) ) chunk_specs = resolve_batched(self.array_bytes_codec, chunk_specs) for bb_codec in self.bytes_bytes_codecs: chunk_bytes_batch = await bb_codec.encode( zip(chunk_bytes_batch, chunk_specs, strict=False) ) chunk_specs = resolve_batched(bb_codec, chunk_specs) return chunk_bytes_batch async def encode_partial_batch( self, batch_info: Iterable[tuple[ByteSetter, NDBuffer, SelectorTuple, ArraySpec]], ) -> None: codec = self.array_bytes_codec if not self.supports_partial_encode or not isinstance( codec, ArrayBytesCodecPartialEncodeMixin ): raise ValueError("This codec pipeline does not support partial encoding.") await codec.encode_partial(batch_info) async def read_batch( self, batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], out: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> tuple[GetResult, ...]: if self.supports_partial_decode: results: list[GetResult] = [] batch_info_list = list(batch_info) chunk_array_batch = await self.decode_partial_batch( [ (byte_getter, chunk_selection, chunk_spec) for byte_getter, chunk_spec, chunk_selection, *_ in batch_info_list ] ) for chunk_array, (_, chunk_spec, _, out_selection, _) in zip( chunk_array_batch, batch_info_list, strict=False ): if chunk_array is not None: if drop_axes: chunk_array = chunk_array.squeeze(axis=drop_axes) out[out_selection] = chunk_array results.append(GetResult(status="present")) else: out[out_selection] = fill_value_or_default(chunk_spec) results.append(GetResult(status="missing")) return tuple(results) return await _async_read_fallback(self, list(batch_info), out, drop_axes) async def write_batch( self, batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], value: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: if self.supports_partial_encode: # Pass scalar values as is if len(value.shape) == 0: await self.encode_partial_batch( [ (byte_setter, value, chunk_selection, chunk_spec) for byte_setter, chunk_spec, chunk_selection, out_selection, _ in batch_info ], ) else: await self.encode_partial_batch( [ (byte_setter, value[out_selection], chunk_selection, chunk_spec) for byte_setter, chunk_spec, chunk_selection, out_selection, _ in batch_info ], ) return await _async_write_fallback(self, list(batch_info), value, drop_axes) async def decode( self, chunk_bytes_and_specs: Iterable[tuple[Buffer | None, ArraySpec]], ) -> Iterable[NDBuffer | None]: output: list[NDBuffer | None] = [] for batch_info in batched(chunk_bytes_and_specs, self.batch_size): output.extend(await self.decode_batch(batch_info)) return output async def encode( self, chunk_arrays_and_specs: Iterable[tuple[NDBuffer | None, ArraySpec]], ) -> Iterable[Buffer | None]: output: list[Buffer | None] = [] for single_batch_info in batched(chunk_arrays_and_specs, self.batch_size): output.extend(await self.encode_batch(single_batch_info)) return output async def read( self, batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], out: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> tuple[GetResult, ...]: batch_results = await concurrent_map( [ (single_batch_info, out, drop_axes) for single_batch_info in batched(batch_info, self.batch_size) ], self.read_batch, config.get("async.concurrency"), ) results: list[GetResult] = [] for batch in batch_results: results.extend(batch) return tuple(results) async def write( self, batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], value: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: await concurrent_map( [ (single_batch_info, value, drop_axes) for single_batch_info in batched(batch_info, self.batch_size) ], self.write_batch, config.get("async.concurrency"), ) def codecs_from_list( codecs: Iterable[Codec], ) -> tuple[tuple[ArrayArrayCodec, ...], ArrayBytesCodec, tuple[BytesBytesCodec, ...]]: """Split `codecs` into `(array_array, array_bytes, bytes_bytes)`, validating order. Emits user-facing advisory warnings about the codec chain (e.g. sharding's "disables partial reads" warning). Use this for the FIRST construction of a codec chain from user-supplied codecs. Use `codecs_from_list_unchecked` when re-splitting a chain that was already validated and warned about by a prior `codecs_from_list` call (e.g. `evolve_from_array_spec` re-splitting the same codecs against an evolved spec) — re-warning there would fire the same advisory once per reconstruction instead of once per user-facing chain. """ from zarr.codecs.sharding import ShardingCodec codecs = tuple(codecs) # materialize to avoid generator consumption issues if any(isinstance(codec, ShardingCodec) for codec in codecs) and len(codecs) > 1: warn( "Combining a `sharding_indexed` codec disables partial reads and " "writes, which may lead to inefficient performance.", category=ZarrUserWarning, stacklevel=3, ) return codecs_from_list_unchecked(codecs) def codecs_from_list_unchecked( codecs: Iterable[Codec], ) -> tuple[tuple[ArrayArrayCodec, ...], ArrayBytesCodec, tuple[BytesBytesCodec, ...]]: """Split `codecs` into `(array_array, array_bytes, bytes_bytes)`, validating order. Same structural validation as `codecs_from_list` (raises on bad codec ordering or a missing/duplicate array-bytes codec) but does NOT emit user-facing advisory warnings. See `codecs_from_list` for when to use each. """ codecs = tuple(codecs) # materialize to avoid generator consumption issues array_array: tuple[ArrayArrayCodec, ...] = () array_bytes_maybe: ArrayBytesCodec | None = None bytes_bytes: tuple[BytesBytesCodec, ...] = () for prev_codec, cur_codec in pairwise((None, *codecs)): if isinstance(cur_codec, ArrayArrayCodec): if isinstance(prev_codec, ArrayBytesCodec | BytesBytesCodec): msg = ( f"Invalid codec order. ArrayArrayCodec {cur_codec}" "must be preceded by another ArrayArrayCodec. " f"Got {type(prev_codec)} instead." ) raise TypeError(msg) array_array += (cur_codec,) elif isinstance(cur_codec, ArrayBytesCodec): if isinstance(prev_codec, BytesBytesCodec): msg = ( f"Invalid codec order. ArrayBytes codec {cur_codec}" f" must be preceded by an ArrayArrayCodec. Got {type(prev_codec)} instead." ) raise TypeError(msg) if array_bytes_maybe is not None: msg = ( f"Got two instances of ArrayBytesCodec: {array_bytes_maybe} and {cur_codec}. " "Only one array-to-bytes codec is allowed." ) raise ValueError(msg) array_bytes_maybe = cur_codec elif isinstance(cur_codec, BytesBytesCodec): if isinstance(prev_codec, ArrayArrayCodec): msg = ( f"Invalid codec order. BytesBytesCodec {cur_codec}" "must be preceded by either another BytesBytesCodec, or an ArrayBytesCodec. " f"Got {type(prev_codec)} instead." ) raise TypeError(msg) bytes_bytes += (cur_codec,) else: raise TypeError if array_bytes_maybe is None: raise ValueError("Required ArrayBytesCodec was not found.") else: return array_array, array_bytes_maybe, bytes_bytes register_pipeline(BatchedCodecPipeline) @dataclass(frozen=True) class FusedCodecPipeline(CodecPipeline): """Codec pipeline that runs codec compute synchronously, in bulk. This is an opt-in alternative to `BatchedCodecPipeline`. The win is NOT "separating IO from compute" — the codecs (notably `ShardingCodec`) still perform their own storage IO. The win is replacing the batched pipeline's per-chunk *async scheduling* (≈one coroutine per chunk, which dominates real codec work) with synchronous, batched/coalesced execution: 1. When every codec implements `SupportsSyncCodec`, a `ChunkTransform` runs the codec chain synchronously (no event loop, no per-chunk coroutine) — optionally across a thread pool for CPU-heavy decode/encode. 2. Sharded reads use the codec's synchronous IO methods: byte-range reads coalesced via `Store.get_ranges_sync`, and a vectorized whole-shard bulk decode for dense, fixed-size, uncompressed shards. Sharded writes go through the codec's synchronous full-shard-rewrite path. 3. When the store lacks synchronous IO (e.g. ZipStore) the pipeline falls back to the async path, equivalent to `BatchedCodecPipeline`. IO ownership: the sharding codec holds the byte getter/setter and reads/ writes storage directly (the same model as zarrs; unlike tensorstore, which keeps codecs storage-free). A storage-free codec is a possible future direction (see the pure-codec design notes) but is explicitly NOT what this pipeline does. """ codecs: tuple[Codec, ...] array_array_codecs: tuple[ArrayArrayCodec, ...] array_bytes_codec: ArrayBytesCodec bytes_bytes_codecs: tuple[BytesBytesCodec, ...] sync_transform: ChunkTransform | None batch_size: int @classmethod def from_codecs(cls, codecs: Iterable[Codec], *, batch_size: int | None = None) -> Self: codec_list = tuple(codecs) aa, ab, bb = codecs_from_list(codec_list) if batch_size is None: batch_size = config.get("codec_pipeline.batch_size") return cls( codecs=codec_list, array_array_codecs=aa, array_bytes_codec=ab, bytes_bytes_codecs=bb, sync_transform=None, batch_size=batch_size, ) def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: # Re-splits an already-`codecs_from_list`-validated (and warned-about) # chain against the evolved spec, so this uses the quiet variant to # avoid re-emitting the same advisory warning on every array open. evolved_codecs = evolve_codecs(self.codecs, array_spec) aa, ab, bb = codecs_from_list_unchecked(evolved_codecs) try: sync_transform: ChunkTransform | None = ChunkTransform(codecs=evolved_codecs) except TypeError: sync_transform = None return type(self)( codecs=evolved_codecs, array_array_codecs=aa, array_bytes_codec=ab, bytes_bytes_codecs=bb, sync_transform=sync_transform, batch_size=self.batch_size, ) def __iter__(self) -> Iterator[Codec]: return iter(self.codecs) @property def supports_partial_decode(self) -> bool: return pipeline_supports_partial_decode( self.array_bytes_codec, array_array_codecs=self.array_array_codecs, bytes_bytes_codecs=self.bytes_bytes_codecs, require_no_aa_bb=True, ) @property def supports_partial_encode(self) -> bool: return pipeline_supports_partial_encode( self.array_bytes_codec, array_array_codecs=self.array_array_codecs, bytes_bytes_codecs=self.bytes_bytes_codecs, require_no_aa_bb=True, ) def validate( self, *, shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGridMetadata, ) -> None: for codec in self.codecs: codec.validate(shape=shape, dtype=dtype, chunk_grid=chunk_grid) def compute_encoded_size(self, byte_length: int, array_spec: ArraySpec) -> int: for codec in self: byte_length = codec.compute_encoded_size(byte_length, array_spec) array_spec = codec.resolve_metadata(array_spec) return byte_length # -- async decode/encode (required by ABC) and sync versions -- async def decode( self, chunk_bytes_and_specs: Iterable[tuple[Buffer | None, ArraySpec]], ) -> Iterable[NDBuffer | None]: # Decode each chunk through AsyncChunkTransform, which threads the # per-stage spec correctly (via resolve_aa_specs). This is the single # source of truth for async per-chunk decode; earlier this method # reused one flat `chunk_specs` across every codec stage, which silently # corrupted/crashed spec-changing codecs (transpose/cast/scale_offset) # on the async fallback path. async_transform = AsyncChunkTransform(codecs=self.codecs) out: list[NDBuffer | None] = [] for chunk_bytes, chunk_spec in chunk_bytes_and_specs: if chunk_bytes is None: out.append(None) else: out.append(await async_transform.decode_chunk(chunk_bytes, chunk_spec)) return out async def encode( self, chunk_arrays_and_specs: Iterable[tuple[NDBuffer | None, ArraySpec]], ) -> Iterable[Buffer | None]: async_transform = AsyncChunkTransform(codecs=self.codecs) out: list[Buffer | None] = [] for chunk_array, chunk_spec in chunk_arrays_and_specs: if chunk_array is None: out.append(None) else: out.append(await async_transform.encode_chunk(chunk_array, chunk_spec)) return out # -- sync read/write -- def read_sync( self, batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], out: NDBuffer, drop_axes: tuple[int, ...] = (), max_workers: int = 1, ) -> tuple[GetResult, ...]: """Synchronous read: fetch -> decode -> scatter, per chunk. When `max_workers > 1` and there are multiple chunks, each chunk's full lifecycle (fetch + decode + scatter) runs as one task on a thread pool sized to `max_workers` — overlapping IO of one chunk with decode/scatter of another. Scatter is thread-safe because the chunks have non-overlapping output selections. `max_workers=1` runs everything sequentially in the calling thread (no pool involvement). Mirrors `BatchedCodecPipeline.read_batch`: when the AB codec supports partial decoding (e.g. sharding), the codec handles its own IO and only fetches the inner-chunk byte ranges that overlap the read selection. Otherwise the pipeline fetches the full blob and decodes the whole chunk. """ transform = self.sync_transform if transform is None: raise RuntimeError( "This codec pipeline contains codecs without synchronous support; " "use the async read/write path instead." ) batch = list(batch_info) if not batch: return () # Partial-decode fast path: the AB codec owns IO (read only the # byte ranges needed for the requested selection). Same condition # and dispatch as BatchedCodecPipeline.read_batch, plus a gate on the # sync partial method: the public partial-decode contract # (`ArrayBytesCodecPartialDecodeMixin`) only requires the async # `_decode_partial_single`, so a codec may support partial decode # without `_decode_partial_sync` — such codecs take the full-chunk # path below instead. codec = self.array_bytes_codec if self.supports_partial_decode and hasattr(codec, "_decode_partial_sync"): def _read_one( item: tuple[Any, ArraySpec, SelectorTuple, SelectorTuple, bool], ) -> GetResult: byte_getter, chunk_spec, chunk_selection, out_selection, _ = item # the partial decode returns the already-selected region decoded = codec._decode_partial_sync(byte_getter, chunk_selection, chunk_spec) return scatter_chunk( decoded, out, chunk_spec=chunk_spec, out_selection=out_selection, drop_axes=drop_axes, ) else: # Per-chunk fused path: fetch + decode + scatter as one task. def _read_one( item: tuple[Any, ArraySpec, SelectorTuple, SelectorTuple, bool], ) -> GetResult: byte_getter, chunk_spec, chunk_selection, out_selection, _ = item raw = byte_getter.get_sync(prototype=chunk_spec.prototype) return decode_and_scatter_chunk( raw, out, chunk_spec=chunk_spec, chunk_selection=chunk_selection, out_selection=out_selection, drop_axes=drop_axes, decode=transform.decode_chunk, ) if max_workers > 1 and len(batch) > 1: pool = _get_pool(max_workers) return tuple(pool.map(_read_one, batch)) return tuple(_read_one(item) for item in batch) def write_sync( self, batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], value: NDBuffer, drop_axes: tuple[int, ...] = (), max_workers: int = 1, ) -> None: """Synchronous write: fetch existing -> merge+encode -> store. When `max_workers > 1` and there are multiple chunks, each chunk's full lifecycle (get-existing + merge + encode + set/delete) runs as one task on a thread pool sized to `max_workers` — overlapping IO of one chunk with compute of another. `max_workers=1` runs everything sequentially in the calling thread (no pool involvement). When the codec pipeline supports partial encoding (e.g. a sharding codec with no outer AA/BB codecs), the AB codec handles the full write cycle — reading existing data, merging, encoding, and writing — matching the async `BatchedCodecPipeline` path. """ transform = self.sync_transform if transform is None: raise RuntimeError( "This codec pipeline contains codecs without synchronous support; " "use the async read/write path instead." ) batch = list(batch_info) if not batch: return # Partial-encode path: the AB codec owns IO (read, merge, encode, # write). Same condition and calling convention as # BatchedCodecPipeline.write_batch, plus a gate on the sync partial # method: the public partial-encode contract # (`ArrayBytesCodecPartialEncodeMixin`) only requires the async # `_encode_partial_single`, so a codec may support partial encode # without `_encode_partial_sync` — such codecs take the full-chunk # path below instead. codec = self.array_bytes_codec if self.supports_partial_encode and hasattr(codec, "_encode_partial_sync"): scalar = len(value.shape) == 0 def _write_one( item: tuple[Any, ArraySpec, SelectorTuple, SelectorTuple, bool], ) -> None: bs, chunk_spec, chunk_selection, out_selection, _is_complete = item chunk_value = value if scalar else value[out_selection] codec._encode_partial_sync(bs, chunk_value, chunk_selection, chunk_spec) else: # Per-chunk fused path: get-existing + merge + encode + set/delete as one task. def _write_one( item: tuple[Any, ArraySpec, SelectorTuple, SelectorTuple, bool], ) -> None: bs, chunk_spec, chunk_selection, out_selection, is_complete = item existing_bytes: Buffer | None = None if not is_complete: existing_bytes = bs.get_sync(prototype=chunk_spec.prototype) encoded = merge_and_encode_chunk( existing_bytes, value, chunk_spec=chunk_spec, chunk_selection=chunk_selection, out_selection=out_selection, is_complete=is_complete, drop_axes=drop_axes, decode=transform.decode_chunk, encode=transform.encode_chunk, ) if encoded is None: bs.delete_sync() else: bs.set_sync(encoded) if max_workers > 1 and len(batch) > 1: pool = _get_pool(max_workers) list(pool.map(_write_one, batch)) else: for item in batch: _write_one(item) # -- async read/write -- async def read( self, batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], out: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> tuple[GetResult, ...]: batch = list(batch_info) if not batch: return () # Fast path: sync transform plus synchronous IO. For StorePath the gate # is the STORE's sync-IO capability (`_store_supports_sync_io`) (StorePath always has a # get_sync method, but it only works when its store implements the full # sync surface); for other byte getters (e.g. the sharding codec's # in-memory _ShardingByteGetter) the SyncByteGetter protocol is the # gate. from zarr.abc.store import SyncByteGetter, _store_supports_sync_io from zarr.storage._common import StorePath first_bg = batch[0][0] if self.sync_transform is not None and ( (isinstance(first_bg, StorePath) and _store_supports_sync_io(first_bg.store)) or (not isinstance(first_bg, StorePath) and isinstance(first_bg, SyncByteGetter)) ): # One thread hop for the WHOLE batch — not per chunk, so the fused # design's win over per-chunk async scheduling is preserved. Running # read_sync inline here would block the event loop for the duration # of the batch's IO+compute; every sync-API call from every user # thread shares this one loop, so inline execution serializes # concurrent callers behind each other's codec compute. return await asyncio.to_thread( self.read_sync, batch, out, drop_axes, max_workers=_resolve_max_workers() ) # Non-sync store (e.g. ZipStore): can't use the sync fast path. But if the # array-bytes codec supports partial decoding (sharding), still route # through the async partial-decode path — it fetches only the needed # inner-chunk byte ranges (coalesced via get_ranges), matching # BatchedCodecPipeline. Without this, the whole-shard _async_read_fallback # below would over-read and diverge from the batched pipeline's IO. codec = self.array_bytes_codec if self.supports_partial_decode and isinstance(codec, ArrayBytesCodecPartialDecodeMixin): chunk_array_batch = await codec.decode_partial( [ (byte_getter, chunk_selection, chunk_spec) for byte_getter, chunk_spec, chunk_selection, *_ in batch ] ) results: list[GetResult] = [] for chunk_array, (_, chunk_spec, _, out_selection, _) in zip( chunk_array_batch, batch, strict=False ): if chunk_array is not None: if drop_axes: chunk_array = chunk_array.squeeze(axis=drop_axes) out[out_selection] = chunk_array results.append(GetResult(status="present")) else: out[out_selection] = fill_value_or_default(chunk_spec) results.append(GetResult(status="missing")) return tuple(results) return await _async_read_fallback(self, batch, out, drop_axes) async def write( self, batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], value: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: batch = list(batch_info) if not batch: return # Fast path: sync transform plus synchronous IO. Mirrors `read`: gate # StorePath on the store's sync-IO capability (`_store_supports_sync_io`) — write_sync # needs the FULL sync surface (get_sync for partial-chunk # read-modify-write, delete_sync for all-fill chunks), not just # set_sync — and other byte setters (e.g. the sharding codec's # in-memory _ShardingByteSetter) on SyncByteSetter. from zarr.abc.store import SyncByteSetter, _store_supports_sync_io from zarr.storage._common import StorePath first_bs = batch[0][0] if self.sync_transform is not None and ( (isinstance(first_bs, StorePath) and _store_supports_sync_io(first_bs.store)) or (not isinstance(first_bs, StorePath) and isinstance(first_bs, SyncByteSetter)) ): # One thread hop for the whole batch; see the matching comment in # `read` for why write_sync must not run inline on the event loop. await asyncio.to_thread( self.write_sync, batch, value, drop_axes, max_workers=_resolve_max_workers() ) return await _async_write_fallback(self, batch, value, drop_axes) register_pipeline(FusedCodecPipeline) zarr-developers-zarr-python-a2f2415/src/zarr/core/common.py000066400000000000000000000376421527014137600240400ustar00rootroot00000000000000from __future__ import annotations import asyncio import math import warnings from collections.abc import Iterable, Mapping, Sequence from enum import Enum from typing import ( TYPE_CHECKING, Any, Final, Literal, NotRequired, TypedDict, cast, overload, ) import numpy as np from typing_extensions import ReadOnly from zarr.core.config import config as zarr_config from zarr.core.json_parse import convert, parse_field from zarr.errors import ZarrRuntimeWarning if TYPE_CHECKING: from collections.abc import Awaitable, Callable, Iterator from zarr.core.metadata.v3 import ChunkGridMetadata ZARR_JSON = "zarr.json" ZARRAY_JSON = ".zarray" ZGROUP_JSON = ".zgroup" ZATTRS_JSON = ".zattrs" ZMETADATA_V2_JSON = ".zmetadata" BytesLike = bytes | bytearray | memoryview ShapeLike = Iterable[int | np.integer[Any]] | int | np.integer[Any] # Per-dimension chunk specs may mix a bare int (uniform chunk size, the # rectilinear spec's step-size shorthand) with explicit edge-length sequences. # A stored chunk grid (`ChunkGridMetadata`) is also accepted and is used # verbatim, under the tolerant stored-metadata validation rules (e.g. trailing # edges beyond the array extent, as left behind by a shrinking resize). type ChunksLike = ShapeLike | Iterable[int | Iterable[int]] | ChunkGridMetadata # For backwards compatibility ChunkCoords = tuple[int, ...] ZarrFormat = Literal[2, 3] NodeType = Literal["array", "group"] JSON = str | int | float | bool | Mapping[str, "JSON"] | Sequence["JSON"] | None MemoryOrder = Literal["C", "F"] AccessModeLiteral = Literal["r", "r+", "a", "w", "w-"] ANY_ACCESS_MODE: Final = "r", "r+", "a", "w", "w-" DimensionNamesLike = Iterable[str | None] | None DimensionNames = DimensionNamesLike # for backwards compatibility class NamedConfig[TName: str, TConfig: Mapping[str, object]](TypedDict): """ A typed dictionary representing an object with a name and configuration, where the configuration is an optional mapping of string keys to values, e.g. another typed dictionary or a JSON object. This class is generic with two type parameters: the type of the name (``TName``) and the type of the configuration (``TConfig``). """ name: ReadOnly[TName] """The name of the object.""" configuration: NotRequired[ReadOnly[TConfig]] """The configuration of the object. Not required.""" class NamedRequiredConfig[TName: str, TConfig: Mapping[str, object]](TypedDict): """ A typed dictionary representing an object with a name and configuration, where the configuration is a mapping of string keys to values, e.g. another typed dictionary or a JSON object. This class is generic with two type parameters: the type of the name (``TName``) and the type of the configuration (``TConfig``). """ name: ReadOnly[TName] """The name of the object.""" configuration: ReadOnly[TConfig] """The configuration of the object.""" def product(tup: tuple[int, ...]) -> int: return math.prod(tup) def ceildiv(a: float, b: float) -> int: """Ceiling of ``a / b`` using floating-point division; zero when ``a`` is zero.""" if a == 0: return 0 return math.ceil(a / b) def ceildiv_int(a: int, b: int) -> int: """Ceiling of integer division using exact Python integer arithmetic.""" return -(-int(a) // int(b)) def concurrent_iter[T: tuple[Any, ...], V]( items: Iterable[T], func: Callable[..., Awaitable[V]], limit: int | None = None, ) -> list[asyncio.Task[V]]: """Launch `func(*item)` for each item concurrently, returning the tasks. When `limit` is set, no more than `limit` calls are in flight at once. Tasks are returned in input order; callers that want completion order should wrap the result in `asyncio.as_completed`. Every task is scheduled (via `ensure_future`) before this function returns, not on first iteration of the result. That matters for callers that await the returned tasks one at a time — without eager scheduling, each coroutine would only start when individually awaited, serializing the work and defeating the semaphore. It also makes the return type honest (real `Task`s support `.cancel()`, `.done()`, callbacks) rather than bare coroutines. See https://docs.python.org/3/library/asyncio-task.html#coroutines: "Note that simply calling a coroutine will not schedule it to be executed:" """ if limit is None: return [asyncio.ensure_future(func(*item)) for item in items] sem = asyncio.Semaphore(limit) async def run(item: T) -> V: async with sem: return await func(*item) return [asyncio.ensure_future(run(item)) for item in items] async def concurrent_map[T: tuple[Any, ...], V]( items: Iterable[T], func: Callable[..., Awaitable[V]], limit: int | None = None, ) -> list[V]: return await asyncio.gather(*concurrent_iter(items, func, limit)) async def concurrent_foreach[T: tuple[Any, ...]]( items: Iterable[T], func: Callable[..., Awaitable[object]], limit: int, ) -> None: """Run `func(*item)` for every item with at most `limit` calls in flight. Unlike `concurrent_map`, `items` is consumed lazily: `limit` workers share one iterator and each takes the next item as soon as it finishes the previous one, so auxiliary memory is bounded by `limit`, not by the number of items. Results are discarded. Once a call fails, no further items are taken and the calls still in flight are cancelled. The failure is re-raised as itself, so callers catch the store's own exception type. If other in-flight calls failed in the same event-loop iteration, the exception raised is the one from the lowest-numbered worker and the `BaseExceptionGroup` holding every failure is attached as its `__cause__`, so nothing is lost from tracebacks and logs. Cancellation of the caller propagates as `CancelledError`. """ if limit < 1: raise ValueError(f"limit must be at least 1, got {limit}") shared = iter(items) failed = False async def worker() -> None: nonlocal failed # `next()` on the shared iterator is synchronous, so workers never # re-enter it: a worker only yields to the loop while awaiting `func`. while not failed: try: item = next(shared) except StopIteration: return try: await func(*item) except BaseException: failed = True raise try: async with asyncio.TaskGroup() as tg: for _ in range(limit): tg.create_task(worker()) except BaseExceptionGroup as eg: if len(eg.exceptions) == 1: raise eg.exceptions[0] from None raise eg.exceptions[0] from eg def enum_names[E: Enum](enum: type[E]) -> Iterator[str]: for item in enum: yield item.name def parse_enum[E: Enum](data: object, cls: type[E]) -> E: if isinstance(data, cls): return data if not isinstance(data, str): raise TypeError(f"Expected str, got {type(data)}") if data in enum_names(cls): return cls(data) raise ValueError(f"Value must be one of {list(enum_names(cls))!r}. Got {data} instead.") def parse_name(data: JSON, expected: str | None = None) -> str: try: data = cast("str", convert(data, str)) except (ValueError, TypeError) as exc: raise TypeError(f"Expected a string, got an instance of {type(data)}.") from exc if expected is None or data == expected: return data raise ValueError(f"Expected '{expected}'. Got {data} instead.") def parse_configuration(data: JSON) -> JSON: if not isinstance(data, dict): raise TypeError(f"Expected dict, got {type(data)}") return data @overload def parse_named_configuration( data: JSON | NamedConfig[str, Any], expected_name: str | None = None ) -> tuple[str, dict[str, JSON]]: ... @overload def parse_named_configuration( data: JSON | NamedConfig[str, Any], expected_name: str | None = None, *, require_configuration: bool = True, ) -> tuple[str, dict[str, JSON] | None]: ... def parse_named_configuration( data: JSON | NamedConfig[str, Any], expected_name: str | None = None, *, require_configuration: bool = True, ) -> tuple[str, JSON | None]: if not isinstance(data, dict): raise TypeError(f"Expected dict, got {type(data)}") if "name" not in data: raise ValueError(f"Named configuration does not have a 'name' key. Got {data}.") name_parsed = parse_name(data["name"], expected_name) if "configuration" in data: configuration_parsed = parse_configuration(data["configuration"]) elif require_configuration: raise ValueError(f"Named configuration does not have a 'configuration' key. Got {data}.") else: configuration_parsed = None return name_parsed, configuration_parsed def parse_shapelike(data: ShapeLike) -> tuple[int, ...]: """ Parse a shape-like input into an explicit shape. """ if isinstance(data, int | np.integer): if data < 0: raise ValueError(f"Expected a non-negative integer. Got {data} instead") return (int(data),) try: data_tuple = tuple(data) except TypeError as e: msg = f"Expected an integer or an iterable of integers. Got {data} instead." raise TypeError(msg) from e if not all(isinstance(v, int | np.integer) for v in data_tuple): msg = f"Expected an iterable of integers. Got {data} instead." raise TypeError(msg) if not all(v > -1 for v in data_tuple): msg = f"Expected all values to be non-negative. Got {data} instead." raise ValueError(msg) # cast NumPy scalars to plain python ints return tuple(int(x) for x in data_tuple) def parse_fill_value(data: Any) -> Any: # todo: real validation return data def parse_order(data: Any) -> Literal["C", "F"]: return cast("Literal['C', 'F']", parse_field(data, Literal["C", "F"], "order")) def parse_bool(data: Any) -> bool: return cast("bool", convert(data, bool)) def parse_int(data: Any) -> int: if isinstance(data, int) and not isinstance(data, bool): return data raise ValueError(f"Expected int, got {data} instead.") def _warn_write_empty_chunks_kwarg() -> None: # TODO: link to docs page on array configuration in this message msg = ( "The `write_empty_chunks` keyword argument is deprecated and will be removed in future versions. " "To control whether empty chunks are written to storage, either use the `config` keyword " "argument, as in `config={'write_empty_chunks': True}`," "or change the global 'array.write_empty_chunks' configuration variable." ) warnings.warn(msg, ZarrRuntimeWarning, stacklevel=2) def _warn_order_kwarg() -> None: # TODO: link to docs page on array configuration in this message msg = ( "The `order` keyword argument has no effect for Zarr format 3 arrays. " "To control the memory layout of the array, either use the `config` keyword " "argument, as in `config={'order': 'C'}`," "or change the global 'array.order' configuration variable." ) warnings.warn(msg, ZarrRuntimeWarning, stacklevel=2) def _default_zarr_format() -> ZarrFormat: """Return the default zarr_format.""" return cast("ZarrFormat", int(zarr_config.get("default_zarr_format", 3))) def _subject(name: str, axis: int | None) -> str: """`name` as the subject of an error message, prefixed by the dimension `axis`.""" return name[0].upper() + name[1:] if axis is None else f"Dimension {axis}: {name}" def _parse_positive_int(value: object, name: str, axis: int | None) -> int: """`value` as an `int` of at least 1. A `bool` is read as the `int` it equals; any other type, a NumPy integer or a float (even an integral one: stored documents with integral floats are read by `zarr.core.metadata.repair`), is rejected.""" subject = _subject(name, axis) if not isinstance(value, int): raise TypeError(f"{subject} must be an int, got {value!r}") if value < 1: raise ValueError(f"{subject} must be >= 1, got {value!r}") return int(value) def parse_chunk_edge(size: object, axis: int | None = None) -> int: """Check that `size` is a chunk edge length: an `int` of at least 1 (a `bool` is read as the `int` it equals). This is the one rule for chunk edge lengths in metadata: bare chunk sizes, explicit edges and run-length encoded sizes. `axis`, when given, is named in the error. """ return _parse_positive_int(size, "chunk edge length", axis) def parse_chunk_shape(data: object) -> tuple[int, ...]: """Check a regular chunk shape: an iterable, other than a string or a mapping, of one chunk edge length per axis (see `parse_chunk_edge`).""" match data: case str() | Mapping(): pass case Iterable(): return tuple(parse_chunk_edge(size, axis) for axis, size in enumerate(data)) raise TypeError(f"A chunk shape must be an iterable of chunk edge lengths, got {data!r}") def expand_rle(data: Sequence[object], axis: int | None = None) -> list[int]: """Expand a mixed array of bare integers and RLE pairs, the edges of dimension `axis` (named in errors, when given). Per the rectilinear chunk grid spec, each element can be: - a bare integer (an explicit edge length) - a two-element array ``[value, count]`` (run-length encoded) """ result: list[int] = [] for item in data: if isinstance(item, list): if len(item) != 2: subject = _subject("RLE entries", axis) raise ValueError(f"{subject} must be an integer or [size, count], got {item}") size, count = item repeat = _parse_positive_int(count, "RLE repeat count", axis) result.extend([parse_chunk_edge(size, axis)] * repeat) else: result.append(parse_chunk_edge(item, axis)) return result def compress_rle(sizes: Sequence[int]) -> list[int | list[int]]: """Compress chunk sizes to mixed RLE format per the rectilinear spec. Runs of length > 1 are emitted as ``[value, count]`` pairs; runs of length 1 are emitted as bare integers:: [10, 10, 10, 5] -> [[10, 3], 5] """ if not sizes: return [] result: list[int | list[int]] = [] current = sizes[0] count = 1 for s in sizes[1:]: if s == current: count += 1 else: result.append([current, count] if count > 1 else current) current = s count = 1 result.append([current, count] if count > 1 else current) return result def validate_rectilinear_kind(kind: str | None) -> None: """Validate the ``kind`` field of a rectilinear chunk grid configuration. The rectilinear spec requires ``kind: "inline"``. """ if kind is None: raise ValueError( "Rectilinear chunk grid configuration requires a 'kind' field. " "Only 'inline' is currently supported." ) if kind != "inline": raise ValueError( f"Unsupported rectilinear chunk grid kind: {kind!r}. " "Only 'inline' is currently supported." ) def validate_rectilinear_edges( chunk_shapes: Sequence[int | Sequence[int]], array_shape: Sequence[int] ) -> None: """Validate that rectilinear chunk edges cover the array extent per dimension. Bare-int dimensions (regular step) always cover any extent, so they are skipped. Explicit edge lists must sum to at least the array extent. """ for i, (dim_spec, extent) in enumerate(zip(chunk_shapes, array_shape, strict=True)): if isinstance(dim_spec, int): continue edge_sum = sum(dim_spec) if edge_sum < extent: raise ValueError( f"Rectilinear chunk edges for dimension {i} sum to {edge_sum} " f"but array shape extent is {extent} (edge sum must be >= extent)" ) zarr-developers-zarr-python-a2f2415/src/zarr/core/config.py000066400000000000000000000157621527014137600240140ustar00rootroot00000000000000""" The config module is responsible for managing the configuration of zarr and is based on the Donfig python library. For selecting custom implementations of codecs, pipelines, buffers and ndbuffers, first register the implementations in the registry and then select them in the config. Example: An implementation of the bytes codec in a class ``your.module.NewBytesCodec`` requires the value of ``codecs.bytes`` to be ``your.module.NewBytesCodec``. Donfig can be configured programmatically, by environment variables, or from YAML files in standard locations. ```python from your.module import NewBytesCodec from zarr.core.config import register_codec, config register_codec("bytes", NewBytesCodec) config.set({"codecs.bytes": "your.module.NewBytesCodec"}) ``` Instead of setting the value programmatically with ``config.set``, you can also set the value with an environment variable. The environment variable ``ZARR_CODECS__BYTES`` can be set to ``your.module.NewBytesCodec``. The double underscore ``__`` is used to indicate nested access. ```bash export ZARR_CODECS__BYTES="your.module.NewBytesCodec" ``` For more information, see the Donfig documentation at https://github.com/pytroll/donfig. """ from __future__ import annotations from typing import TYPE_CHECKING, Any, Literal, cast from donfig import Config as DConfig from zarr.core.json_parse import parse_field if TYPE_CHECKING: from donfig.config_obj import ConfigSet class BadConfigError(ValueError): _msg = "bad Config: %r" class Config(DConfig): # type: ignore[misc] """The Config will collect configuration from config files and environment variables Example environment variables: Grabs environment variables of the form "ZARR_FOO__BAR_BAZ=123" and turns these into config variables of the form ``{"foo": {"bar-baz": 123}}`` It transforms the key and value in the following way: - Lower-cases the key text - Treats ``__`` (double-underscore) as nested access - Calls ``ast.literal_eval`` on the value """ def reset(self) -> None: self.clear() self.refresh() def enable_gpu(self) -> ConfigSet: """ Configure Zarr to use GPUs where possible. """ return self.set( {"buffer": "zarr.buffer.gpu.Buffer", "ndbuffer": "zarr.buffer.gpu.NDBuffer"} ) # these keys were removed from the config as part of the 3.1.0 release. # these deprecations should be removed in 3.1.1 or thereabouts. deprecations = { "array.v2_default_compressor.numeric": None, "array.v2_default_compressor.string": None, "array.v2_default_compressor.bytes": None, "array.v2_default_filters.string": None, "array.v2_default_filters.bytes": None, "array.v3_default_filters.numeric": None, "array.v3_default_filters.raw": None, "array.v3_default_filters.bytes": None, "array.v3_default_serializer.numeric": None, "array.v3_default_serializer.string": None, "array.v3_default_serializer.bytes": None, "array.v3_default_compressors.string": None, "array.v3_default_compressors.bytes": None, "array.v3_default_compressors": None, } # The default configuration for zarr config = Config( "zarr", defaults=[ { "default_zarr_format": 3, "array": { "order": "C", "write_empty_chunks": False, "read_missing_chunks": True, "target_shard_size_bytes": None, "rectilinear_chunks": False, "sharding_coalesce_max_gap_bytes": 1 << 20, # 1 MiB "sharding_coalesce_max_bytes": 16 << 20, # 16 MiB # Decoding data with the pickle codec can execute arbitrary code, # so it must be explicitly enabled. "allow_unsafe_pickle_decode": False, }, "async": {"concurrency": 10, "timeout": None}, "threading": {"max_workers": None}, "json_indent": 2, "codec_pipeline": { # FusedCodecPipeline is the faster synchronous pipeline, but it stays # opt-in for now so behavior is unchanged for existing users. Early # adopters can switch with # zarr.config.set( # {"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"} # ) "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", "batch_size": 1, # Only read by FusedCodecPipeline (BatchedCodecPipeline ignores it). "max_workers": None, }, "codecs": { "blosc": "zarr.codecs.blosc.BloscCodec", "gzip": "zarr.codecs.gzip.GzipCodec", "zstd": "zarr.codecs.zstd.ZstdCodec", "bytes": "zarr.codecs.bytes.BytesCodec", "endian": "zarr.codecs.bytes.BytesCodec", # compatibility with earlier versions of ZEP1 "crc32c": "zarr.codecs.crc32c_.Crc32cCodec", "sharding_indexed": "zarr.codecs.sharding.ShardingCodec", "transpose": "zarr.codecs.transpose.TransposeCodec", "vlen-utf8": "zarr.codecs.vlen_utf8.VLenUTF8Codec", "vlen-bytes": "zarr.codecs.vlen_utf8.VLenBytesCodec", "numcodecs.bz2": "zarr.codecs.numcodecs.BZ2", "numcodecs.crc32": "zarr.codecs.numcodecs.CRC32", "numcodecs.crc32c": "zarr.codecs.numcodecs.CRC32C", "numcodecs.lz4": "zarr.codecs.numcodecs.LZ4", "numcodecs.lzma": "zarr.codecs.numcodecs.LZMA", "numcodecs.zfpy": "zarr.codecs.numcodecs.ZFPY", "numcodecs.adler32": "zarr.codecs.numcodecs.Adler32", "numcodecs.astype": "zarr.codecs.numcodecs.AsType", "numcodecs.bitround": "zarr.codecs.numcodecs.BitRound", "numcodecs.blosc": "zarr.codecs.numcodecs.Blosc", "numcodecs.delta": "zarr.codecs.numcodecs.Delta", "numcodecs.fixedscaleoffset": "zarr.codecs.numcodecs.FixedScaleOffset", "numcodecs.fletcher32": "zarr.codecs.numcodecs.Fletcher32", "numcodecs.gzip": "zarr.codecs.numcodecs.GZip", "numcodecs.jenkins_lookup3": "zarr.codecs.numcodecs.JenkinsLookup3", "numcodecs.pcodec": "zarr.codecs.numcodecs.PCodec", "numcodecs.packbits": "zarr.codecs.numcodecs.PackBits", "numcodecs.shuffle": "zarr.codecs.numcodecs.Shuffle", "numcodecs.quantize": "zarr.codecs.numcodecs.Quantize", "numcodecs.zlib": "zarr.codecs.numcodecs.Zlib", "numcodecs.zstd": "zarr.codecs.numcodecs.Zstd", }, "buffer": "zarr.buffer.cpu.Buffer", "ndbuffer": "zarr.buffer.cpu.NDBuffer", } ], deprecations=deprecations, ) def parse_indexing_order(data: Any) -> Literal["C", "F"]: return cast("Literal['C', 'F']", parse_field(data, Literal["C", "F"], "order")) zarr-developers-zarr-python-a2f2415/src/zarr/core/context.py000066400000000000000000000066331527014137600242300ustar00rootroot00000000000000""" The extensions that Zarr metadata is read with, and the resolver that reads with them. A `Context` is the set of extensions a read may resolve, by kind. It is a value: the same for a whole read. A `Resolver` is a context positioned in a document: the context, the Zarr format of the document, and the location in the document. It is re-positioned at every nested step, and is threaded through the read by zarr. (The pair mirrors `referencing.Registry` and `referencing.Resolver` in python-jsonschema.) Both are internal: nothing in the public API takes a context yet. """ from __future__ import annotations from dataclasses import dataclass, replace from typing import TYPE_CHECKING if TYPE_CHECKING: from zarr.core.common import ZarrFormat from zarr.core.dtype.common import DTypeJSON from zarr.core.dtype.registry import DataTypeRegistry from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType @dataclass(frozen=True, kw_only=True) class Context: """ The extensions that Zarr metadata is read with, by kind. Attributes ---------- data_types : DataTypeRegistry The data types a read may resolve. """ data_types: DataTypeRegistry @classmethod def default(cls) -> Context: """ The context of the default registries, such as `zarr.dtype.data_type_registry`. It refers to the registries themselves, so it includes extensions registered after it is created. """ # avoid circular import from zarr.core.dtype import data_type_registry return cls(data_types=data_type_registry) type JSONLocation = tuple[str | int, ...] def json_pointer(loc: JSONLocation) -> str: """ The JSON Pointer (RFC 6901) of a location in a JSON document, e.g. `/codecs/0/configuration`. """ return "".join("/" + str(key).replace("~", "~0").replace("/", "~1") for key in loc) @dataclass(frozen=True, kw_only=True) class Resolver: """ Resolves the extensions in a Zarr metadata document from a context, at a location in the document. An extension that contains other extensions, such as a structured data type, resolves each of them with the resolver returned by `at`, so that they come from the same context, use the same Zarr format, and report errors with their location. Attributes ---------- context : Context The extensions the read may resolve. zarr_format : ZarrFormat The Zarr format of the document. loc : tuple[str | int, ...] The location in the document of the JSON being read, as keys and indices. A Zarr V2 data type is located by its name as written in the document (the value of `"dtype"`), not by the `{"name": ..., "object_codec_id": ...}` form zarr reads it in. """ context: Context zarr_format: ZarrFormat loc: JSONLocation = () def at(self, *keys: str | int) -> Resolver: """ The resolver for the JSON at `keys` within the JSON at this resolver's location. """ return replace(self, loc=(*self.loc, *keys)) def resolve_data_type(self, data: DTypeJSON) -> ZDType[TBaseDType, TBaseScalar]: """ Resolve the JSON representation of a data type at this resolver's location. Raises ------ ValueError If no data type in the context matches `data`. """ return self.context.data_types._match_json(data, resolver=self) zarr-developers-zarr-python-a2f2415/src/zarr/core/dtype/000077500000000000000000000000001527014137600233075ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/core/dtype/__init__.py000066400000000000000000000226301527014137600254230ustar00rootroot00000000000000from __future__ import annotations from collections.abc import Sequence from typing import TYPE_CHECKING, Final from zarr.core.dtype.npy.bool import Bool from zarr.core.dtype.npy.bytes import ( NullTerminatedBytes, NullterminatedBytesJSON_V2, NullTerminatedBytesJSON_V3, RawBytes, RawBytesJSON_V2, RawBytesJSON_V3, VariableLengthBytes, VariableLengthBytesJSON_V2, ) from zarr.core.dtype.npy.complex import Complex64, Complex128 from zarr.core.dtype.npy.float import Float16, Float32, Float64 from zarr.core.dtype.npy.int import Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64 from zarr.core.dtype.npy.structured import ( Struct, StructJSON_V3, Structured, StructuredJSON_V2, StructuredJSON_V3, ) from zarr.core.dtype.npy.time import ( DateTime64, DateTime64JSON_V2, DateTime64JSON_V3, TimeDelta64, TimeDelta64JSON_V2, TimeDelta64JSON_V3, ) if TYPE_CHECKING: from zarr.core.common import ZarrFormat from zarr.core.dtype.common import DTypeJSON from collections.abc import Mapping import numpy as np import numpy.typing as npt from zarr.core.common import JSON from zarr.core.dtype.npy.string import ( FixedLengthUTF32, FixedLengthUTF32JSON_V2, FixedLengthUTF32JSON_V3, VariableLengthUTF8, VariableLengthUTF8JSON_V2, ) from zarr.core.dtype.registry import DataTypeRegistry from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType __all__ = [ "Bool", "Complex64", "Complex128", "DataTypeRegistry", "DateTime64", "DateTime64JSON_V2", "DateTime64JSON_V3", "FixedLengthUTF32", "FixedLengthUTF32JSON_V2", "FixedLengthUTF32JSON_V3", "Float16", "Float32", "Float64", "Int8", "Int16", "Int32", "Int64", "NullTerminatedBytes", "NullTerminatedBytesJSON_V3", "NullterminatedBytesJSON_V2", "RawBytes", "RawBytesJSON_V2", "RawBytesJSON_V3", "Struct", "StructJSON_V3", "Structured", "StructuredJSON_V2", "StructuredJSON_V3", "TBaseDType", "TBaseScalar", "TimeDelta64", "TimeDelta64JSON_V2", "TimeDelta64JSON_V3", "UInt8", "UInt16", "UInt32", "UInt64", "VariableLengthBytes", "VariableLengthBytesJSON_V2", "VariableLengthUTF8", "VariableLengthUTF8JSON_V2", "ZDType", "data_type_registry", "parse_data_type", "parse_dtype", ] data_type_registry = DataTypeRegistry() IntegerDType = Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 | UInt64 INTEGER_DTYPE: Final = Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64 FloatDType = Float16 | Float32 | Float64 FLOAT_DTYPE: Final = Float16, Float32, Float64 ComplexFloatDType = Complex64 | Complex128 COMPLEX_FLOAT_DTYPE: Final = Complex64, Complex128 StringDType = FixedLengthUTF32 | VariableLengthUTF8 STRING_DTYPE: Final = FixedLengthUTF32, VariableLengthUTF8 TimeDType = DateTime64 | TimeDelta64 TIME_DTYPE: Final = DateTime64, TimeDelta64 BytesDType = RawBytes | NullTerminatedBytes | VariableLengthBytes BYTES_DTYPE: Final = RawBytes, NullTerminatedBytes, VariableLengthBytes AnyDType = ( Bool | IntegerDType | FloatDType | ComplexFloatDType | StringDType | BytesDType | Struct | TimeDType | VariableLengthBytes ) # mypy has trouble inferring the type of variablelengthstring dtype, because its class definition # depends on the installed numpy version. That's why the type: ignore statement is needed here. ANY_DTYPE: Final = ( Bool, *INTEGER_DTYPE, *FLOAT_DTYPE, *COMPLEX_FLOAT_DTYPE, *STRING_DTYPE, *BYTES_DTYPE, Struct, *TIME_DTYPE, VariableLengthBytes, ) # These are aliases for variable-length UTF-8 strings # We handle them when a user requests a data type instead of using NumPy's dtype inferece because # the default NumPy behavior -- to inspect the user-provided array data and choose # an appropriately sized U dtype -- is unworkable for Zarr. VLEN_UTF8_ALIAS: Final = ("str", str, "string") # This type models inputs that can be coerced to a ZDType type ZDTypeLike = npt.DTypeLike | ZDType[TBaseDType, TBaseScalar] | Mapping[str, JSON] | str for dtype in ANY_DTYPE: # mypy does not know that all the elements of ANY_DTYPE are subclasses of ZDType data_type_registry.register(dtype._zarr_v3_name, dtype) # type: ignore[arg-type] # TODO: find a better name for this function def get_data_type_from_native_dtype(dtype: npt.DTypeLike) -> ZDType[TBaseDType, TBaseScalar]: """ Get a data type wrapper (an instance of ``ZDType``) from a native data type, e.g. a numpy dtype. """ if not isinstance(dtype, np.dtype): na_dtype: np.dtype[np.generic] if isinstance(dtype, list): # this is a valid _VoidDTypeLike check na_dtype = np.dtype([tuple(d) for d in dtype]) else: na_dtype = np.dtype(dtype) else: na_dtype = dtype return data_type_registry.match_dtype(dtype=na_dtype) def get_data_type_from_json( dtype_spec: DTypeJSON, *, zarr_format: ZarrFormat ) -> ZDType[TBaseDType, TBaseScalar]: """ Given a JSON representation of a data type and a Zarr format version, attempt to create a ZDType instance from the registered ZDType classes. """ return data_type_registry.match_json(dtype_spec, zarr_format=zarr_format) def parse_data_type( dtype_spec: ZDTypeLike, *, zarr_format: ZarrFormat, ) -> ZDType[TBaseDType, TBaseScalar]: """ Interpret the input as a ZDType. This function wraps ``parse_dtype``. The only difference is the function name. This function may be deprecated in a future version of Zarr Python in favor of ``parse_dtype``. Parameters ---------- dtype_spec : ZDTypeLike The input to be interpreted as a ZDType. This could be a ZDType, which will be returned directly, or a JSON representation of a ZDType, or a native dtype, or a python object that can be converted into a native dtype. zarr_format : ZarrFormat The Zarr format version. This parameter is required because this function will attempt to parse the JSON representation of a data type, and the JSON representation of data types varies between Zarr 2 and Zarr 3. Returns ------- ZDType[TBaseDType, TBaseScalar] The ZDType corresponding to the input. Examples -------- ```python from zarr.dtype import parse_data_type import numpy as np parse_data_type("int32", zarr_format=2) # Int32(endianness='little') parse_data_type(np.dtype('S10'), zarr_format=2) # NullTerminatedBytes(length=10) parse_data_type({"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 10}}, zarr_format=3) # DateTime64(endianness='little', scale_factor=10, unit='s') ``` """ return parse_dtype(dtype_spec, zarr_format=zarr_format) def parse_dtype( dtype_spec: ZDTypeLike, *, zarr_format: ZarrFormat, ) -> ZDType[TBaseDType, TBaseScalar]: """ Convert the input as a ZDType. Parameters ---------- dtype_spec : ZDTypeLike The input to be converted to a ZDType. This could be a ZDType, which will be returned directly, or a JSON representation of a ZDType, or a numpy dtype, or a python object that can be converted into a native dtype. zarr_format : ZarrFormat The Zarr format version. This parameter is required because this function will attempt to parse the JSON representation of a data type, and the JSON representation of data types varies between Zarr 2 and Zarr 3. Returns ------- ZDType[TBaseDType, TBaseScalar] The ZDType corresponding to the input. Examples -------- ```python from zarr.dtype import parse_dtype import numpy as np parse_dtype("int32", zarr_format=2) # Int32(endianness='little') parse_dtype(np.dtype('S10'), zarr_format=2) # NullTerminatedBytes(length=10) parse_dtype({"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 10}}, zarr_format=3) # DateTime64(endianness='little', scale_factor=10, unit='s') ``` """ if isinstance(dtype_spec, ZDType): return dtype_spec # First attempt to interpret the input as JSON if isinstance(dtype_spec, Mapping | str | Sequence): try: return get_data_type_from_json(dtype_spec, zarr_format=zarr_format) except ValueError: # no data type matched this JSON-like input pass if dtype_spec in VLEN_UTF8_ALIAS: # If the dtype request is one of the aliases for variable-length UTF-8 strings, # return that dtype. return VariableLengthUTF8() # type: ignore[return-value] # otherwise, we have either a numpy dtype string, or a zarr v3 dtype string, and in either case # we can create a native dtype from it, and do the dtype inference from that return get_data_type_from_native_dtype(dtype_spec) # type: ignore[arg-type] def __getattr__(name: str) -> object: if name == "DataTypeValidationError": import warnings from zarr.errors import DataTypeValidationError, ZarrDeprecationWarning warnings.warn( "Importing DataTypeValidationError from zarr.core.dtype is deprecated. " "Use zarr.errors.DataTypeValidationError instead.", ZarrDeprecationWarning, stacklevel=2, ) return DataTypeValidationError raise AttributeError(f"module {__name__!r} has no attribute {name!r}") zarr-developers-zarr-python-a2f2415/src/zarr/core/dtype/common.py000066400000000000000000000235201527014137600251530ustar00rootroot00000000000000from __future__ import annotations import sys import warnings from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import ( ClassVar, Final, Literal, TypedDict, TypeGuard, ) from typing_extensions import ReadOnly from zarr.core.common import NamedConfig from zarr.errors import UnstableSpecificationWarning EndiannessStr = Literal["little", "big"] ENDIANNESS_STR: Final = "little", "big" SpecialFloatStrings = Literal["NaN", "Infinity", "-Infinity"] SPECIAL_FLOAT_STRINGS: Final = ("NaN", "Infinity", "-Infinity") JSONFloatV2 = float | SpecialFloatStrings JSONFloatV3 = float | SpecialFloatStrings | str ObjectCodecID = Literal["vlen-utf8", "vlen-bytes", "vlen-array", "pickle", "json2", "msgpack2"] # These are the ids of the known object codecs for zarr v2. OBJECT_CODEC_IDS: Final = ("vlen-utf8", "vlen-bytes", "vlen-array", "pickle", "json2", "msgpack2") # This is a wider type than our standard JSON type because we need # to work with typeddict objects which are assignable to Mapping[str, object] DTypeJSON = str | int | float | Sequence["DTypeJSON"] | None | Mapping[str, object] # The DTypeJSON_V2 type exists because ZDType.from_json takes a single argument, which must contain # all the information necessary to decode the data type. Zarr v2 supports multiple distinct # data types that all used the "|O" data type identifier. These data types can only be # discriminated on the basis of their "object codec", i.e. a special data type specific # compressor or filter. So to figure out what data type a zarr v2 array has, we need the # data type identifier from metadata, as well as an object codec id if the data type identifier # is "|O". # So we will pack the name of the dtype alongside the name of the object codec id, if applicable, # in a single dict, and pass that to the data type inference logic. # These type variables have a very wide bound because the individual zdtype # classes can perform a very specific type check. # This is the JSON representation of a structured dtype in zarr v2 StructuredName_V2 = Sequence["str | StructuredName_V2"] # This models the type of the name a dtype might have in zarr v2 array metadata DTypeName_V2 = StructuredName_V2 | str class DTypeConfig_V2[TDTypeNameV2: DTypeName_V2, TObjectCodecID: str | None](TypedDict): name: ReadOnly[TDTypeNameV2] object_codec_id: ReadOnly[TObjectCodecID] DTypeSpec_V2 = DTypeConfig_V2[DTypeName_V2, None | str] def check_structured_dtype_v2_inner(data: object) -> TypeGuard[StructuredName_V2]: """ A type guard for the inner elements of a structured dtype. This is a recursive check because the type is itself recursive. This check ensures that all the elements are 2-element sequences beginning with a string and ending with either another string or another 2-element sequence beginning with a string and ending with another instance of that type. """ if isinstance(data, (str, Mapping)): return False if not isinstance(data, Sequence): return False if len(data) != 2: return False if not (isinstance(data[0], str)): return False if isinstance(data[-1], str): return True elif isinstance(data[-1], Sequence): # A nested structured dtype's field has the form [name, [[sub_name, sub_dtype], ...]], # i.e. the last element is itself a sequence of field pairs rather than a single # [name, dtype] pair, so it must be validated as a list of fields, not a single field. return check_structured_dtype_name_v2(data[-1]) return False def check_structured_dtype_name_v2(data: Sequence[object]) -> TypeGuard[StructuredName_V2]: """ Check that all the elements of a sequence are valid zarr v2 structured dtype identifiers """ return all(check_structured_dtype_v2_inner(d) for d in data) def check_dtype_name_v2(data: object) -> TypeGuard[DTypeName_V2]: """ Type guard for narrowing the type of a python object to a valid zarr v2 dtype name. """ if isinstance(data, str): return True elif isinstance(data, Sequence): return check_structured_dtype_name_v2(data) return False def check_dtype_spec_v2(data: object) -> TypeGuard[DTypeSpec_V2]: """ Type guard for narrowing a python object to an instance of DTypeSpec_V2 """ if not isinstance(data, Mapping): return False if set(data.keys()) != {"name", "object_codec_id"}: return False if not check_dtype_name_v2(data["name"]): return False return isinstance(data["object_codec_id"], str | None) def check_dtype_spec_no_object_codec_v2( data: object, ) -> TypeGuard[DTypeConfig_V2[DTypeName_V2, None]]: """ Type guard for narrowing a python object to a Zarr V2 data type without an object codec. Only the data types stored with the NumPy "O" data type have an object codec, so every other data type should check its Zarr V2 JSON with this function rather than [`check_dtype_spec_v2`][zarr.dtype.check_dtype_spec_v2]. """ return check_dtype_spec_v2(data) and data["object_codec_id"] is None # By comparison, The JSON representation of a dtype in zarr v3 is much simpler. # It's either a string, or a structured dict DTypeSpec_V3 = str | NamedConfig[str, Mapping[str, object]] def check_dtype_spec_v3(data: object) -> TypeGuard[DTypeSpec_V3]: """ Type guard for narrowing the type of a python object to an instance of DTypeSpec_V3, i.e either a string or a dict with a "name" field that's a string and a "configuration" field that's a mapping with string keys. """ if isinstance(data, str) or ( # noqa: SIM103 isinstance(data, Mapping) and set(data.keys()) == {"name", "configuration"} and isinstance(data["configuration"], Mapping) and all(isinstance(k, str) for k in data["configuration"]) ): return True return False def unpack_dtype_json(data: DTypeSpec_V2 | DTypeSpec_V3) -> DTypeJSON: """ Return the array metadata form of the dtype JSON representation. For the Zarr V3 form of dtype metadata, this is a no-op. For the Zarr V2 form of dtype metadata, this unpacks the dtype name. """ if isinstance(data, Mapping) and set(data.keys()) == {"name", "object_codec_id"}: return data["name"] return data def __getattr__(name: str) -> object: if name == "DataTypeValidationError": import warnings from zarr.errors import DataTypeValidationError, ZarrDeprecationWarning warnings.warn( "Importing DataTypeValidationError from zarr.core.dtype.common is deprecated. " "Use zarr.errors.DataTypeValidationError or zarr.dtype.DataTypeValidationError instead.", ZarrDeprecationWarning, stacklevel=2, ) return DataTypeValidationError raise AttributeError(f"module {__name__!r} has no attribute {name!r}") class ScalarTypeValidationError(ValueError): ... @dataclass(frozen=True, kw_only=True) class HasLength: """ A mix-in class for data types with a length attribute, such as fixed-size collections of unicode strings, or bytes. Attributes ---------- length : int The length of the scalars belonging to this data type. Note that this class does not assign a unit to the length. Child classes may assign units. """ length: int @dataclass(frozen=True, kw_only=True) class HasEndianness: """ A mix-in class for data types with an endianness attribute. The endianness is the byte order of the in-memory array, not the byte order of stored chunks, which the `bytes` codec sets. Zarr V3 data type metadata carries no byte order, so it defaults to the byte order of the host. """ endianness: EndiannessStr = sys.byteorder @dataclass(frozen=True, kw_only=True) class HasItemSize: """ A mix-in class for data types with an item size attribute. This mix-in bears a property ``item_size``, which denotes the size of each element of the data type, in bytes. """ @property def item_size(self) -> int: raise NotImplementedError @dataclass(frozen=True, kw_only=True) class HasObjectCodec: """ A mix-in class for data types that require an object codec id. This class bears the property ``object_codec_id``, which is the string name of an object codec that is required to encode and decode the data type. In zarr-python 2.x certain data types like variable-length strings or variable-length arrays used the catch-all numpy "object" data type for their in-memory representation. But these data types cannot be stored as numpy object data types, because the object data type does not define a fixed memory layout. So these data types required a special codec, called an "object codec", that effectively defined a compact representation for the data type, which was used to encode and decode the data type. Zarr-python 2.x would not allow the creation of arrays with the "object" data type if an object codec was not specified, and thus the name of the object codec is effectively part of the data type model. """ object_codec_id: ClassVar[str] def v3_unstable_dtype_warning(dtype: object) -> None: """ Emit this warning when a data type does not have a stable zarr v3 spec """ msg = ( f"The data type ({dtype}) does not have a Zarr V3 specification. " "That means that the representation of arrays saved with this data type may change without " "warning in a future version of Zarr Python. " "Arrays stored with this data type may be unreadable by other Zarr libraries. " "Use this data type at your own risk! " "Check https://github.com/zarr-developers/zarr-extensions/tree/main/data-types for the " "status of data type specifications for Zarr V3." ) warnings.warn(msg, category=UnstableSpecificationWarning, stacklevel=2) zarr-developers-zarr-python-a2f2415/src/zarr/core/dtype/npy/000077500000000000000000000000001527014137600241155ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/core/dtype/npy/__init__.py000066400000000000000000000000001527014137600262140ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/core/dtype/npy/bool.py000066400000000000000000000212531527014137600254250ustar00rootroot00000000000000from __future__ import annotations from dataclasses import dataclass from typing import TYPE_CHECKING, ClassVar, Literal, Self, TypeGuard, overload import numpy as np from zarr.core.dtype.common import ( DTypeConfig_V2, DTypeJSON, HasItemSize, check_dtype_spec_no_object_codec_v2, ) from zarr.core.dtype.wrapper import TBaseDType, ZDType from zarr.errors import DataTypeValidationError if TYPE_CHECKING: from zarr.core.common import JSON, ZarrFormat @dataclass(frozen=True, kw_only=True, slots=True) class Bool(ZDType[np.dtypes.BoolDType, np.bool_], HasItemSize): """ A Zarr data type for arrays containing booleans. Wraps the [`np.dtypes.BoolDType`][numpy.dtypes.BoolDType] data type. Scalars for this data type are instances of [`np.bool_`][numpy.bool_]. Attributes ---------- _zarr_v3_name : Literal["bool"] = "bool" The Zarr v3 name of the dtype. _zarr_v2_name : ``Literal["|b1"]`` = ``"|b1"`` The Zarr v2 name of the dtype, which is also a string representation of the boolean dtype used by NumPy. dtype_cls : ClassVar[type[np.dtypes.BoolDType]] = np.dtypes.BoolDType The NumPy dtype class. References ---------- This class implements the boolean data type defined in Zarr V2 and V3. See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding)and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. """ _zarr_v3_name: ClassVar[Literal["bool"]] = "bool" _zarr_v2_name: ClassVar[Literal["|b1"]] = "|b1" dtype_cls = np.dtypes.BoolDType @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ Create an instance of Bool from an instance of np.dtypes.BoolDType. Parameters ---------- dtype : TBaseDType The NumPy boolean dtype instance to convert. Returns ------- Bool An instance of Bool. Raises ------ DataTypeValidationError If the provided dtype is not compatible with this ZDType. """ if cls._check_native_dtype(dtype): return cls() raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) def to_native_dtype(self: Self) -> np.dtypes.BoolDType: """ Create a NumPy boolean dtype instance from this ZDType. Returns ------- np.dtypes.BoolDType The NumPy boolean dtype. """ return self.dtype_cls() @classmethod def _check_json_v2( cls, data: DTypeJSON, ) -> TypeGuard[DTypeConfig_V2[Literal["|b1"], None]]: """ Check that the input is a valid JSON representation of a Bool. Parameters ---------- data : DTypeJSON The JSON data to check. Returns ------- ``TypeGuard[DTypeConfig_V2[Literal["|b1"], None]]`` True if the input is a valid JSON representation, False otherwise. """ return check_dtype_spec_no_object_codec_v2(data) and data["name"] == cls._zarr_v2_name @classmethod def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[Literal["bool"]]: """ Check that the input is a valid JSON representation of this class in Zarr V3. Parameters ---------- data : DTypeJSON The JSON data to check. Returns ------- bool True if the input is a valid JSON representation, False otherwise. """ return data == cls._zarr_v3_name @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create an instance of Bool from Zarr V2-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Bool An instance of Bool. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class. """ if cls._check_json_v2(data): return cls() msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v2_name!r}" raise DataTypeValidationError(msg) @classmethod def _from_json_v3(cls: type[Self], data: DTypeJSON) -> Self: """ Create an instance of Bool from Zarr V3-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Bool An instance of Bool. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class. """ if cls._check_json_v3(data): return cls() msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal["|b1"], None]: ... @overload def to_json(self, zarr_format: Literal[3]) -> Literal["bool"]: ... def to_json( self, zarr_format: ZarrFormat ) -> DTypeConfig_V2[Literal["|b1"], None] | Literal["bool"]: """ Serialize this Bool instance to JSON. Parameters ---------- zarr_format : ZarrFormat The Zarr format version (2 or 3). Returns ------- ``DTypeConfig_V2[Literal["|b1"], None] | Literal["bool"]`` The JSON representation of the Bool instance. Raises ------ ValueError If the zarr_format is not 2 or 3. """ if zarr_format == 2: return {"name": self._zarr_v2_name, "object_codec_id": None} elif zarr_format == 3: return self._zarr_v3_name raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover def _check_scalar(self, data: object) -> bool: """ Check if the input can be cast to a boolean scalar. Parameters ---------- data : object The data to check. Returns ------- bool True if the input can be cast to a boolean scalar, False otherwise. """ return True def cast_scalar(self, data: object) -> np.bool_: """ Cast the input to a numpy boolean scalar. Parameters ---------- data : object The data to cast. Returns ------- bool : np.bool_ The numpy boolean scalar. Raises ------ TypeError If the input cannot be converted to a numpy boolean. """ if self._check_scalar(data): return np.bool_(data) msg = ( # pragma: no cover f"Cannot convert object {data!r} with type {type(data)} to a scalar compatible with the " f"data type {self}." ) raise TypeError(msg) # pragma: no cover def default_scalar(self) -> np.bool_: """ Get the default value for the boolean dtype. Returns ------- bool : np.bool_ The default value. """ return np.False_ def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> bool: """ Convert a scalar to a python bool. Parameters ---------- data : object The value to convert. zarr_format : ZarrFormat The zarr format version. Returns ------- bool The JSON-serializable format. """ return bool(data) def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.bool_: """ Read a JSON-serializable value as a numpy boolean scalar. Parameters ---------- data : JSON The JSON-serializable value. zarr_format : ZarrFormat The zarr format version. Returns ------- bool : np.bool_ The numpy boolean scalar. Raises ------ TypeError If the input is not a valid boolean type. """ if self._check_scalar(data): return np.bool_(data) raise TypeError(f"Invalid type: {data}. Expected a boolean.") # pragma: no cover @property def item_size(self) -> int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return 1 zarr-developers-zarr-python-a2f2415/src/zarr/core/dtype/npy/bytes.py000066400000000000000000001160411527014137600256200ustar00rootroot00000000000000from __future__ import annotations import base64 import re from dataclasses import dataclass from typing import ClassVar, Literal, Self, TypedDict, TypeGuard, cast, overload import numpy as np from zarr.core.common import JSON, NamedConfig, ZarrFormat from zarr.core.dtype.common import ( DTypeConfig_V2, DTypeJSON, HasItemSize, HasLength, HasObjectCodec, check_dtype_spec_no_object_codec_v2, check_dtype_spec_v2, v3_unstable_dtype_warning, ) from zarr.core.dtype.npy.common import check_json_str from zarr.core.dtype.wrapper import TBaseDType, ZDType from zarr.errors import DataTypeValidationError BytesLike = np.bytes_ | str | bytes | int class FixedLengthBytesConfig(TypedDict): """ A configuration for a data type that takes a ``length_bytes`` parameter. Attributes ---------- length_bytes : int The length in bytes of the data associated with this configuration. Examples -------- ```python { "length_bytes": 12 } ``` """ length_bytes: int class NullterminatedBytesJSON_V2(DTypeConfig_V2[str, None]): """ A wrapper around the JSON representation of the ``NullTerminatedBytes`` data type in Zarr V2. The ``name`` field of this class contains the value that would appear under the ``dtype`` field in Zarr V2 array metadata. References ---------- The structure of the ``name`` field is defined in the Zarr V2 [specification document](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding). Examples -------- ```python { "name": "|S10", "object_codec_id": None } ``` """ class NullTerminatedBytesJSON_V3( NamedConfig[Literal["null_terminated_bytes"], FixedLengthBytesConfig] ): """ The JSON representation of the ``NullTerminatedBytes`` data type in Zarr V3. References ---------- This representation is not currently defined in an external specification. Examples -------- ```python { "name": "null_terminated_bytes", "configuration": { "length_bytes": 12 } } ``` """ class RawBytesJSON_V2(DTypeConfig_V2[str, None]): """ A wrapper around the JSON representation of the ``RawBytes`` data type in Zarr V2. The ``name`` field of this class contains the value that would appear under the ``dtype`` field in Zarr V2 array metadata. References ---------- The structure of the ``name`` field is defined in the Zarr V2 [specification document](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding). Examples -------- ```python { "name": "|V10", "object_codec_id": None } ``` """ class RawBytesJSON_V3(NamedConfig[Literal["raw_bytes"], FixedLengthBytesConfig]): """ The JSON representation of the ``RawBytes`` data type in Zarr V3. References ---------- This representation is not currently defined in an external specification. Examples -------- ```python { "name": "raw_bytes", "configuration": { "length_bytes": 12 } } ``` """ class VariableLengthBytesJSON_V2(DTypeConfig_V2[Literal["|O"], Literal["vlen-bytes"]]): """ A wrapper around the JSON representation of the ``VariableLengthBytes`` data type in Zarr V2. The ``name`` field of this class contains the value that would appear under the ``dtype`` field in Zarr V2 array metadata. The ``object_codec_id`` field is always ``"vlen-bytes"`` References ---------- The structure of the ``name`` field is defined in the Zarr V2 [specification document](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding). Examples -------- ```python { "name": "|O", "object_codec_id": "vlen-bytes" } ``` """ @dataclass(frozen=True, kw_only=True) class NullTerminatedBytes(ZDType[np.dtypes.BytesDType[int], np.bytes_], HasLength, HasItemSize): """ A Zarr data type for arrays containing fixed-length null-terminated byte sequences. Wraps the [`np.dtypes.BytesDType`][numpy.dtypes.BytesDType] data type. Scalars for this data type are instances of [`np.bytes_`][numpy.bytes_]. This data type is parametrized by an integral length which specifies size in bytes of each scalar. Because this data type uses null-terminated semantics, indexing into NumPy arrays with this data type may return fewer than ``length`` bytes. Attributes ---------- dtype_cls: ClassVar[type[np.dtypes.BytesDType[int]]] = np.dtypes.BytesDType The NumPy data type wrapped by this ZDType. _zarr_v3_name : ClassVar[Literal["null_terminated_bytes"]] length : int The length of the bytes. Notes ----- This data type is designed for compatibility with NumPy arrays that use the NumPy ``bytes`` data type. It may not be desirable for usage outside of that context. If compatibility with the NumPy ``bytes`` data type is not essential, consider using the ``RawBytes`` or ``VariableLengthBytes`` data types instead. """ dtype_cls = np.dtypes.BytesDType _zarr_v3_name: ClassVar[Literal["null_terminated_bytes"]] = "null_terminated_bytes" def __post_init__(self) -> None: """ We don't allow instances of this class with length less than 1 because there is no way such a data type can contain actual data. """ if self.length < 1: raise ValueError(f"length must be >= 1, got {self.length}.") @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ Create an instance of NullTerminatedBytes from an instance of np.dtypes.BytesDType. This method checks if the provided data type is an instance of np.dtypes.BytesDType. If so, it returns a new instance of NullTerminatedBytes with a length equal to the length of input data type. Parameters ---------- dtype : TBaseDType The native dtype to convert. Returns ------- NullTerminatedBytes An instance of NullTerminatedBytes with the specified length. Raises ------ DataTypeValidationError If the dtype is not compatible with NullTerminatedBytes. """ if cls._check_native_dtype(dtype): return cls(length=dtype.itemsize) raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) def to_native_dtype(self) -> np.dtypes.BytesDType[int]: """ Create a NumPy bytes dtype from this NullTerminatedBytes ZDType. Returns ------- np.dtypes.BytesDType[int] A NumPy data type object representing null-terminated bytes with a specified length. """ return self.dtype_cls(self.length) @classmethod def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[NullterminatedBytesJSON_V2]: """ Check that the input is a valid JSON representation of NullTerminatedBytes in Zarr V2. The input data must be a mapping that contains a "name" key that matches the pattern "|S" and an "object_codec_id" key that is None. Parameters ---------- data : DTypeJSON The JSON data to check. Returns ------- bool True if the input data is a valid representation, False otherwise. """ return ( check_dtype_spec_no_object_codec_v2(data) and isinstance(data["name"], str) and re.match(r"^\|S\d+$", data["name"]) is not None ) @classmethod def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[NullTerminatedBytesJSON_V3]: """ Check that the input is a valid JSON representation of this class in Zarr V3. Parameters ---------- data : DTypeJSON The JSON data to check. Returns ------- TypeGuard[NullTerminatedBytesJSON_V3] True if the input is a valid representation of this class in Zarr V3, False otherwise. """ return ( isinstance(data, dict) and set(data.keys()) == {"name", "configuration"} and data["name"] == cls._zarr_v3_name and isinstance(data["configuration"], dict) and "length_bytes" in data["configuration"] and isinstance(data["configuration"]["length_bytes"], int) ) @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create an instance of this class from Zarr V2-flavored JSON. This method checks if the input data is a valid representation of this class in Zarr V2. If so, it returns a new instance of this class with a ``length`` as specified in the input data. Parameters ---------- data : DTypeJSON The JSON data to parse. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input data is not a valid representation of this class. """ if cls._check_json_v2(data): name = data["name"] return cls(length=int(name[2:])) msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string like '|S1', '|S2', etc" raise DataTypeValidationError(msg) @classmethod def _from_json_v3(cls, data: DTypeJSON) -> Self: """ Create an instance of this class from Zarr V3-flavored JSON. This method checks if the input data is a valid representation of this class in Zarr V3. If so, it returns a new instance of this class with a ``length`` as specified in the input data. Parameters ---------- data : DTypeJSON The JSON data to parse. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input data is not a valid representation of this class. """ if cls._check_json_v3(data): return cls(length=data["configuration"]["length_bytes"]) msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> NullterminatedBytesJSON_V2: ... @overload def to_json(self, zarr_format: Literal[3]) -> NullTerminatedBytesJSON_V3: ... def to_json( self, zarr_format: ZarrFormat ) -> DTypeConfig_V2[str, None] | NullTerminatedBytesJSON_V3: """ Generate a JSON representation of this data type. Parameters ---------- zarr_format : ZarrFormat The zarr format version. Returns ------- NullterminatedBytesJSON_V2 | NullTerminatedBytesJSON_V3 The JSON-serializable representation of the data type """ if zarr_format == 2: return {"name": self.to_native_dtype().str, "object_codec_id": None} elif zarr_format == 3: v3_unstable_dtype_warning(self) return { "name": self._zarr_v3_name, "configuration": {"length_bytes": self.length}, } raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover def _check_scalar(self, data: object) -> TypeGuard[BytesLike]: """ Check if the provided data is of type BytesLike. This method is used to verify if the input data can be considered as a scalar of bytes-like type, which includes NumPy bytes, strings, bytes, and integers. Parameters ---------- data : object The data to check. Returns ------- TypeGuard[BytesLike] True if the data is bytes-like, False otherwise. """ return isinstance(data, BytesLike) def _cast_scalar_unchecked(self, data: BytesLike) -> np.bytes_: """ Cast the provided scalar data to [`np.bytes_`][numpy.bytes_], truncating if necessary. Parameters ---------- data : BytesLike The data to cast. Returns ------- bytes : [`np.bytes_`][numpy.bytes_] The casted data as a NumPy bytes scalar. Notes ----- This method does not perform any type checking. The input data must be bytes-like. """ if isinstance(data, int): return self.to_native_dtype().type(str(data)[: self.length]) else: return self.to_native_dtype().type(data[: self.length]) def cast_scalar(self, data: object) -> np.bytes_: """ Attempt to cast a given object to a NumPy bytes scalar. This method first checks if the provided data is a valid scalar that can be converted to a NumPy bytes scalar. If the check succeeds, the unchecked casting operation is performed. If the data is not valid, a TypeError is raised. Parameters ---------- data : object The data to be cast to a NumPy bytes scalar. Returns ------- bytes : [`np.bytes_`][numpy.bytes_] The data cast as a NumPy bytes scalar. Raises ------ TypeError If the data cannot be converted to a NumPy bytes scalar. """ if self._check_scalar(data): return self._cast_scalar_unchecked(data) msg = ( f"Cannot convert object {data!r} with type {type(data)} to a scalar compatible with the " f"data type {self}." ) raise TypeError(msg) def default_scalar(self) -> np.bytes_: """ Return a default scalar value, which for this data type is an empty byte string. Returns ------- bytes : [`np.bytes_`][numpy.bytes_] The default scalar value. """ return np.bytes_(b"") def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: """ Convert a scalar to a JSON-serializable string representation. This method encodes the given scalar as a NumPy bytes scalar and then encodes the bytes as a base64-encoded string. Parameters ---------- data : object The scalar to convert. zarr_format : ZarrFormat The zarr format version. Returns ------- str A string representation of the scalar. """ as_bytes = self.cast_scalar(data) return base64.standard_b64encode(as_bytes).decode("ascii") def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.bytes_: """ Read a JSON-serializable value as [`np.bytes_`][numpy.bytes_]. Parameters ---------- data : JSON The JSON-serializable base64-encoded string. zarr_format : ZarrFormat The zarr format version. Returns ------- bytes : [`np.bytes_`][numpy.bytes_] The NumPy bytes scalar obtained from decoding the base64 string. Raises ------ TypeError If the input data is not a base64-encoded string. """ if check_json_str(data): return self.to_native_dtype().type(base64.standard_b64decode(data.encode("ascii"))) raise TypeError( f"Invalid type: {data}. Expected a base64-encoded string." ) # pragma: no cover @property def item_size(self) -> int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return self.length @dataclass(frozen=True, kw_only=True) class RawBytes(ZDType[np.dtypes.VoidDType[int], np.void], HasLength, HasItemSize): """ A Zarr data type for arrays containing fixed-length sequences of raw bytes. Wraps the NumPy ``void`` data type. Scalars for this data type are instances of [`np.void`][numpy.void]. This data type is parametrized by an integral length which specifies size in bytes of each scalar belonging to this data type. Attributes ---------- dtype_cls: ClassVar[type[np.dtypes.VoidDType[int]]] = np.dtypes.VoidDtype The NumPy data type wrapped by this ZDType. _zarr_v3_name : ClassVar[Literal["raw_bytes"]] length : int The length of the bytes. Notes ----- Although the NumPy "Void" data type is used to create "structured" data types in NumPy, this class does not support structured data types. See the ``Structured`` data type for this functionality. """ # np.dtypes.VoidDType is specified in an odd way in NumPy # it cannot be used to create instances of the dtype # so we have to tell mypy to ignore this here dtype_cls = np.dtypes.VoidDType # type: ignore[assignment] _zarr_v3_name: ClassVar[Literal["raw_bytes"]] = "raw_bytes" def __post_init__(self) -> None: """ We don't allow instances of this class with length less than 1 because there is no way such a data type can contain actual data. """ if self.length < 1: raise ValueError(f"length must be >= 1, got {self.length}.") @classmethod def _check_native_dtype( cls: type[Self], dtype: TBaseDType ) -> TypeGuard[np.dtypes.VoidDType[int]]: """ Check that the input is a NumPy void dtype with no fields. Numpy void dtype comes in two forms: * If the ``fields`` attribute is ``None``, then the dtype represents N raw bytes. * If the ``fields`` attribute is not ``None``, then the dtype represents a structured dtype, In this check we ensure that ``fields`` is ``None``. Parameters ---------- dtype : TDBaseDType The dtype to check. Returns ------- Bool True if the dtype is an instance of np.dtypes.VoidDType with no fields, False otherwise. """ return cls.dtype_cls is type(dtype) and dtype.fields is None @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ Create an instance of RawBytes from an instance of np.dtypes.VoidDType. This method checks if the provided data type is compatible with RawBytes. The input must be an instance of np.dtypes.VoidDType, and have no fields. If the input is compatible, this method returns an instance of RawBytes with the specified length. Parameters ---------- dtype : TBaseDType The native dtype to convert. Returns ------- RawBytes An instance of RawBytes with the specified length. Raises ------ DataTypeValidationError If the dtype is not compatible with RawBytes. """ if cls._check_native_dtype(dtype): return cls(length=dtype.itemsize) raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) def to_native_dtype(self) -> np.dtypes.VoidDType[int]: """ Create a NumPy void dtype from this RawBytes ZDType. Returns ------- np.dtypes.VoidDType[int] A NumPy data type object representing raw bytes with a specified length. """ # Numpy does not allow creating a void type # by invoking np.dtypes.VoidDType directly return cast("np.dtypes.VoidDType[int]", np.dtype(f"V{self.length}")) @classmethod def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[RawBytesJSON_V2]: """ Check that the input is a valid representation of this class in Zarr V2. Parameters ---------- data : DTypeJSON The JSON data to check. Returns ------- True if the input is a valid representation of this class in Zarr V3, False otherwise. """ return ( check_dtype_spec_no_object_codec_v2(data) and isinstance(data["name"], str) and re.match(r"^\|V\d+$", data["name"]) is not None ) @classmethod def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[RawBytesJSON_V3]: """ Check that the input is a valid JSON representation of this class in Zarr V3. Parameters ---------- data : DTypeJSON The JSON data to check. Returns ------- TypeGuard[RawBytesJSON_V3] True if the input is a valid representation of this class in Zarr V3, False otherwise. """ return ( isinstance(data, dict) and set(data.keys()) == {"name", "configuration"} and data["name"] == cls._zarr_v3_name and isinstance(data["configuration"], dict) and set(data["configuration"].keys()) == {"length_bytes"} and isinstance(data["configuration"]["length_bytes"], int) ) @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create an instance of RawBytes from Zarr V2-flavored JSON. This method checks if the input data is a valid representation of RawBytes in Zarr V2. If so, it returns a new instance of RawBytes with a ``length`` as specified in the input data. Parameters ---------- data : DTypeJSON The JSON data to parse. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input data is not a valid representation of this class. """ if cls._check_json_v2(data): name = data["name"] return cls(length=int(name[2:])) msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string like '|V1', '|V2', etc" raise DataTypeValidationError(msg) @classmethod def _from_json_v3(cls, data: DTypeJSON) -> Self: """ Create an instance of RawBytes from Zarr V3-flavored JSON. This method checks if the input data is a valid representation of RawBytes in Zarr V3. If so, it returns a new instance of RawBytes with a ``length`` as specified in the input data. Parameters ---------- data : DTypeJSON The JSON data to parse. Returns ------- RawBytes An instance of RawBytes. Raises ------ DataTypeValidationError If the input data is not a valid representation of this class. """ if cls._check_json_v3(data): return cls(length=data["configuration"]["length_bytes"]) msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> RawBytesJSON_V2: ... @overload def to_json(self, zarr_format: Literal[3]) -> RawBytesJSON_V3: ... def to_json(self, zarr_format: ZarrFormat) -> RawBytesJSON_V2 | RawBytesJSON_V3: """ Generate a JSON representation of this data type. Parameters ---------- zarr_format : ZarrFormat The zarr format version. Returns ------- RawBytesJSON_V2 | RawBytesJSON_V3 The JSON-serializable representation of the data type. """ if zarr_format == 2: return {"name": self.to_native_dtype().str, "object_codec_id": None} elif zarr_format == 3: v3_unstable_dtype_warning(self) return {"name": self._zarr_v3_name, "configuration": {"length_bytes": self.length}} raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover def _check_scalar(self, data: object) -> TypeGuard[np.bytes_ | str | bytes | np.void]: """ Check if the provided data can be cast to np.void. This method is used to verify if the input data can be considered as a scalar of bytes-like type, which includes np.bytes_, np.void, strings, and bytes objects. Parameters ---------- data : object The data to check. Returns ------- TypeGuard[np.bytes_ | str | bytes | np.void] True if the data is void-scalar-like, False otherwise. """ return isinstance(data, np.bytes_ | str | bytes | np.void) def _cast_scalar_unchecked(self, data: object) -> np.void: """ Cast the provided scalar data to np.void. Parameters ---------- data : BytesLike The data to cast. Returns ------- np.void The casted data as a NumPy void scalar. Notes ----- This method does not perform any type checking. The input data must be castable to np.void. """ native_dtype = self.to_native_dtype() # Without the second argument, NumPy will return a void scalar for dtype V1. # The second argument ensures that, if native_dtype is something like V10, # the result will actually be a V10 scalar. return native_dtype.type(data, native_dtype) def cast_scalar(self, data: object) -> np.void: """ Attempt to cast a given object to a NumPy void scalar. This method first checks if the provided data is a valid scalar that can be converted to a NumPy void scalar. If the check succeeds, the unchecked casting operation is performed. If the data is not valid, a TypeError is raised. Parameters ---------- data : object The data to be cast to a NumPy void scalar. Returns ------- np.void The data cast as a NumPy void scalar. Raises ------ TypeError If the data cannot be converted to a NumPy void scalar. """ if self._check_scalar(data): return self._cast_scalar_unchecked(data) msg = ( f"Cannot convert object {data!r} with type {type(data)} to a scalar compatible with the " f"data type {self}." ) raise TypeError(msg) def default_scalar(self) -> np.void: """ Return the default scalar value for this data type. The default scalar is a NumPy void scalar of the same length as the data type, filled with zero bytes. Returns ------- np.void The default scalar value. """ return self.to_native_dtype().type(("\x00" * self.length).encode("ascii")) def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: """ Convert a scalar to a JSON-serializable string representation. This method converts the given scalar to bytes and then encodes the bytes as a base64-encoded string. Parameters ---------- data : object The scalar to convert. zarr_format : ZarrFormat The zarr format version. Returns ------- str A string representation of the scalar. """ as_bytes = self.cast_scalar(data) return base64.standard_b64encode(as_bytes.tobytes()).decode("ascii") def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.void: """ Read a JSON-serializable value as an np.void. Parameters ---------- data : JSON The JSON-serializable value. zarr_format : ZarrFormat The zarr format version. Returns ------- np.void The NumPy void scalar. Raises ------ TypeError If the data is not a string, or if the string is not a valid base64 encoding. """ if check_json_str(data): return self.to_native_dtype().type(base64.standard_b64decode(data)) raise TypeError(f"Invalid type: {data}. Expected a string.") # pragma: no cover @property def item_size(self) -> int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return self.length @dataclass(frozen=True, kw_only=True) class VariableLengthBytes(ZDType[np.dtypes.ObjectDType, bytes], HasObjectCodec): """ A Zarr data type for arrays containing variable-length sequences of bytes. Wraps the NumPy "object" data type. Scalars for this data type are instances of ``bytes``. Attributes ---------- dtype_cls: ClassVar[type[np.dtypes.ObjectDType]] = np.dtypes.ObjectDType The NumPy data type wrapped by this ZDType. _zarr_v3_name: ClassVar[Literal["variable_length_bytes"]] = "variable_length_bytes" The name of this data type in Zarr V3. object_codec_id: ClassVar[Literal["vlen-bytes"]] = "vlen-bytes" The object codec ID for this data type. Notes ----- Because this data type uses the NumPy "object" data type, it does not guarantee a compact memory representation of array data. Therefore a "vlen-bytes" codec is needed to ensure that the array data can be persisted to storage. """ dtype_cls = np.dtypes.ObjectDType _zarr_v3_name: ClassVar[Literal["variable_length_bytes"]] = "variable_length_bytes" object_codec_id: ClassVar[Literal["vlen-bytes"]] = "vlen-bytes" @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ Create an instance of VariableLengthBytes from an instance of np.dtypes.ObjectDType. This method checks if the provided data type is an instance of np.dtypes.ObjectDType. If so, it returns an instance of VariableLengthBytes. Parameters ---------- dtype : TBaseDType The native dtype to convert. Returns ------- VariableLengthBytes An instance of VariableLengthBytes. Raises ------ DataTypeValidationError If the dtype is not compatible with VariableLengthBytes. """ if cls._check_native_dtype(dtype): return cls() raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) def to_native_dtype(self) -> np.dtypes.ObjectDType: """ Create a NumPy object dtype from this VariableLengthBytes ZDType. Returns ------- np.dtypes.ObjectDType A NumPy data type object representing variable-length bytes. """ return self.dtype_cls() @classmethod def _check_json_v2( cls, data: DTypeJSON, ) -> TypeGuard[VariableLengthBytesJSON_V2]: """ Check that the input is a valid JSON representation of a NumPy O dtype, and that the object codec id is appropriate for variable-length bytes strings. Parameters ---------- data : DTypeJSON The JSON data to check. Returns ------- True if the input is a valid representation of this class in Zarr V2, False otherwise. """ # Check that the input is a valid JSON representation of a Zarr v2 data type spec. if not check_dtype_spec_v2(data): return False # Check that the object codec id is appropriate for variable-length bytes strings. if data["name"] != "|O": return False return data["object_codec_id"] == cls.object_codec_id @classmethod def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[Literal["variable_length_bytes"]]: """ Check that the input is a valid JSON representation of this class in Zarr V3. Parameters ---------- data : DTypeJSON The JSON data to check. Returns ------- TypeGuard[Literal["variable_length_bytes"]] True if the input is a valid representation of this class in Zarr V3, False otherwise. """ return data in (cls._zarr_v3_name, "bytes") @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create an instance of this VariableLengthBytes from Zarr V2-flavored JSON. This method checks if the input data is a valid representation of this class in Zarr V2. If so, it returns a new instance this class. Parameters ---------- data : DTypeJSON The JSON data to parse. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input data is not a valid representation of this class. """ if cls._check_json_v2(data): return cls() msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string '|O' and an object_codec_id of {cls.object_codec_id}" raise DataTypeValidationError(msg) @classmethod def _from_json_v3(cls, data: DTypeJSON) -> Self: """ Create an instance of VariableLengthBytes from Zarr V3-flavored JSON. This method checks if the input data is a valid representation of VariableLengthBytes in Zarr V3. If so, it returns a new instance of VariableLengthBytes. Parameters ---------- data : DTypeJSON The JSON data to parse. Returns ------- VariableLengthBytes An instance of VariableLengthBytes. Raises ------ DataTypeValidationError If the input data is not a valid representation of this class. """ if cls._check_json_v3(data): return cls() msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> VariableLengthBytesJSON_V2: ... @overload def to_json(self, zarr_format: Literal[3]) -> Literal["variable_length_bytes"]: ... def to_json( self, zarr_format: ZarrFormat ) -> VariableLengthBytesJSON_V2 | Literal["variable_length_bytes"]: """ Convert the variable-length bytes data type to a JSON-serializable form. Parameters ---------- zarr_format : ZarrFormat The zarr format version. Accepted values are 2 and 3. Returns ------- ``DTypeConfig_V2[Literal["|O"], Literal["vlen-bytes"]] | Literal["variable_length_bytes"]`` The JSON-serializable representation of the variable-length bytes data type. For zarr_format 2, returns a dictionary with "name" and "object_codec_id". For zarr_format 3, returns a string identifier "variable_length_bytes". Raises ------ ValueError If zarr_format is not 2 or 3. """ if zarr_format == 2: return {"name": "|O", "object_codec_id": self.object_codec_id} elif zarr_format == 3: v3_unstable_dtype_warning(self) return self._zarr_v3_name raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover def default_scalar(self) -> bytes: """ Return the default scalar value for the variable-length bytes data type. Returns ------- bytes The default scalar value, which is an empty byte string. """ return b"" def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: """ Convert a scalar to a JSON-serializable string representation. This method encodes the given scalar as bytes and then encodes the bytes as a base64-encoded string. Parameters ---------- data : object The scalar to convert. zarr_format : ZarrFormat The zarr format version. Returns ------- str A string representation of the scalar. """ return base64.standard_b64encode(data).decode("ascii") # type: ignore[arg-type] def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> bytes: """ Decode a base64-encoded JSON string to bytes. Parameters ---------- data : JSON The JSON-serializable base64-encoded string. zarr_format : ZarrFormat The zarr format version. Returns ------- bytes The decoded bytes from the base64 string. Raises ------ TypeError If the input data is not a base64-encoded string. """ if check_json_str(data): return base64.standard_b64decode(data.encode("ascii")) raise TypeError(f"Invalid type: {data}. Expected a string.") # pragma: no cover def _check_scalar(self, data: object) -> TypeGuard[BytesLike]: """ Check if the provided data is of type BytesLike. This method is used to verify if the input data can be considered as a scalar of bytes-like type, which includes NumPy bytes, strings, bytes, and integers. Parameters ---------- data : object The data to check. Returns ------- TypeGuard[BytesLike] True if the data is bytes-like, False otherwise. """ return isinstance(data, BytesLike) def _cast_scalar_unchecked(self, data: BytesLike) -> bytes: """ Cast the provided scalar data to bytes. Parameters ---------- data : BytesLike The data to cast. Returns ------- bytes The casted data as bytes. Notes ----- This method does not perform any type checking. The input data must be bytes-like. """ if isinstance(data, str): return bytes(data, encoding="utf-8") return bytes(data) def cast_scalar(self, data: object) -> bytes: """ Attempt to cast a given object to a bytes scalar. This method first checks if the provided data is a valid scalar that can be converted to a bytes scalar. If the check succeeds, the unchecked casting operation is performed. If the data is not valid, a TypeError is raised. Parameters ---------- data : object The data to be cast to a bytes scalar. Returns ------- bytes The data cast as a bytes scalar. Raises ------ TypeError If the data cannot be converted to a bytes scalar. """ if self._check_scalar(data): return self._cast_scalar_unchecked(data) msg = ( f"Cannot convert object {data!r} with type {type(data)} to a scalar compatible with the " f"data type {self}." ) raise TypeError(msg) zarr-developers-zarr-python-a2f2415/src/zarr/core/dtype/npy/common.py000066400000000000000000000327541527014137600257720ustar00rootroot00000000000000from __future__ import annotations import base64 import struct import sys from collections.abc import Sequence from typing import ( TYPE_CHECKING, Any, Final, Literal, NewType, SupportsComplex, SupportsFloat, SupportsIndex, SupportsInt, TypeGuard, ) import numpy as np from zarr.core.dtype.common import ( ENDIANNESS_STR, SPECIAL_FLOAT_STRINGS, EndiannessStr, JSONFloatV2, JSONFloatV3, ) if TYPE_CHECKING: from zarr.core.common import JSON, ZarrFormat IntLike = SupportsInt | SupportsIndex | bytes | str FloatLike = SupportsIndex | SupportsFloat | bytes | str ComplexLike = SupportsFloat | SupportsIndex | SupportsComplex | bytes | str | None DateTimeUnit = Literal[ "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", "generic" ] DATETIME_UNIT: Final = ( "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", "generic", ) IntishFloat = NewType("IntishFloat", float) """A type for floats that represent integers, like 1.0 (but not 1.1).""" IntishStr = NewType("IntishStr", str) """A type for strings that represent integers, like "0" or "42".""" FloatishStr = NewType("FloatishStr", str) """A type for strings that represent floats, like "3.14" or "-2.5".""" NumpyEndiannessStr = Literal[">", "<", "="] NUMPY_ENDIANNESS_STR: Final = ">", "<", "=" def endianness_from_numpy_str(endianness: NumpyEndiannessStr) -> EndiannessStr: """ Convert a numpy endianness string literal to a human-readable literal value. Parameters ---------- endianness : Literal[">", "<", "="] The numpy string representation of the endianness. Returns ------- Endianness The human-readable representation of the endianness. Raises ------ ValueError If the endianness is invalid. """ match endianness: case "=": # Use the local system endianness return sys.byteorder case "<": return "little" case ">": return "big" raise ValueError(f"Invalid endianness: {endianness!r}. Expected one of {NUMPY_ENDIANNESS_STR}") def endianness_to_numpy_str(endianness: EndiannessStr) -> NumpyEndiannessStr: """ Convert an endianness literal to its numpy string representation. Parameters ---------- endianness : Endianness The endianness to convert. Returns ------- Literal[">", "<"] The numpy string representation of the endianness. Raises ------ ValueError If the endianness is invalid. """ match endianness: case "little": return "<" case "big": return ">" raise ValueError( f"Invalid endianness: {endianness!r}. Expected one of {ENDIANNESS_STR} or None" ) def get_endianness_from_numpy_dtype(dtype: np.dtype[np.generic]) -> EndiannessStr: """ Gets the endianness from a numpy dtype that has an endianness. This function will raise a ValueError if the numpy data type does not have a concrete endianness. """ endianness = dtype.byteorder if dtype.byteorder in NUMPY_ENDIANNESS_STR: return endianness_from_numpy_str(endianness) # type: ignore [arg-type] raise ValueError(f"The dtype {dtype} has an unsupported endianness: {endianness}") def float_from_json_v2(data: JSONFloatV2) -> float: """ Convert a JSON float to a float (Zarr v2). Parameters ---------- data : JSONFloat The JSON float to convert. Returns ------- float The float value. """ match data: case "NaN": return float("nan") case "Infinity": return float("inf") case "-Infinity": return float("-inf") case _: return float(data) def float_from_json_v3(data: JSONFloatV3) -> float: """ Convert a JSON float to a float (v3). Parameters ---------- data : JSONFloat The JSON float to convert. Returns ------- float The float value. Notes ----- Zarr V3 allows floats to be stored as hex strings. To quote the spec: "...for float32, "NaN" is equivalent to "0x7fc00000". This representation is the only way to specify a NaN value other than the specific NaN value denoted by "NaN"." """ if isinstance(data, str): if data in SPECIAL_FLOAT_STRINGS: return float_from_json_v2(data) # type: ignore[arg-type] if not data.startswith("0x"): msg = ( f"Invalid float value: {data!r}. Expected a string starting with the hex prefix" " '0x', or one of 'NaN', 'Infinity', or '-Infinity'." ) raise ValueError(msg) if len(data[2:]) == 4: dtype_code = ">e" elif len(data[2:]) == 8: dtype_code = ">f" elif len(data[2:]) == 16: dtype_code = ">d" else: msg = ( f"Invalid hexadecimal float value: {data!r}. " "Expected the '0x' prefix to be followed by 4, 8, or 16 numeral characters" ) raise ValueError(msg) return float(struct.unpack(dtype_code, bytes.fromhex(data[2:]))[0]) return float_from_json_v2(data) def bytes_from_json(data: str, *, zarr_format: ZarrFormat) -> bytes: """ Convert a JSON string to bytes Parameters ---------- data : str The JSON string to convert. zarr_format : ZarrFormat The zarr format version. Returns ------- bytes The bytes. """ if zarr_format == 2: return base64.b64decode(data.encode("ascii")) # TODO: differentiate these as needed. This is a spec question. if zarr_format == 3: return base64.b64decode(data.encode("ascii")) raise ValueError(f"Invalid zarr format: {zarr_format}. Expected 2 or 3.") # pragma: no cover def bytes_to_json(data: bytes, zarr_format: ZarrFormat) -> str: """ Convert bytes to JSON. Parameters ---------- data : bytes The bytes to store. zarr_format : ZarrFormat The zarr format version. Returns ------- str The bytes encoded as ascii using the base64 alphabet. """ # TODO: decide if we are going to make this implementation zarr format-specific return base64.b64encode(data).decode("ascii") def float_to_json_v2(data: float | np.floating[Any]) -> JSONFloatV2: """ Convert a float to JSON (v2). Parameters ---------- data : float or np.floating The float value to convert. Returns ------- JSONFloat The JSON representation of the float. """ if np.isnan(data): return "NaN" elif np.isinf(data): return "Infinity" if data > 0 else "-Infinity" return float(data) def float_to_json_v3(data: float | np.floating[Any]) -> JSONFloatV3: """ Convert a float to JSON (v3). Parameters ---------- data : float or np.floating The float value to convert. Returns ------- JSONFloat The JSON representation of the float. """ # v3 can in principle handle distinct NaN values, but numpy does not represent these explicitly # so we just reuse the v2 routine here return float_to_json_v2(data) def complex_float_to_json_v3( data: complex | np.complexfloating[Any, Any], ) -> tuple[JSONFloatV3, JSONFloatV3]: """ Convert a complex number to JSON as defined by the Zarr V3 spec. Parameters ---------- data : complex or np.complexfloating The complex value to convert. Returns ------- tuple[JSONFloat, JSONFloat] The JSON representation of the complex number. """ return float_to_json_v3(data.real), float_to_json_v3(data.imag) def complex_float_to_json_v2( data: complex | np.complexfloating[Any, Any], ) -> tuple[JSONFloatV2, JSONFloatV2]: """ Convert a complex number to JSON as defined by the Zarr V2 spec. Parameters ---------- data : complex | np.complexfloating The complex value to convert. Returns ------- tuple[JSONFloat, JSONFloat] The JSON representation of the complex number. """ return float_to_json_v2(data.real), float_to_json_v2(data.imag) def complex_float_from_json_v2(data: tuple[JSONFloatV2, JSONFloatV2]) -> complex: """ Convert a JSON complex float to a complex number (v2). Parameters ---------- data : tuple[JSONFloat, JSONFloat] The JSON complex float to convert. Returns ------- np.complexfloating The complex number. """ return complex(float_from_json_v2(data[0]), float_from_json_v2(data[1])) def complex_float_from_json_v3(data: tuple[JSONFloatV3, JSONFloatV3]) -> complex: """ Convert a JSON complex float to a complex number (v3). Parameters ---------- data : tuple[JSONFloat, JSONFloat] The JSON complex float to convert. Returns ------- np.complexfloating The complex number. """ return complex(float_from_json_v3(data[0]), float_from_json_v3(data[1])) def check_json_float_v2(data: JSON) -> TypeGuard[JSONFloatV2]: """ Check if a JSON value represents a float (v2). Parameters ---------- data : JSON The JSON value to check. Returns ------- Bool True if the data is a float, False otherwise. """ return data in ("NaN", "Infinity", "-Infinity") or isinstance(data, float | int) def check_json_float_v3(data: JSON) -> TypeGuard[JSONFloatV3]: """ Check if a JSON value represents a float (v3). Parameters ---------- data : JSON The JSON value to check. Returns ------- Bool True if the data is a float, False otherwise. """ return check_json_float_v2(data) or (isinstance(data, str) and data.startswith("0x")) def check_json_complex_float_v2(data: JSON) -> TypeGuard[tuple[JSONFloatV2, JSONFloatV2]]: """ Check if a JSON value represents a complex float, as per the behavior of zarr-python 2.x Parameters ---------- data : JSON The JSON value to check. Returns ------- Bool True if the data is a complex float, False otherwise. """ return ( not isinstance(data, str) and isinstance(data, Sequence) and len(data) == 2 and check_json_float_v2(data[0]) and check_json_float_v2(data[1]) ) def check_json_complex_float_v3(data: JSON) -> TypeGuard[tuple[JSONFloatV3, JSONFloatV3]]: """ Check if a JSON value represents a complex float, as per the zarr v3 spec Parameters ---------- data : JSON The JSON value to check. Returns ------- Bool True if the data is a complex float, False otherwise. """ return ( not isinstance(data, str) and isinstance(data, Sequence) and len(data) == 2 and check_json_float_v3(data[0]) and check_json_float_v3(data[1]) ) def check_json_int(data: JSON) -> TypeGuard[int]: """ Check if a JSON value is an integer. Parameters ---------- data : JSON The JSON value to check. Returns ------- Bool True if the data is an integer, False otherwise. """ return bool(isinstance(data, int)) def check_json_intish_float(data: JSON) -> TypeGuard[IntishFloat]: """ Check if a JSON value is an "intish float", i.e. a float that represents an integer, like 0.0. Parameters ---------- data : JSON The JSON value to check. Returns ------- Bool True if the data is an intish float, False otherwise. """ return isinstance(data, float) and data.is_integer() def check_json_intish_str(data: JSON) -> TypeGuard[IntishStr]: """ Check if a JSON value is a string that represents an integer, like "0", "42", or "-5". Parameters ---------- data : JSON The JSON value to check. Returns ------- bool True if the data is a string representing an integer, False otherwise. """ if not isinstance(data, str): return False try: int(data) except ValueError: return False else: return True def check_json_floatish_str(data: JSON) -> TypeGuard[FloatishStr]: """ Check if a JSON value is a string that represents a float, like "3.14", "-2.5", or "0.0". Note: This function is intended to be used AFTER check_json_float_v2/v3, so it only handles regular string representations that those functions don't cover. Parameters ---------- data : JSON The JSON value to check. Returns ------- bool True if the data is a string representing a regular float, False otherwise. """ if not isinstance(data, str): return False try: float(data) except ValueError: return False else: return True def check_json_str(data: JSON) -> TypeGuard[str]: """ Check if a JSON value is a string. Parameters ---------- data : JSON The JSON value to check. Returns ------- Bool True if the data is a string, False otherwise. """ return bool(isinstance(data, str)) def check_json_bool(data: JSON) -> TypeGuard[bool]: """ Check if a JSON value is a boolean. Parameters ---------- data : JSON The JSON value to check. Returns ------- Bool True if the data is a boolean, False otherwise. """ return isinstance(data, bool) zarr-developers-zarr-python-a2f2415/src/zarr/core/dtype/npy/complex.py000066400000000000000000000310751527014137600261440ustar00rootroot00000000000000from __future__ import annotations from dataclasses import dataclass from typing import ( TYPE_CHECKING, ClassVar, Literal, Self, TypeGuard, overload, ) import numpy as np from zarr.core.dtype.common import ( DTypeConfig_V2, DTypeJSON, HasEndianness, HasItemSize, check_dtype_spec_no_object_codec_v2, ) from zarr.core.dtype.npy.common import ( ComplexLike, check_json_complex_float_v2, check_json_complex_float_v3, complex_float_from_json_v2, complex_float_from_json_v3, complex_float_to_json_v2, complex_float_to_json_v3, endianness_to_numpy_str, get_endianness_from_numpy_dtype, ) from zarr.core.dtype.wrapper import TBaseDType, ZDType from zarr.errors import DataTypeValidationError if TYPE_CHECKING: from zarr.core.common import JSON, ZarrFormat @dataclass(frozen=True) class BaseComplex[ DType: np.dtypes.Complex64DType | np.dtypes.Complex128DType, Scalar: np.complex64 | np.complex128, ](ZDType[DType, Scalar], HasEndianness, HasItemSize): """ A base class for Zarr data types that wrap NumPy complex float data types. """ # This attribute holds the possible zarr v2 JSON names for the data type _zarr_v2_names: ClassVar[tuple[str, ...]] @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ Create an instance of this data type from a NumPy complex dtype. Parameters ---------- dtype : TBaseDType The native dtype to convert. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the dtype is not compatible with this data type. """ if cls._check_native_dtype(dtype): return cls(endianness=get_endianness_from_numpy_dtype(dtype)) raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) def to_native_dtype(self) -> DType: """ Convert this class to a NumPy complex dtype with the appropriate byte order. Returns ------- DType A NumPy data type object representing the complex data type with the specified byte order. """ byte_order = endianness_to_numpy_str(self.endianness) return self.dtype_cls().newbyteorder(byte_order) # type: ignore[no-any-return,call-overload] @classmethod def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]: """ Check that the input is a valid JSON representation of this data type. The input data must be a mapping that contains a "name" key that is one of the strings from cls._zarr_v2_names and an "object_codec_id" key that is None. Parameters ---------- data : DTypeJSON The JSON data to check. Returns ------- bool True if the input is a valid JSON representation, False otherwise. """ return check_dtype_spec_no_object_codec_v2(data) and data["name"] in cls._zarr_v2_names @classmethod def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[str]: """ Check that the input is a valid JSON representation of this data type in Zarr V3. This method verifies that the provided data matches the expected Zarr V3 representation, which is the string specified by the class-level attribute _zarr_v3_name. Parameters ---------- data : DTypeJSON The JSON data to check. Returns ------- TypeGuard[str] True if the input is a valid representation of this class in Zarr V3, False otherwise. """ return data == cls._zarr_v3_name @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create an instance of this class from Zarr V2-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this class. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class. """ if cls._check_json_v2(data): # Going via numpy ensures that we get the endianness correct without # annoying string parsing. name = data["name"] return cls.from_native_dtype(np.dtype(name)) msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." raise DataTypeValidationError(msg) @classmethod def _from_json_v3(cls, data: DTypeJSON) -> Self: """ Create an instance of this class from Zarr V3-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class. """ if cls._check_json_v3(data): return cls() msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected {cls._zarr_v3_name}." raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... @overload def to_json(self, zarr_format: Literal[3]) -> str: ... def to_json(self, zarr_format: ZarrFormat) -> DTypeConfig_V2[str, None] | str: """ Serialize this object to a JSON-serializable representation. Parameters ---------- zarr_format : ZarrFormat The Zarr format version. Supported values are 2 and 3. Returns ------- DTypeConfig_V2[str, None] | str If ``zarr_format`` is 2, a dictionary with ``"name"`` and ``"object_codec_id"`` keys is returned. If ``zarr_format`` is 3, a string representation of the complex data type is returned. Raises ------ ValueError If `zarr_format` is not 2 or 3. """ if zarr_format == 2: return {"name": self.to_native_dtype().str, "object_codec_id": None} elif zarr_format == 3: return self._zarr_v3_name raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover def _check_scalar(self, data: object) -> TypeGuard[ComplexLike]: """ Check that the input is a scalar complex value. Parameters ---------- data : object The value to check. Returns ------- TypeGuard[ComplexLike] True if the input is a scalar complex value, False otherwise. """ return isinstance(data, ComplexLike) def _cast_scalar_unchecked(self, data: ComplexLike) -> Scalar: """ Cast the provided scalar data to the native scalar type of this class. Parameters ---------- data : ComplexLike The data to cast. Returns ------- Scalar The casted data as a numpy complex scalar. Notes ----- This method does not perform any type checking. The input data must be a scalar complex value. """ return self.to_native_dtype().type(data) # type: ignore[return-value] def cast_scalar(self, data: object) -> Scalar: """ Attempt to cast a given object to a numpy complex scalar. Parameters ---------- data : object The data to be cast to a numpy complex scalar. Returns ------- Scalar The data cast as a numpy complex scalar. Raises ------ TypeError If the data cannot be converted to a numpy complex scalar. """ if self._check_scalar(data): return self._cast_scalar_unchecked(data) msg = ( f"Cannot convert object {data!r} with type {type(data)} to a scalar compatible with the " f"data type {self}." ) raise TypeError(msg) def default_scalar(self) -> Scalar: """ Get the default value, which is 0 cast to this dtype Returns ------- Int scalar The default value. """ return self._cast_scalar_unchecked(0) def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> Scalar: """ Read a JSON-serializable value as a numpy float. Parameters ---------- data : JSON The JSON-serializable value. zarr_format : ZarrFormat The zarr format version. Returns ------- Scalar The numpy float. """ if zarr_format == 2: if check_json_complex_float_v2(data): return self._cast_scalar_unchecked(complex_float_from_json_v2(data)) raise TypeError( f"Invalid type: {data}. Expected a float or a special string encoding of a float." ) elif zarr_format == 3: if check_json_complex_float_v3(data): return self._cast_scalar_unchecked(complex_float_from_json_v3(data)) raise TypeError( f"Invalid type: {data}. Expected a float or a special string encoding of a float." ) raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> JSON: """ Convert an object to a JSON-serializable float. Parameters ---------- data : _BaseScalar The value to convert. zarr_format : ZarrFormat The zarr format version. Returns ------- JSON The JSON-serializable form of the complex number, which is a list of two floats, each of which is encoding according to a zarr-format-specific encoding. """ if zarr_format == 2: return complex_float_to_json_v2(self.cast_scalar(data)) elif zarr_format == 3: return complex_float_to_json_v3(self.cast_scalar(data)) raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover @dataclass(frozen=True, kw_only=True) class Complex64(BaseComplex[np.dtypes.Complex64DType, np.complex64]): """ A Zarr data type for arrays containing 64 bit complex floats. Wraps the [`np.dtypes.Complex64DType`][numpy.dtypes.Complex64DType] data type. Scalars for this data type are instances of [`np.complex64`][numpy.complex64]. Attributes ---------- dtype_cls : Type[np.dtypes.Complex64DType] The numpy dtype class for this data type. _zarr_v3_name : ClassVar[Literal["complex64"]] The name of this data type in Zarr V3. _zarr_v2_names : ClassVar[tuple[Literal[">c8"], Literal["c8"], Literal["c8", " int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return 8 @dataclass(frozen=True, kw_only=True) class Complex128(BaseComplex[np.dtypes.Complex128DType, np.complex128], HasEndianness): """ A Zarr data type for arrays containing 64 bit complex floats. Wraps the [`np.dtypes.Complex128DType`][numpy.dtypes.Complex128DType] data type. Scalars for this data type are instances of [`np.complex128`][numpy.complex128]. Attributes ---------- dtype_cls : Type[np.dtypes.Complex128DType] The numpy dtype class for this data type. _zarr_v3_name : ClassVar[Literal["complex128"]] The name of this data type in Zarr V3. _zarr_v2_names : ClassVar[tuple[Literal[">c16"], Literal["c16"], Literal["c16", " int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return 16 zarr-developers-zarr-python-a2f2415/src/zarr/core/dtype/npy/float.py000066400000000000000000000324421527014137600256010ustar00rootroot00000000000000from __future__ import annotations from dataclasses import dataclass from typing import TYPE_CHECKING, ClassVar, Literal, Self, TypeGuard, overload import numpy as np from zarr.core.dtype.common import ( DTypeConfig_V2, DTypeJSON, HasEndianness, HasItemSize, check_dtype_spec_no_object_codec_v2, ) from zarr.core.dtype.npy.common import ( FloatLike, check_json_float_v2, check_json_float_v3, check_json_floatish_str, endianness_to_numpy_str, float_from_json_v2, float_from_json_v3, float_to_json_v2, float_to_json_v3, get_endianness_from_numpy_dtype, ) from zarr.core.dtype.wrapper import TBaseDType, ZDType from zarr.errors import DataTypeValidationError if TYPE_CHECKING: from zarr.core.common import JSON, ZarrFormat @dataclass(frozen=True) class BaseFloat[ DType: np.dtypes.Float16DType | np.dtypes.Float32DType | np.dtypes.Float64DType, Scalar: np.float16 | np.float32 | np.float64, ](ZDType[DType, Scalar], HasEndianness, HasItemSize): """ A base class for Zarr data types that wrap NumPy float data types. """ # This attribute holds the possible zarr v2 JSON names for the data type _zarr_v2_names: ClassVar[tuple[str, ...]] @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ Create an instance of this ZDType from a NumPy data type. Parameters ---------- dtype : TBaseDType The NumPy data type. Returns ------- Self An instance of this data type. """ if cls._check_native_dtype(dtype): return cls(endianness=get_endianness_from_numpy_dtype(dtype)) raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) def to_native_dtype(self) -> DType: """ Convert the wrapped data type to a NumPy data type. Returns ------- DType The NumPy data type. """ byte_order = endianness_to_numpy_str(self.endianness) return self.dtype_cls().newbyteorder(byte_order) # type: ignore[no-any-return,call-overload] @classmethod def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]: """ Check that the input is a valid JSON representation of this data type. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- TypeGuard[DTypeConfig_V2[str, None]] True if the input is a valid JSON representation of this data type, False otherwise. """ return check_dtype_spec_no_object_codec_v2(data) and data["name"] in cls._zarr_v2_names @classmethod def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[str]: """ Check that the input is a valid JSON representation of this class in Zarr V3. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- TypeGuard[str] True if the input is a valid JSON representation of this class, False otherwise. """ return data == cls._zarr_v3_name @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create an instance of this ZDType from Zarr v2-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. """ if cls._check_json_v2(data): # Going via NumPy ensures that we get the endianness correct without # annoying string parsing. name = data["name"] return cls.from_native_dtype(np.dtype(name)) msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." raise DataTypeValidationError(msg) @classmethod def _from_json_v3(cls, data: DTypeJSON) -> Self: """ Create an instance of this ZDType from Zarr v3-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. """ if cls._check_json_v3(data): return cls() msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected {cls._zarr_v3_name}." raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... @overload def to_json(self, zarr_format: Literal[3]) -> str: ... def to_json(self, zarr_format: ZarrFormat) -> DTypeConfig_V2[str, None] | str: """ Convert the wrapped data type to a JSON-serializable form. Parameters ---------- zarr_format : ZarrFormat The zarr format version. Returns ------- DTypeConfig_V2[str, None] or str The JSON-serializable representation of the wrapped data type. Raises ------ ValueError If zarr_format is not 2 or 3. """ if zarr_format == 2: return {"name": self.to_native_dtype().str, "object_codec_id": None} elif zarr_format == 3: return self._zarr_v3_name raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover def _check_scalar(self, data: object) -> TypeGuard[FloatLike]: """ Check that the input is a valid scalar value. Parameters ---------- data : object The input to check. Returns ------- TypeGuard[FloatLike] True if the input is a valid scalar value, False otherwise. """ if isinstance(data, str): # Only accept strings that are valid float representations (e.g. "NaN", "inf"). # Plain strings that cannot be converted should return False so that cast_scalar # raises TypeError rather than a confusing ValueError. try: self.to_native_dtype().type(data) except (ValueError, OverflowError): return False else: return True return isinstance(data, FloatLike) def _cast_scalar_unchecked(self, data: FloatLike) -> Scalar: """ Cast a scalar value to a NumPy float scalar. Parameters ---------- data : FloatLike The scalar value to cast. Returns ------- Scalar The NumPy float scalar. """ return self.to_native_dtype().type(data) # type: ignore[return-value] def cast_scalar(self, data: object) -> Scalar: """ Cast a scalar value to a NumPy float scalar. Parameters ---------- data : object The scalar value to cast. Returns ------- Scalar The NumPy float scalar. """ if self._check_scalar(data): return self._cast_scalar_unchecked(data) msg = ( f"Cannot convert object {data!r} with type {type(data)} to a scalar compatible with the " f"data type {self}." ) raise TypeError(msg) def default_scalar(self) -> Scalar: """ Get the default value, which is 0 cast to this zdtype. Returns ------- Scalar The default value. """ return self._cast_scalar_unchecked(0) def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> Scalar: """ Read a JSON-serializable value as a NumPy float scalar. Parameters ---------- data : JSON The JSON-serializable value. zarr_format : ZarrFormat The zarr format version. Returns ------- Scalar The NumPy float scalar. """ if zarr_format == 2: if check_json_float_v2(data): return self._cast_scalar_unchecked(float_from_json_v2(data)) elif check_json_floatish_str(data): return self._cast_scalar_unchecked(float(data)) else: raise TypeError( f"Invalid type: {data}. Expected a float or a special string encoding of a float." ) elif zarr_format == 3: if check_json_float_v3(data): return self._cast_scalar_unchecked(float_from_json_v3(data)) elif check_json_floatish_str(data): return self._cast_scalar_unchecked(float(data)) else: raise TypeError( f"Invalid type: {data}. Expected a float or a special string encoding of a float." ) else: raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> float | str: """ Convert an object to a JSON-serializable float. Parameters ---------- data : _BaseScalar The value to convert. zarr_format : ZarrFormat The zarr format version. Returns ------- JSON The JSON-serializable form of the float, which is potentially a number or a string. See the zarr specifications for details on the JSON encoding for floats. """ if zarr_format == 2: return float_to_json_v2(self.cast_scalar(data)) elif zarr_format == 3: return float_to_json_v3(self.cast_scalar(data)) else: raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover @dataclass(frozen=True, kw_only=True) class Float16(BaseFloat[np.dtypes.Float16DType, np.float16]): """ A Zarr data type for arrays containing 16-bit floating point numbers. Wraps the [`np.dtypes.Float16DType`][numpy.dtypes.Float16DType] data type. Scalars for this data type are instances of [`np.float16`][numpy.float16]. Attributes ---------- dtype_cls : Type[np.dtypes.Float16DType] The NumPy dtype class for this data type. References ---------- This class implements the float16 data type defined in Zarr V2 and V3. See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. """ dtype_cls = np.dtypes.Float16DType _zarr_v3_name = "float16" _zarr_v2_names: ClassVar[tuple[Literal[">f2"], Literal["f2", " int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return 2 @dataclass(frozen=True, kw_only=True) class Float32(BaseFloat[np.dtypes.Float32DType, np.float32]): """ A Zarr data type for arrays containing 32-bit floating point numbers. Wraps the [`np.dtypes.Float32DType`][numpy.dtypes.Float32DType] data type. Scalars for this data type are instances of [`np.float32`][numpy.float32]. Attributes ---------- dtype_cls : Type[np.dtypes.Float32DType] The NumPy dtype class for this data type. References ---------- This class implements the float32 data type defined in Zarr V2 and V3. See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. """ dtype_cls = np.dtypes.Float32DType _zarr_v3_name = "float32" _zarr_v2_names: ClassVar[tuple[Literal[">f4"], Literal["f4", " int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return 4 @dataclass(frozen=True, kw_only=True) class Float64(BaseFloat[np.dtypes.Float64DType, np.float64]): """ A Zarr data type for arrays containing 64-bit floating point numbers. Wraps the [`np.dtypes.Float64DType`][numpy.dtypes.Float64DType] data type. Scalars for this data type are instances of [`np.float64`][numpy.float64]. Attributes ---------- dtype_cls : Type[np.dtypes.Float64DType] The NumPy dtype class for this data type. References ---------- This class implements the float64 data type defined in Zarr V2 and V3. See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. """ dtype_cls = np.dtypes.Float64DType _zarr_v3_name = "float64" _zarr_v2_names: ClassVar[tuple[Literal[">f8"], Literal["f8", " int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return 8 zarr-developers-zarr-python-a2f2415/src/zarr/core/dtype/npy/int.py000066400000000000000000001370671527014137600252770ustar00rootroot00000000000000from __future__ import annotations from dataclasses import dataclass from typing import ( TYPE_CHECKING, ClassVar, Literal, Self, SupportsIndex, SupportsInt, TypeGuard, overload, ) import numpy as np from zarr.core.dtype.common import ( DTypeConfig_V2, DTypeJSON, HasEndianness, HasItemSize, check_dtype_spec_no_object_codec_v2, ) from zarr.core.dtype.npy.common import ( check_json_int, check_json_intish_float, check_json_intish_str, endianness_to_numpy_str, get_endianness_from_numpy_dtype, ) from zarr.core.dtype.wrapper import TBaseDType, ZDType from zarr.errors import DataTypeValidationError if TYPE_CHECKING: from zarr.core.common import JSON, ZarrFormat _NumpyIntDType = ( np.dtypes.Int8DType | np.dtypes.Int16DType | np.dtypes.Int32DType | np.dtypes.Int64DType | np.dtypes.UInt8DType | np.dtypes.UInt16DType | np.dtypes.UInt32DType | np.dtypes.UInt64DType ) _NumpyIntScalar = ( np.int8 | np.int16 | np.int32 | np.int64 | np.uint8 | np.uint16 | np.uint32 | np.uint64 ) IntLike = SupportsInt | SupportsIndex | bytes | str @dataclass(frozen=True) class BaseInt[ DType: _NumpyIntDType, Scalar: np.int8 | np.int16 | np.int32 | np.int64 | np.uint8 | np.uint16 | np.uint32 | np.uint64, ](ZDType[DType, Scalar], HasItemSize): """ A base class for integer data types in Zarr. This class provides methods for serialization and deserialization of integer types in both Zarr v2 and v3 formats, as well as methods for checking and casting scalars. """ _zarr_v2_names: ClassVar[tuple[str, ...]] @classmethod def _check_json_v2(cls, data: object) -> TypeGuard[DTypeConfig_V2[str, None]]: """ Check that the input is a valid JSON representation of this integer data type in Zarr V2. This method verifies that the provided data matches the expected Zarr V2 representation for this data type. The input data must be a mapping that contains a "name" key that is one of the strings from cls._zarr_v2_names and an "object_codec_id" key that is None. Parameters ---------- data : object The JSON data to check. Returns ------- TypeGuard[DTypeConfig_V2[str, None]] True if the input is a valid representation of this class in Zarr V2, False otherwise. """ return check_dtype_spec_no_object_codec_v2(data) and data["name"] in cls._zarr_v2_names @classmethod def _check_json_v3(cls, data: object) -> TypeGuard[str]: """ Check that the input is a valid JSON representation of this class in Zarr V3. Parameters ---------- data : object The JSON data to check. Returns ------- TypeGuard[str] True if the input is a valid representation of this class in Zarr v3, False otherwise. """ return data == cls._zarr_v3_name def _check_scalar(self, data: object) -> TypeGuard[IntLike]: """ Check if the input object is of an IntLike type. This method verifies whether the provided data can be considered as an integer-like value, which includes objects supporting integer conversion. Parameters ---------- data : object The data to check. Returns ------- TypeGuard[IntLike] True if the data is IntLike, False otherwise. """ return isinstance(data, IntLike) def _cast_scalar_unchecked(self, data: IntLike) -> Scalar: """ Casts a given scalar value to the native integer scalar type without type checking. Parameters ---------- data : IntLike The scalar value to cast. Returns ------- Scalar The casted integer scalar of the native dtype. """ return self.to_native_dtype().type(data) # type: ignore[return-value] def cast_scalar(self, data: object) -> Scalar: """ Attempt to cast a given object to a NumPy integer scalar. Parameters ---------- data : object The data to be cast to a NumPy integer scalar. Returns ------- Scalar The data cast as a NumPy integer scalar. Raises ------ TypeError If the data cannot be converted to a NumPy integer scalar. """ if self._check_scalar(data): return self._cast_scalar_unchecked(data) msg = ( f"Cannot convert object {data!r} with type {type(data)} to a scalar compatible with the " f"data type {self}." ) raise TypeError(msg) def default_scalar(self) -> Scalar: """ Get the default value, which is 0 cast to this dtype. Returns ------- Scalar The default value. """ return self._cast_scalar_unchecked(0) def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> Scalar: """ Read a JSON-serializable value as a NumPy int scalar. Parameters ---------- data : JSON The JSON-serializable value. zarr_format : ZarrFormat The Zarr format version. Returns ------- Scalar The NumPy int scalar. Raises ------ TypeError If the input is not a valid integer type. """ if check_json_int(data): return self._cast_scalar_unchecked(data) if check_json_intish_float(data): return self._cast_scalar_unchecked(int(data)) if check_json_intish_str(data): return self._cast_scalar_unchecked(int(data)) raise TypeError(f"Invalid type: {data}. Expected an integer.") def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> int: """ Convert an object to a JSON serializable scalar. For the integer data types, the JSON form is a plain integer. Parameters ---------- data : object The value to convert. zarr_format : ZarrFormat The Zarr format version. Returns ------- int The JSON-serializable form of the scalar. """ return int(self.cast_scalar(data)) @dataclass(frozen=True, kw_only=True) class Int8(BaseInt[np.dtypes.Int8DType, np.int8]): """ A Zarr data type for arrays containing 8-bit signed integers. Wraps the [`np.dtypes.Int8DType`][numpy.dtypes.Int8DType] data type. Scalars for this data type are instances of [`np.int8`][numpy.int8]. Attributes ---------- dtype_cls : np.dtypes.Int8DType The class of the underlying NumPy dtype. References ---------- This class implements the 8-bit signed integer data type defined in Zarr V2 and V3. See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. """ dtype_cls = np.dtypes.Int8DType _zarr_v3_name: ClassVar[Literal["int8"]] = "int8" _zarr_v2_names: ClassVar[tuple[Literal["|i1"]]] = ("|i1",) @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ Create an Int8 from an np.dtype('int8') instance. Parameters ---------- dtype : TBaseDType The np.dtype('int8') instance. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input data type is not a valid representation of this class Int8. """ if cls._check_native_dtype(dtype): return cls() raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) def to_native_dtype(self: Self) -> np.dtypes.Int8DType: """ Convert the Int8 instance to an np.dtype('int8') instance. Returns ------- np.dtypes.Int8DType The np.dtype('int8') instance. """ return self.dtype_cls() @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create an Int8 from Zarr V2-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class Int8. """ if cls._check_json_v2(data): return cls() msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v2_names[0]!r}" raise DataTypeValidationError(msg) @classmethod def _from_json_v3(cls, data: DTypeJSON) -> Self: """ Create an Int8 from Zarr V3-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class Int8. """ if cls._check_json_v3(data): return cls() msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal["|i1"], None]: ... @overload def to_json(self, zarr_format: Literal[3]) -> Literal["int8"]: ... def to_json( self, zarr_format: ZarrFormat ) -> DTypeConfig_V2[Literal["|i1"], None] | Literal["int8"]: """ Convert the data type to a JSON-serializable form. Parameters ---------- zarr_format : ZarrFormat The Zarr format version. Returns ------- ``DTypeConfig_V2[Literal["|i1"], None] | Literal["int8"]`` The JSON-serializable representation of the data type. Raises ------ ValueError If the zarr_format is not 2 or 3. """ if zarr_format == 2: return {"name": self._zarr_v2_names[0], "object_codec_id": None} elif zarr_format == 3: return self._zarr_v3_name raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover @property def item_size(self) -> int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return 1 @dataclass(frozen=True, kw_only=True) class UInt8(BaseInt[np.dtypes.UInt8DType, np.uint8]): """ A Zarr data type for arrays containing 8-bit unsigned integers. Wraps the [`np.dtypes.UInt8DType`][numpy.dtypes.UInt8DType] data type. Scalars for this data type are instances of [`np.uint8`][numpy.uint8]. Attributes ---------- dtype_cls : np.dtypes.UInt8DType The class of the underlying NumPy dtype. References ---------- This class implements the 8-bit unsigned integer data type defined in Zarr V2 and V3. See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. """ dtype_cls = np.dtypes.UInt8DType _zarr_v3_name: ClassVar[Literal["uint8"]] = "uint8" _zarr_v2_names: ClassVar[tuple[Literal["|u1"]]] = ("|u1",) @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ Create a UInt8 from an np.dtype('uint8') instance. """ if cls._check_native_dtype(dtype): return cls() raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) def to_native_dtype(self: Self) -> np.dtypes.UInt8DType: """ Create a NumPy unsigned 8-bit integer dtype instance from this UInt8 ZDType. Returns ------- np.dtypes.UInt8DType The NumPy unsigned 8-bit integer dtype. """ return self.dtype_cls() @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create an instance of this data type from Zarr V2-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class. """ if cls._check_json_v2(data): return cls() msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v2_names[0]!r}" raise DataTypeValidationError(msg) @classmethod def _from_json_v3(cls, data: DTypeJSON) -> Self: """ Create an instance of this data type from Zarr V3-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class. """ if cls._check_json_v3(data): return cls() msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal["|u1"], None]: ... @overload def to_json(self, zarr_format: Literal[3]) -> Literal["uint8"]: ... def to_json( self, zarr_format: ZarrFormat ) -> DTypeConfig_V2[Literal["|u1"], None] | Literal["uint8"]: """ Convert the data type to a JSON-serializable form. Parameters ---------- zarr_format : ZarrFormat The Zarr format version. Supported values are 2 and 3. Returns ------- ``DTypeConfig_V2[Literal["|u1"], None] | Literal["uint8"]`` The JSON-serializable representation of the data type. Raises ------ ValueError If `zarr_format` is not 2 or 3. """ if zarr_format == 2: # For Zarr format version 2, return a dictionary with the name and object codec ID. return {"name": self._zarr_v2_names[0], "object_codec_id": None} elif zarr_format == 3: # For Zarr format version 3, return the v3 name as a string. return self._zarr_v3_name # Raise an error if the zarr_format is neither 2 nor 3. raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover @property def item_size(self) -> int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return 1 @dataclass(frozen=True, kw_only=True) class Int16(BaseInt[np.dtypes.Int16DType, np.int16], HasEndianness): """ A Zarr data type for arrays containing 16-bit signed integers. Wraps the [`np.dtypes.Int16DType`][numpy.dtypes.Int16DType] data type. Scalars for this data type are instances of [`np.int16`][numpy.int16]. Attributes ---------- dtype_cls : np.dtypes.Int16DType The class of the underlying NumPy dtype. References ---------- This class implements the 16-bit signed integer data type defined in Zarr V2 and V3. See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. """ dtype_cls = np.dtypes.Int16DType _zarr_v3_name: ClassVar[Literal["int16"]] = "int16" _zarr_v2_names: ClassVar[tuple[Literal[">i2"], Literal["i2", " Self: """ Create an instance of this data type from an np.dtype('int16') instance. Parameters ---------- dtype : np.dtype The instance of np.dtype('int16') to create from. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input data type is not an instance of np.dtype('int16'). """ if cls._check_native_dtype(dtype): return cls(endianness=get_endianness_from_numpy_dtype(dtype)) raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) def to_native_dtype(self) -> np.dtypes.Int16DType: """ Convert the data type to an np.dtype('int16') instance. Returns ------- np.dtype The np.dtype('int16') instance. """ byte_order = endianness_to_numpy_str(self.endianness) # numpy 2.x stub: newbyteorder widens to base dtype, runtime preserves the concrete subclass return self.dtype_cls().newbyteorder(byte_order) # type: ignore[return-value] @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create an instance of this data type from Zarr V2-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class. """ if cls._check_json_v2(data): # Going via NumPy ensures that we get the endianness correct without # annoying string parsing. name = data["name"] return cls.from_native_dtype(np.dtype(name)) msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names!r}." raise DataTypeValidationError(msg) @classmethod def _from_json_v3(cls, data: DTypeJSON) -> Self: """ Create an instance of this data type from Zarr V3-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class. """ if cls._check_json_v3(data): return cls() msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">i2", " Literal["int16"]: ... def to_json( self, zarr_format: ZarrFormat ) -> DTypeConfig_V2[Literal[">i2", "i2", " int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return 2 @dataclass(frozen=True, kw_only=True) class UInt16(BaseInt[np.dtypes.UInt16DType, np.uint16], HasEndianness): """ A Zarr data type for arrays containing 16-bit unsigned integers. Wraps the [`np.dtypes.UInt16DType`][numpy.dtypes.UInt16DType] data type. Scalars for this data type are instances of [`np.uint16`][numpy.uint16]. Attributes ---------- dtype_cls : np.dtypes.UInt16DType The class of the underlying NumPy dtype. References ---------- This class implements the unsigned 16-bit unsigned integer data type defined in Zarr V2 and V3. See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. """ dtype_cls = np.dtypes.UInt16DType _zarr_v3_name: ClassVar[Literal["uint16"]] = "uint16" _zarr_v2_names: ClassVar[tuple[Literal[">u2"], Literal["u2", " Self: """ Create an instance of this data type from an np.dtype('uint16') instance. Parameters ---------- dtype : np.dtype The NumPy data type. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input data type is not an instance of np.dtype('uint16'). """ if cls._check_native_dtype(dtype): return cls(endianness=get_endianness_from_numpy_dtype(dtype)) raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) def to_native_dtype(self) -> np.dtypes.UInt16DType: """ Convert the data type to an np.dtype('uint16') instance. Returns ------- np.dtype The np.dtype('uint16') instance. """ byte_order = endianness_to_numpy_str(self.endianness) # numpy 2.x stub: newbyteorder widens to base dtype, runtime preserves the concrete subclass return self.dtype_cls().newbyteorder(byte_order) # type: ignore[return-value] @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create an instance of this data type from Zarr V2-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class. """ if cls._check_json_v2(data): # Going via NumPy ensures that we get the endianness correct without # annoying string parsing. name = data["name"] return cls.from_native_dtype(np.dtype(name)) msg = f"Invalid JSON representation of UInt16. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." raise DataTypeValidationError(msg) @classmethod def _from_json_v3(cls, data: DTypeJSON) -> Self: """ Create an instance of this data type from Zarr V3-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class. """ if cls._check_json_v3(data): return cls() msg = f"Invalid JSON representation of UInt16. Got {data!r}, expected the string {cls._zarr_v3_name!r}" raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">u2", " Literal["uint16"]: ... def to_json( self, zarr_format: ZarrFormat ) -> DTypeConfig_V2[Literal[">u2", "u2", " int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return 2 @dataclass(frozen=True, kw_only=True) class Int32(BaseInt[np.dtypes.Int32DType, np.int32], HasEndianness): """ A Zarr data type for arrays containing 32-bit signed integers. Wraps the [`np.dtypes.Int32DType`][numpy.dtypes.Int32DType] data type. Scalars for this data type are instances of [`np.int32`][numpy.int32]. Attributes ---------- dtype_cls : np.dtypes.Int32DType The class of the underlying NumPy dtype. References ---------- This class implements the 32-bit signed integer data type defined in Zarr V2 and V3. See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. """ dtype_cls = np.dtypes.Int32DType _zarr_v3_name: ClassVar[Literal["int32"]] = "int32" _zarr_v2_names: ClassVar[tuple[Literal[">i4"], Literal["i4", " TypeGuard[np.dtypes.Int32DType]: """ A type guard that checks if the input is assignable to the type of ``cls.dtype_class`` This method is overridden for this particular data type because of a Windows-specific issue where np.dtype('i') creates an instance of ``np.dtypes.IntDType``, rather than an instance of ``np.dtypes.Int32DType``, even though both represent 32-bit signed integers. Parameters ---------- dtype : TDType The dtype to check. Returns ------- Bool True if the dtype matches, False otherwise. """ return super()._check_native_dtype(dtype) or dtype == np.dtypes.Int32DType() @classmethod def from_native_dtype(cls: type[Self], dtype: TBaseDType) -> Self: """ Create an Int32 from an np.dtype('int32') instance. Parameters ---------- dtype : TBaseDType The np.dtype('int32') instance. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class Int32. """ if cls._check_native_dtype(dtype): return cls(endianness=get_endianness_from_numpy_dtype(dtype)) raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) def to_native_dtype(self: Self) -> np.dtypes.Int32DType: """ Convert the Int32 instance to an np.dtype('int32') instance. Returns ------- np.dtypes.Int32DType The np.dtype('int32') instance. """ byte_order = endianness_to_numpy_str(self.endianness) # numpy 2.x stub: newbyteorder widens to base dtype, runtime preserves the concrete subclass return self.dtype_cls().newbyteorder(byte_order) # type: ignore[return-value] @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create an Int32 from Zarr V2-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class Int32. """ if cls._check_json_v2(data): # Going via NumPy ensures that we get the endianness correct without # annoying string parsing. name = data["name"] return cls.from_native_dtype(np.dtype(name)) msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names!r}." raise DataTypeValidationError(msg) @classmethod def _from_json_v3(cls, data: DTypeJSON) -> Self: """ Create an Int32 from Zarr V3-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class Int32. """ if cls._check_json_v3(data): return cls() msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">i4", " Literal["int32"]: ... def to_json( self, zarr_format: ZarrFormat ) -> DTypeConfig_V2[Literal[">i4", "i4", " int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return 4 @dataclass(frozen=True, kw_only=True) class UInt32(BaseInt[np.dtypes.UInt32DType, np.uint32], HasEndianness): """ A Zarr data type for arrays containing 32-bit unsigned integers. Wraps the [`np.dtypes.UInt32DType`][numpy.dtypes.UInt32DType] data type. Scalars for this data type are instances of [`np.uint32`][numpy.uint32]. Attributes ---------- dtype_cls : np.dtypes.UInt32DType The class of the underlying NumPy dtype. References ---------- This class implements the 32-bit unsigned integer data type defined in Zarr V2 and V3. See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. """ dtype_cls = np.dtypes.UInt32DType _zarr_v3_name: ClassVar[Literal["uint32"]] = "uint32" _zarr_v2_names: ClassVar[tuple[Literal[">u4"], Literal["u4", " TypeGuard[np.dtypes.UInt32DType]: """ A type guard that checks if the input is assignable to the type of ``cls.dtype_class`` This method is overridden for this particular data type because of a Windows-specific issue where ``np.array([1], dtype=np.uint32) & 1`` creates an instance of ``np.dtypes.UIntDType``, rather than an instance of ``np.dtypes.UInt32DType``, even though both represent 32-bit unsigned integers. (In contrast to ``np.dtype('i')``, ``np.dtype('u')`` raises an error.) Parameters ---------- dtype : TDType The dtype to check. Returns ------- Bool True if the dtype matches, False otherwise. """ return super()._check_native_dtype(dtype) or dtype == np.dtypes.UInt32DType() @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ Create a UInt32 from an np.dtype('uint32') instance. Parameters ---------- dtype : TBaseDType The NumPy data type. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input data type is not a valid representation of this class 32-bit unsigned integer. """ if cls._check_native_dtype(dtype): return cls(endianness=get_endianness_from_numpy_dtype(dtype)) raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) def to_native_dtype(self) -> np.dtypes.UInt32DType: """ Create a NumPy unsigned 32-bit integer dtype instance from this UInt32 ZDType. Returns ------- np.dtypes.UInt32DType The NumPy unsigned 32-bit integer dtype. """ byte_order = endianness_to_numpy_str(self.endianness) # numpy 2.x stub: newbyteorder widens to base dtype, runtime preserves the concrete subclass return self.dtype_cls().newbyteorder(byte_order) # type: ignore[return-value] @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create an instance of this data type from Zarr V2-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class 32-bit unsigned integer. """ if cls._check_json_v2(data): # Going via NumPy ensures that we get the endianness correct without # annoying string parsing. name = data["name"] return cls.from_native_dtype(np.dtype(name)) msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." raise DataTypeValidationError(msg) @classmethod def _from_json_v3(cls, data: DTypeJSON) -> Self: """ Create an instance of this data type from Zarr V3-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class 32-bit unsigned integer. """ if cls._check_json_v3(data): return cls() msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">u4", " Literal["uint32"]: ... def to_json( self, zarr_format: ZarrFormat ) -> DTypeConfig_V2[Literal[">u4", "u4", " int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return 4 @dataclass(frozen=True, kw_only=True) class Int64(BaseInt[np.dtypes.Int64DType, np.int64], HasEndianness): """ A Zarr data type for arrays containing 64-bit signed integers. Wraps the [`np.dtypes.Int64DType`][numpy.dtypes.Int64DType] data type. Scalars for this data type are instances of [`np.int64`][numpy.int64]. Attributes ---------- dtype_cls : np.dtypes.Int64DType The class of the underlying NumPy dtype. References ---------- This class implements the 64-bit signed integer data type defined in Zarr V2 and V3. See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. """ dtype_cls = np.dtypes.Int64DType _zarr_v3_name: ClassVar[Literal["int64"]] = "int64" _zarr_v2_names: ClassVar[tuple[Literal[">i8"], Literal["i8", " Self: """ Create an Int64 from an np.dtype('int64') instance. Parameters ---------- dtype : TBaseDType The NumPy data type. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input data type is not a valid representation of this class 64-bit signed integer. """ if cls._check_native_dtype(dtype): return cls(endianness=get_endianness_from_numpy_dtype(dtype)) raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) def to_native_dtype(self) -> np.dtypes.Int64DType: """ Create a NumPy signed 64-bit integer dtype instance from this Int64 ZDType. Returns ------- np.dtypes.Int64DType The NumPy signed 64-bit integer dtype. """ byte_order = endianness_to_numpy_str(self.endianness) # numpy 2.x stub: newbyteorder widens to base dtype, runtime preserves the concrete subclass return self.dtype_cls().newbyteorder(byte_order) # type: ignore[return-value] @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create an instance of this data type from Zarr V2-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class 64-bit signed integer. """ if cls._check_json_v2(data): # Going via NumPy ensures that we get the endianness correct without # annoying string parsing. name = data["name"] return cls.from_native_dtype(np.dtype(name)) msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." raise DataTypeValidationError(msg) @classmethod def _from_json_v3(cls, data: DTypeJSON) -> Self: """ Create an instance of this data type from Zarr V3-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class 64-bit signed integer. """ if cls._check_json_v3(data): return cls() msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">i8", " Literal["int64"]: ... def to_json( self, zarr_format: ZarrFormat ) -> DTypeConfig_V2[Literal[">i8", "i8", " int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return 8 @dataclass(frozen=True, kw_only=True) class UInt64(BaseInt[np.dtypes.UInt64DType, np.uint64], HasEndianness): """ A Zarr data type for arrays containing 64-bit unsigned integers. Wraps the [`np.dtypes.UInt64DType`][numpy.dtypes.UInt64DType] data type. Scalars for this data type are instances of [`np.uint64`][numpy.uint64]. Attributes ---------- dtype_cls: np.dtypes.UInt64DType The class of the underlying NumPy dtype. References ---------- This class implements the unsigned 64-bit integer data type defined in Zarr V2 and V3. See the [Zarr V2](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding) and [Zarr V3](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/data-types/index.rst) specification documents for details. """ dtype_cls = np.dtypes.UInt64DType _zarr_v3_name: ClassVar[Literal["uint64"]] = "uint64" _zarr_v2_names: ClassVar[tuple[Literal[">u8"], Literal["u8", " np.dtypes.UInt64DType: """ Convert the data type to a native NumPy dtype. Returns ------- np.dtypes.UInt64DType The native NumPy dtype.eeeeeeeeeeeeeeeee """ byte_order = endianness_to_numpy_str(self.endianness) # numpy 2.x stub: newbyteorder widens to base dtype, runtime preserves the concrete subclass return self.dtype_cls().newbyteorder(byte_order) # type: ignore[return-value] @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create an instance of this data type from Zarr V2-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class unsigned 64-bit integer. """ if cls._check_json_v2(data): # Going via NumPy ensures that we get the endianness correct without # annoying string parsing. name = data["name"] return cls.from_native_dtype(np.dtype(name)) msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." raise DataTypeValidationError(msg) @classmethod def _from_json_v3(cls, data: DTypeJSON) -> Self: """ Create an instance of this data type from Zarr V3-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class unsigned 64-bit integer. """ if cls._check_json_v3(data): return cls() msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">u8", " Literal["uint64"]: ... def to_json( self, zarr_format: ZarrFormat ) -> DTypeConfig_V2[Literal[">u8", "u8", " Self: """ Create an instance of this data type from a native NumPy dtype. Parameters ---------- dtype : TBaseDType The native NumPy dtype. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input dtype is not a valid representation of this class unsigned 64-bit integer. """ if cls._check_native_dtype(dtype): return cls(endianness=get_endianness_from_numpy_dtype(dtype)) raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) @property def item_size(self) -> int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return 8 zarr-developers-zarr-python-a2f2415/src/zarr/core/dtype/npy/string.py000066400000000000000000000532631527014137600260060ustar00rootroot00000000000000from __future__ import annotations import re from dataclasses import dataclass from typing import ( TYPE_CHECKING, ClassVar, Literal, Protocol, Self, TypedDict, TypeGuard, overload, runtime_checkable, ) import numpy as np from zarr.core.common import NamedConfig from zarr.core.dtype.common import ( DTypeConfig_V2, DTypeJSON, HasEndianness, HasItemSize, HasLength, HasObjectCodec, check_dtype_spec_no_object_codec_v2, check_dtype_spec_v2, ) from zarr.core.dtype.npy.common import ( check_json_str, endianness_to_numpy_str, get_endianness_from_numpy_dtype, ) from zarr.core.dtype.wrapper import ZDType from zarr.errors import DataTypeValidationError if TYPE_CHECKING: from zarr.core.common import JSON, ZarrFormat from zarr.core.dtype.wrapper import TBaseDType @runtime_checkable class SupportsStr(Protocol): def __str__(self) -> str: ... class LengthBytesConfig(TypedDict): """ Configuration for a fixed-length string data type in Zarr V3. Attributes ---------- length_bytes : int The length in bytes of the data associated with this configuration. """ length_bytes: int class FixedLengthUTF32JSON_V2(DTypeConfig_V2[str, None]): """ A wrapper around the JSON representation of the ``FixedLengthUTF32`` data type in Zarr V2. The ``name`` field of this class contains the value that would appear under the ``dtype`` field in Zarr V2 array metadata. References ---------- The structure of the ``name`` field is defined in the Zarr V2 [specification document](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding). Examples -------- ```python { "name": " None: """ We don't allow instances of this class with length less than 1 because there is no way such a data type can contain actual data. """ if self.length < 1: raise ValueError(f"length must be >= 1, got {self.length}.") @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ Create a FixedLengthUTF32 from a NumPy data type. Parameters ---------- dtype : TBaseDType The NumPy data type. Returns ------- Self An instance of this data type. """ if cls._check_native_dtype(dtype): endianness = get_endianness_from_numpy_dtype(dtype) return cls( length=dtype.itemsize // (cls.code_point_bytes), endianness=endianness, ) raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) def to_native_dtype(self) -> np.dtypes.StrDType[int]: """ Convert the FixedLengthUTF32 instance to a NumPy data type. Returns ------- np.dtypes.StrDType[int] The NumPy data type. """ byte_order = endianness_to_numpy_str(self.endianness) # numpy 2.x stub: newbyteorder widens to base dtype, runtime preserves the concrete subclass return self.dtype_cls(self.length).newbyteorder(byte_order) # type: ignore[return-value] @classmethod def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[FixedLengthUTF32JSON_V2]: """ Check that the input is a valid JSON representation of a NumPy U dtype. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- TypeGuard[FixedLengthUTF32JSON_V2] Whether the input is a valid JSON representation of a NumPy U dtype. """ return ( check_dtype_spec_no_object_codec_v2(data) and isinstance(data["name"], str) and re.match(r"^[><]U\d+$", data["name"]) is not None ) @classmethod def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[FixedLengthUTF32JSON_V3]: """ Check that the input is a valid JSON representation of this class in Zarr V3. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- TypeGuard[FixedLengthUTF32JSONV3] Whether the input is a valid JSON representation of a NumPy U dtype. """ return ( isinstance(data, dict) and set(data.keys()) == {"name", "configuration"} and data["name"] == cls._zarr_v3_name and "configuration" in data and isinstance(data["configuration"], dict) and set(data["configuration"].keys()) == {"length_bytes"} and isinstance(data["configuration"]["length_bytes"], int) ) @overload def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... @overload def to_json(self, zarr_format: Literal[3]) -> FixedLengthUTF32JSON_V3: ... def to_json( self, zarr_format: ZarrFormat ) -> DTypeConfig_V2[str, None] | FixedLengthUTF32JSON_V3: """ Convert the FixedLengthUTF32 instance to a JSON representation. Parameters ---------- zarr_format : ZarrFormat The Zarr format to use. Returns ------- DTypeConfig_V2[str, None] | FixedLengthUTF32JSON_V3 The JSON representation of the data type. """ if zarr_format == 2: return {"name": self.to_native_dtype().str, "object_codec_id": None} elif zarr_format == 3: return { "name": self._zarr_v3_name, "configuration": {"length_bytes": self.length * self.code_point_bytes}, } raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create a FixedLengthUTF32 from a JSON representation of a NumPy U dtype. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. """ if cls._check_json_v2(data): # Construct the NumPy dtype instead of string parsing. name = data["name"] return cls.from_native_dtype(np.dtype(name)) raise DataTypeValidationError( f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string representation of a NumPy U dtype." ) @classmethod def _from_json_v3(cls, data: DTypeJSON) -> Self: """ Create a FixedLengthUTF32 from a JSON representation of a NumPy U dtype. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- Self An instance of this data type. """ if cls._check_json_v3(data): return cls(length=data["configuration"]["length_bytes"] // cls.code_point_bytes) msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected {cls._zarr_v3_name}." raise DataTypeValidationError(msg) def default_scalar(self) -> np.str_: """ Return the default scalar value for this data type. Returns ------- ``np.str_`` The default scalar value. """ return np.str_("") def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: """ Convert the scalar value to a JSON representation. Parameters ---------- data : object The scalar value. zarr_format : ZarrFormat The Zarr format to use. Returns ------- str The JSON representation of the scalar value. """ return str(data) def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.str_: """ Convert the JSON representation of a scalar value to the native scalar value. Parameters ---------- data : JSON The JSON data. zarr_format : ZarrFormat The Zarr format to use. Returns ------- ``np.str_`` The native scalar value. """ if check_json_str(data): return self.to_native_dtype().type(data) raise TypeError(f"Invalid type: {data}. Expected a string.") # pragma: no cover def _check_scalar(self, data: object) -> TypeGuard[SupportsStr]: """ Check that the input is a valid scalar value for this data type. Parameters ---------- data : object The scalar value. Returns ------- TypeGuard[SupportsStr] Whether the input is a valid scalar value for this data type. """ # this is generous for backwards compatibility return isinstance(data, SupportsStr) def cast_scalar(self, data: object) -> np.str_: """ Cast the scalar value to the native scalar value. Parameters ---------- data : object The scalar value. Returns ------- ``np.str_`` The native scalar value. """ if self._check_scalar(data): # We explicitly truncate before casting because of the following NumPy behavior: # >>> x = np.dtype('U3').type('hello world') # >>> x # np.str_('hello world') # >>> x.dtype # dtype('U11') return self.to_native_dtype().type(str(data)[: self.length]) msg = ( # pragma: no cover f"Cannot convert object {data!r} with type {type(data)} to a scalar compatible with the " f"data type {self}." ) raise TypeError(msg) # pragma: no-cover @property def item_size(self) -> int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return self.length * self.code_point_bytes def check_vlen_string_json_scalar(data: object) -> TypeGuard[int | str | float]: """ Check if the input is a valid JSON scalar for a variable-length string. This function is generous for backwards compatibility, as Zarr Python v2 would use ints for variable-length string fill values. Parameters ---------- data : object The JSON value to check. Returns ------- TypeGuard[int | str | float] True if the input is a valid scalar for a variable-length string. """ return isinstance(data, int | str | float) class VariableLengthUTF8JSON_V2(DTypeConfig_V2[Literal["|O"], Literal["vlen-utf8"]]): """ A wrapper around the JSON representation of the ``VariableLengthUTF8`` data type in Zarr V2. The ``name`` field of this class contains the value that would appear under the ``dtype`` field in Zarr V2 array metadata. The ``object_codec_id`` field is always ``"vlen-utf8"``. References ---------- The structure of the ``name`` field is defined in the Zarr V2 [specification document](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding). Examples -------- ```python { "name": "|O", "object_codec_id": "vlen-utf8" } ``` """ @dataclass(frozen=True, kw_only=True) class VariableLengthUTF8(ZDType[np.dtypes.StringDType, str], HasObjectCodec): # type: ignore[type-var] """ A Zarr data type for arrays containing variable-length UTF-8 strings. Wraps the ``np.dtypes.StringDType`` data type. Scalars for this data type are instances of ``str``. Attributes ---------- dtype_cls : Type[np.dtypes.StringDType] The NumPy dtype class for this data type. _zarr_v3_name : ClassVar[Literal["variable_length_utf8"]] = "variable_length_utf8" The name of this data type in Zarr V3. object_codec_id : ClassVar[Literal["vlen-utf8"]] = "vlen-utf8" The object codec ID for this data type. References ---------- https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/string """ dtype_cls = np.dtypes.StringDType # type: ignore[assignment] _zarr_v3_name: ClassVar[Literal["string"]] = "string" object_codec_id: ClassVar[Literal["vlen-utf8"]] = "vlen-utf8" @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ Create an instance of this data type from a compatible NumPy data type. We reject NumPy StringDType instances that have the `na_object` field set, because this is not representable by the Zarr `string` data type. Parameters ---------- dtype : TBaseDType The native data type. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input is not compatible with this data type. ValueError If the input is `numpy.dtypes.StringDType` and has `na_object` set. """ if cls._check_native_dtype(dtype): if hasattr(dtype, "na_object"): msg = ( f"Zarr data type resolution from {dtype} failed. " "Attempted to resolve a zarr data type from a `numpy.dtypes.StringDType` " "with `na_object` set, which is not supported." ) raise ValueError(msg) return cls() raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) def to_native_dtype(self) -> np.dtypes.StringDType: """ Create a NumPy string dtype from this VariableLengthUTF8 ZDType. Returns ------- np.dtypes.StringDType The NumPy string dtype. """ return self.dtype_cls() @classmethod def _check_json_v2( cls, data: DTypeJSON, ) -> TypeGuard[VariableLengthUTF8JSON_V2]: """ "Check if the input is a valid JSON representation of a variable-length UTF-8 string dtype for Zarr v2." Parameters ---------- data : DTypeJSON The JSON data to check. Returns ------- ``TypeGuard[VariableLengthUTF8JSON_V2]`` Whether the input is a valid JSON representation of a NumPy "object" data type, and that the object codec id is appropriate for variable-length UTF-8 strings. """ return ( check_dtype_spec_v2(data) and data["name"] == "|O" and data["object_codec_id"] == cls.object_codec_id ) @classmethod def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[Literal["variable_length_utf8"]]: """ Check that the input is a valid JSON representation of this class in Zarr V3. Parameters ---------- data : DTypeJSON The JSON data to check. Returns ------- TypeGuard[Literal["variable_length_utf8"]] Whether the input is a valid JSON representation of a variable length UTF-8 string data type. """ return data == cls._zarr_v3_name @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create an instance of this class from a JSON representation of a NumPy "object" dtype. Parameters ---------- data : DTypeJSON The JSON data to create an instance from. Returns ------- Self An instance of this data type. """ if cls._check_json_v2(data): return cls() msg = ( f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string '|O'" ) raise DataTypeValidationError(msg) @classmethod def _from_json_v3(cls, data: DTypeJSON) -> Self: """ Create an instance of this class from a JSON representation of a variable length UTF-8 string data type. Parameters ---------- data : DTypeJSON The JSON data to create an instance from. Returns ------- Self An instance of this data type. """ if cls._check_json_v3(data): return cls() msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected {cls._zarr_v3_name}." raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> VariableLengthUTF8JSON_V2: ... @overload def to_json(self, zarr_format: Literal[3]) -> Literal["string"]: ... def to_json(self, zarr_format: ZarrFormat) -> VariableLengthUTF8JSON_V2 | Literal["string"]: """ Convert this data type to a JSON representation. Parameters ---------- zarr_format : int The zarr format to use for the JSON representation. Returns ------- ``VariableLengthUTF8JSON_V2 | Literal["string"]`` The JSON representation of this data type. """ if zarr_format == 2: return {"name": "|O", "object_codec_id": self.object_codec_id} elif zarr_format == 3: return self._zarr_v3_name raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover def default_scalar(self) -> str: """ Return the default scalar value for this data type. Returns ------- str The default scalar value. """ return "" def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: """ Convert a scalar value to a JSON representation. Parameters ---------- data : object The scalar value to convert. zarr_format : int The zarr format to use for the JSON representation. Returns ------- str The JSON representation of the scalar value. """ if self._check_scalar(data): return self._cast_scalar_unchecked(data) raise TypeError(f"Invalid type: {data}. Expected a string.") def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> str: """ Convert a JSON representation of a scalar value to the native scalar type. Parameters ---------- data : JSON The JSON representation of the scalar value. zarr_format : int The zarr format to use for the JSON representation. Returns ------- str The native scalar type of the scalar value. """ if not check_vlen_string_json_scalar(data): raise TypeError(f"Invalid type: {data}. Expected a string or number.") return str(data) def _check_scalar(self, data: object) -> TypeGuard[SupportsStr]: """ Check that the input is a valid scalar value for this data type. Parameters ---------- data : object The scalar value to check. Returns ------- TypeGuard[SupportsStr] Whether the input is a valid scalar value for this data type. """ return isinstance(data, SupportsStr) def _cast_scalar_unchecked(self, data: SupportsStr) -> str: """ Cast a scalar value to a string. Parameters ---------- data : object The scalar value to cast. Returns ------- str The string representation of the scalar value. """ return str(data) def cast_scalar(self, data: object) -> str: """ Cast an object to a string. Parameters ---------- data : object The value to cast. Returns ------- str The input cast to str. """ if self._check_scalar(data): return self._cast_scalar_unchecked(data) msg = ( # pragma: no cover f"Cannot convert object {data!r} with type {type(data)} to a scalar compatible with the " f"data type {self}." ) raise TypeError(msg) # pragma: no cover zarr-developers-zarr-python-a2f2415/src/zarr/core/dtype/npy/structured.py000066400000000000000000000743371527014137600267110ustar00rootroot00000000000000from __future__ import annotations import warnings from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, ClassVar, Literal, Self, TypeGuard, cast, overload import numpy as np from numpy.lib.recfunctions import repack_fields from zarr.core.common import NamedConfig from zarr.core.context import Context, Resolver from zarr.core.dtype.common import ( DTypeConfig_V2, DTypeJSON, HasItemSize, StructuredName_V2, check_dtype_spec_no_object_codec_v2, check_structured_dtype_name_v2, v3_unstable_dtype_warning, ) from zarr.core.dtype.npy.common import ( bytes_from_json, bytes_to_json, check_json_str, ) from zarr.core.dtype.registry import _NestedDataTypeValidationError from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType from zarr.errors import DataTypeValidationError, ZarrUserWarning if TYPE_CHECKING: from zarr.core.common import JSON, ZarrFormat StructuredScalarLike = list[object] | tuple[object, ...] | bytes | int # The root of the zarr package, used to attribute layout warnings to the caller outside zarr. _ZARR_PACKAGE_ROOT = str(Path(__file__).parents[3]) def _unsupported_field_feature(dtype: np.dtype[np.void]) -> str | None: """ Check for field features unsupported by this implementation's native dtype conversion. `Structured.fields` stores `(name, ZDType)` pairs and `to_native_dtype` packs them contiguously. This conversion does not preserve NumPy field titles or subarray shapes. Reject those features, including in nested fields, rather than silently losing them: - field titles, e.g. `np.dtype([(("title", "name"), "i4")])` - subarray fields, e.g. `np.dtype([("name", "i4", (2,))])` These are implementation restrictions. In particular, Zarr V2 supports subarray field descriptors; this implementation does not preserve them. Returns ------- str | None `None` if these field features are supported, otherwise a description of the problem. Notes ----- This is an implementation limitation, not a statement about the V2 format, which has an encoding for subarray fields. """ names = dtype.names fields = dtype.fields if names is None or fields is None: # pragma: no cover - only called on structured dtypes return None for name in names: field_dtype, _offset, *title = fields[name] if title: return f"field {name!r} has a title ({title[0]!r})" if field_dtype.subdtype is not None: return f"field {name!r} is a subarray with shape {field_dtype.shape}" if field_dtype.names is not None: reason = _unsupported_field_feature(field_dtype) if reason is not None: return f"within field {name!r}: {reason}" return None def _default_resolver(*, zarr_format: ZarrFormat) -> Resolver: """A resolver for the default registry, at the top of a document.""" return Resolver(context=Context.default(), zarr_format=zarr_format) def _resolve_field(data: DTypeJSON, resolver: Resolver) -> ZDType[TBaseDType, TBaseScalar]: """ Resolve the data type of a structured field. A field data type that matches nothing makes the structured data type invalid, rather than a different data type. """ try: return resolver.resolve_data_type(data) except DataTypeValidationError: raise except ValueError as e: raise _NestedDataTypeValidationError(str(e)) from e class StructuredJSON_V2(DTypeConfig_V2[StructuredName_V2, None]): """ A wrapper around the JSON representation of the ``Structured`` data type in Zarr V2. The ``name`` field is a sequence of sequences, where each inner sequence has two values: the field name and the data type name for that field (which could be another sequence). The data type names are strings, and the object codec ID is always None. References ---------- The structure of the ``name`` field is defined in the Zarr V2 [specification document](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding). Examples -------- ```python { "name": [ ["f0", " None: if len(self.fields) < 1: raise ValueError(f"must have at least one field. Got {self.fields!r}") @classmethod def _check_native_dtype(cls, dtype: TBaseDType) -> TypeGuard[np.dtypes.VoidDType[int]]: """ Check that this dtype is a numpy structured dtype Parameters ---------- dtype : np.dtypes.DTypeLike The dtype to check. Returns ------- TypeGuard[np.dtypes.VoidDType] True if the dtype matches, False otherwise. """ return isinstance(dtype, cls.dtype_cls) and dtype.fields is not None @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ Create a Structured ZDType from a native NumPy data type. Parameters ---------- dtype : TBaseDType The native data type. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input data type is not an instance of np.dtypes.VoidDType with a non-null ``fields`` attribute. ValueError If the input has field titles or subarray fields, which this implementation's native dtype conversion does not support. Warns ----- ZarrUserWarning If a non-default field layout is converted to a packed layout. Field values are preserved when writing arrays, but offsets and itemsize may change. Notes ----- This method attempts to resolve the fields of the structured dtype using the data type registry. """ from zarr.core.dtype import get_data_type_from_native_dtype fields: list[tuple[str, ZDType[TBaseDType, TBaseScalar]]] = [] if cls._check_native_dtype(dtype): reason = _unsupported_field_feature(dtype) if reason is not None: # NOTE: this is a ValueError rather than a DataTypeValidationError on purpose. # The data type registry suppresses DataTypeValidationError (treating it as # "this dtype does not match"), but a dtype with an unsupported field feature # *does* match this dtype class -- it simply cannot be represented faithfully -- # so we must raise an error the registry propagates to the caller. raise ValueError( f"Cannot convert the structured data type {dtype}: {reason}. " "Zarr-Python's current structured dtype conversion does not support " "field titles or subarray fields." ) # Repack once, at the top level, before resolving the fields. Nested fields then # reach the registry already packed, so a padded nested field warns exactly once, # here, rather than once per level of nesting. packed = cast("np.dtype[np.void]", repack_fields(dtype, recurse=True)) if packed != dtype: warnings.warn( "The structured dtype is converted to a packed field layout. " "Field values are preserved when writing arrays, but field offsets and " "itemsize may change.", ZarrUserWarning, # Attribute the warning to the first frame outside the zarr package, since # the depth of the call chain that leads here varies by entry point. This # reaches the caller for direct uses of the dtype API; the synchronous array # API runs on the event loop thread, where no caller frame is available. skip_file_prefixes=(_ZARR_PACKAGE_ROOT,), ) # Iterate over `names` rather than `fields`: the `fields` mapping also # contains an entry for every field title, which would duplicate titled fields. for key in packed.names: # type: ignore[union-attr] dtype_wrapped = get_data_type_from_native_dtype(packed.fields[key][0]) # type: ignore[index] fields.append((key, dtype_wrapped)) return cls(fields=tuple(fields)) raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) def to_native_dtype(self) -> np.dtypes.VoidDType[int]: """ Convert the structured Zarr data type to a native NumPy void dtype. This method constructs a NumPy dtype with fields corresponding to the fields of the structured Zarr data type, by converting each field's data type to its native dtype representation. Returns ------- np.dtypes.VoidDType[int] The native NumPy void dtype representing the structured data type. """ return cast( "np.dtypes.VoidDType[int]", np.dtype([(key, dtype.to_native_dtype()) for (key, dtype) in self.fields]), ) @classmethod def _check_json_v2( cls, data: DTypeJSON, ) -> TypeGuard[StructuredJSON_V2]: """ Check if the input is a valid JSON representation of a Structured data type for Zarr V2. The input data must be a mapping that contains a "name" key that is not a str, and an "object_codec_id" key that is None. Parameters ---------- data : DTypeJSON The JSON data to check. Returns ------- TypeGuard[StructuredJSON_V2] True if the input is a valid JSON representation of a Structured data type for Zarr V2, False otherwise. """ return ( check_dtype_spec_no_object_codec_v2(data) and not isinstance(data["name"], str) and check_structured_dtype_name_v2(data["name"]) ) @classmethod def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[StructuredJSON_V3]: """ Check that the input is a valid JSON representation of this class in Zarr V3. Parameters ---------- data : DTypeJSON The JSON data to check. Returns ------- TypeGuard[StructuredJSON_V3] True if the input is a valid JSON representation of a structured data type for Zarr V3, False otherwise. """ return ( isinstance(data, dict) and set(data.keys()) == {"name", "configuration"} and data["name"] == cls._zarr_v3_name and isinstance(data["configuration"], dict) and set(data["configuration"].keys()) == {"fields"} ) @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: # the fields are resolved from the default registry return cls._from_json_resolved_v2(data, resolver=_default_resolver(zarr_format=2)) @classmethod def _from_json_v3(cls, data: DTypeJSON) -> Self: # the fields are resolved from the default registry return cls._from_json_resolved_v3(data, resolver=_default_resolver(zarr_format=3)) @classmethod def _from_json_resolved(cls, data: DTypeJSON, *, resolver: Resolver) -> Self: if resolver.zarr_format == 2: return cls._from_json_resolved_v2(data, resolver=resolver) if resolver.zarr_format == 3: return cls._from_json_resolved_v3(data, resolver=resolver) raise ValueError( f"zarr_format must be 2 or 3, got {resolver.zarr_format}" ) # pragma: no cover @classmethod def _from_json_resolved_v2(cls, data: DTypeJSON, *, resolver: Resolver) -> Self: if cls._check_json_v2(data): # structured dtypes are constructed directly from a list of lists # note that we do not handle the object codec here! this will prevent structured # dtypes from containing object dtypes. name = data["name"] return cls( fields=tuple( ( # type: ignore[misc] f_name, _resolve_field( {"name": f_dtype, "object_codec_id": None}, # the location in the data type as written in the document, which # is the "name" of the Zarr V2 data type JSON resolver.at(index, 1), ), ) for index, (f_name, f_dtype) in enumerate(name) ) ) msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a JSON array of arrays" raise DataTypeValidationError(msg) @classmethod def _from_json_resolved_v3(cls, data: DTypeJSON, *, resolver: Resolver) -> Self: if cls._check_json_v3(data): config = data["configuration"] meta_fields = config["fields"] return cls( fields=tuple( ( # type: ignore[misc] f_name, _resolve_field(f_dtype, resolver.at("configuration", "fields", index, 1)), ) for index, (f_name, f_dtype) in enumerate(meta_fields) ) ) msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a JSON object with the key {cls._zarr_v3_name!r}" raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> StructuredJSON_V2: ... @overload def to_json(self, zarr_format: Literal[3]) -> StructuredJSON_V3: ... def to_json(self, zarr_format: ZarrFormat) -> StructuredJSON_V2 | StructuredJSON_V3: """ Convert the structured data type to a JSON-serializable form. Parameters ---------- zarr_format : ZarrFormat The Zarr format version. Accepted values are 2 and 3. Returns ------- StructuredJSON_V2 | StructuredJSON_V3 The JSON representation of the structured data type. Raises ------ ValueError If the zarr_format is not 2 or 3. """ if zarr_format == 2: fields = [ [f_name, f_dtype.to_json(zarr_format=zarr_format)["name"]] for f_name, f_dtype in self.fields ] return {"name": fields, "object_codec_id": None} elif zarr_format == 3: v3_unstable_dtype_warning(self) fields = [ [f_name, f_dtype.to_json(zarr_format=zarr_format)] # type: ignore[list-item] for f_name, f_dtype in self.fields ] base_dict = { "name": self._zarr_v3_name, "configuration": {"fields": fields}, } return cast("StructuredJSON_V3", base_dict) raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover def _check_scalar(self, data: object) -> TypeGuard[StructuredScalarLike]: # TODO: implement something more precise here! """ Check that the input is a valid scalar value for this structured data type. Parameters ---------- data : object The scalar value to check. Returns ------- TypeGuard[StructuredScalarLike] Whether the input is a valid scalar value for this structured data type. """ return isinstance(data, (bytes, list, tuple, int, np.void)) def _cast_scalar_unchecked(self, data: StructuredScalarLike) -> np.void: """ Cast a python object to a numpy structured scalar without type checking. Parameters ---------- data : StructuredScalarLike The data to cast. Returns ------- np.void The casted data as a numpy structured scalar. Notes ----- This method does not perform any type checking. The input data must be castable to a numpy structured scalar. """ na_dtype = self.to_native_dtype() if isinstance(data, bytes): res = np.frombuffer(data, dtype=na_dtype)[0] elif isinstance(data, list | tuple): res = np.array([tuple(data)], dtype=na_dtype)[0] else: res = np.array([data], dtype=na_dtype)[0] return cast("np.void", res) def cast_scalar(self, data: object) -> np.void: """ Cast a Python object to a NumPy structured scalar. This function attempts to cast the provided data to a NumPy structured scalar. If the data is compatible with the structured scalar type, it is cast without type checking. Otherwise, a TypeError is raised. Parameters ---------- data : object The data to be cast to a NumPy structured scalar. Returns ------- np.void The data cast as a NumPy structured scalar. Raises ------ TypeError If the data cannot be converted to a NumPy structured scalar. """ if self._check_scalar(data): return self._cast_scalar_unchecked(data) msg = ( f"Cannot convert object {data!r} with type {type(data)} to a scalar compatible with the " f"data type {self}." ) raise TypeError(msg) def default_scalar(self) -> np.void: """ Get the default scalar value for this structured data type. Returns ------- np.void The default scalar value, which is the scalar representation of 0 cast to this structured data type. """ values: list[object] = [] for _, field in self.fields: dtype = field.to_native_dtype() if isinstance(field, Structured): value = field.default_scalar() elif ( isinstance(dtype, np.dtypes.DateTime64DType) and np.datetime_data(cast("np.dtypes.DateTime64DType", dtype))[0] == "generic" ): # NumPy rejects casting integer zero to generic datetime, but a # zero count is representable by viewing the integer storage. value = np.zeros(1, dtype=dtype.byteorder + "i8").view(dtype)[0] else: value = np.zeros(1, dtype=dtype)[0] values.append(value) return self._cast_scalar_unchecked(tuple(values)) def _scalar_bytes_to_void(self, data: bytes, zarr_format: ZarrFormat) -> np.void: """Read the raw bytes of a base64 fill value as a scalar of this data type. Zarr V2 metadata spells out the byte order of every field. Zarr V3 metadata has none, so V3 fill value bytes are little-endian, like the default `bytes` codec. """ dtype = self.to_native_dtype() stored = dtype if zarr_format == 2 else dtype.newbyteorder("<") return cast("np.void", np.frombuffer(data, dtype=stored).astype(dtype)[0]) def _void_to_scalar_bytes(self, data: np.void, zarr_format: ZarrFormat) -> bytes: """Inverse of `_scalar_bytes_to_void`.""" dtype = self.to_native_dtype() stored = dtype if zarr_format == 2 else dtype.newbyteorder("<") return np.asarray(data).astype(stored).tobytes() def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.void: """ Read a JSON-serializable value as a NumPy structured scalar. Parameters ---------- data : JSON The JSON-serializable value. zarr_format : ZarrFormat The zarr format version. Returns ------- np.void The NumPy structured scalar. Raises ------ TypeError If the input is not a base64-encoded string. """ if check_json_str(data): as_bytes = bytes_from_json(data, zarr_format=zarr_format) return self._scalar_bytes_to_void(as_bytes, zarr_format) raise TypeError(f"Invalid type: {data}. Expected a string.") def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str | dict[str, JSON]: """ Convert a scalar to a JSON-serializable string representation. Parameters ---------- data : object The scalar to convert. zarr_format : ZarrFormat The zarr format version. Returns ------- str | dict[str, JSON] A string representation of the scalar, which is a base64-encoded string of the bytes that make up the scalar. Subclasses may return a dict for V3 format. """ return bytes_to_json( self._void_to_scalar_bytes(self.cast_scalar(data), zarr_format), zarr_format ) @property def item_size(self) -> int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return self.to_native_dtype().itemsize def has_multi_byte_fields(self) -> bool: """ Check if this structured dtype has any fields with item_size > 1. Returns ------- bool True if any field has item_size > 1, False otherwise. """ return any( isinstance(field_dtype, HasItemSize) and field_dtype.item_size > 1 for _, field_dtype in self.fields ) @dataclass(frozen=True, kw_only=True) class Struct(Structured): """ A Zarr data type for arrays containing structured scalars, AKA "record arrays". Wraps the NumPy `np.dtypes.VoidDType` if the data type has fields. Scalars for this data type are instances of `np.void`, with a ``fields`` attribute. This is the canonical data type registered for structured arrays. It reads both the canonical ``"struct"`` format (object-style fields) and the legacy ``"structured"`` format (tuple-style fields), but always writes the canonical ``"struct"`` format. Attributes ---------- fields : Sequence[tuple[str, ZDType]] The fields of the structured dtype. References ---------- The Zarr V3 specification for this data type is defined in the zarr-extensions repository: https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/struct The Zarr V2 data type specification can be found [here](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding). """ _zarr_v3_name: ClassVar[Literal["struct"]] = "struct" # type: ignore[assignment] @classmethod def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[StructJSON_V3]: # type: ignore[override] return ( isinstance(data, dict) and set(data.keys()) == {"name", "configuration"} and data["name"] in ("struct", "structured") and isinstance(data["configuration"], dict) and set(data["configuration"].keys()) == {"fields"} ) @classmethod def _from_json_resolved_v3(cls, data: DTypeJSON, *, resolver: Resolver) -> Self: if cls._check_json_v3(data): config = data["configuration"] meta_fields = config["fields"] parsed_fields: list[tuple[str, ZDType[TBaseDType, TBaseScalar]]] = [] for index, field in enumerate(meta_fields): if isinstance(field, dict): f_name = field["name"] f_dtype = field["data_type"] f_resolver = resolver.at("configuration", "fields", index, "data_type") else: # Legacy tuple-style field format from "structured" dtype f_name, f_dtype = field # type: ignore[unreachable] f_resolver = resolver.at("configuration", "fields", index, 1) parsed_fields.append((f_name, _resolve_field(f_dtype, f_resolver))) # type: ignore[arg-type] return cls(fields=tuple(parsed_fields)) msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a JSON object with the key {cls._zarr_v3_name!r}" raise DataTypeValidationError(msg) @overload # type: ignore[override] def to_json(self, zarr_format: Literal[2]) -> StructuredJSON_V2: ... @overload def to_json(self, zarr_format: Literal[3]) -> StructJSON_V3: ... def to_json(self, zarr_format: ZarrFormat) -> StructuredJSON_V2 | StructJSON_V3: if zarr_format == 2: fields_v2 = [ [f_name, f_dtype.to_json(zarr_format=zarr_format)["name"]] for f_name, f_dtype in self.fields ] return {"name": fields_v2, "object_codec_id": None} elif zarr_format == 3: # The "struct" data type has a stable Zarr V3 specification # (https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/struct), # so unlike the legacy "structured" alias it does not emit an unstable-spec warning. fields_v3 = [ {"name": f_name, "data_type": f_dtype.to_json(zarr_format=zarr_format)} for f_name, f_dtype in self.fields ] return cast( "StructJSON_V3", {"name": self._zarr_v3_name, "configuration": {"fields": fields_v3}}, ) raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.void: """ Read a JSON-serializable value as a NumPy structured scalar. Parameters ---------- data : JSON The JSON-serializable value. Can be either: - A dict mapping field names to values (primary format for V3) - A base64-encoded string (legacy format, for backward compatibility) zarr_format : ZarrFormat The zarr format version. Returns ------- np.void The NumPy structured scalar. Raises ------ TypeError If the input is not a dict or base64-encoded string. """ if isinstance(data, dict): field_values = [] for field_name, field_dtype in self.fields: if field_name in data: field_values.append( field_dtype.from_json_scalar(data[field_name], zarr_format=zarr_format) ) else: field_values.append(field_dtype.default_scalar()) return self._cast_scalar_unchecked(tuple(field_values)) elif check_json_str(data): as_bytes = bytes_from_json(data, zarr_format=zarr_format) return self._scalar_bytes_to_void(as_bytes, zarr_format) raise TypeError(f"Invalid type: {data}. Expected a dict or base64-encoded string.") def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str | dict[str, JSON]: """ Convert a scalar to a JSON-serializable representation. Parameters ---------- data : object The scalar to convert. zarr_format : ZarrFormat The zarr format version. Returns ------- str | dict[str, JSON] For V2: A base64-encoded string of the bytes that make up the scalar. For V3: A dict mapping field names to their JSON-serialized values. """ scalar = self.cast_scalar(data) if zarr_format == 2: return bytes_to_json(self._void_to_scalar_bytes(scalar, zarr_format), zarr_format) result: dict[str, JSON] = {} for field_name, field_dtype in self.fields: result[field_name] = field_dtype.to_json_scalar( scalar[field_name], zarr_format=zarr_format ) return result zarr-developers-zarr-python-a2f2415/src/zarr/core/dtype/npy/time.py000066400000000000000000000720101527014137600254250ustar00rootroot00000000000000from __future__ import annotations from dataclasses import dataclass from datetime import datetime, timedelta from typing import ( TYPE_CHECKING, ClassVar, Literal, Self, TypedDict, TypeGuard, cast, get_args, overload, ) import numpy as np from typing_extensions import ReadOnly from zarr.core.common import NamedRequiredConfig from zarr.core.dtype.common import ( DTypeConfig_V2, DTypeJSON, HasEndianness, HasItemSize, check_dtype_spec_no_object_codec_v2, ) from zarr.core.dtype.npy.common import ( DATETIME_UNIT, DateTimeUnit, check_json_int, endianness_to_numpy_str, get_endianness_from_numpy_dtype, ) from zarr.core.dtype.wrapper import TBaseDType, ZDType from zarr.errors import DataTypeValidationError if TYPE_CHECKING: from zarr.core.common import JSON, ZarrFormat TimeDeltaLike = str | int | bytes | np.timedelta64 | timedelta | None DateTimeLike = str | int | bytes | np.datetime64 | datetime | None def datetime_from_int(data: int, *, unit: DateTimeUnit, scale_factor: int) -> np.datetime64: """ Convert an integer to a datetime64. Parameters ---------- data : int The integer to convert. unit : DateTimeUnit The unit of the datetime64. scale_factor : int The scale factor of the datetime64. Returns ------- numpy.datetime64 The datetime64 value. """ dtype_name = f"datetime64[{scale_factor}{unit}]" return cast("np.datetime64", np.int64(data).view(dtype_name)) def datetimelike_to_int(data: np.datetime64 | np.timedelta64) -> int: """ Convert a datetime64 or a timedelta64 to an integer. Parameters ---------- data : np.datetime64 | numpy.timedelta64 The value to convert. Returns ------- int An integer representation of the scalar. """ return data.view(np.int64).item() def check_json_time(data: JSON) -> TypeGuard[Literal["NaT"] | int]: """ Type guard to check if the input JSON data is the literal string "NaT" or an integer. """ return check_json_int(data) or data == "NaT" class TimeConfig(TypedDict): """ The configuration for the numpy.timedelta64 or numpy.datetime64 data type in Zarr V3. Attributes ---------- unit : ReadOnly[DateTimeUnit] A string encoding a unit of time. scale_factor : ReadOnly[int] A scale factor. Examples -------- ```python {"unit": "ms", "scale_factor": 1} ``` """ unit: ReadOnly[DateTimeUnit] scale_factor: ReadOnly[int] class DateTime64JSON_V3(NamedRequiredConfig[Literal["numpy.datetime64"], TimeConfig]): """ The JSON representation of the ``numpy.datetime64`` data type in Zarr V3. References ---------- This representation is defined in the ``numpy.datetime64`` [specification document](https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/numpy.datetime64). Examples -------- ```python { "name": "numpy.datetime64", "configuration": { "unit": "ms", "scale_factor": 1 } } ``` """ class TimeDelta64JSON_V3(NamedRequiredConfig[Literal["numpy.timedelta64"], TimeConfig]): """ The JSON representation of the ``TimeDelta64`` data type in Zarr V3. References ---------- This representation is defined in the numpy.timedelta64 [specification document](https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/numpy.timedelta64). Examples -------- ```python { "name": "numpy.timedelta64", "configuration": { "unit": "ms", "scale_factor": 1 } } ``` """ class TimeDelta64JSON_V2(DTypeConfig_V2[str, None]): """ A wrapper around the JSON representation of the ``TimeDelta64`` data type in Zarr V2. The ``name`` field of this class contains the value that would appear under the ``dtype`` field in Zarr V2 array metadata. References ---------- The structure of the ``name`` field is defined in the Zarr V2 [specification document](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding). Examples -------- ```python { "name": " None: if self.scale_factor < 1: raise ValueError(f"scale_factor must be > 0, got {self.scale_factor}.") if self.scale_factor >= 2**31: raise ValueError(f"scale_factor must be < 2147483648, got {self.scale_factor}.") if self.unit not in get_args(DateTimeUnit): raise ValueError(f"unit must be one of {get_args(DateTimeUnit)}, got {self.unit!r}.") if self.unit == "μs": # NumPy spells the microsecond unit "us" and resolves "μs" to it, so an # instance built with "μs" would not round-trip through to_native_dtype(). # Store the NumPy spelling; "μs" stays accepted as input and in stored metadata. object.__setattr__(self, "unit", "us") @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ Create an instance of this class from a native NumPy data type. Parameters ---------- dtype : TBaseDType The native NumPy dtype to convert. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the dtype is not a valid representation of this class. """ if cls._check_native_dtype(dtype): unit, scale_factor = np.datetime_data(dtype) unit = cast("DateTimeUnit", unit) return cls( unit=unit, scale_factor=scale_factor, endianness=get_endianness_from_numpy_dtype(dtype), ) raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) def to_native_dtype(self) -> DType: # Numpy does not allow creating datetime64 or timedelta64 via # np.dtypes.{dtype_name}() # so we use np.dtype with a formatted string. """ Convert this data type to a NumPy temporal data type with the appropriate unit and scale factor. Returns ------- DType A NumPy data type object representing the time data type with the specified unit, scale factor, and byte order. """ dtype_string = f"{self._numpy_name}[{self.scale_factor}{self.unit}]" return np.dtype(dtype_string).newbyteorder(endianness_to_numpy_str(self.endianness)) # type: ignore[return-value] def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> int: """ Convert a python object to a JSON representation of a datetime64 or timedelta64 scalar. Parameters ---------- data : object The python object to convert. zarr_format : ZarrFormat The Zarr format version (2 or 3). Returns ------- int The JSON representation of the scalar. """ return datetimelike_to_int(data) # type: ignore[arg-type] @property def item_size(self) -> int: """ The size of a single scalar in bytes. Returns ------- int The size of a single scalar in bytes. """ return 8 @dataclass(frozen=True, kw_only=True, slots=True) class TimeDelta64(TimeDTypeBase[np.dtypes.TimeDelta64DType, np.timedelta64], HasEndianness): """ A Zarr data type for arrays containing NumPy TimeDelta64 data. Wraps the ``np.dtypesTimeDelta64DType`` data type. Scalars for this data type are instances of `np.timedelta64`. Attributes ---------- dtype_cls : Type[np.dtypesTimeDelta64DType] The NumPy dtype class for this data type. scale_factor : int The scale factor for this data type. unit : DateTimeUnit The unit for this data type. References ---------- The Zarr V2 representation of this data type is defined in the Zarr V2 [specification document](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding). The Zarr V3 representation of this data type is defined in the ``numpy.timedelta64`` [specification document](https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/numpy.timedelta64) """ # mypy infers the type of np.dtypes.TimeDelta64DType to be # "Callable[[Literal['Y', 'M', 'W', 'D'] | Literal['h', 'm', 's', 'ms', 'us', 'ns', 'ps', 'fs', 'as']], Never]" dtype_cls = np.dtypes.TimeDelta64DType # type: ignore[assignment] unit: DateTimeUnit = "generic" scale_factor: int = 1 _zarr_v3_name: ClassVar[Literal["numpy.timedelta64"]] = "numpy.timedelta64" _zarr_v2_names: ClassVar[tuple[Literal[">m8"], Literal["m8", " TypeGuard[TimeDelta64JSON_V2]: """ Validate that the provided JSON input accurately represents a NumPy timedelta64 data type, which could be in the form of strings like "m8[10s]". This method serves as a type guard, helping to refine the type of unknown JSON input by confirming its adherence to the expected format for NumPy timedelta64 data types. The JSON input should contain a "name" key with a value that matches the expected string pattern for NumPy timedelta64 data types. The pattern includes an optional unit enclosed within square brackets, following the base type identifier. Returns ------- bool True if the JSON input is a valid representation of this class, otherwise False. """ if not check_dtype_spec_no_object_codec_v2(data): return False name = data["name"] # match m[M], etc # consider making this a standalone function if not isinstance(name, str): return False if not name.startswith(cls._zarr_v2_names): return False if len(name) == 3: # no unit, and # we already checked that this string is either m8 return True else: return name[4:-1].endswith(DATETIME_UNIT) and name[-1] == "]" @classmethod def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[DateTime64JSON_V3]: """ Check that the input is a valid JSON representation of this class in Zarr V3. Returns ------- TypeGuard[DateTime64JSON_V3] True if the JSON input is a valid representation of this class, otherwise False. """ return ( isinstance(data, dict) and set(data.keys()) == {"name", "configuration"} and data["name"] == cls._zarr_v3_name and isinstance(data["configuration"], dict) and set(data["configuration"].keys()) == {"unit", "scale_factor"} ) @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create a TimeDelta64 from a Zarr V2-flavored JSON. Parameters ---------- data : DTypeJSON The JSON data. Returns ------- TimeDelta64 An instance of TimeDelta64. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class. """ if cls._check_json_v2(data): name = data["name"] return cls.from_native_dtype(np.dtype(name)) msg = ( f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string " f"representation of an instance of {cls.dtype_cls}" ) raise DataTypeValidationError(msg) @classmethod def _from_json_v3(cls, data: DTypeJSON) -> Self: """ Create a TimeDelta64 from a Zarr V3-flavored JSON. The JSON representation of a TimeDelta64 in Zarr V3 is a dict with a 'name' key with the value 'numpy.timedelta64', and a 'configuration' key with a value of a dict with a 'unit' key and a 'scale_factor' key. For example: ```json { "name": "numpy.timedelta64", "configuration": { "unit": "generic", "scale_factor": 1 } } ``` """ if cls._check_json_v3(data): unit = data["configuration"]["unit"] scale_factor = data["configuration"]["scale_factor"] return cls(unit=unit, scale_factor=scale_factor) msg = ( f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a dict " f"with a 'name' key with the value 'numpy.timedelta64', " "and a 'configuration' key with a value of a dict with a 'unit' key and a " "'scale_factor' key" ) raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> TimeDelta64JSON_V2: ... @overload def to_json(self, zarr_format: Literal[3]) -> TimeDelta64JSON_V3: ... def to_json(self, zarr_format: ZarrFormat) -> TimeDelta64JSON_V2 | TimeDelta64JSON_V3: """ Serialize this data type to JSON. Parameters ---------- zarr_format : ZarrFormat The Zarr format version (2 or 3). Returns ------- TimeDelta64JSON_V2 | TimeDelta64JSON_V3 The JSON representation of the data type. Raises ------ ValueError If the zarr_format is not 2 or 3. """ if zarr_format == 2: name: str = self.to_native_dtype().str if self.unit == "generic" and self.scale_factor != 1: # NumPy omits generic scale from dtype.str; preserve it explicitly. name += f"[{self.scale_factor}generic]" return {"name": name, "object_codec_id": None} elif zarr_format == 3: return { "name": self._zarr_v3_name, "configuration": {"unit": self.unit, "scale_factor": self.scale_factor}, } raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover def _check_scalar(self, data: object) -> TypeGuard[TimeDeltaLike]: """ Check if the input is a scalar of this data type. Parameters ---------- data : object The object to check. Returns ------- TypeGuard[TimeDeltaLike] True if the input is a scalar of this data type, False otherwise. """ if data is None: return True return isinstance(data, str | int | bytes | np.timedelta64 | timedelta) def _cast_scalar_unchecked(self, data: TimeDeltaLike) -> np.timedelta64: """ Cast the provided scalar input to a numpy timedelta64 without any type checking. This method assumes that the input data is already a valid scalar of this data type, and does not perform any validation or type checks. It directly casts the input to a numpy timedelta64 scalar using the unit and scale factor defined in the class. Parameters ---------- data : TimeDeltaLike The scalar input data to cast. Returns ------- numpy.timedelta64 The input data cast as a numpy timedelta64 scalar. """ # numpy 2.x stub: timedelta64(scalar, formatted_unit_str) is runtime-valid # but no overload matches the dynamic f-string unit argument. return self.to_native_dtype().type(data, f"{self.scale_factor}{self.unit}") # type: ignore[call-overload, no-any-return] def cast_scalar(self, data: object) -> np.timedelta64: """ Cast the input to a numpy timedelta64 scalar. If the input is not a scalar of this data type, raise a TypeError. """ if self._check_scalar(data): if isinstance(data, np.timedelta64) and np.isnat(data): return self._cast_scalar_unchecked("NaT") return self._cast_scalar_unchecked(data) msg = ( f"Cannot convert object {data!r} with type {type(data)} to a scalar compatible with the " f"data type {self}." ) raise TypeError(msg) def default_scalar(self) -> np.timedelta64: """ Return a default scalar of this data type. This method provides a default value for the timedelta64 scalar, which is a 'Not-a-Time' (NaT) value. """ # Carry the scale factor: NumPy < 2.2 turns a timedelta NaT into a count when # casting it between scales, e.g. NaT[us] into a [2us] array. return self._cast_scalar_unchecked("NaT") def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.timedelta64: """ Create a scalar of this data type from JSON input. Parameters ---------- data : JSON The JSON representation of the scalar value. zarr_format : int The zarr format to use for the JSON representation. Returns ------- numpy.timedelta64 The scalar value of this data type. Raises ------ TypeError If the input JSON is not a valid representation of a scalar for this data type. """ if check_json_time(data): # numpy 2.x stub: timedelta64(scalar, formatted_unit_str) is runtime-valid # but no overload matches the dynamic f-string unit argument. return self.to_native_dtype().type(data, f"{self.scale_factor}{self.unit}") # type: ignore[call-overload, no-any-return] raise TypeError(f"Invalid type: {data}. Expected an integer.") # pragma: no cover @dataclass(frozen=True, kw_only=True, slots=True) class DateTime64(TimeDTypeBase[np.dtypes.DateTime64DType, np.datetime64], HasEndianness): """ A Zarr data type for arrays containing NumPy Datetime64 data. Wraps the ``np.dtypes.TimeDelta64DType`` data type. Scalars for this data type are instances of ``np.datetime64``. Attributes ---------- dtype_cls : Type[np.dtypesTimeDelta64DType] The numpy dtype class for this data type. unit : DateTimeUnit The unit of time for this data type. scale_factor : int The scale factor for the time unit. References ---------- The Zarr V2 representation of this data type is defined in the Zarr V2 [specification document](https://github.com/zarr-developers/zarr-specs/blob/main/docs/v2/v2.0.rst#data-type-encoding). The Zarr V3 representation of this data type is defined in the ``numpy.datetime64`` [specification document](https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/numpy.datetime64) """ dtype_cls = np.dtypes.DateTime64DType # type: ignore[assignment] _zarr_v3_name: ClassVar[Literal["numpy.datetime64"]] = "numpy.datetime64" _zarr_v2_names: ClassVar[tuple[Literal[">M8"], Literal["M8", " TypeGuard[DateTime64JSON_V2]: """ Check that the input is a valid JSON representation of this data type. Parameters ---------- data : DTypeJSON The JSON data to check. Returns ------- TypeGuard[DateTime64JSON_V2] True if the input is a valid JSON representation of a NumPy datetime64 data type, otherwise False. """ if not check_dtype_spec_no_object_codec_v2(data): return False name = data["name"] if not isinstance(name, str): return False if not name.startswith(cls._zarr_v2_names): return False if len(name) == 3: # no unit, and # we already checked that this string is either M8 return True else: return name[4:-1].endswith(DATETIME_UNIT) and name[-1] == "]" @classmethod def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[DateTime64JSON_V3]: """ Check that the input is a valid JSON representation of this class in Zarr V3. Parameters ---------- data : DTypeJSON The JSON data to check. Returns ------- TypeGuard[DateTime64JSON_V3] True if the input is a valid JSON representation of a numpy datetime64 data type in Zarr V3, False otherwise. """ return ( isinstance(data, dict) and set(data.keys()) == {"name", "configuration"} and data["name"] == cls._zarr_v3_name and isinstance(data["configuration"], dict) and set(data["configuration"].keys()) == {"unit", "scale_factor"} ) @classmethod def _from_json_v2(cls, data: DTypeJSON) -> Self: """ Create an instance of this data type from a Zarr V2-flavored JSON representation. This method checks if the provided JSON data is a valid representation of this class. If valid, it creates an instance using the native NumPy dtype. Otherwise, it raises a DataTypeValidationError. Parameters ---------- data : DTypeJSON The JSON data to parse. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class. """ if cls._check_json_v2(data): name = data["name"] return cls.from_native_dtype(np.dtype(name)) msg = ( f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string " f"representation of an instance of {cls.dtype_cls}" ) raise DataTypeValidationError(msg) @classmethod def _from_json_v3(cls, data: DTypeJSON) -> Self: """ Create an instance of this data type from a Zarr V3-flavored JSON representation. This method checks if the provided JSON data is a valid representation of this class. If valid, it creates an instance using the native NumPy dtype. Otherwise, it raises a DataTypeValidationError. Parameters ---------- data : DTypeJSON The JSON data to parse. Returns ------- Self An instance of this data type. Raises ------ DataTypeValidationError If the input JSON is not a valid representation of this class. """ if cls._check_json_v3(data): unit = data["configuration"]["unit"] scale_factor = data["configuration"]["scale_factor"] return cls(unit=unit, scale_factor=scale_factor) msg = ( f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a dict " f"with a 'name' key with the value 'numpy.datetime64', " "and a 'configuration' key with a value of a dict with a 'unit' key and a " "'scale_factor' key" ) raise DataTypeValidationError(msg) @overload def to_json(self, zarr_format: Literal[2]) -> DateTime64JSON_V2: ... @overload def to_json(self, zarr_format: Literal[3]) -> DateTime64JSON_V3: ... def to_json(self, zarr_format: ZarrFormat) -> DateTime64JSON_V2 | DateTime64JSON_V3: """ Serialize this data type to JSON. Parameters ---------- zarr_format : ZarrFormat The Zarr format version (2 or 3). Returns ------- DateTime64JSON_V2 | DateTime64JSON_V3 The JSON representation of the data type. Raises ------ ValueError If the zarr_format is not 2 or 3. """ if zarr_format == 2: name: str = self.to_native_dtype().str if self.unit == "generic" and self.scale_factor != 1: # NumPy omits generic scale from dtype.str; preserve it explicitly. name += f"[{self.scale_factor}generic]" return {"name": name, "object_codec_id": None} elif zarr_format == 3: return { "name": self._zarr_v3_name, "configuration": {"unit": self.unit, "scale_factor": self.scale_factor}, } raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover def _check_scalar(self, data: object) -> TypeGuard[DateTimeLike]: """ Check if the input is convertible to a scalar of this data type. Parameters ---------- data : object The object to check. Returns ------- TypeGuard[DateTimeLike] True if the input is a scalar of this data type, False otherwise. """ if data is None: return True return isinstance(data, str | int | bytes | np.datetime64 | datetime) def _cast_scalar_unchecked(self, data: DateTimeLike) -> np.datetime64: """ Cast the input to a scalar of this data type without any type checking. Parameters ---------- data : DateTimeLike The scalar data to cast. Returns ------- numpy.datetime64 The input cast to a NumPy datetime scalar. """ if isinstance(data, int): # The scalar constructor rejects integer counts with a generic unit. return datetime_from_int(data, unit=self.unit, scale_factor=self.scale_factor) # numpy 2.x stub: datetime64(scalar, formatted_unit_str) is runtime-valid # but no overload matches the dynamic f-string unit argument. return self.to_native_dtype().type(data, f"{self.scale_factor}{self.unit}") # type: ignore[call-overload, no-any-return] def cast_scalar(self, data: object) -> np.datetime64: """ Cast the input to a scalar of this data type after a type check. Parameters ---------- data : object The scalar value to cast. Returns ------- numpy.datetime64 The input cast to a NumPy datetime scalar. Raises ------ TypeError If the data cannot be converted to a numpy datetime scalar. """ if self._check_scalar(data): return self._cast_scalar_unchecked(data) msg = ( f"Cannot convert object {data!r} with type {type(data)} to a scalar compatible with the " f"data type {self}." ) raise TypeError(msg) def default_scalar(self) -> np.datetime64: """ Return the default scalar value for this data type. Returns ------- numpy.datetime64 The default scalar value, which is a 'Not-a-Time' (NaT) value """ return self._cast_scalar_unchecked("NaT") def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.datetime64: """ Read a JSON-serializable value as a scalar. Parameters ---------- data : JSON The JSON-serializable value. zarr_format : ZarrFormat The zarr format version. Returns ------- numpy.datetime64 The numpy datetime scalar. Raises ------ TypeError If the input is not a valid integer type. """ if check_json_time(data): return self._cast_scalar_unchecked(data) raise TypeError(f"Invalid type: {data}. Expected an integer.") # pragma: no cover zarr-developers-zarr-python-a2f2415/src/zarr/core/dtype/registry.py000066400000000000000000000247061527014137600255420ustar00rootroot00000000000000from __future__ import annotations import contextlib from dataclasses import dataclass, field from typing import TYPE_CHECKING, Final, Self import numpy as np from zarr.core.context import Context, Resolver, json_pointer from zarr.errors import DataTypeValidationError if TYPE_CHECKING: from collections.abc import Mapping from importlib.metadata import EntryPoint from zarr.core.common import ZarrFormat from zarr.core.dtype.common import DTypeJSON from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType # Zarr V2 data type names are NumPy typestrs, whose first character is the byte order: "<", ">", or # "|" (not relevant). For the data types below the byte order is not relevant, so their "<" and ">" # spellings name the same data type as the canonical "|" spelling. Other implementations write # them: netCDF-C writes "i1" and ">u1". # Single-byte data types, keyed by their non-canonical spellings. _V2_SINGLE_BYTE_ALIASES: Final[Mapping[str, str]] = { "b1": "|b1", "i1": "|i1", "u1": "|u1", } # Fixed-length bytes data types, whose names are the kind followed by the length in bytes. _V2_BYTES_KINDS: Final = ("V",) def _v2_canonical_name(name: str) -> str: """ The canonical spelling of a Zarr V2 data type name: `"|"` for a byte order that is not relevant, and the name as written otherwise. """ if name in _V2_SINGLE_BYTE_ALIASES: return _V2_SINGLE_BYTE_ALIASES[name] byte_order, kind, length = name[:1], name[1:2], name[2:] if ( byte_order in ("<", ">") and kind in _V2_BYTES_KINDS and length.isascii() and length.isdigit() ): return f"|{kind}{length}" return name def _v2_spellings(data: DTypeJSON) -> tuple[DTypeJSON, ...]: """ The spellings of a Zarr V2 data type JSON to match, in order: the name as written, then its canonical spelling if it has another. A data type that declares a non-canonical spelling (e.g. `">S1"`) takes precedence over the canonical alias, because the name as written is tried first. """ if ( isinstance(data, dict) and isinstance(name := data.get("name"), str) and (canonical := _v2_canonical_name(name)) != name ): return (data, {**data, "name": canonical}) return (data,) class _NestedDataTypeValidationError(DataTypeValidationError): """ A data type contains a data type that is invalid, such as a structured data type with a field whose data type matches no data type. The JSON has the shape of the containing data type, so a data type registry re-raises this error rather than trying other data types. """ # This class is different from the other registry classes, which inherit from # dict. IMO it's simpler to just do a dataclass. But long-term we should # have just 1 registry class in use. @dataclass(frozen=True, kw_only=True) class DataTypeRegistry: """ A registry for ZDType classes. This registry is a mapping from Zarr data type names to their corresponding ZDType classes. Attributes ---------- contents : dict[str, type[ZDType[TBaseDType, TBaseScalar]]] The mapping from Zarr data type names to their corresponding ZDType classes. """ contents: dict[str, type[ZDType[TBaseDType, TBaseScalar]]] = field( default_factory=dict, init=False ) _lazy_load_list: list[EntryPoint] = field(default_factory=list, init=False) def _lazy_load(self) -> None: """ Load all data types from the lazy load list and register them with the registry. After loading, clear the lazy load list. """ for e in self._lazy_load_list: cls = e.load() self.register(cls._zarr_v3_name, cls) self._lazy_load_list.clear() def register(self: Self, key: str, cls: type[ZDType[TBaseDType, TBaseScalar]]) -> None: """ Register a data type with the registry. Parameters ---------- key : str The Zarr V3 name of the data type. cls : type[ZDType[TBaseDType, TBaseScalar]] The class of the data type to register. Notes ----- This method is idempotent. If the data type is already registered, this method does nothing. """ if key not in self.contents or self.contents[key] != cls: self.contents[key] = cls def unregister(self, key: str) -> None: """ Unregister a data type from the registry. Parameters ---------- key : str The key associated with the ZDType class to be unregistered. Returns ------- None Raises ------ KeyError If the data type is not found in the registry. """ if key in self.contents: del self.contents[key] else: raise KeyError(f"Data type '{key}' not found in registry.") def get(self, key: str) -> type[ZDType[TBaseDType, TBaseScalar]]: """ Retrieve a registered ZDType class by its key. Parameters ---------- key : str The key associated with the desired ZDType class. Returns ------- type[ZDType[TBaseDType, TBaseScalar]] The ZDType class registered under the given key. Raises ------ KeyError If the key is not found in the registry. """ self._lazy_load() return self.contents[key] def match_dtype(self, dtype: TBaseDType) -> ZDType[TBaseDType, TBaseScalar]: """ Match a native data type, e.g. a NumPy data type, to a registered ZDType. Parameters ---------- dtype : TBaseDType The native data type to match. Returns ------- ZDType[TBaseDType, TBaseScalar] The matched ZDType corresponding to the provided NumPy data type. Raises ------ ValueError If the data type is a NumPy "Object" type, which is ambiguous, or if multiple or no Zarr data types are found that match the provided dtype. Notes ----- This function attempts to resolve a Zarr data type from a given native data type. If the dtype is a NumPy "Object" data type, it raises a ValueError, as this type can represent multiple Zarr data types. In such cases, a specific Zarr data type should be explicitly constructed instead of relying on dynamic resolution. If multiple matches are found, it will also raise a ValueError. In this case conflicting data types must be unregistered, or the Zarr data type should be explicitly constructed. """ self._lazy_load() if dtype == np.dtype("O"): msg = ( f"Zarr data type resolution from {dtype} failed. " 'Attempted to resolve a zarr data type from a numpy "Object" data type, which is ' 'ambiguous, as multiple zarr data types can be represented by the numpy "Object" ' "data type. " "In this case you should construct your array by providing a specific Zarr data " 'type. For a list of Zarr data types that are compatible with the numpy "Object"' "data type, see https://github.com/zarr-developers/zarr-python/issues/3117" ) raise ValueError(msg) matched: list[ZDType[TBaseDType, TBaseScalar]] = [] for val in self.contents.values(): # DataTypeValidationError means "this dtype doesn't match me", which is # expected and suppressed. Other exceptions (e.g. ValueError for a dtype # that matches the type but has an invalid configuration) are propagated # to the caller. with contextlib.suppress(DataTypeValidationError): matched.append(val.from_native_dtype(dtype)) if len(matched) == 1: return matched[0] elif len(matched) > 1: msg = ( f"Zarr data type resolution from {dtype} failed. " f"Multiple data type wrappers found that match dtype '{dtype}': {matched}. " "You should unregister one of these data types, or avoid Zarr data type inference " "entirely by providing a specific Zarr data type when creating your array." "For more information, see https://github.com/zarr-developers/zarr-python/issues/3117" ) raise ValueError(msg) raise ValueError(f"No Zarr data type found that matches dtype '{dtype!r}'") def match_json( self, data: DTypeJSON, *, zarr_format: ZarrFormat ) -> ZDType[TBaseDType, TBaseScalar]: """ Match a JSON representation of a data type to a registered ZDType. Parameters ---------- data : DTypeJSON The JSON representation of a data type to match. zarr_format : ZarrFormat The Zarr format version to consider when matching data types. Returns ------- ZDType[TBaseDType, TBaseScalar] The matched ZDType corresponding to the JSON representation. Raises ------ ValueError If no matching Zarr data type is found for the given JSON data. """ resolver = Resolver(context=Context(data_types=self), zarr_format=zarr_format) return self._match_json(data, resolver=resolver) def _match_json( self, data: DTypeJSON, *, resolver: Resolver ) -> ZDType[TBaseDType, TBaseScalar]: """ Match a JSON representation of a data type to a registered ZDType, at `resolver`'s location. """ self._lazy_load() candidates = _v2_spellings(data) if resolver.zarr_format == 2 else (data,) for candidate in candidates: for val in self.contents.values(): try: return val._from_json_resolved(candidate, resolver=resolver) except _NestedDataTypeValidationError: # the JSON is this data type, and a data type it contains is invalid raise except DataTypeValidationError: pass msg = f"No Zarr data type found that matches {data!r}" if resolver.loc: msg += f" at {json_pointer(resolver.loc)}" raise ValueError(msg) zarr-developers-zarr-python-a2f2415/src/zarr/core/dtype/wrapper.py000066400000000000000000000236251527014137600253510ustar00rootroot00000000000000""" Wrapper for native array data types. The ``ZDType`` class is an abstract base class for wrapping native array data types, e.g. NumPy dtypes. ``ZDType`` provides a common interface for working with data types in a way that is independent of the underlying data type system. The wrapper class encapsulates a native data type. Instances of the class can be created from a native data type instance, and a native data type instance can be created from an instance of the wrapper class. The wrapper class is responsible for: - Serializing and deserializing a native data type to Zarr V2 or Zarr V3 metadata. This ensures that the data type can be properly stored and retrieved from array metadata. - Serializing and deserializing scalar values to Zarr V2 or Zarr V3 metadata. This is important for storing a fill value for an array in a manner that is valid for the data type. You can add support for a new data type in Zarr by subclassing ``ZDType`` wrapper class and adapt its methods to support your native data type. The wrapper class must be added to a data type registry (defined elsewhere) before array creation routines or array reading routines can use your new data type. """ from __future__ import annotations from abc import ABC, abstractmethod from dataclasses import dataclass from typing import ( TYPE_CHECKING, ClassVar, Literal, Self, TypeGuard, overload, ) import numpy as np if TYPE_CHECKING: from zarr.core.common import JSON, ZarrFormat from zarr.core.context import Resolver from zarr.core.dtype.common import DTypeJSON, DTypeSpec_V2, DTypeSpec_V3 # This the upper bound for the scalar types we support. It's numpy scalars + str, # because the new variable-length string dtype in numpy does not have a corresponding scalar type type TBaseScalar = np.generic | str | bytes # This is the bound for the dtypes that we support. If we support non-numpy dtypes, # then this bound will need to be widened. type TBaseDType = np.dtype[np.generic] @dataclass(frozen=True, kw_only=True, slots=True) class ZDType[DType: TBaseDType, Scalar: TBaseScalar](ABC): """ Abstract base class for wrapping native array data types, e.g. numpy dtypes Attributes ---------- dtype_cls : ClassVar[type[TDType]] The wrapped dtype class. This is a class variable. _zarr_v3_name : ClassVar[str] The name given to the data type by a Zarr v3 data type specification. This is a class variable, and it should generally be unique across different data types. """ # this class will create a native data type dtype_cls: ClassVar[type[TBaseDType]] _zarr_v3_name: ClassVar[str] @classmethod def _check_native_dtype(cls: type[Self], dtype: TBaseDType) -> TypeGuard[DType]: """ Check that a native data type matches the dtype_cls class attribute. Used as a type guard. Parameters ---------- dtype : TDType The dtype to check. Returns ------- Bool True if the dtype matches, False otherwise. """ return type(dtype) is cls.dtype_cls @classmethod @abstractmethod def from_native_dtype(cls: type[Self], dtype: TBaseDType) -> Self: """ Create a ZDType instance from a native data type. This method is used when taking a user-provided native data type, like a NumPy data type, and creating the corresponding ZDType instance from them. Parameters ---------- dtype : TDType The native data type object to wrap. Returns ------- Self The ZDType that wraps the native data type. Raises ------ TypeError If the native data type is not consistent with the wrapped data type. """ raise NotImplementedError # pragma: no cover @abstractmethod def to_native_dtype(self: Self) -> DType: """ Return an instance of the wrapped data type. This operation inverts ``from_native_dtype``. Returns ------- TDType The native data type wrapped by this ZDType. """ raise NotImplementedError # pragma: no cover @classmethod @abstractmethod def _from_json_v2(cls: type[Self], data: DTypeJSON) -> Self: raise NotImplementedError # pragma: no cover @classmethod @abstractmethod def _from_json_v3(cls: type[Self], data: DTypeJSON) -> Self: raise NotImplementedError # pragma: no cover @classmethod def from_json(cls: type[Self], data: DTypeJSON, *, zarr_format: ZarrFormat) -> Self: """ Create an instance of this ZDType from JSON data. Parameters ---------- data : DTypeJSON The JSON representation of the data type. zarr_format : ZarrFormat The zarr format version. Returns ------- Self An instance of this data type. """ if zarr_format == 2: return cls._from_json_v2(data) if zarr_format == 3: return cls._from_json_v3(data) raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover @classmethod def _from_json_resolved(cls: type[Self], data: DTypeJSON, *, resolver: Resolver) -> Self: """ Create an instance of this ZDType from JSON data at `resolver`'s location. A data type registry creates every data type with this method. A data type that contains other data types, such as a structured data type, overrides it to resolve each of them with `resolver.at(...).resolve_data_type(...)`, so that they come from the same context. Other data types ignore the resolver: the default is `from_json` in its Zarr format. """ return cls.from_json(data, zarr_format=resolver.zarr_format) @overload def to_json(self, zarr_format: Literal[2]) -> DTypeSpec_V2: ... @overload def to_json(self, zarr_format: Literal[3]) -> DTypeSpec_V3: ... @abstractmethod def to_json(self, zarr_format: ZarrFormat) -> DTypeSpec_V2 | DTypeSpec_V3: """ Serialize this ZDType to JSON. Parameters ---------- zarr_format : ZarrFormat The zarr format version. Returns ------- DTypeJSON_V2 | DTypeJSON_V3 The JSON-serializable representation of the wrapped data type """ raise NotImplementedError # pragma: no cover @abstractmethod def _check_scalar(self, data: object) -> bool: """ Check that a python object is a valid scalar value for the wrapped data type. Parameters ---------- data : object A value to check. Returns ------- Bool True if the object is valid, False otherwise. """ raise NotImplementedError # pragma: no cover @abstractmethod def cast_scalar(self, data: object) -> Scalar: """ Cast a python object to the wrapped scalar type. The type of the provided scalar is first checked for compatibility. If it's incompatible with the associated scalar type, a ``TypeError`` will be raised. Parameters ---------- data : object The python object to cast. Returns ------- TScalar The cast value. """ raise NotImplementedError # pragma: no cover @abstractmethod def default_scalar(self) -> Scalar: """ Get the default scalar value for the wrapped data type. This is a method, rather than an attribute, because the default value for some data types depends on parameters that are not known until a concrete data type is wrapped. For example, data types parametrized by a length like fixed-length strings or bytes will generate scalars consistent with that length. Returns ------- TScalar The default value for this data type. """ raise NotImplementedError # pragma: no cover @abstractmethod def from_json_scalar(self: Self, data: JSON, *, zarr_format: ZarrFormat) -> Scalar: """ Read a JSON-serializable value as a scalar. Parameters ---------- data : JSON A JSON representation of a scalar value. zarr_format : ZarrFormat The zarr format version. This is specified because the JSON serialization of scalars differs between Zarr V2 and Zarr V3. Returns ------- TScalar The deserialized scalar value. """ raise NotImplementedError # pragma: no cover @abstractmethod def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> JSON: """ Serialize a python object to the JSON representation of a scalar. The value will first be cast to the scalar type associated with this ZDType, then serialized to JSON. Parameters ---------- data : object The value to convert. zarr_format : ZarrFormat The zarr format version. This is specified because the JSON serialization of scalars differs between Zarr V2 and Zarr V3. Returns ------- JSON The JSON-serialized scalar. """ raise NotImplementedError # pragma: no cover def scalar_failed_type_check_msg( cls_instance: ZDType[TBaseDType, TBaseScalar], bad_scalar: object ) -> str: """ Generate an error message reporting that a particular value failed a type check when attempting to cast that value to a scalar. """ return ( f"The value {bad_scalar!r} failed a type check. " f"It cannot be safely cast to a scalar compatible with {cls_instance}. " f"Consult the documentation for {cls_instance} to determine the possible values that can " "be cast to scalars of the wrapped data type." ) zarr-developers-zarr-python-a2f2415/src/zarr/core/group.py000066400000000000000000004531161527014137600237020ustar00rootroot00000000000000from __future__ import annotations import asyncio import logging import unicodedata import warnings from collections import defaultdict from dataclasses import asdict, dataclass, field, replace from itertools import accumulate from typing import TYPE_CHECKING, Final, Literal, assert_never, cast, overload import numpy as np import zarr.api.asynchronous as async_api from zarr.abc.metadata import Metadata from zarr.abc.store import Store, set_or_delete from zarr.core._info import GroupInfo from zarr.core._json import buffer_to_json_object, json_to_buffer from zarr.core.array import ( DEFAULT_FILL_VALUE, Array, AsyncArray, CompressorLike, CompressorsLike, FiltersLike, SerializerLike, ShardsLike, _parse_deprecated_compressor, create_array, ) from zarr.core.array_spec import parse_array_config from zarr.core.attributes import Attributes from zarr.core.buffer import default_buffer_prototype from zarr.core.common import ( JSON, ZARR_JSON, ZARRAY_JSON, ZATTRS_JSON, ZGROUP_JSON, ZMETADATA_V2_JSON, ChunksLike, DimensionNamesLike, NodeType, ShapeLike, ZarrFormat, parse_shapelike, ) from zarr.core.config import config from zarr.core.dtype import parse_data_type from zarr.core.json_parse import parse_field from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.core.metadata.io import ( encode_documents, save_metadata, save_new_metadata, store_documents, ) from zarr.core.metadata.v3 import AllowedExtraField, check_storable, parse_extra_fields from zarr.core.sync import SyncMixin, sync from zarr.errors import ( ArrayNotFoundError, ContainsArrayError, ContainsGroupError, GroupNotFoundError, MetadataValidationError, NodeTypeValidationError, ZarrUserWarning, ) from zarr.storage import StoreLike, StorePath from zarr.storage._common import make_store_path from zarr.storage._utils import _join_paths, _normalize_path_keys, normalize_path if TYPE_CHECKING: from collections.abc import ( AsyncGenerator, AsyncIterator, Coroutine, Generator, Iterable, Iterator, Mapping, ) from typing import Any from zarr.core.array_spec import ArrayConfigLike from zarr.core.buffer import Buffer, BufferPrototype from zarr.core.chunk_key_encodings import ChunkKeyEncodingLike from zarr.core.common import MemoryOrder from zarr.core.dtype import ZDTypeLike from zarr.types import AnyArray, AnyAsyncArray, ArrayV2, ArrayV3, AsyncArrayV2, AsyncArrayV3 logger = logging.getLogger("zarr.group") def parse_zarr_format(data: Any) -> ZarrFormat: """Parse the zarr_format field from metadata.""" return cast("ZarrFormat", parse_field(data, Literal[2, 3], "zarr_format")) def parse_node_type(data: Any) -> NodeType: """Parse the node_type field from metadata.""" return cast( "Literal['array', 'group']", parse_field(data, Literal["array", "group"], "node_type", error=MetadataValidationError), ) # todo: convert None to empty dict def parse_attributes(data: Any) -> dict[str, Any]: """Parse the attributes field from metadata.""" if data is None: return {} elif isinstance(data, dict) and all(isinstance(k, str) for k in data): return dict(data) msg = f"Expected dict with string keys. Got {type(data)} instead." raise TypeError(msg) @overload def _parse_async_node(node: AsyncArrayV3) -> ArrayV3: ... @overload def _parse_async_node(node: AsyncArrayV2) -> ArrayV2: ... @overload def _parse_async_node(node: AsyncGroup) -> Group: ... def _parse_async_node( node: AnyAsyncArray | AsyncGroup, ) -> AnyArray | Group: """Wrap an AsyncArray in an Array, or an AsyncGroup in a Group.""" if isinstance(node, AsyncArray): return Array(node) elif isinstance(node, AsyncGroup): return Group(node) else: raise TypeError(f"Unknown node type, got {type(node)}") @dataclass(frozen=True) class ConsolidatedMetadata: """ Consolidated Metadata for this Group. This stores the metadata of child nodes below this group. Any child groups will have their consolidated metadata set appropriately. """ metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] kind: Literal["inline"] = "inline" must_understand: Literal[False] = False def to_dict(self) -> dict[str, JSON]: """The consolidated metadata document. An array read from a stored document that had to be repaired is written as that document was stored, so every reader of the consolidated metadata reads it as repaired again (see `mark_repaired`).""" return { "kind": self.kind, "must_understand": self.must_understand, "metadata": { k: v.to_dict() if isinstance(v, GroupMetadata) or v._stored_document is None else dict(v._stored_document) for k, v in sorted( self.flattened_metadata.items(), key=lambda item: ( item[0].count("/"), unicodedata.normalize("NFKC", item[0]).casefold(), ), ) }, } @classmethod def from_dict(cls, data: dict[str, JSON], *, path: str | None = None) -> ConsolidatedMetadata: """Read consolidated metadata, naming each member by its path under the group at `path` (or relative to that group, when `path` is not given) in warnings.""" data = dict(data) kind = data.get("kind") if kind != "inline": raise ValueError(f"Consolidated metadata kind='{kind}' is not supported.") raw_metadata = data.get("metadata") if not isinstance(raw_metadata, dict): raise TypeError(f"Unexpected type for 'metadata': {type(raw_metadata)}") metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} if raw_metadata: for k, v in raw_metadata.items(): member = k if path is None else _join_paths([path, k]) if not isinstance(v, dict): raise TypeError( f"Invalid value for metadata items. key='{k}', type='{type(v).__name__}'" ) # zarr_format is present in v2 and v3. zarr_format = parse_zarr_format(v["zarr_format"]) if zarr_format == 3: node_type = parse_node_type(v.get("node_type", None)) if node_type == "group": metadata[k] = GroupMetadata.from_dict(v, path=member) elif node_type == "array": metadata[k] = ArrayV3Metadata.from_dict(v, path=member) else: assert_never(node_type) elif zarr_format == 2: if "shape" in v: metadata[k] = ArrayV2Metadata.from_dict(v, path=member) else: metadata[k] = GroupMetadata.from_dict(v, path=member) else: assert_never(zarr_format) cls._flat_to_nested(metadata) return cls(metadata=metadata) @staticmethod def _flat_to_nested( metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], ) -> None: """ Convert a flat metadata representation to a nested one. Notes ----- Flat metadata is used when persisting the consolidated metadata. The keys include the full path, not just the node name. The key prefixes can be used to determine which nodes are children of which other nodes. Nested metadata is used in-memory. The outermost level will only have the *immediate* children of the Group. All nested child groups will be stored under the consolidated metadata of their immediate parent. """ # We have a flat mapping from {k: v} where the keys include the *full* # path segment: # { # "/a/b": { group_metadata }, # "/a/b/array-0": { array_metadata }, # "/a/b/array-1": { array_metadata }, # } # # We want to reorganize the metadata such that each Group contains the # array metadata of its immediate children. # In the example, the group at `/a/b` will have consolidated metadata # for its children `array-0` and `array-1`. # Group keys by their parent path. This must not rely on same-parent keys # being adjacent: the persisted key order is arbitrary, so accumulate # instead of using itertools.groupby, which only groups consecutive runs. grouped: dict[str, list[str]] = defaultdict(list) for k in sorted(metadata, key=lambda k: k.count("/")): grouped[k.rsplit("/", 1)[0]].append(k) # we go top down and directly manipulate metadata. for key, children_keys in grouped.items(): # key is a key like "a", "a/b", "a/b/c" # The basic idea is to find the immediate parent (so "", "a", or "a/b") # and update that node's consolidated metadata to include the metadata # in children_keys *prefixes, name = key.split("/") parent = metadata while prefixes: # e.g. a/b/c has a parent "a/b". Walk through to get # metadata["a"]["b"] part = prefixes.pop(0) # we can assume that parent[part] here is a group # otherwise we wouldn't have a node with this `part` prefix. # We can also assume that the parent node will have consolidated metadata, # because we're walking top to bottom. parent = parent[part].consolidated_metadata.metadata # type: ignore[union-attr] node = parent[name] children_keys = list(children_keys) if isinstance(node, ArrayV2Metadata | ArrayV3Metadata): # These are already present, either thanks to being an array in the # root, or by being collected as a child in the else clause continue children: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} # We pop from metadata, since we're *moving* this under group. # While doing this, normalize leaf groups to carry empty consolidated metadata. for child_key in children_keys: if child_key == key: continue child = metadata.pop(child_key) if isinstance(child, GroupMetadata) and child.consolidated_metadata is None: child = replace(child, consolidated_metadata=ConsolidatedMetadata(metadata={})) children[child_key.split("/")[-1]] = child parent[name] = replace( node, consolidated_metadata=ConsolidatedMetadata(metadata=children) ) @property def flattened_metadata(self) -> dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata]: """ Return the flattened representation of Consolidated Metadata. The returned dictionary will have a key for each child node in the hierarchy under this group. Under the default (nested) representation available through ``self.metadata``, the dictionary only contains keys for immediate children. The keys of the dictionary will include the full path to a child node from the current group, where segments are joined by ``/``. Examples -------- ```python from zarr.core.group import ConsolidatedMetadata, GroupMetadata cm = ConsolidatedMetadata( metadata={ "group-0": GroupMetadata( consolidated_metadata=ConsolidatedMetadata( { "group-0-0": GroupMetadata(), } ) ), "group-1": GroupMetadata(), } ) # {'group-0': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group', extra_fields={}), # 'group-0/group-0-0': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group', extra_fields={}), # 'group-1': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group', extra_fields={})} ``` """ metadata = {} def flatten( key: str, group: GroupMetadata | ArrayV2Metadata | ArrayV3Metadata ) -> dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata]: children: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} if isinstance(group, ArrayV2Metadata | ArrayV3Metadata): children[key] = group else: if group.consolidated_metadata and group.consolidated_metadata.metadata is not None: children[key] = replace( group, consolidated_metadata=ConsolidatedMetadata(metadata={}) ) for name, val in group.consolidated_metadata.metadata.items(): full_key = f"{key}/{name}" if isinstance(val, GroupMetadata): children.update(flatten(full_key, val)) else: children[full_key] = val else: children[key] = replace(group, consolidated_metadata=None) return children for k, v in self.metadata.items(): metadata.update(flatten(k, v)) return metadata GROUP_METADATA_KEYS: Final[set[str]] = { "zarr_format", "node_type", "attributes", "consolidated_metadata", } """ The names of the fields of the group metadata document. `consolidated_metadata` is not in the Zarr V3 spec; Zarr-Python writes it. """ @dataclass(frozen=True) class GroupMetadata(Metadata): """ Metadata for a Group. """ attributes: dict[str, Any] = field(default_factory=dict) zarr_format: ZarrFormat = 3 consolidated_metadata: ConsolidatedMetadata | None = None node_type: Literal["group"] = field(default="group", init=False) extra_fields: dict[str, AllowedExtraField] = field(default_factory=dict) def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: if self.consolidated_metadata is not None: for path, member in self.consolidated_metadata.flattened_metadata.items(): # A member read from a document that had to be repaired is stored as it # was stored (see `ConsolidatedMetadata.to_dict`). if isinstance(member, ArrayV3Metadata) and member._stored_document is None: try: check_storable(member) except ValueError as e: e.add_note(f"Array {path!r} in the consolidated metadata.") raise indent = config.get("json_indent") if self.zarr_format == 3: return {ZARR_JSON: json_to_buffer(self.to_dict(), prototype=prototype, indent=indent)} else: items = { ZGROUP_JSON: json_to_buffer( {"zarr_format": self.zarr_format}, prototype=prototype, indent=indent ), ZATTRS_JSON: json_to_buffer(self.attributes, prototype=prototype, indent=indent), } if self.consolidated_metadata: d = { ZGROUP_JSON: {"zarr_format": self.zarr_format}, ZATTRS_JSON: self.attributes, } consolidated_metadata = self.consolidated_metadata.to_dict()["metadata"] if not isinstance(consolidated_metadata, dict): raise TypeError( "Expected consolidated metadata to serialize to a dict, " f"got {type(consolidated_metadata).__name__}." ) for k, v in consolidated_metadata.items(): attrs = v.pop("attributes", {}) d[f"{k}/{ZATTRS_JSON}"] = attrs if "shape" in v: # it's an array d[f"{k}/{ZARRAY_JSON}"] = v else: d[f"{k}/{ZGROUP_JSON}"] = { "zarr_format": self.zarr_format, "consolidated_metadata": { "metadata": {}, "must_understand": False, "kind": "inline", }, } # The consolidated metadata blob is written compactly (no indent). items[ZMETADATA_V2_JSON] = json_to_buffer( {"metadata": d, "zarr_consolidated_format": 1}, prototype=prototype ) return items def __init__( self, attributes: dict[str, Any] | None = None, zarr_format: ZarrFormat = 3, consolidated_metadata: ConsolidatedMetadata | None = None, extra_fields: Mapping[str, AllowedExtraField] | None = None, ) -> None: attributes_parsed = parse_attributes(attributes) zarr_format_parsed = parse_zarr_format(zarr_format) if zarr_format_parsed == 2 and extra_fields: raise ValueError( "Invalid extra fields. Zarr format 2 group metadata does not support extra fields." ) extra_fields_parsed = parse_extra_fields( extra_fields, reserved_keys=GROUP_METADATA_KEYS, node_type="group" ) object.__setattr__(self, "attributes", attributes_parsed) object.__setattr__(self, "zarr_format", zarr_format_parsed) object.__setattr__(self, "consolidated_metadata", consolidated_metadata) object.__setattr__(self, "extra_fields", extra_fields_parsed) @classmethod def from_dict(cls, data: dict[str, Any], *, path: str | None = None) -> GroupMetadata: """Read a stored group document; `path` names the group in warnings about the consolidated metadata it holds.""" data = dict(data) node_type = data.pop("node_type", None) if node_type not in ("group", None): raise NodeTypeValidationError( f"Invalid value for 'node_type'. Expected 'group' or None. Got {node_type!r}." ) consolidated_metadata = data.pop("consolidated_metadata", None) if consolidated_metadata: data["consolidated_metadata"] = ConsolidatedMetadata.from_dict( consolidated_metadata, path=path ) zarr_format = data.get("zarr_format") if zarr_format == 3: # Zarr v3 allows an extra key only if it is an object with "must_understand": false. extra_fields = parse_extra_fields( {k: v for k, v in data.items() if k not in GROUP_METADATA_KEYS}, reserved_keys=GROUP_METADATA_KEYS, node_type="group", ) else: # Zarr v2 allowed arbitrary keys; they are dropped rather than kept. extra_fields = {} data = {k: v for k, v in data.items() if k in GROUP_METADATA_KEYS} return cls(**data, extra_fields=extra_fields) def to_dict(self) -> dict[str, Any]: result = asdict(replace(self, consolidated_metadata=None)) result.update(result.pop("extra_fields")) if self.consolidated_metadata is not None: result["consolidated_metadata"] = self.consolidated_metadata.to_dict() else: # Leave consolidated metadata unset if it's None result.pop("consolidated_metadata") return result @dataclass(frozen=True) class ImplicitGroupMarker(GroupMetadata): """ Marker for an implicit group. Instances of this class are only used in the context of group creation as a placeholder to represent groups that should only be created if they do not already exist in storage """ @dataclass(frozen=True) class AsyncGroup: """ Asynchronous Group object. """ metadata: GroupMetadata store_path: StorePath # TODO: make this correct and work # TODO: ensure that this can be bound properly to subclass of AsyncGroup @classmethod async def from_store( cls, store: StoreLike, *, attributes: dict[str, Any] | None = None, overwrite: bool = False, zarr_format: ZarrFormat = 3, ) -> AsyncGroup: store_path = await make_store_path(store) attributes = attributes or {} group = cls( metadata=GroupMetadata(attributes=attributes, zarr_format=zarr_format), store_path=store_path, ) await save_new_metadata( store_path, group.metadata, overwrite=overwrite, ensure_parents=True ) return group @classmethod async def open( cls, store: StoreLike, zarr_format: ZarrFormat | None = 3, use_consolidated: bool | str | None = None, ) -> AsyncGroup: """Open a new AsyncGroup Parameters ---------- store : StoreLike zarr_format : {2, 3}, optional use_consolidated : bool or str, default None Whether to use consolidated metadata. By default, consolidated metadata is used if it's present in the store (in the ``zarr.json`` for Zarr format 3 and in the ``.zmetadata`` file for Zarr format 2) and the Store supports it. To explicitly require consolidated metadata, set ``use_consolidated=True``. In this case, if the Store doesn't support consolidation or consolidated metadata is not found, a ``ValueError`` exception is raised. To explicitly *not* use consolidated metadata, set ``use_consolidated=False``, which will fall back to using the regular, non consolidated metadata. Zarr format 2 allowed configuring the key storing the consolidated metadata (``.zmetadata`` by default). Specify the custom key as ``use_consolidated`` to load consolidated metadata from a non-default key. """ store_path = await make_store_path(store) if not store_path.store.supports_consolidated_metadata: # Fail if consolidated metadata was requested but the Store doesn't support it if use_consolidated: store_name = type(store_path.store).__name__ raise ValueError( f"The Zarr store in use ({store_name}) doesn't support consolidated metadata." ) # if use_consolidated was None (optional), the Store dictates it doesn't want consolidation use_consolidated = False consolidated_key = ZMETADATA_V2_JSON if (zarr_format == 2 or zarr_format is None) and isinstance(use_consolidated, str): consolidated_key = use_consolidated if zarr_format == 2: paths = [store_path / ZGROUP_JSON, store_path / ZATTRS_JSON] if use_consolidated or use_consolidated is None: paths.append(store_path / consolidated_key) zgroup_bytes, zattrs_bytes, *rest = await asyncio.gather( *[path.get() for path in paths] ) if zgroup_bytes is None: raise FileNotFoundError(store_path) if use_consolidated or use_consolidated is None: maybe_consolidated_metadata_bytes = rest[0] else: maybe_consolidated_metadata_bytes = None elif zarr_format == 3: zarr_json_bytes = await (store_path / ZARR_JSON).get() if zarr_json_bytes is None: raise FileNotFoundError(store_path) elif zarr_format is None: ( zarr_json_bytes, zgroup_bytes, zattrs_bytes, maybe_consolidated_metadata_bytes, ) = await asyncio.gather( (store_path / ZARR_JSON).get(), (store_path / ZGROUP_JSON).get(), (store_path / ZATTRS_JSON).get(), (store_path / str(consolidated_key)).get(), ) if zarr_json_bytes is not None and zgroup_bytes is not None: # warn and favor v3 msg = f"Both zarr.json (Zarr format 3) and .zgroup (Zarr format 2) metadata objects exist at {store_path}. Zarr format 3 will be used." warnings.warn(msg, category=ZarrUserWarning, stacklevel=1) if zarr_json_bytes is None and zgroup_bytes is None: raise FileNotFoundError( f"could not find zarr.json or .zgroup objects in {store_path}" ) # set zarr_format based on which keys were found if zarr_json_bytes is not None: zarr_format = 3 else: zarr_format = 2 else: msg = f"Invalid value for 'zarr_format'. Expected 2, 3, or None. Got '{zarr_format}'." # type: ignore[unreachable] raise MetadataValidationError(msg) if zarr_format == 2: if zgroup_bytes is None: raise FileNotFoundError(store_path) if use_consolidated and maybe_consolidated_metadata_bytes is None: # the user requested consolidated metadata, but it was missing raise ValueError(consolidated_key) elif use_consolidated is False: # the user explicitly opted out of consolidated_metadata. # Discard anything we might have read. maybe_consolidated_metadata_bytes = None return cls._from_bytes_v2( store_path, zgroup_bytes, zattrs_bytes, maybe_consolidated_metadata_bytes ) else: # V3 groups are comprised of a zarr.json object if zarr_json_bytes is None: raise FileNotFoundError(store_path) if not isinstance(use_consolidated, bool | None): raise TypeError("use_consolidated must be a bool or None for Zarr format 3.") return cls._from_bytes_v3( store_path, zarr_json_bytes, use_consolidated=use_consolidated, ) @classmethod def _from_bytes_v2( cls, store_path: StorePath, zgroup_bytes: Buffer, zattrs_bytes: Buffer | None, consolidated_metadata_bytes: Buffer | None, ) -> AsyncGroup: # V2 groups are comprised of a .zgroup and .zattrs objects zgroup = buffer_to_json_object(zgroup_bytes) zattrs = buffer_to_json_object(zattrs_bytes) if zattrs_bytes is not None else {} group_metadata: dict[str, Any] = {**zgroup, "attributes": zattrs} if consolidated_metadata_bytes is not None: v2_consolidated_doc = buffer_to_json_object(consolidated_metadata_bytes) v2_consolidated_metadata = cast("dict[str, Any]", v2_consolidated_doc["metadata"]) # We already read zattrs and zgroup. Should we ignore these? v2_consolidated_metadata.pop(".zattrs", None) v2_consolidated_metadata.pop(".zgroup", None) consolidated_metadata: defaultdict[str, dict[str, Any]] = defaultdict(dict) # keys like air/.zarray, air/.zattrs for k, v in v2_consolidated_metadata.items(): path, kind = k.rsplit("/.", 1) if kind == "zarray": consolidated_metadata[path].update(v) elif kind == "zattrs": consolidated_metadata[path]["attributes"] = v elif kind == "zgroup": consolidated_metadata[path].update(v) else: raise ValueError(f"Invalid file type '{kind}' at path '{path}") group_metadata["consolidated_metadata"] = { "metadata": dict(consolidated_metadata), "kind": "inline", "must_understand": False, } return cls.from_dict(store_path, group_metadata) @classmethod def _from_bytes_v3( cls, store_path: StorePath, zarr_json_bytes: Buffer, use_consolidated: bool | None, ) -> AsyncGroup: group_metadata = buffer_to_json_object(zarr_json_bytes) if use_consolidated and group_metadata.get("consolidated_metadata") is None: msg = f"Consolidated metadata requested with 'use_consolidated=True' but not found in '{store_path.path}'." raise ValueError(msg) elif use_consolidated is False: # Drop consolidated metadata if it's there. group_metadata.pop("consolidated_metadata", None) return cls.from_dict(store_path, group_metadata) @classmethod def from_dict( cls, store_path: StorePath, data: dict[str, Any], ) -> AsyncGroup: node_type = data.pop("node_type", None) if node_type == "array": msg = f"An array already exists in store {store_path.store} at path {store_path.path}." raise ContainsArrayError(msg) elif node_type not in ("group", None): msg = f"Node type in metadata ({node_type}) is not 'group'" raise GroupNotFoundError(msg) return cls( metadata=GroupMetadata.from_dict(data, path=str(store_path)), store_path=store_path, ) async def setitem(self, key: str, value: Any) -> None: """ Fastpath for creating a new array New arrays will be created with default array settings for the array type. Parameters ---------- key : str Array name value : array-like Array data """ path = self.store_path / key await async_api.save_array( store=path, arr=value, zarr_format=self.metadata.zarr_format, overwrite=True ) async def getitem( self, key: str, ) -> AnyAsyncArray | AsyncGroup: """ Get a subarray or subgroup from the group. Parameters ---------- key : str Array or group name Returns ------- AsyncArray or AsyncGroup """ store_path = self.store_path / key logger.debug("key=%s, store_path=%s", key, store_path) # Consolidated metadata lets us avoid some I/O operations so try that first. if self.metadata.consolidated_metadata is not None: return self._getitem_consolidated(store_path, key, prefix=self.name) try: return await get_node( store=store_path.store, path=store_path.path, zarr_format=self.metadata.zarr_format ) except FileNotFoundError as e: raise KeyError(key) from e def _getitem_consolidated( self, store_path: StorePath, key: str, prefix: str ) -> AnyAsyncArray | AsyncGroup: # getitem, in the special case where we have consolidated metadata. # Note that this is a regular def (non async) function. # This shouldn't do any additional I/O. # we support nested getitems like group/subgroup/array indexers = normalize_path(key).split("/") indexers.reverse() metadata: ArrayV2Metadata | ArrayV3Metadata | GroupMetadata = self.metadata while indexers: indexer = indexers.pop() if isinstance(metadata, ArrayV2Metadata | ArrayV3Metadata): # we've indexed into an array with group["array/subarray"]. Invalid. raise KeyError(key) if metadata.consolidated_metadata is None: # we've indexed into a group without consolidated metadata. # This isn't normal; typically, consolidated metadata # will include explicit markers for when there are no child # nodes as metadata={}. # We have some freedom in exactly how we interpret this case. # For now, we treat None as the same as {}, i.e. we don't # have any children. raise KeyError(key) try: metadata = metadata.consolidated_metadata.metadata[indexer] except KeyError as e: # The Group Metadata has consolidated metadata, but the key # isn't present. We trust this to mean that the key isn't in # the hierarchy, and *don't* fall back to checking the store. msg = f"'{key}' not found in consolidated metadata." raise KeyError(msg) from e # update store_path to ensure that AsyncArray/Group.name is correct if prefix != "/": key = "/".join([prefix.lstrip("/"), key]) store_path = StorePath(store=store_path.store, path=key) if isinstance(metadata, GroupMetadata): return AsyncGroup(metadata=metadata, store_path=store_path) else: return AsyncArray(metadata=metadata, store_path=store_path) async def delitem(self, key: str) -> None: """Delete a group member. Parameters ---------- key : str Array or group name """ store_path = self.store_path / key consolidated = self.metadata.consolidated_metadata if consolidated is None: await store_path.delete_dir() return # Encode the group metadata without the member before deleting it: metadata that # cannot be stored then fails with the store and this group untouched. What is # stored is encoded after the deletion, from the metadata as it then is, so # concurrent deletions each store the deletions made before them. members = {name: node for name, node in consolidated.metadata.items() if name != key} encode_documents( self.store_path, replace(self.metadata, consolidated_metadata=replace(consolidated, metadata=members)), ) await store_path.delete_dir() # In place, so every handle sharing this consolidated metadata (a parent's or a # subgroup's) sees the deletion. consolidated.metadata.pop(key, None) await store_documents(self.store_path, encode_documents(self.store_path, self.metadata)) async def get[DefaultT]( self, key: str, default: DefaultT | None = None ) -> AnyAsyncArray | AsyncGroup | DefaultT | None: """Obtain a group member, returning default if not found. Parameters ---------- key : str Group member name. default : object Default value to return if key is not found (default: None). Returns ------- object Group member (AsyncArray or AsyncGroup) or default if not found. """ try: return await self.getitem(key) except KeyError: return default async def get_array(self, path: str) -> AnyAsyncArray: """Obtain an array member of this group, raising if it is absent or not an array. Parameters ---------- path : str Path of the array relative to this group. May contain `/` to reference a member of a subgroup, e.g. `subgroup/subarray`. Returns ------- AsyncArray The array at the given path. Raises ------ ArrayNotFoundError If no node exists at the given path. ContainsGroupError If the node at the given path is a group rather than an array. """ store_path = self.store_path / path try: node = await self.getitem(path) except KeyError as e: msg = f"No array found in store {store_path.store!r} at path {store_path.path!r}" raise ArrayNotFoundError(msg) from e if isinstance(node, AsyncGroup): msg = f"A group exists in store {store_path.store!r} at path {store_path.path!r}." raise ContainsGroupError(msg) return node async def get_group(self, path: str) -> AsyncGroup: """Obtain a group member of this group, raising if it is absent or not a group. Parameters ---------- path : str Path of the group relative to this group. May contain `/` to reference a member of a subgroup, e.g. `subgroup/subsubgroup`. Returns ------- AsyncGroup The group at the given path. Raises ------ GroupNotFoundError If no node exists at the given path. ContainsArrayError If the node at the given path is an array rather than a group. """ store_path = self.store_path / path try: node = await self.getitem(path) except KeyError as e: msg = f"No group found in store {store_path.store!r} at path {store_path.path!r}" raise GroupNotFoundError(msg) from e if isinstance(node, AsyncArray): msg = f"An array exists in store {store_path.store!r} at path {store_path.path!r}." raise ContainsArrayError(msg) return node async def _save_metadata(self, ensure_parents: bool = False) -> None: await save_metadata(self.store_path, self.metadata, ensure_parents=ensure_parents) @property def path(self) -> str: """Storage path.""" return self.store_path.path @property def name(self) -> str: """Group name following h5py convention.""" if self.path: # follow h5py convention: add leading slash name = self.path if name[0] != "/": name = f"/{name}" return name return "/" @property def basename(self) -> str: """Final component of name.""" return self.name.split("/")[-1] @property def attrs(self) -> dict[str, Any]: return self.metadata.attributes @property def info(self) -> Any: """ Return a visual representation of the statically known information about a group. Note that this doesn't include dynamic information, like the number of child Groups or Arrays. Returns ------- GroupInfo Related ------- [zarr.AsyncGroup.info_complete][] All information about a group, including dynamic information """ if self.metadata.consolidated_metadata: members = list(self.metadata.consolidated_metadata.flattened_metadata.values()) else: members = None return self._info(members=members) async def info_complete(self) -> Any: """ Return all the information for a group. This includes dynamic information like the number of child Groups or Arrays. If this group doesn't contain consolidated metadata then this will need to read from the backing Store. Returns ------- GroupInfo Related ------- [zarr.AsyncGroup.info][] """ members = [x[1].metadata async for x in self.members(max_depth=None)] return self._info(members=members) def _info( self, members: list[ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] | None = None ) -> Any: kwargs = {} if members is not None: kwargs["_count_members"] = len(members) count_arrays = 0 count_groups = 0 for member in members: if isinstance(member, GroupMetadata): count_groups += 1 else: count_arrays += 1 kwargs["_count_arrays"] = count_arrays kwargs["_count_groups"] = count_groups return GroupInfo( _name=self.store_path.path, _read_only=self.read_only, _store_type=type(self.store_path.store).__name__, _zarr_format=self.metadata.zarr_format, # maybe do a typeddict **kwargs, # type: ignore[arg-type] ) @property def store(self) -> Store: return self.store_path.store @property def read_only(self) -> bool: # Backwards compatibility for 2.x return self.store_path.read_only @property def synchronizer(self) -> None: # Backwards compatibility for 2.x # Not implemented in 3.x yet. return None async def create_group( self, name: str, *, overwrite: bool = False, attributes: dict[str, Any] | None = None, ) -> AsyncGroup: """Create a sub-group. Parameters ---------- name : str Group name. overwrite : bool, optional If True, do not raise an error if the group already exists. attributes : dict, optional Group attributes. Returns ------- g : AsyncGroup """ attributes = attributes or {} return await type(self).from_store( self.store_path / name, attributes=attributes, overwrite=overwrite, zarr_format=self.metadata.zarr_format, ) async def require_group(self, name: str, overwrite: bool = False) -> AsyncGroup: """Obtain a sub-group, creating one if it doesn't exist. Parameters ---------- name : str Group name. overwrite : bool, optional Overwrite any existing group with given `name` if present. Returns ------- g : AsyncGroup """ if overwrite: # TODO: check that overwrite=True errors if an array exists where the group is being created grp = await self.create_group(name, overwrite=True) else: try: item: AsyncGroup | AnyAsyncArray = await self.getitem(name) if not isinstance(item, AsyncGroup): raise TypeError( f"Incompatible object ({item.__class__.__name__}) already exists" ) grp = item except KeyError: grp = await self.create_group(name) return grp async def require_groups(self, *names: str) -> tuple[AsyncGroup, ...]: """Convenience method to require multiple groups in a single call. Parameters ---------- *names : str Group names. Returns ------- Tuple[AsyncGroup, ...] """ if not names: return () return tuple(await asyncio.gather(*(self.require_group(name) for name in names))) async def create_array( self, name: str, *, shape: ShapeLike | None = None, dtype: ZDTypeLike | None = None, data: np.ndarray[Any, np.dtype[Any]] | None = None, chunks: ChunksLike | Literal["auto"] = "auto", shards: ShardsLike | None = None, filters: FiltersLike = "auto", compressors: CompressorsLike = "auto", compressor: CompressorLike = "auto", serializer: SerializerLike = "auto", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfigLike | None = None, write_data: bool = True, ) -> AnyAsyncArray: """Create an array within this group. This method lightly wraps [zarr.core.array.create_array][]. Parameters ---------- name : str The name of the array relative to the group. If `path` is `None`, the array will be located at the root of the store. shape : ShapeLike, optional Shape of the array. Must be `None` if `data` is provided. dtype : ZDTypeLike | None Data type of the array. Must be `None` if `data` is provided. data : np.ndarray, optional Array-like data to use for initializing the array. If this parameter is provided, the `shape` and `dtype` parameters must be `None`. chunks : tuple[int, ...] | Sequence[Sequence[int]] | Literal["auto"], default="auto" Chunk shape of the array. If chunks is "auto", a chunk shape is guessed based on the shape of the array and the dtype. A nested list of per-dimension edge sizes creates a rectilinear grid. Rectilinear chunk grids are experimental and must be explicitly enabled with `zarr.config.set({'array.rectilinear_chunks': True})` while the feature is stabilizing. shards : tuple[int, ...], optional Shard shape of the array. The default value of `None` results in no sharding at all. filters : Iterable[Codec] | Literal["auto"], optional Iterable of filters to apply to each chunk of the array, in order, before serializing that chunk to bytes. For Zarr format 3, a "filter" is a codec that takes an array and returns an array, and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the order of your filters is consistent with the behavior of each filter. The default value of `"auto"` instructs Zarr to use a default based on the data type of the array and the Zarr format specified. For all data types in Zarr V3, and most data types in Zarr V2, the default filters are empty. The only cases where default filters are not empty is when the Zarr format is 2, and the data type is a variable-length data type like [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, the default filters contains a single element which is a codec specific to that particular data type. To create an array with no filters, provide an empty iterable or the value `None`. compressors : Iterable[Codec], optional List of compressors to apply to the array. Compressors are applied in order, and after any filters are applied (if any are specified) and the data is serialized into bytes. For Zarr format 3, a "compressor" is a codec that takes a bytestream, and returns another bytestream. These values must be instances of [`zarr.abc.codec.BytesBytesCodec`][], or dict representations of [`zarr.abc.codec.BytesBytesCodec`][]. Multiple compressors may be provided for Zarr format 3. Codecs that take an array and return bytes are serializers and must be supplied with `serializer` instead. If no `compressors` are provided, the default for Zarr format 3 is `(ZstdCodec(),)`. Use `None` to omit default compressors. For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may be provided for Zarr format 2. If no `compressor` is provided, a default compressor will be used. This default can be changed by modifying the value of `array.v2_default_compressor` in [`zarr.config`][zarr.config]. Use `None` to omit the default compressor. compressor : Codec, optional Deprecated in favor of `compressors`. serializer : dict[str, JSON] | ArrayBytesCodec, optional Array-to-bytes codec to use for encoding the array data. Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. Codecs that are instances of [`zarr.abc.codec.ArrayBytesCodec`][] must be supplied here, not with `compressors`. If no `serializer` is provided, a default serializer is chosen based on the data type: `VLenUTF8Codec` for variable-length strings, `VLenBytesCodec` for variable-length bytes, and `BytesCodec` otherwise. fill_value : Any, optional Fill value for the array. order : {"C", "F"}, optional The memory order of the array (default is "C"). For Zarr format 2, this parameter sets the memory order of the array. For Zarr format 3, this parameter is deprecated, because memory order is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. If no `order` is provided, a default order will be used. This default can be changed by modifying the value of `array.order` in [`zarr.config`][zarr.config]. attributes : dict, optional Attributes for the array. chunk_key_encoding : ChunkKeyEncodingLike, optional A specification of how the chunk keys are represented in storage. For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. dimension_names : Iterable[str], optional The names of the dimensions (default is None). Zarr format 3 only. Zarr format 2 arrays should not use this parameter. storage_options : dict, optional If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. overwrite : bool, default False Whether to overwrite an array with the same name in the store, if one exists. If `True`, any existing keys under that path are deleted first. config : ArrayConfigLike, optional Runtime configuration for the array. write_data : bool If a pre-existing array-like object was provided to this function via the `data` parameter then `write_data` determines whether the values in that array-like object should be written to the Zarr array created by this function. If `write_data` is `False`, then the array will be left empty. Returns ------- AsyncArray """ compressors = _parse_deprecated_compressor( compressor, compressors, zarr_format=self.metadata.zarr_format ) return await create_array( store=self.store_path, name=name, shape=shape, dtype=dtype, data=data, chunks=chunks, shards=shards, filters=filters, compressors=compressors, serializer=serializer, fill_value=fill_value, order=order, zarr_format=self.metadata.zarr_format, attributes=attributes, chunk_key_encoding=chunk_key_encoding, dimension_names=dimension_names, storage_options=storage_options, overwrite=overwrite, config=config, write_data=write_data, ) async def require_array( self, name: str, *, shape: ShapeLike, dtype: ZDTypeLike | None = None, exact: bool = False, config: ArrayConfigLike | None = None, **kwargs: Any, ) -> AnyAsyncArray: """Obtain an array, creating if it doesn't exist. Other `kwargs` are as per [zarr.AsyncGroup.create_array][]. Parameters ---------- name : str Array name. shape : int or tuple of ints Array shape. dtype : ZDTypeLike, optional The data type of the array, given as a string, a NumPy dtype, or a Zarr data type. exact : bool, optional If True, require `dtype` to match exactly. If false, require `dtype` can be cast from array dtype. config : ArrayConfigLike or None, default=None Runtime configuration for the array, whether it is created or already exists. Keys not specified are taken from the global configuration. Returns ------- a : AsyncArray """ try: ds = await self.getitem(name) if not isinstance(ds, AsyncArray): raise TypeError(f"Incompatible object ({ds.__class__.__name__}) already exists") shape = parse_shapelike(shape) if shape != ds.shape: raise TypeError(f"Incompatible shape ({ds.shape} vs {shape})") # `np.dtype(None)` used to resolve to float64 here; keep that default. dtype = parse_data_type( "float64" if dtype is None else dtype, zarr_format=self.metadata.zarr_format, ).to_native_dtype() if exact: if ds.dtype != dtype: raise TypeError(f"Incompatible dtype ({ds.dtype} vs {dtype})") else: if not np.can_cast(ds.dtype, dtype): raise TypeError(f"Incompatible dtype ({ds.dtype} vs {dtype})") except KeyError: return await self.create_array(name, shape=shape, dtype=dtype, config=config, **kwargs) # `config` is the runtime configuration of the returned array, not stored metadata, # so it applies to an existing array too. It is parsed the same way as on the create # branch: missing keys come from the global configuration, unknown keys raise. if config is not None: return ds.with_config(parse_array_config(config)) return ds async def update_attributes(self, new_attributes: dict[str, Any]) -> AsyncGroup: """Update group attributes. Parameters ---------- new_attributes : dict New attributes to set on the group. Returns ------- self : AsyncGroup """ self.metadata.attributes.update(new_attributes) # Write new metadata await self._save_metadata() return self def __repr__(self) -> str: return f"" async def nmembers( self, max_depth: int | None = 0, ) -> int: """Count the number of members in this group. Parameters ---------- max_depth : int, default 0 The maximum number of levels of the hierarchy to include. By default, (``max_depth=0``) only immediate children are included. Set ``max_depth=None`` to include all nodes, and some positive integer to consider children within that many levels of the root Group. Returns ------- count : int """ # check if we can use consolidated metadata, which requires that we have non-None # consolidated metadata at all points in the hierarchy. if self.metadata.consolidated_metadata is not None: if max_depth is not None and max_depth < 0: raise ValueError(f"max_depth must be None or >= 0. Got '{max_depth}' instead") if max_depth is None: return len(self.metadata.consolidated_metadata.flattened_metadata) else: return len( [ x for x in self.metadata.consolidated_metadata.flattened_metadata if x.count("/") <= max_depth ] ) # TODO: consider using aioitertools.builtins.sum for this # return await aioitertools.builtins.sum((1 async for _ in self.members()), start=0) n = 0 async for _ in self.members(max_depth=max_depth): n += 1 return n async def members( self, max_depth: int | None = 0, *, use_consolidated_for_children: bool = True, ) -> AsyncGenerator[ tuple[str, AnyAsyncArray | AsyncGroup], None, ]: """ Returns an AsyncGenerator over the arrays and groups contained in this group. This method requires that `store_path.store` supports directory listing. The results are not guaranteed to be ordered. Parameters ---------- max_depth : int, default 0 The maximum number of levels of the hierarchy to include. By default, (``max_depth=0``) only immediate children are included. Set ``max_depth=None`` to include all nodes, and some positive integer to consider children within that many levels of the root Group. use_consolidated_for_children : bool, default True Whether to use the consolidated metadata of child groups loaded from the store. Note that this only affects groups loaded from the store. If the current Group already has consolidated metadata, it will always be used. Returns ------- path: A string giving the path to the target, relative to the Group ``self``. value: AsyncArray or AsyncGroup The AsyncArray or AsyncGroup that is a child of ``self``. """ if max_depth is not None and max_depth < 0: raise ValueError(f"max_depth must be None or >= 0. Got '{max_depth}' instead") async for item in self._members( max_depth=max_depth, use_consolidated_for_children=use_consolidated_for_children ): yield item def _members_consolidated( self, max_depth: int | None, prefix: str = "" ) -> Generator[ tuple[str, AnyAsyncArray | AsyncGroup], None, ]: consolidated_metadata = self.metadata.consolidated_metadata do_recursion = max_depth is None or max_depth > 0 # we kind of just want the top-level keys. if consolidated_metadata is not None: for key in consolidated_metadata.metadata: obj = self._getitem_consolidated( self.store_path, key, prefix=self.name ) # Metadata -> Group/Array key = f"{prefix}/{key}".lstrip("/") yield key, obj if do_recursion and isinstance(obj, AsyncGroup): if max_depth is None: new_depth = None else: new_depth = max_depth - 1 yield from obj._members_consolidated(new_depth, prefix=key) async def _members( self, max_depth: int | None, *, use_consolidated_for_children: bool = True ) -> AsyncGenerator[tuple[str, AnyAsyncArray | AsyncGroup], None]: skip_keys: tuple[str, ...] if self.metadata.zarr_format == 2: skip_keys = (".zattrs", ".zgroup", ".zarray", ".zmetadata") elif self.metadata.zarr_format == 3: skip_keys = ("zarr.json",) else: raise ValueError(f"Unknown Zarr format: {self.metadata.zarr_format}") if self.metadata.consolidated_metadata is not None: members = self._members_consolidated(max_depth=max_depth) for member in members: yield member return if not self.store_path.store.supports_listing: msg = ( f"The store associated with this group ({type(self.store_path.store)}) " "does not support listing, " "specifically via the `list_dir` method. " "This function requires a store that supports listing." ) raise ValueError(msg) # enforce a concurrency limit by passing a semaphore to all the recursive functions semaphore = asyncio.Semaphore(config.get("async.concurrency")) async for member in _iter_members_deep( self, max_depth=max_depth, skip_keys=skip_keys, semaphore=semaphore, use_consolidated_for_children=use_consolidated_for_children, ): yield member async def create_hierarchy( self, nodes: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], *, overwrite: bool = False, ) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: """ Create a hierarchy of arrays or groups rooted at this group. This function will parse its input to ensure that the hierarchy is complete. Any implicit groups will be inserted as needed. For example, an input like ```{'a/b': GroupMetadata}``` will be parsed to ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}```. Explicitly specifying a root group, e.g. with ``nodes = {'': GroupMetadata()}`` is an error because this group instance is the root group. After input parsing, this function then creates all the nodes in the hierarchy concurrently. Arrays and Groups are yielded in the order they are created. This order is not stable and should not be relied on. Parameters ---------- nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, relative to the path of the group. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that all values must have the same ``zarr_format`` as the parent group -- it is an error to mix zarr versions in the same hierarchy. Leading "/" characters from keys will be removed. overwrite : bool Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is raised instead of overwriting an existing array or group. This function will not erase an existing group unless that group is explicitly named in ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a group already exists at path ``a``, then this function will leave the group at ``a`` as-is. Yields ------ tuple[str, AsyncArray | AsyncGroup]. """ # check that all the nodes have the same zarr_format as Self prefix = self.path nodes_parsed = {} for key, value in nodes.items(): if value.zarr_format != self.metadata.zarr_format: msg = ( "The zarr_format of the nodes must be the same as the parent group. " f"The node at {key} has zarr_format {value.zarr_format}, but the parent group" f" has zarr_format {self.metadata.zarr_format}." ) raise ValueError(msg) if normalize_path(key) == "": msg = ( "The input defines a root node, but a root node already exists, namely this Group instance." "It is an error to use this method to create a root node. " "Remove the root node from the input dict, or use a function like " "create_rooted_hierarchy to create a rooted hierarchy." ) raise ValueError(msg) else: nodes_parsed[_join_paths([prefix, key])] = value async for key, node in create_hierarchy( store=self.store, nodes=nodes_parsed, overwrite=overwrite, ): if prefix == "": out_key = key else: out_key = key.removeprefix(prefix + "/") yield out_key, node async def keys(self) -> AsyncGenerator[str, None]: """Iterate over member names.""" async for key, _ in self.members(): yield key async def contains(self, member: str) -> bool: """Check if a member exists in the group. Parameters ---------- member : str Member name. Returns ------- bool """ # TODO: this can be made more efficient. try: await self.getitem(member) except KeyError: return False else: return True async def groups(self) -> AsyncGenerator[tuple[str, AsyncGroup], None]: """Iterate over subgroups.""" async for name, value in self.members(): if isinstance(value, AsyncGroup): yield name, value async def group_keys(self) -> AsyncGenerator[str, None]: """Iterate over group names.""" async for key, _ in self.groups(): yield key async def group_values(self) -> AsyncGenerator[AsyncGroup, None]: """Iterate over group values.""" async for _, group in self.groups(): yield group async def arrays( self, ) -> AsyncGenerator[tuple[str, AnyAsyncArray], None]: """Iterate over arrays.""" async for key, value in self.members(): if isinstance(value, AsyncArray): yield key, value async def array_keys(self) -> AsyncGenerator[str, None]: """Iterate over array names.""" async for key, _ in self.arrays(): yield key async def array_values( self, ) -> AsyncGenerator[AnyAsyncArray, None]: """Iterate over array values.""" async for _, array in self.arrays(): yield array async def tree( self, expand: bool | None = None, level: int | None = None, *, max_nodes: int = 500, plain: bool = False, ) -> Any: """ Return a tree-like representation of a hierarchy. Parameters ---------- expand : bool, optional This keyword is not yet supported. A NotImplementedError is raised if it's used. level : int, optional The maximum depth below this Group to display in the tree. max_nodes : int Maximum number of nodes to display before truncating. Default is 500. plain : bool, optional If True, return a plain-text tree without ANSI styling. This is useful when the output will be consumed by an LLM or written to a file. Default is False. Returns ------- TreeRepr A pretty-printable object displaying the hierarchy. """ from zarr.core._tree import group_tree_async if expand is not None: raise NotImplementedError("'expand' is not yet implemented.") return await group_tree_async(self, max_depth=level, max_nodes=max_nodes, plain=plain) def _member_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat: """ The zarr format of a new array in this group, which is the format of the group. A group only lists members of its own format, so an array of another format would be written into the group without becoming a member of it. """ if zarr_format is not None and zarr_format != self.metadata.zarr_format: raise ValueError( f"Cannot create a zarr_format={zarr_format} array in a " f"zarr_format={self.metadata.zarr_format} group." ) return self.metadata.zarr_format async def empty(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: """Create an empty array with the specified shape in this Group. The contents will be filled with the array's fill value or zeros if no fill value is provided. Parameters ---------- name : str Name of the array. shape : int or tuple of int Shape of the empty array. **kwargs Keyword arguments passed to [zarr.api.asynchronous.create][]. Notes ----- The contents of an empty Zarr array are not defined. On attempting to retrieve data from an empty Zarr array, any values may be returned, and these are not guaranteed to be stable from one access to the next. """ return await async_api.empty( shape=shape, store=self.store_path, path=name, zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), **kwargs, ) async def zeros(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: """Create an array, with zero being used as the default value for uninitialized portions of the array. Parameters ---------- name : str Name of the array. shape : int or tuple of int Shape of the empty array. **kwargs Keyword arguments passed to [zarr.api.asynchronous.create][]. Returns ------- AsyncArray The new array. """ return await async_api.zeros( shape=shape, store=self.store_path, path=name, zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), **kwargs, ) async def ones(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: """Create an array, with one being used as the default value for uninitialized portions of the array. Parameters ---------- name : str Name of the array. shape : int or tuple of int Shape of the empty array. **kwargs Keyword arguments passed to [zarr.api.asynchronous.create][]. Returns ------- AsyncArray The new array. """ return await async_api.ones( shape=shape, store=self.store_path, path=name, zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), **kwargs, ) async def full( self, *, name: str, shape: tuple[int, ...], fill_value: Any | None, **kwargs: Any ) -> AnyAsyncArray: """Create an array, with "fill_value" being used as the default value for uninitialized portions of the array. Parameters ---------- name : str Name of the array. shape : int or tuple of int Shape of the empty array. fill_value : scalar Value to fill the array with. **kwargs Keyword arguments passed to [zarr.api.asynchronous.create][]. Returns ------- AsyncArray The new array. """ return await async_api.full( shape=shape, fill_value=fill_value, store=self.store_path, path=name, zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), **kwargs, ) async def empty_like( self, *, name: str, data: async_api.ArrayLike, **kwargs: Any ) -> AnyAsyncArray: """Create an empty sub-array like `data`. The contents will be filled with the array's fill value or zeros if no fill value is provided. Parameters ---------- name : str Name of the array. data : array-like The array to create an empty array like. **kwargs Keyword arguments passed to [zarr.api.asynchronous.create][]. Returns ------- AsyncArray The new array. """ return await async_api.empty_like( a=data, store=self.store_path, path=name, zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), **kwargs, ) async def zeros_like( self, *, name: str, data: async_api.ArrayLike, **kwargs: Any ) -> AnyAsyncArray: """Create a sub-array of zeros like `data`. Parameters ---------- name : str Name of the array. data : array-like The array to create the new array like. **kwargs Keyword arguments passed to [zarr.api.asynchronous.create][]. Returns ------- AsyncArray The new array. """ return await async_api.zeros_like( a=data, store=self.store_path, path=name, zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), **kwargs, ) async def ones_like( self, *, name: str, data: async_api.ArrayLike, **kwargs: Any ) -> AnyAsyncArray: """Create a sub-array of ones like `data`. Parameters ---------- name : str Name of the array. data : array-like The array to create the new array like. **kwargs Keyword arguments passed to [zarr.api.asynchronous.create][]. Returns ------- AsyncArray The new array. """ return await async_api.ones_like( a=data, store=self.store_path, path=name, zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), **kwargs, ) async def full_like( self, *, name: str, data: async_api.ArrayLike, **kwargs: Any ) -> AnyAsyncArray: """Create a sub-array like `data` filled with the `fill_value` of `data` . Parameters ---------- name : str Name of the array. data : array-like The array to create the new array like. **kwargs Keyword arguments passed to [zarr.api.asynchronous.create][]. Returns ------- AsyncArray The new array. """ return await async_api.full_like( a=data, store=self.store_path, path=name, zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), **kwargs, ) async def move(self, source: str, dest: str) -> None: """Move a sub-group or sub-array from one path to another. Notes ----- Not implemented """ raise NotImplementedError @dataclass(frozen=True) class Group(SyncMixin): """ A Zarr group. """ _async_group: AsyncGroup @classmethod def from_store( cls, store: StoreLike, *, attributes: dict[str, Any] | None = None, zarr_format: ZarrFormat = 3, overwrite: bool = False, ) -> Group: """Instantiate a group from an initialized store. Parameters ---------- store : StoreLike StoreLike containing the Group. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. attributes : dict, optional A dictionary of JSON-serializable values with user-defined attributes. zarr_format : {2, 3}, optional Zarr storage format version. overwrite : bool, optional If True, do not raise an error if the group already exists. Returns ------- Group Group instantiated from the store. Raises ------ ContainsArrayError, ContainsGroupError, ContainsArrayAndGroupError """ attributes = attributes or {} obj = sync( AsyncGroup.from_store( store, attributes=attributes, overwrite=overwrite, zarr_format=zarr_format, ), ) return cls(obj) @classmethod def open( cls, store: StoreLike, zarr_format: ZarrFormat | None = 3, ) -> Group: """Open a group from an initialized store. Parameters ---------- store : StoreLike Store containing the Group. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. zarr_format : {2, 3, None}, optional Zarr storage format version. Returns ------- Group Group instantiated from the store. """ obj = sync(AsyncGroup.open(store, zarr_format=zarr_format)) return cls(obj) def __getitem__(self, path: str) -> AnyArray | Group: """Obtain a group member. Parameters ---------- path : str Group member name. Returns ------- Array | Group Group member (Array or Group) at the specified key Examples -------- ```python import zarr from zarr.core.group import Group group = Group.from_store(zarr.storage.MemoryStore()) group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") group.create_group(name="subgroup").create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") group["subarray"] # group["subgroup"] # group["subgroup"]["subarray"] # ``` """ obj = self._sync(self._async_group.getitem(path)) if isinstance(obj, AsyncArray): return Array(obj) else: return Group(obj) def get[DefaultT]( self, path: str, default: DefaultT | None = None ) -> AnyArray | Group | DefaultT | None: """Obtain a group member, returning default if not found. Parameters ---------- path : str Group member name. default : object Default value to return if key is not found (default: None). Returns ------- object Group member (Array or Group) or default if not found. Examples -------- ```python import zarr from zarr.core.group import Group group = Group.from_store(zarr.storage.MemoryStore()) group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") group.create_group(name="subgroup") group.get("subarray") # group.get("subgroup") # group.get("nonexistent", None) # None ``` """ try: return self[path] except KeyError: return default def get_array(self, path: str) -> AnyArray: """Obtain an array member of this group, raising if it is absent or not an array. Parameters ---------- path : str Path of the array relative to this group. May contain `/` to reference a member of a subgroup, e.g. `subgroup/subarray`. Returns ------- Array The array at the given path. Raises ------ ArrayNotFoundError If no node exists at the given path. ContainsGroupError If the node at the given path is a group rather than an array. Examples -------- ```python import zarr from zarr.core.group import Group group = Group.from_store(zarr.storage.MemoryStore()) group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") group.get_array("subarray") # ``` """ return Array(self._sync(self._async_group.get_array(path))) def get_group(self, path: str) -> Group: """Obtain a group member of this group, raising if it is absent or not a group. Parameters ---------- path : str Path of the group relative to this group. May contain `/` to reference a member of a subgroup, e.g. `subgroup/subsubgroup`. Returns ------- Group The group at the given path. Raises ------ GroupNotFoundError If no node exists at the given path. ContainsArrayError If the node at the given path is an array rather than a group. Examples -------- ```python import zarr from zarr.core.group import Group group = Group.from_store(zarr.storage.MemoryStore()) group.create_group(name="subgroup") group.get_group("subgroup") # ``` """ return Group(self._sync(self._async_group.get_group(path))) def __delitem__(self, key: str) -> None: """Delete a group member. Parameters ---------- key : str Group member name. Examples -------- >>> import zarr >>> group = Group.from_store(zarr.storage.MemoryStore()) >>> a = group.create_array(name="subarray", dtype="i1", shape=(10,), chunks=(10,)) >>> del group["subarray"] >>> "subarray" in group False """ self._sync(self._async_group.delitem(key)) def __iter__(self) -> Iterator[str]: """Return an iterator over group member names. Examples -------- >>> import zarr >>> g1 = zarr.group() >>> g2 = g1.create_group('foo') >>> g3 = g1.create_group('bar') >>> d1 = g1.create_array('baz', dtype="i1", shape=(10,), chunks=(10,)) >>> d2 = g1.create_array('quux', dtype="i1", shape=(10,), chunks=(10,)) >>> sorted(g1) ['bar', 'baz', 'foo', 'quux'] """ yield from self.keys() def __len__(self) -> int: """Number of members.""" return self.nmembers() def __setitem__(self, key: str, value: Any) -> None: """Fastpath for creating a new array. New arrays will be created using default settings for the array type. If you need to create an array with custom settings, use the `create_array` method. Parameters ---------- key : str Array name. value : Any Array data. Examples -------- >>> import numpy as np >>> import zarr >>> group = zarr.group() >>> group["foo"] = np.array(zarr.zeros((10,))) >>> group["foo"] """ self._sync(self._async_group.setitem(key, value)) def __repr__(self) -> str: return f"" async def update_attributes_async(self, new_attributes: dict[str, Any]) -> Group: """Update the attributes of this group. Examples -------- >>> async def example(): ... import zarr ... ... group = zarr.group() ... new_group = await group.update_attributes_async({"foo": "bar"}) ... return new_group.attrs.asdict() >>> import asyncio >>> asyncio.run(example()) {'foo': 'bar'} """ new_metadata = replace(self.metadata, attributes=new_attributes) # Write new metadata to_save = new_metadata.to_buffer_dict(default_buffer_prototype()) awaitables = [set_or_delete(self.store_path / key, value) for key, value in to_save.items()] await asyncio.gather(*awaitables) async_group = replace(self._async_group, metadata=new_metadata) return replace(self, _async_group=async_group) @property def store_path(self) -> StorePath: """Path-like interface for the Store.""" return self._async_group.store_path @property def metadata(self) -> GroupMetadata: """Group metadata.""" return self._async_group.metadata @property def path(self) -> str: """Storage path.""" return self._async_group.path @property def name(self) -> str: """Group name following h5py convention.""" return self._async_group.name @property def basename(self) -> str: """Final component of name.""" return self._async_group.basename @property def attrs(self) -> Attributes: """Attributes of this Group""" return Attributes(self) @property def info(self) -> Any: """ Return the statically known information for a group. Returns ------- GroupInfo Related ------- [zarr.Group.info_complete][] All information about a group, including dynamic information like the children members. """ return self._async_group.info def info_complete(self) -> Any: """ Return information for a group. If this group doesn't contain consolidated metadata then this will need to read from the backing Store. Returns ------- GroupInfo Related ------- [zarr.Group.info][] """ return self._sync(self._async_group.info_complete()) @property def store(self) -> Store: # Backwards compatibility for 2.x return self._async_group.store @property def read_only(self) -> bool: # Backwards compatibility for 2.x return self._async_group.read_only @property def synchronizer(self) -> None: # Backwards compatibility for 2.x # Not implemented in 3.x yet. return self._async_group.synchronizer def update_attributes(self, new_attributes: dict[str, Any]) -> Group: """Update the attributes of this group. Examples -------- >>> import zarr >>> group = zarr.group().update_attributes({"foo": "bar"}) >>> group.attrs.asdict() {'foo': 'bar'} """ self._sync(self._async_group.update_attributes(new_attributes)) return self def nmembers(self, max_depth: int | None = 0) -> int: """Count the number of members in this group. Parameters ---------- max_depth : int, default 0 The maximum number of levels of the hierarchy to include. By default, (``max_depth=0``) only immediate children are included. Set ``max_depth=None`` to include all nodes, and some positive integer to consider children within that many levels of the root Group. Returns ------- count : int """ return self._sync(self._async_group.nmembers(max_depth=max_depth)) def members( self, max_depth: int | None = 0, *, use_consolidated_for_children: bool = True ) -> tuple[tuple[str, AnyArray | Group], ...]: """ Returns an AsyncGenerator over the arrays and groups contained in this group. This method requires that `store_path.store` supports directory listing. The results are not guaranteed to be ordered. Parameters ---------- max_depth : int, default 0 The maximum number of levels of the hierarchy to include. By default, (``max_depth=0``) only immediate children are included. Set ``max_depth=None`` to include all nodes, and some positive integer to consider children within that many levels of the root Group. use_consolidated_for_children : bool, default True Whether to use the consolidated metadata of child groups loaded from the store. Note that this only affects groups loaded from the store. If the current Group already has consolidated metadata, it will always be used. Returns ------- path: A string giving the path to the target, relative to the Group ``self``. value: AsyncArray or AsyncGroup The AsyncArray or AsyncGroup that is a child of ``self``. """ _members = self._sync_iter(self._async_group.members(max_depth=max_depth)) return tuple((kv[0], _parse_async_node(kv[1])) for kv in _members) def create_hierarchy( self, nodes: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], *, overwrite: bool = False, ) -> Iterator[tuple[str, Group | AnyArray]]: """ Create a hierarchy of arrays or groups rooted at this group. This function will parse its input to ensure that the hierarchy is complete. Any implicit groups will be inserted as needed. For example, an input like ```{'a/b': GroupMetadata}``` will be parsed to ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}```. Explicitly specifying a root group, e.g. with ``nodes = {'': GroupMetadata()}`` is an error because this group instance is the root group. After input parsing, this function then creates all the nodes in the hierarchy concurrently. Arrays and Groups are yielded in the order they are created. This order is not stable and should not be relied on. Parameters ---------- nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, relative to the path of the group. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that all values must have the same ``zarr_format`` as the parent group -- it is an error to mix zarr versions in the same hierarchy. Leading "/" characters from keys will be removed. overwrite : bool Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is raised instead of overwriting an existing array or group. This function will not erase an existing group unless that group is explicitly named in ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a group already exists at path ``a``, then this function will leave the group at ``a`` as-is. Yields ------ tuple[str, Array | Group]. Examples -------- >>> import zarr >>> from zarr.core.group import GroupMetadata >>> root = zarr.create_group(store={}) >>> sorted(root.create_hierarchy({'a/b/c': GroupMetadata()})) [('a', ), ('a/b', ), ('a/b/c', )] """ for key, node in self._sync_iter( self._async_group.create_hierarchy(nodes, overwrite=overwrite) ): yield (key, _parse_async_node(node)) def keys(self) -> Generator[str]: """Return an iterator over group member names. Examples -------- >>> import zarr >>> g1 = zarr.group() >>> g2 = g1.create_group('foo') >>> g3 = g1.create_group('bar') >>> d1 = g1.create_array('baz', dtype="i1", shape=(10,), chunks=(10,)) >>> d2 = g1.create_array('quux', dtype="i1", shape=(10,), chunks=(10,)) >>> sorted(g1.keys()) ['bar', 'baz', 'foo', 'quux'] """ yield from self._sync_iter(self._async_group.keys()) def __contains__(self, member: str) -> bool: """Test for group membership. Examples -------- >>> import zarr >>> g1 = zarr.group() >>> g2 = g1.create_group('foo') >>> d1 = g1.create_array('bar', dtype="i1", shape=(10,), chunks=(10,)) >>> 'foo' in g1 True >>> 'bar' in g1 True >>> 'baz' in g1 False """ return self._sync(self._async_group.contains(member)) def groups(self) -> Generator[tuple[str, Group], None]: """Return the sub-groups of this group as a generator of (name, group) pairs. Examples -------- >>> import zarr >>> group = zarr.group() >>> subgroup = group.create_group("subgroup") >>> list(group.groups()) [('subgroup', )] """ for name, async_group in self._sync_iter(self._async_group.groups()): yield name, Group(async_group) def group_keys(self) -> Generator[str, None]: """Return an iterator over group member names. Examples -------- >>> import zarr >>> group = zarr.group() >>> subgroup = group.create_group("subgroup") >>> list(group.group_keys()) ['subgroup'] """ for name, _ in self.groups(): yield name def group_values(self) -> Generator[Group, None]: """Return an iterator over group members. Examples -------- >>> import zarr >>> group = zarr.group() >>> subgroup = group.create_group("subgroup") >>> list(group.group_values()) [] """ for _, group in self.groups(): yield group def arrays(self) -> Generator[tuple[str, AnyArray], None]: """Return the sub-arrays of this group as a generator of (name, array) pairs Examples -------- >>> import zarr >>> group = zarr.group() >>> subarray = group.create_array("subarray", dtype="i1", shape=(10,), chunks=(10,)) >>> list(group.arrays()) [('subarray', )] """ for name, async_array in self._sync_iter(self._async_group.arrays()): yield name, Array(async_array) def array_keys(self) -> Generator[str, None]: """Return an iterator over group member names. Examples -------- >>> import zarr >>> group = zarr.group() >>> subarray = group.create_array("subarray", dtype="i1", shape=(10,), chunks=(10,)) >>> list(group.array_keys()) ['subarray'] """ for name, _ in self.arrays(): yield name def array_values(self) -> Generator[AnyArray, None]: """Return an iterator over group members. Examples -------- >>> import zarr >>> group = zarr.group() >>> subarray = group.create_array("subarray", dtype="i1", shape=(10,), chunks=(10,)) >>> list(group.array_values()) [] """ for _, array in self.arrays(): yield array def tree( self, expand: bool | None = None, level: int | None = None, *, max_nodes: int = 500, plain: bool = False, ) -> Any: """ Return a tree-like representation of a hierarchy. Parameters ---------- expand : bool, optional This keyword is not yet supported. A NotImplementedError is raised if it's used. level : int, optional The maximum depth below this Group to display in the tree. max_nodes : int Maximum number of nodes to display before truncating. Default is 500. plain : bool, optional If True, return a plain-text tree without ANSI styling. This is useful when the output will be consumed by an LLM or written to a file. Default is False. Returns ------- TreeRepr A pretty-printable object displaying the hierarchy. """ return self._sync( self._async_group.tree(expand=expand, level=level, max_nodes=max_nodes, plain=plain) ) def create_group(self, name: str, **kwargs: Any) -> Group: """Create a sub-group. Parameters ---------- name : str Name of the new subgroup. Returns ------- Group Examples -------- >>> import zarr >>> group = zarr.group() >>> subgroup = group.create_group("subgroup") >>> subgroup """ return Group(self._sync(self._async_group.create_group(name, **kwargs))) def require_group(self, name: str, **kwargs: Any) -> Group: """Obtain a sub-group, creating one if it doesn't exist. Parameters ---------- name : str Group name. Returns ------- g : Group """ return Group(self._sync(self._async_group.require_group(name, **kwargs))) def require_groups(self, *names: str) -> tuple[Group, ...]: """Convenience method to require multiple groups in a single call. Parameters ---------- *names : str Group names. Returns ------- groups : tuple of Groups """ return tuple(map(Group, self._sync(self._async_group.require_groups(*names)))) def create( self, name: str, *, shape: ShapeLike | None = None, dtype: ZDTypeLike | None = None, data: np.ndarray[Any, np.dtype[Any]] | None = None, chunks: ChunksLike | Literal["auto"] = "auto", shards: ShardsLike | None = None, filters: FiltersLike = "auto", compressors: CompressorsLike = "auto", compressor: CompressorLike = "auto", serializer: SerializerLike = "auto", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfigLike | None = None, write_data: bool = True, ) -> AnyArray: """Create an array within this group. This method lightly wraps [`zarr.core.array.create_array`][]. Parameters ---------- name : str The name of the array relative to the group. If `path` is `None`, the array will be located at the root of the store. shape : ShapeLike, optional Shape of the array. Must be `None` if `data` is provided. dtype : ZDTypeLike | None Data type of the array. Must be `None` if `data` is provided. data : np.ndarray, optional Array-like data to use for initializing the array. If this parameter is provided, the `shape` and `dtype` parameters must be `None`. chunks : tuple[int, ...] | Sequence[Sequence[int]] | Literal["auto"], default="auto" Chunk shape of the array. If chunks is "auto", a chunk shape is guessed based on the shape of the array and the dtype. A nested list of per-dimension edge sizes creates a rectilinear grid. Rectilinear chunk grids are experimental and must be explicitly enabled with `zarr.config.set({'array.rectilinear_chunks': True})` while the feature is stabilizing. shards : tuple[int, ...], optional Shard shape of the array. The default value of `None` results in no sharding at all. filters : Iterable[Codec] | Literal["auto"], optional Iterable of filters to apply to each chunk of the array, in order, before serializing that chunk to bytes. For Zarr format 3, a "filter" is a codec that takes an array and returns an array, and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the order of your filters is consistent with the behavior of each filter. The default value of `"auto"` instructs Zarr to use a default based on the data type of the array and the Zarr format specified. For all data types in Zarr V3, and most data types in Zarr V2, the default filters are empty. The only cases where default filters are not empty is when the Zarr format is 2, and the data type is a variable-length data type like [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, the default filters contains a single element which is a codec specific to that particular data type. To create an array with no filters, provide an empty iterable or the value `None`. compressors : Iterable[Codec], optional List of compressors to apply to the array. Compressors are applied in order, and after any filters are applied (if any are specified) and the data is serialized into bytes. For Zarr format 3, a "compressor" is a codec that takes a bytestream, and returns another bytestream. These values must be instances of [`zarr.abc.codec.BytesBytesCodec`][], or dict representations of [`zarr.abc.codec.BytesBytesCodec`][]. Multiple compressors may be provided for Zarr format 3. Codecs that take an array and return bytes are serializers and must be supplied with `serializer` instead. If no `compressors` are provided, the default for Zarr format 3 is `(ZstdCodec(),)`. Use `None` to omit default compressors. For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may be provided for Zarr format 2. If no `compressor` is provided, a default compressor will be used. This default can be changed by modifying the value of `array.v2_default_compressor` in [`zarr.config`][zarr.config]. Use `None` to omit the default compressor. compressor : Codec, optional Deprecated in favor of `compressors`. serializer : dict[str, JSON] | ArrayBytesCodec, optional Array-to-bytes codec to use for encoding the array data. Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. Codecs that are instances of [`zarr.abc.codec.ArrayBytesCodec`][] must be supplied here, not with `compressors`. If no `serializer` is provided, a default serializer is chosen based on the data type: `VLenUTF8Codec` for variable-length strings, `VLenBytesCodec` for variable-length bytes, and `BytesCodec` otherwise. fill_value : Any, optional Fill value for the array. order : {"C", "F"}, optional The memory order of the array (default is "C"). For Zarr format 2, this parameter sets the memory order of the array. For Zarr format 3, this parameter is deprecated, because memory order is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. If no `order` is provided, a default order will be used. This default can be changed by modifying the value of `array.order` in [`zarr.config`][zarr.config]. attributes : dict, optional Attributes for the array. chunk_key_encoding : ChunkKeyEncodingLike, optional A specification of how the chunk keys are represented in storage. For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. dimension_names : Iterable[str], optional The names of the dimensions (default is None). Zarr format 3 only. Zarr format 2 arrays should not use this parameter. storage_options : dict, optional If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. overwrite : bool, default False Whether to overwrite an array with the same name in the store, if one exists. If `True`, any existing keys under that path are deleted first. config : ArrayConfigLike, optional Runtime configuration for the array. write_data : bool If a pre-existing array-like object was provided to this function via the `data` parameter then `write_data` determines whether the values in that array-like object should be written to the Zarr array created by this function. If `write_data` is `False`, then the array will be left empty. Returns ------- AsyncArray """ return self.create_array( name, shape=shape, dtype=dtype, data=data, chunks=chunks, shards=shards, filters=filters, compressors=compressors, compressor=compressor, serializer=serializer, fill_value=fill_value, order=order, attributes=attributes, chunk_key_encoding=chunk_key_encoding, dimension_names=dimension_names, storage_options=storage_options, overwrite=overwrite, config=config, write_data=write_data, ) def create_array( self, name: str, *, shape: ShapeLike | None = None, dtype: ZDTypeLike | None = None, data: np.ndarray[Any, np.dtype[Any]] | None = None, chunks: ChunksLike | Literal["auto"] = "auto", shards: ShardsLike | None = None, filters: FiltersLike = "auto", compressors: CompressorsLike = "auto", compressor: CompressorLike = "auto", serializer: SerializerLike = "auto", fill_value: Any | None = DEFAULT_FILL_VALUE, order: MemoryOrder | None = None, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, storage_options: dict[str, Any] | None = None, overwrite: bool = False, config: ArrayConfigLike | None = None, write_data: bool = True, ) -> AnyArray: """Create an array within this group. This method lightly wraps [zarr.core.array.create_array][]. Parameters ---------- name : str The name of the array relative to the group. If `path` is `None`, the array will be located at the root of the store. shape : ShapeLike, optional Shape of the array. Must be `None` if `data` is provided. dtype : ZDTypeLike | None Data type of the array. Must be `None` if `data` is provided. data : np.ndarray, optional Array-like data to use for initializing the array. If this parameter is provided, the `shape` and `dtype` parameters must be `None`. chunks : tuple[int, ...] | Sequence[Sequence[int]] | Literal["auto"], default="auto" Chunk shape of the array. If chunks is "auto", a chunk shape is guessed based on the shape of the array and the dtype. A nested list of per-dimension edge sizes creates a rectilinear grid. Rectilinear chunk grids are experimental and must be explicitly enabled with `zarr.config.set({'array.rectilinear_chunks': True})` while the feature is stabilizing. shards : tuple[int, ...], optional Shard shape of the array. The default value of `None` results in no sharding at all. filters : Iterable[Codec] | Literal["auto"], optional Iterable of filters to apply to each chunk of the array, in order, before serializing that chunk to bytes. For Zarr format 3, a "filter" is a codec that takes an array and returns an array, and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a dict representations of [`zarr.abc.codec.ArrayArrayCodec`][]. For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the order of your filters is consistent with the behavior of each filter. The default value of `"auto"` instructs Zarr to use a default based on the data type of the array and the Zarr format specified. For all data types in Zarr V3, and most data types in Zarr V2, the default filters are empty. The only cases where default filters are not empty is when the Zarr format is 2, and the data type is a variable-length data type like [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases, the default filters contains a single element which is a codec specific to that particular data type. To create an array with no filters, provide an empty iterable or the value `None`. compressors : Iterable[Codec], optional List of compressors to apply to the array. Compressors are applied in order, and after any filters are applied (if any are specified) and the data is serialized into bytes. For Zarr format 3, a "compressor" is a codec that takes a bytestream, and returns another bytestream. These values must be instances of [`zarr.abc.codec.BytesBytesCodec`][], or dict representations of [`zarr.abc.codec.BytesBytesCodec`][]. Multiple compressors may be provided for Zarr format 3. Codecs that take an array and return bytes are serializers and must be supplied with `serializer` instead. If no `compressors` are provided, the default for Zarr format 3 is `(ZstdCodec(),)`. Use `None` to omit default compressors. For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may be provided for Zarr format 2. If no `compressor` is provided, a default compressor will be used. This default can be changed by modifying the value of `array.v2_default_compressor` in [`zarr.config`][zarr.config]. Use `None` to omit the default compressor. compressor : Codec, optional Deprecated in favor of `compressors`. serializer : dict[str, JSON] | ArrayBytesCodec, optional Array-to-bytes codec to use for encoding the array data. Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. Codecs that are instances of [`zarr.abc.codec.ArrayBytesCodec`][] must be supplied here, not with `compressors`. If no `serializer` is provided, a default serializer is chosen based on the data type: `VLenUTF8Codec` for variable-length strings, `VLenBytesCodec` for variable-length bytes, and `BytesCodec` otherwise. fill_value : Any, optional Fill value for the array. order : {"C", "F"}, optional The memory order of the array (default is "C"). For Zarr format 2, this parameter sets the memory order of the array. For Zarr format 3, this parameter is deprecated, because memory order is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory order for Zarr format 3 arrays is via the `config` parameter, e.g. `{'config': 'C'}`. If no `order` is provided, a default order will be used. This default can be changed by modifying the value of `array.order` in [`zarr.config`][zarr.config]. attributes : dict, optional Attributes for the array. chunk_key_encoding : ChunkKeyEncodingLike, optional A specification of how the chunk keys are represented in storage. For Zarr format 3, the default is `{"name": "default", "separator": "/"}`. For Zarr format 2, the default is `{"name": "v2", "separator": "."}`. dimension_names : Iterable[str], optional The names of the dimensions (default is None). Zarr format 3 only. Zarr format 2 arrays should not use this parameter. storage_options : dict, optional If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. overwrite : bool, default False Whether to overwrite an array with the same name in the store, if one exists. If `True`, any existing keys under that path are deleted first. config : ArrayConfigLike, optional Runtime configuration for the array. write_data : bool If a pre-existing array-like object was provided to this function via the `data` parameter then `write_data` determines whether the values in that array-like object should be written to the Zarr array created by this function. If `write_data` is `False`, then the array will be left empty. Returns ------- AsyncArray """ compressors = _parse_deprecated_compressor( compressor, compressors, zarr_format=self.metadata.zarr_format ) return Array( self._sync( self._async_group.create_array( name=name, shape=shape, dtype=dtype, data=data, chunks=chunks, shards=shards, fill_value=fill_value, attributes=attributes, chunk_key_encoding=chunk_key_encoding, compressors=compressors, serializer=serializer, dimension_names=dimension_names, order=order, filters=filters, overwrite=overwrite, storage_options=storage_options, config=config, write_data=write_data, ) ) ) def require_array( self, name: str, *, shape: ShapeLike, config: ArrayConfigLike | None = None, **kwargs: Any, ) -> AnyArray: """Obtain an array, creating if it doesn't exist. Other `kwargs` are as per [zarr.Group.create_array][]. Parameters ---------- name : str Array name. config : ArrayConfigLike or None, default=None Runtime configuration for the array, whether it is created or already exists. Keys not specified are taken from the global configuration. **kwargs : See [zarr.Group.create_array][]. Returns ------- a : Array """ return Array( self._sync(self._async_group.require_array(name, shape=shape, config=config, **kwargs)) ) def empty(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray: """Create an empty array with the specified shape in this Group. The contents will be filled with the array's fill value or zeros if no fill value is provided. Parameters ---------- name : str Name of the array. shape : int or tuple of int Shape of the empty array. **kwargs Keyword arguments passed to [zarr.api.asynchronous.create][]. Notes ----- The contents of an empty Zarr array are not defined. On attempting to retrieve data from an empty Zarr array, any values may be returned, and these are not guaranteed to be stable from one access to the next. """ return Array(self._sync(self._async_group.empty(name=name, shape=shape, **kwargs))) def zeros(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray: """Create an array, with zero being used as the default value for uninitialized portions of the array. Parameters ---------- name : str Name of the array. shape : int or tuple of int Shape of the empty array. **kwargs Keyword arguments passed to [zarr.api.asynchronous.create][]. Returns ------- Array The new array. """ return Array(self._sync(self._async_group.zeros(name=name, shape=shape, **kwargs))) def ones(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray: """Create an array, with one being used as the default value for uninitialized portions of the array. Parameters ---------- name : str Name of the array. shape : int or tuple of int Shape of the empty array. **kwargs Keyword arguments passed to [zarr.api.asynchronous.create][]. Returns ------- Array The new array. """ return Array(self._sync(self._async_group.ones(name=name, shape=shape, **kwargs))) def full( self, *, name: str, shape: tuple[int, ...], fill_value: Any | None, **kwargs: Any ) -> AnyArray: """Create an array, with "fill_value" being used as the default value for uninitialized portions of the array. Parameters ---------- name : str Name of the array. shape : int or tuple of int Shape of the empty array. fill_value : scalar Value to fill the array with. **kwargs Keyword arguments passed to [zarr.api.asynchronous.create][]. Returns ------- Array The new array. """ return Array( self._sync( self._async_group.full(name=name, shape=shape, fill_value=fill_value, **kwargs) ) ) def empty_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: """Create an empty sub-array like `data`. The contents will be filled with the array's fill value or zeros if no fill value is provided. Parameters ---------- name : str Name of the array. data : array-like The array to create an empty array like. **kwargs Keyword arguments passed to [zarr.api.asynchronous.create][]. Returns ------- Array The new array. Notes ----- The contents of an empty Zarr array are not defined. On attempting to retrieve data from an empty Zarr array, any values may be returned, and these are not guaranteed to be stable from one access to the next. """ return Array(self._sync(self._async_group.empty_like(name=name, data=data, **kwargs))) def zeros_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: """Create a sub-array of zeros like `data`. Parameters ---------- name : str Name of the array. data : array-like The array to create the new array like. **kwargs Keyword arguments passed to [zarr.api.asynchronous.create][]. Returns ------- Array The new array. """ return Array(self._sync(self._async_group.zeros_like(name=name, data=data, **kwargs))) def ones_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: """Create a sub-array of ones like `data`. Parameters ---------- name : str Name of the array. data : array-like The array to create the new array like. **kwargs Keyword arguments passed to [zarr.api.asynchronous.create][]. Returns ------- Array The new array. """ return Array(self._sync(self._async_group.ones_like(name=name, data=data, **kwargs))) def full_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> AnyArray: """Create a sub-array like `data` filled with the `fill_value` of `data` . Parameters ---------- name : str Name of the array. data : array-like The array to create the new array like. **kwargs Keyword arguments passed to [zarr.api.asynchronous.create][]. Returns ------- Array The new array. """ return Array(self._sync(self._async_group.full_like(name=name, data=data, **kwargs))) def move(self, source: str, dest: str) -> None: """Move a sub-group or sub-array from one path to another. Notes ----- Not implemented """ return self._sync(self._async_group.move(source, dest)) async def create_hierarchy( *, store: Store, nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], overwrite: bool = False, ) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: """ Create a complete zarr hierarchy from a collection of metadata objects. This function will parse its input to ensure that the hierarchy is complete. Any implicit groups will be inserted as needed. For example, an input like ```{'a/b': GroupMetadata}``` will be parsed to ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}``` After input parsing, this function then creates all the nodes in the hierarchy concurrently. Arrays and Groups are yielded in the order they are created. This order is not stable and should not be relied on. Parameters ---------- store : Store The storage backend to use. nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, relative to the root of the ``Store``. The root of the store can be specified with the empty string ``''``. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that all values must have the same ``zarr_format`` -- it is an error to mix zarr versions in the same hierarchy. Leading "/" characters from keys will be removed. overwrite : bool Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is raised instead of overwriting an existing array or group. This function will not erase an existing group unless that group is explicitly named in ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a group already exists at path ``a``, then this function will leave the group at ``a`` as-is. Yields ------ tuple[str, AsyncGroup | AsyncArray] Yields (path, node) pairs, in the order the nodes were created. Examples -------- >>> async def example(): ... from zarr.api.asynchronous import create_hierarchy ... from zarr.core.group import GroupMetadata ... from zarr.storage import MemoryStore ... ... store = MemoryStore() ... nodes = {'a': GroupMetadata(attributes={'name': 'leaf'})} ... return sorted([x async for x in create_hierarchy(store=store, nodes=nodes)]) >>> import asyncio >>> asyncio.run(example()) [('', ), ('a', )] """ # normalize the keys to be valid paths nodes_normed_keys = _normalize_path_keys(nodes) # ensure that all nodes have the same zarr_format, and add implicit groups as needed nodes_parsed = _parse_hierarchy_dict(data=nodes_normed_keys) redundant_implicit_groups = [] to_delete_keys: list[str] = [] # empty hierarchies should be a no-op if len(nodes_parsed) > 0: # figure out which zarr format we are using zarr_format = next(iter(nodes_parsed.values())).zarr_format # check which implicit groups will require materialization implicit_group_keys = tuple( filter(lambda k: isinstance(nodes_parsed[k], ImplicitGroupMarker), nodes_parsed) ) # read potential group metadata for each implicit group maybe_extant_group_coros = ( _read_group_metadata(store, k, zarr_format=zarr_format) for k in implicit_group_keys ) maybe_extant_groups = await asyncio.gather( *maybe_extant_group_coros, return_exceptions=True ) for key, value in zip(implicit_group_keys, maybe_extant_groups, strict=True): if isinstance(value, BaseException): if isinstance(value, FileNotFoundError): # this is fine -- there was no group there, so we will create one pass else: raise value else: # a loop exists already at ``key``, so we can avoid creating anything there redundant_implicit_groups.append(key) if overwrite: # we will remove any nodes that collide with arrays and non-implicit groups defined in # nodes to_delete_keys = [k for k in nodes_parsed if k not in implicit_group_keys] else: # This type is long. coros: ( Generator[Coroutine[Any, Any, ArrayV2Metadata | GroupMetadata], None, None] | Generator[Coroutine[Any, Any, ArrayV3Metadata | GroupMetadata], None, None] ) if zarr_format == 2: coros = (_read_metadata_v2(store=store, path=key) for key in nodes_parsed) elif zarr_format == 3: coros = (_read_metadata_v3(store=store, path=key) for key in nodes_parsed) else: # pragma: no cover raise ValueError(f"Invalid zarr_format: {zarr_format}") # pragma: no cover extant_node_query = dict( zip( nodes_parsed.keys(), await asyncio.gather(*coros, return_exceptions=True), strict=False, ) ) # iterate over the existing arrays / groups and figure out which of them conflict # with the arrays / groups we want to create for key, extant_node in extant_node_query.items(): proposed_node = nodes_parsed[key] if isinstance(extant_node, BaseException): if isinstance(extant_node, FileNotFoundError): # ignore FileNotFoundError, because they represent nodes we can safely create pass else: # Any other exception is a real error raise extant_node else: # this is a node that already exists, but a node with the same key was specified # in nodes_parsed. if isinstance(extant_node, GroupMetadata): # a group already exists where we want to create a group if isinstance(proposed_node, ImplicitGroupMarker): # we have proposed an implicit group, which is OK -- we will just skip # creating this particular metadata document redundant_implicit_groups.append(key) else: # we have proposed an explicit group, which is an error, given that a # group already exists. msg = f"A group exists in store {store!r} at path {key!r}." raise ContainsGroupError(msg) elif isinstance(extant_node, ArrayV2Metadata | ArrayV3Metadata): # we are trying to overwrite an existing array. this is an error. msg = f"An array exists in store {store!r} at path {key!r}." raise ContainsArrayError(msg) nodes_explicit: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] = {} for k, v in nodes_parsed.items(): if k not in redundant_implicit_groups: if isinstance(v, ImplicitGroupMarker): nodes_explicit[k] = GroupMetadata(zarr_format=v.zarr_format) else: nodes_explicit[k] = v # Build and encode every node before deleting anything: a node that cannot be built # or whose metadata cannot be stored then fails with the store untouched. built, documents = _prepare_nodes(store, nodes_explicit) await asyncio.gather(*(store.delete_dir(key) for key in to_delete_keys)) async for key, node in _store_nodes(store, nodes_explicit, built, documents): yield key, node async def create_nodes( *, store: Store, nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], ) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: """Create a collection of arrays and / or groups concurrently. Note: no attempt is made to validate that these arrays and / or groups collectively form a valid Zarr hierarchy. It is the responsibility of the caller of this function to ensure that the ``nodes`` parameter satisfies any correctness constraints. Parameters ---------- store : Store The storage backend to use. nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, and the values are the metadata of the nodes. The metadata must be either an instance of GroupMetadata, ArrayV3Metadata or ArrayV2Metadata. Yields ------ AsyncGroup | AsyncArray The created nodes in the order they are created. """ async for key, node in _store_nodes(store, nodes, *_prepare_nodes(store, nodes)): yield key, node def _prepare_nodes( store: Store, nodes: Mapping[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] ) -> tuple[dict[str, AsyncGroup | AnyAsyncArray], dict[str, Buffer]]: """The array or group each of `nodes` describes, at its path in `store`, and the metadata documents of `nodes`, by their keys in the store: every node is built and encoded before anything is stored.""" built = { path: _build_node(store=store, path=path, metadata=meta) for path, meta in nodes.items() } documents = { _join_paths([path, key]): value for path, metadata in nodes.items() for key, value in encode_documents(StorePath(store, path), metadata).items() } return built, documents async def _store_nodes( store: Store, nodes: Mapping[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], built: Mapping[str, AsyncGroup | AnyAsyncArray], documents: Mapping[str, Buffer], ) -> AsyncIterator[tuple[str, AsyncGroup | AnyAsyncArray]]: """Store the `documents` encoded from `nodes` and yield the nodes built from them, as `_prepare_nodes` returns them (see `create_nodes`).""" # Note: the only way to alter this value is via the config. If that's undesirable for some reason, # then we should consider adding a keyword argument to this function semaphore = asyncio.Semaphore(config.get("async.concurrency")) create_tasks = [ _set_return_key(store=store, key=key, value=value, semaphore=semaphore) for key, value in documents.items() ] created_object_keys = [] for coro in asyncio.as_completed(create_tasks): created_key = await coro # we need this to track which metadata documents were written so that we can yield a # complete v2 Array / Group class after both .zattrs and the metadata JSON was created. created_object_keys.append(created_key) # get the node name from the object key if len(created_key.split("/")) == 1: # this is the root node meta_out = nodes[""] node_name = "" else: # turn "foo/" into "foo" node_name = created_key[: created_key.rfind("/")] meta_out = nodes[node_name] if meta_out.zarr_format == 3: yield node_name, built[node_name] else: # For zarr v2 # we only want to yield when both the metadata and attributes are created # so we track which keys have been created, and wait for both the meta key and # the attrs key to be created before yielding back the AsyncArray / AsyncGroup attrs_done = _join_paths([node_name, ZATTRS_JSON]) in created_object_keys if isinstance(meta_out, GroupMetadata): meta_done = _join_paths([node_name, ZGROUP_JSON]) in created_object_keys else: meta_done = _join_paths([node_name, ZARRAY_JSON]) in created_object_keys if meta_done and attrs_done: yield node_name, built[node_name] continue def _get_roots( data: Iterable[str], ) -> tuple[str, ...]: """ Return the keys of the root(s) of the hierarchy. A root is a key with the fewest number of path segments. """ if "" in data: return ("",) keys_split = sorted((key.split("/") for key in data), key=len) groups: defaultdict[int, list[str]] = defaultdict(list) for key_split in keys_split: groups[len(key_split)].append("/".join(key_split)) return tuple(groups[min(groups.keys())]) def _parse_hierarchy_dict( *, data: Mapping[str, ImplicitGroupMarker | GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], ) -> dict[str, ImplicitGroupMarker | GroupMetadata | ArrayV2Metadata | ArrayV3Metadata]: """ Take an input with type Mapping[str, ArrayMetadata | GroupMetadata] and parse it into a dict of str: node pairs that models a valid, complete Zarr hierarchy. If the input represents a complete Zarr hierarchy, i.e. one with no implicit groups, then return a dict with the exact same data as the input. Otherwise, return a dict derived from the input with GroupMetadata inserted as needed to make the hierarchy complete. For example, an input of {'a/b': ArrayMetadata} is incomplete, because it references two groups (the root group '' and a group at 'a') that are not specified in the input. Applying this function to that input will result in a return value of {'': GroupMetadata, 'a': GroupMetadata, 'a/b': ArrayMetadata}, i.e. the implied groups were added. The input is also checked for the following conditions; an error is raised if any are violated: - No arrays can contain group or arrays (i.e., all arrays must be leaf nodes). - All arrays and groups must have the same ``zarr_format`` value. This function ensures that the input is transformed into a specification of a complete and valid Zarr hierarchy. """ # ensure that all nodes have the same zarr format data_purified = _ensure_consistent_zarr_format(data) # ensure that keys are normalized to zarr paths data_normed_keys = _normalize_path_keys(data_purified) # insert an implicit root group if a root was not specified # but not if an empty dict was provided, because any empty hierarchy has no nodes if len(data_normed_keys) > 0 and "" not in data_normed_keys: z_format = next(iter(data_normed_keys.values())).zarr_format data_normed_keys = data_normed_keys | {"": ImplicitGroupMarker(zarr_format=z_format)} out: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] = {**data_normed_keys} for k, v in data_normed_keys.items(): key_split = k.split("/") # get every parent path *subpaths, _ = accumulate(key_split, lambda a, b: _join_paths([a, b])) for subpath in subpaths: # If a component is not already in the output dict, add ImplicitGroupMetadata if subpath not in out: out[subpath] = ImplicitGroupMarker(zarr_format=v.zarr_format) else: if not isinstance(out[subpath], GroupMetadata | ImplicitGroupMarker): msg = ( f"The node at {subpath} contains other nodes, but it is not a Zarr group. " "This is invalid. Only Zarr groups can contain other nodes." ) raise ValueError(msg) return out def _ensure_consistent_zarr_format( data: Mapping[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], ) -> Mapping[str, GroupMetadata | ArrayV2Metadata] | Mapping[str, GroupMetadata | ArrayV3Metadata]: """ Ensure that all values of the input dict have the same zarr format. If any do not, then a value error is raised. """ observed_zarr_formats: dict[ZarrFormat, list[str]] = {2: [], 3: []} for k, v in data.items(): observed_zarr_formats[v.zarr_format].append(k) if len(observed_zarr_formats[2]) > 0 and len(observed_zarr_formats[3]) > 0: msg = ( "Got data with both Zarr v2 and Zarr v3 nodes, which is invalid. " f"The following keys map to Zarr v2 nodes: {observed_zarr_formats.get(2)}. " f"The following keys map to Zarr v3 nodes: {observed_zarr_formats.get(3)}." "Ensure that all nodes have the same Zarr format." ) raise ValueError(msg) return cast( "Mapping[str, GroupMetadata | ArrayV2Metadata] | Mapping[str, GroupMetadata | ArrayV3Metadata]", data, ) async def _getitem_semaphore( node: AsyncGroup, key: str, semaphore: asyncio.Semaphore | None ) -> AnyAsyncArray | AsyncGroup: """ Wrap Group.getitem with an optional semaphore. If the semaphore parameter is an asyncio.Semaphore instance, then the getitem operation is performed inside an async context manager provided by that semaphore. If the semaphore parameter is None, then getitem is invoked without a context manager. """ if semaphore is not None: async with semaphore: return await node.getitem(key) else: return await node.getitem(key) async def _iter_members( node: AsyncGroup, skip_keys: tuple[str, ...], semaphore: asyncio.Semaphore | None, ) -> AsyncGenerator[tuple[str, AnyAsyncArray | AsyncGroup], None]: """ Iterate over the arrays and groups contained in a group. Parameters ---------- node : AsyncGroup The group to traverse. skip_keys : tuple[str, ...] A tuple of keys to skip when iterating over the possible members of the group. semaphore : asyncio.Semaphore | None An optional semaphore to use for concurrency control. Yields ------ tuple[str, AnyAsyncArray | AsyncGroup] """ # retrieve keys from storage keys = [key async for key in node.store.list_dir(node.path)] keys_filtered = tuple(filter(lambda v: v not in skip_keys, keys)) node_tasks = tuple( asyncio.create_task(_getitem_semaphore(node, key, semaphore), name=key) for key in keys_filtered ) for fetched_node_coro in asyncio.as_completed(node_tasks): try: fetched_node = await fetched_node_coro except KeyError as e: # keyerror is raised when `key` names an object (in the object storage sense), # as opposed to a prefix, in the store under the prefix associated with this group # in which case `key` cannot be the name of a sub-array or sub-group. warnings.warn( f"Object at {e.args[0]} is not recognized as a component of a Zarr hierarchy.", ZarrUserWarning, stacklevel=1, ) continue match fetched_node: case AsyncArray() | AsyncGroup(): yield fetched_node.basename, fetched_node case _: raise ValueError(f"Unexpected type: {type(fetched_node)}") async def _iter_members_deep( group: AsyncGroup, *, max_depth: int | None, skip_keys: tuple[str, ...], semaphore: asyncio.Semaphore | None = None, use_consolidated_for_children: bool = True, ) -> AsyncGenerator[tuple[str, AnyAsyncArray | AsyncGroup], None]: """ Iterate over the arrays and groups contained in a group, and optionally the arrays and groups contained in those groups. Parameters ---------- group : AsyncGroup The group to traverse. max_depth : int | None The maximum depth of recursion. skip_keys : tuple[str, ...] A tuple of keys to skip when iterating over the possible members of the group. semaphore : asyncio.Semaphore | None An optional semaphore to use for concurrency control. use_consolidated_for_children : bool, default True Whether to use the consolidated metadata of child groups loaded from the store. Note that this only affects groups loaded from the store. If the current Group already has consolidated metadata, it will always be used. Yields ------ tuple[str, AnyAsyncArray | AsyncGroup] """ to_recurse = {} do_recursion = max_depth is None or max_depth > 0 if max_depth is None: new_depth = None else: new_depth = max_depth - 1 async for name, node in _iter_members(group, skip_keys=skip_keys, semaphore=semaphore): is_group = isinstance(node, AsyncGroup) if ( is_group and not use_consolidated_for_children and node.metadata.consolidated_metadata is not None ): node = cast("AsyncGroup", node) # We've decided not to trust consolidated metadata at this point, because we're # reconsolidating the metadata, for example. node = replace(node, metadata=replace(node.metadata, consolidated_metadata=None)) yield name, node if is_group and do_recursion: node = cast("AsyncGroup", node) to_recurse[name] = _iter_members_deep( node, max_depth=new_depth, skip_keys=skip_keys, semaphore=semaphore ) for prefix, subgroup_iter in to_recurse.items(): async for name, node in subgroup_iter: key = f"{prefix}/{name}".lstrip("/") yield key, node async def _read_metadata_v3(store: Store, path: str) -> ArrayV3Metadata | GroupMetadata: """ Given a store_path, return ArrayV3Metadata or GroupMetadata defined by the metadata document stored at store_path.path / zarr.json. If no such document is found, raise a FileNotFoundError. """ zarr_json_bytes = await store.get( _join_paths([path, ZARR_JSON]), prototype=default_buffer_prototype() ) if zarr_json_bytes is None: raise FileNotFoundError(path) return _build_metadata_v3( buffer_to_json_object(zarr_json_bytes), path=str(StorePath(store, path)) ) async def _read_metadata_v2(store: Store, path: str) -> ArrayV2Metadata | GroupMetadata: """ Given a store_path, return ArrayV2Metadata or GroupMetadata defined by the metadata document stored at store_path.path / (.zgroup | .zarray). If no such document is found, raise a FileNotFoundError. """ # TODO: consider first fetching array metadata, and only fetching group metadata when we don't # find an array zarray_bytes, zgroup_bytes, zattrs_bytes = await asyncio.gather( store.get(_join_paths([path, ZARRAY_JSON]), prototype=default_buffer_prototype()), store.get(_join_paths([path, ZGROUP_JSON]), prototype=default_buffer_prototype()), store.get(_join_paths([path, ZATTRS_JSON]), prototype=default_buffer_prototype()), ) zattrs: dict[str, JSON] if zattrs_bytes is None: zattrs = {} else: zattrs = buffer_to_json_object(zattrs_bytes) # TODO: decide how to handle finding both array and group metadata. The spec does not seem to # consider this situation. A practical approach would be to ignore that combination, and only # return the array metadata. if zarray_bytes is not None: zmeta = buffer_to_json_object(zarray_bytes) else: if zgroup_bytes is None: # neither .zarray or .zgroup were found results in KeyError raise FileNotFoundError(path) else: zmeta = buffer_to_json_object(zgroup_bytes) return _build_metadata_v2(zmeta, zattrs, path=str(StorePath(store, path))) async def _read_group_metadata_v2(store: Store, path: str) -> GroupMetadata: """ Read group metadata or error """ meta = await _read_metadata_v2(store=store, path=path) if not isinstance(meta, GroupMetadata): raise FileNotFoundError(f"Group metadata was not found in {store} at {path}") return meta async def _read_group_metadata_v3(store: Store, path: str) -> GroupMetadata: """ Read group metadata or error """ meta = await _read_metadata_v3(store=store, path=path) if not isinstance(meta, GroupMetadata): raise FileNotFoundError(f"Group metadata was not found in {store} at {path}") return meta async def _read_group_metadata( store: Store, path: str, *, zarr_format: ZarrFormat ) -> GroupMetadata: if zarr_format == 2: return await _read_group_metadata_v2(store=store, path=path) return await _read_group_metadata_v3(store=store, path=path) def _build_metadata_v3( zarr_json: dict[str, JSON], *, path: str | None = None ) -> ArrayV3Metadata | GroupMetadata: """ Convert a dict representation of Zarr V3 metadata into the corresponding metadata class. """ if "node_type" not in zarr_json: msg = "Required key 'node_type' is missing from the provided metadata document." raise MetadataValidationError(msg) match zarr_json: case {"node_type": "array"}: return ArrayV3Metadata.from_dict(zarr_json, path=path) case {"node_type": "group"}: return GroupMetadata.from_dict(zarr_json, path=path) case _: # pragma: no cover raise ValueError( "invalid value for `node_type` key in metadata document" ) # pragma: no cover def _build_metadata_v2( zarr_json: dict[str, JSON], attrs_json: dict[str, JSON], *, path: str | None = None ) -> ArrayV2Metadata | GroupMetadata: """ Convert a dict representation of Zarr V2 metadata into the corresponding metadata class. """ match zarr_json: case {"shape": _}: return ArrayV2Metadata.from_dict(zarr_json | {"attributes": attrs_json}, path=path) case _: # pragma: no cover return GroupMetadata.from_dict(zarr_json | {"attributes": attrs_json}, path=path) @overload def _build_node(*, store: Store, path: str, metadata: ArrayV2Metadata) -> AsyncArrayV2: ... @overload def _build_node(*, store: Store, path: str, metadata: ArrayV3Metadata) -> AsyncArrayV3: ... @overload def _build_node(*, store: Store, path: str, metadata: GroupMetadata) -> AsyncGroup: ... def _build_node( *, store: Store, path: str, metadata: ArrayV3Metadata | ArrayV2Metadata | GroupMetadata ) -> AnyAsyncArray | AsyncGroup: """ Take a metadata object and return a node (AsyncArray or AsyncGroup). """ store_path = StorePath(store=store, path=path) match metadata: case ArrayV2Metadata() | ArrayV3Metadata(): return AsyncArray(metadata, store_path=store_path) case GroupMetadata(): return AsyncGroup(metadata, store_path=store_path) case _: # pragma: no cover raise ValueError(f"Unexpected metadata type: {type(metadata)}") # pragma: no cover async def _get_node_v2(store: Store, path: str) -> AsyncArrayV2 | AsyncGroup: """ Read a Zarr v2 AsyncArray or AsyncGroup from a path in a Store. Parameters ---------- store : Store The store-like object to read from. path : str The path to the node to read. Returns ------- AsyncArray | AsyncGroup """ metadata = await _read_metadata_v2(store=store, path=path) return _build_node(store=store, path=path, metadata=metadata) async def _get_node_v3(store: Store, path: str) -> AsyncArrayV3 | AsyncGroup: """ Read a Zarr v3 AsyncArray or AsyncGroup from a path in a Store. Parameters ---------- store : Store The store-like object to read from. path : str The path to the node to read. Returns ------- AsyncArray | AsyncGroup """ metadata = await _read_metadata_v3(store=store, path=path) return _build_node(store=store, path=path, metadata=metadata) async def get_node(store: Store, path: str, zarr_format: ZarrFormat) -> AnyAsyncArray | AsyncGroup: """ Get an AsyncArray or AsyncGroup from a path in a Store. Parameters ---------- store : Store The store-like object to read from. path : str The path to the node to read. zarr_format : {2, 3} The zarr format of the node to read. Returns ------- AsyncArray | AsyncGroup """ match zarr_format: case 2: return await _get_node_v2(store=store, path=path) case 3: return await _get_node_v3(store=store, path=path) case _: # pragma: no cover raise ValueError(f"Unexpected zarr format: {zarr_format}") # pragma: no cover async def _set_return_key( *, store: Store, key: str, value: Buffer, semaphore: asyncio.Semaphore | None = None ) -> str: """ Write a value to storage at the given key. The key is returned. Useful when saving values via routines that return results in execution order, like asyncio.as_completed, because in this case we need to know which key was saved in order to yield the right object to the caller. Parameters ---------- store : Store The store to save the value to. key : str The key to save the value to. value : Buffer The value to save. semaphore : asyncio.Semaphore | None An optional semaphore to use to limit the number of concurrent writes. """ if semaphore is not None: async with semaphore: await store.set(key, value) else: await store.set(key, value) return key async def create_rooted_hierarchy( *, store: Store, nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], overwrite: bool = False, ) -> AsyncGroup | AnyAsyncArray: """ Create an ``AsyncGroup`` or ``AsyncArray`` from a store and a dict of metadata documents. This function ensures that its input contains a specification of a root node, calls ``create_hierarchy`` to create nodes, and returns the root node of the hierarchy. """ roots = _get_roots(nodes.keys()) if len(roots) != 1: msg = ( "The input does not specify a root node. " "This function can only create hierarchies that contain a root node, which is " "defined as a group that is ancestral to all the other arrays and " "groups in the hierarchy, or a single array." ) raise ValueError(msg) else: root_key = roots[0] nodes_created = [ x async for x in create_hierarchy(store=store, nodes=nodes, overwrite=overwrite) ] return dict(nodes_created)[root_key] zarr-developers-zarr-python-a2f2415/src/zarr/core/indexing.py000066400000000000000000002110521527014137600243420ustar00rootroot00000000000000from __future__ import annotations import itertools import numbers from collections.abc import Iterator, Sequence from dataclasses import dataclass, field from enum import Enum from functools import lru_cache from types import EllipsisType from typing import ( TYPE_CHECKING, Any, Literal, NamedTuple, Protocol, TypeGuard, cast, runtime_checkable, ) import numpy as np import numpy.typing as npt from zarr.core.chunk_grids import FixedDimension from zarr.core.common import ceildiv_int, product from zarr.core.metadata.v2 import ArrayV2Metadata from zarr.core.metadata.v3 import ArrayV3Metadata from zarr.errors import ( ArrayIndexError, BoundsCheckError, NegativeStepError, VindexInvalidSelectionError, ) if TYPE_CHECKING: from zarr.core.array import AsyncArray from zarr.core.buffer import NDArrayLikeOrScalar from zarr.core.chunk_grids import ChunkGrid, DimensionGrid from zarr.types import AnyArray IntSequence = list[int] | npt.NDArray[np.intp] ArrayOfIntOrBool = npt.NDArray[np.intp] | npt.NDArray[np.bool_] BasicSelector = int | slice | EllipsisType Selector = BasicSelector | ArrayOfIntOrBool BasicSelection = BasicSelector | tuple[BasicSelector, ...] # also used for BlockIndex CoordinateSelection = IntSequence | tuple[IntSequence, ...] MaskSelection = npt.NDArray[np.bool_] OrthogonalSelection = Selector | tuple[Selector, ...] Selection = BasicSelection | CoordinateSelection | MaskSelection | OrthogonalSelection CoordinateSelectionNormalized = tuple[npt.NDArray[np.intp], ...] SelectionNormalized = tuple[Selector, ...] | ArrayOfIntOrBool SelectionWithFields = Selection | str | Sequence[str] SelectorTuple = tuple[Selector, ...] | npt.NDArray[np.intp] | slice Fields = str | list[str] | tuple[str, ...] def err_too_many_indices(selection: Any, shape: tuple[int, ...]) -> None: raise IndexError(f"too many indices for array; expected {len(shape)}, got {len(selection)}") def _zarr_array_to_int_or_bool_array(arr: AnyArray) -> npt.NDArray[np.intp] | npt.NDArray[np.bool_]: if arr.dtype.kind in ("i", "b"): return np.asarray(arr) else: raise IndexError( f"Invalid array dtype: {arr.dtype}. Arrays used as indices must be of integer or boolean type" ) @runtime_checkable class Indexer(Protocol): shape: tuple[int, ...] drop_axes: tuple[int, ...] def __iter__(self) -> Iterator[ChunkProjection]: ... type _ArrayIndexingOrder = Literal["lexicographic"] def _iter_grid( grid_shape: Sequence[int], *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None, order: _ArrayIndexingOrder = "lexicographic", ) -> Iterator[tuple[int, ...]]: """ Iterate over the elements of grid of integers, with the option to restrict the domain of iteration to a contiguous subregion of that grid. Parameters ---------- grid_shape : Sequence[int] The size of the domain to iterate over. origin : Sequence[int] | None, default=None The first coordinate of the domain to return. selection_shape : Sequence[int] | None, default=None The shape of the selection. order : Literal["lexicographic"], default="lexicographic" The linear indexing order to use. Returns ------- Iterator[tuple[int, ...]] An iterator over tuples of integers Examples -------- ```python from zarr.core.indexing import _iter_grid tuple(_iter_grid((1,))) # ((0,),) tuple(_iter_grid((2,3))) # ((0, 0), (0, 1), (0, 2), (1, 0), (1, 1), (1, 2)) tuple(_iter_grid((2,3), origin=(1,1))) # ((1, 1), (1, 2)) tuple(_iter_grid((2,3), origin=(0,0), selection_shape=(2,2))) # ((0, 0), (0, 1), (1, 0), (1, 1)) ``` """ if origin is None: origin_parsed = (0,) * len(grid_shape) else: if len(origin) != len(grid_shape): msg = ( "Shape and origin parameters must have the same length." f"Got {len(grid_shape)} elements in shape, but {len(origin)} elements in origin." ) raise ValueError(msg) origin_parsed = tuple(origin) if selection_shape is None: selection_shape_parsed = tuple( g - o for o, g in zip(origin_parsed, grid_shape, strict=True) ) else: selection_shape_parsed = tuple(selection_shape) if order == "lexicographic": dimensions: tuple[range, ...] = () for idx, (o, gs, ss) in enumerate( zip(origin_parsed, grid_shape, selection_shape_parsed, strict=True) ): if o + ss > gs: raise IndexError( f"Invalid selection shape ({ss}) for origin ({o}) and grid shape ({gs}) at axis {idx}." ) dimensions += (range(o, o + ss),) return itertools.product(*(dimensions)) else: msg = f"Indexing order {order} is not supported at this time." # type: ignore[unreachable] # pragma: no cover raise NotImplementedError(msg) # pragma: no cover def _iter_regions( domain_shape: Sequence[int], region_shape: Sequence[int], *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None, order: _ArrayIndexingOrder = "lexicographic", trim_excess: bool = True, ) -> Iterator[tuple[slice, ...]]: """ Iterate over contiguous regions on a grid of integers, with the option to restrict the domain of iteration to a contiguous subregion of that grid. Parameters ---------- domain_shape : Sequence[int] The size of the domain to iterate over. region_shape : Sequence[int] The shape of the region to iterate over. origin : Sequence[int] | None, default=None The location, in grid coordinates, of the first region to return. selection_shape : Sequence[int] | None, default=None The shape of the selection, in grid coordinates. order : Literal["lexicographic"], default="lexicographic" The linear indexing order to use. Yields ------ Iterator[tuple[slice, ...]] An iterator over tuples of slices, where each slice spans a separate contiguous region Examples -------- ```python from zarr.core.indexing import _iter_regions tuple(_iter_regions((1,), (1,))) # ((slice(0, 1, 1),),) tuple(_iter_regions((2, 3), (1, 2))) # ((slice(0, 1, 1), slice(0, 2, 1)), (slice(1, 2, 1), slice(0, 2, 1))) tuple(_iter_regions((2,3), (1,2), origin=(1,1))) # ((slice(1, 2, 1), slice(1, 3, 1)), (slice(2, 3, 1), slice(1, 3, 1))) tuple(_iter_regions((2,3), (1,2), origin=(0,0), selection_shape=(2,2))) # ((slice(0, 1, 1), slice(0, 2, 1)), (slice(1, 2, 1), slice(0, 2, 1))) ``` """ grid_shape = tuple(itertools.starmap(ceildiv_int, zip(domain_shape, region_shape, strict=True))) for grid_position in _iter_grid( grid_shape=grid_shape, origin=origin, selection_shape=selection_shape, order=order ): out: list[slice] = [] for g_pos, r_shape, d_shape in zip(grid_position, region_shape, domain_shape, strict=True): start = g_pos * r_shape stop = start + r_shape if trim_excess: stop = min(stop, d_shape) out.append(slice(start, stop, 1)) yield tuple(out) def is_integer(x: Any) -> TypeGuard[int]: """True if x is an integer (both pure Python or NumPy).""" return isinstance(x, numbers.Integral) and not is_bool(x) def is_bool(x: Any) -> TypeGuard[bool | np.bool_]: """True if x is a boolean (both pure Python or NumPy).""" return type(x) in [bool, np.bool_] def is_integer_list(x: Any) -> TypeGuard[list[int]]: """True if x is a list of integers.""" return isinstance(x, list) and len(x) > 0 and all(is_integer(i) for i in x) def is_bool_list(x: Any) -> TypeGuard[list[bool | np.bool_]]: """True if x is a list of boolean.""" return isinstance(x, list) and len(x) > 0 and all(is_bool(i) for i in x) def is_integer_array(x: Any, ndim: int | None = None) -> TypeGuard[npt.NDArray[np.intp]]: t = not np.isscalar(x) and hasattr(x, "shape") and hasattr(x, "dtype") and x.dtype.kind in "ui" if ndim is not None: t = t and hasattr(x, "shape") and len(x.shape) == ndim return t def is_bool_array(x: Any, ndim: int | None = None) -> TypeGuard[npt.NDArray[np.bool_]]: t = hasattr(x, "shape") and hasattr(x, "dtype") and x.dtype == bool if ndim is not None: t = t and hasattr(x, "shape") and len(x.shape) == ndim return t def is_int_or_bool_iterable(x: Any) -> bool: return is_integer_list(x) or is_integer_array(x) or is_bool_array(x) or is_bool_list(x) def is_scalar(value: Any, dtype: np.dtype[Any]) -> bool: if np.isscalar(value): return True if hasattr(value, "shape") and value.shape == (): return True return isinstance(value, tuple) and dtype.names is not None and len(value) == len(dtype.names) def is_pure_fancy_indexing(selection: Any, ndim: int) -> bool: """Check whether a selection contains only scalars or integer/bool array-likes. Parameters ---------- selection : tuple, slice, or scalar A valid selection value for indexing into arrays. Returns ------- is_pure : bool True if the selection is a pure fancy indexing expression (ie not mixed with boolean or slices). """ if is_bool_array(selection): # is mask selection return True if ndim == 1 and ( is_integer_list(selection) or is_integer_array(selection) or is_bool_list(selection) ): return True # if not, we go through the normal path below, because a 1-tuple # of integers is also allowed. no_slicing = ( isinstance(selection, tuple) and len(selection) == ndim and not (any(isinstance(elem, slice) or elem is Ellipsis for elem in selection)) ) return ( no_slicing and all( is_integer(elem) or is_integer_list(elem) or is_integer_array(elem) for elem in selection ) and any(is_integer_list(elem) or is_integer_array(elem) for elem in selection) ) def is_pure_orthogonal_indexing(selection: Selection, ndim: int) -> TypeGuard[OrthogonalSelection]: if not ndim: return False selection_normalized = (selection,) if not isinstance(selection, tuple) else selection # Case 1: Selection contains of iterable of integers or boolean if len(selection_normalized) == ndim and all( is_int_or_bool_iterable(s) for s in selection_normalized ): return True # Case 2: selection contains either zero or one integer iterables. # All other selection elements are slices or integers return ( len(selection_normalized) <= ndim and sum(is_int_or_bool_iterable(s) for s in selection_normalized) <= 1 and all( is_int_or_bool_iterable(s) or isinstance(s, int | slice) for s in selection_normalized ) ) def normalize_integer_selection(dim_sel: int, dim_len: int) -> int: # normalize type to int dim_sel = int(dim_sel) # handle wraparound if dim_sel < 0: dim_sel = dim_len + dim_sel # handle out of bounds if dim_sel >= dim_len or dim_sel < 0: msg = f"index out of bounds for dimension with length {dim_len}" raise BoundsCheckError(msg) return dim_sel class ChunkDimProjection(NamedTuple): """A mapping from chunk to output array for a single dimension. Attributes ---------- dim_chunk_ix Index of chunk. dim_chunk_sel Selection of items from chunk array. dim_out_sel Selection of items in target (output) array. """ dim_chunk_ix: int dim_chunk_sel: Selector dim_out_sel: Selector | None is_complete_chunk: bool @dataclass(frozen=True) class IntDimIndexer: dim_sel: int dim_len: int dim_grid: DimensionGrid nitems: int = 1 def __init__(self, dim_sel: int, dim_len: int, dim_grid: DimensionGrid) -> None: object.__setattr__(self, "dim_sel", normalize_integer_selection(dim_sel, dim_len)) object.__setattr__(self, "dim_len", dim_len) object.__setattr__(self, "dim_grid", dim_grid) def __iter__(self) -> Iterator[ChunkDimProjection]: g = self.dim_grid dim_chunk_ix = g.index_to_chunk(self.dim_sel) dim_offset = g.chunk_offset(dim_chunk_ix) dim_chunk_sel = self.dim_sel - dim_offset dim_out_sel = None is_complete_chunk = g.data_size(dim_chunk_ix) == 1 yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel, is_complete_chunk) @dataclass(frozen=True) class SliceDimIndexer: dim_len: int nitems: int nchunks: int dim_grid: DimensionGrid start: int stop: int step: int def __init__( self, dim_sel: slice, dim_len: int, dim_grid: DimensionGrid, ) -> None: # normalize start, stop, step = dim_sel.indices(dim_len) if step < 1: raise NegativeStepError("only slices with step >= 1 are supported.") object.__setattr__(self, "start", start) object.__setattr__(self, "stop", stop) object.__setattr__(self, "step", step) object.__setattr__(self, "dim_len", dim_len) object.__setattr__(self, "dim_grid", dim_grid) object.__setattr__(self, "nitems", max(0, ceildiv_int((stop - start), step))) object.__setattr__(self, "nchunks", dim_grid.nchunks) def __iter__(self) -> Iterator[ChunkDimProjection]: # figure out the range of chunks we need to visit if self.start >= self.stop: return # empty slice g = self.dim_grid dim_chunk_ix_from = g.index_to_chunk(self.start) if self.start > 0 else 0 dim_chunk_ix_to = g.index_to_chunk(self.stop - 1) + 1 if self.stop > 0 else 0 # iterate over chunks in range for dim_chunk_ix in range(dim_chunk_ix_from, dim_chunk_ix_to): # compute offsets for chunk within overall array dim_offset = g.chunk_offset(dim_chunk_ix) # determine chunk length, accounting for trailing chunk dim_chunk_len = g.data_size(dim_chunk_ix) dim_limit = dim_offset + dim_chunk_len if self.start < dim_offset: # selection starts before current chunk dim_chunk_sel_start = 0 remainder = (dim_offset - self.start) % self.step if remainder: dim_chunk_sel_start += self.step - remainder # compute number of previous items, provides offset into output array dim_out_offset = ceildiv_int((dim_offset - self.start), self.step) else: # selection starts within current chunk dim_chunk_sel_start = self.start - dim_offset dim_out_offset = 0 if self.stop > dim_limit: # selection ends after current chunk dim_chunk_sel_stop = dim_chunk_len else: # selection ends within current chunk dim_chunk_sel_stop = self.stop - dim_offset dim_chunk_sel = slice(dim_chunk_sel_start, dim_chunk_sel_stop, self.step) dim_chunk_nitems = ceildiv_int((dim_chunk_sel_stop - dim_chunk_sel_start), self.step) # If there are no elements on the selection within this chunk, then skip if dim_chunk_nitems == 0: continue dim_out_sel = slice(dim_out_offset, dim_out_offset + dim_chunk_nitems) is_complete_chunk = ( dim_chunk_sel_start == 0 and (self.stop >= dim_limit) and self.step in [1, None] ) yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel, is_complete_chunk) def check_selection_length(selection: SelectionNormalized, shape: tuple[int, ...]) -> None: if len(selection) > len(shape): err_too_many_indices(selection, shape) def replace_ellipsis(selection: Any, shape: tuple[int, ...]) -> SelectionNormalized: selection = ensure_tuple(selection) # count number of ellipsis present n_ellipsis = sum(1 for i in selection if i is Ellipsis) if n_ellipsis > 1: # more than 1 is an error raise IndexError("an index can only have a single ellipsis ('...')") elif n_ellipsis == 1: # locate the ellipsis, count how many items to left and right n_items_l = selection.index(Ellipsis) # items to left of ellipsis n_items_r = len(selection) - (n_items_l + 1) # items to right of ellipsis n_items = len(selection) - 1 # all non-ellipsis items if n_items >= len(shape): # ellipsis does nothing, just remove it selection = tuple(i for i in selection if i != Ellipsis) else: # replace ellipsis with as many slices are needed for number of dims new_item = selection[:n_items_l] + ((slice(None),) * (len(shape) - n_items)) if n_items_r: new_item += selection[-n_items_r:] selection = new_item # fill out selection if not completely specified if len(selection) < len(shape): selection += (slice(None),) * (len(shape) - len(selection)) # check selection not too long check_selection_length(selection, shape) return cast("SelectionNormalized", selection) def replace_lists(selection: SelectionNormalized) -> SelectionNormalized: return tuple( cast("ArrayOfIntOrBool", np.asarray(dim_sel)) if isinstance(dim_sel, list) else dim_sel for dim_sel in selection ) def ensure_tuple(v: Any) -> SelectionNormalized: if not isinstance(v, tuple): v = (v,) return cast("SelectionNormalized", v) class ChunkProjection(NamedTuple): """A mapping of items from chunk to output array. Can be used to extract items from the chunk array for loading into an output array. Can also be used to extract items from a value array for setting/updating in a chunk array. Attributes ---------- chunk_coords Indices of chunk. chunk_selection Selection of items from chunk array. out_selection Selection of items in target (output) array. is_complete_chunk: True if a complete chunk is indexed """ chunk_coords: tuple[int, ...] chunk_selection: tuple[Selector, ...] | npt.NDArray[np.intp] out_selection: tuple[Selector, ...] | npt.NDArray[np.intp] | slice is_complete_chunk: bool def is_slice(s: Any) -> TypeGuard[slice]: return isinstance(s, slice) def is_contiguous_slice(s: Any) -> TypeGuard[slice]: return is_slice(s) and (s.step is None or s.step == 1) def is_positive_slice(s: Any) -> TypeGuard[slice]: return is_slice(s) and (s.step is None or s.step >= 1) def is_contiguous_selection(selection: Any) -> TypeGuard[slice]: selection = ensure_tuple(selection) return all((is_integer_array(s) or is_contiguous_slice(s) or s == Ellipsis) for s in selection) def is_basic_selection(selection: Any) -> TypeGuard[BasicSelection]: selection = ensure_tuple(selection) return all(is_integer(s) or is_positive_slice(s) for s in selection) @dataclass(frozen=True) class BasicIndexer(Indexer): dim_indexers: list[IntDimIndexer | SliceDimIndexer] shape: tuple[int, ...] drop_axes: tuple[int, ...] def __init__( self, selection: BasicSelection, shape: tuple[int, ...], chunk_grid: ChunkGrid, ) -> None: dim_grids = chunk_grid._dimensions # handle ellipsis selection_normalized = replace_ellipsis(selection, shape) # setup per-dimension indexers dim_indexers: list[IntDimIndexer | SliceDimIndexer] = [] for dim_sel, dim_len, dim_grid in zip(selection_normalized, shape, dim_grids, strict=True): dim_indexer: IntDimIndexer | SliceDimIndexer if is_integer(dim_sel): dim_indexer = IntDimIndexer(dim_sel, dim_len, dim_grid) elif is_slice(dim_sel): dim_indexer = SliceDimIndexer(dim_sel, dim_len, dim_grid) else: raise IndexError( "unsupported selection item for basic indexing; " f"expected integer or slice, got {type(dim_sel)!r}" ) dim_indexers.append(dim_indexer) object.__setattr__(self, "dim_indexers", dim_indexers) object.__setattr__( self, "shape", tuple(s.nitems for s in self.dim_indexers if not isinstance(s, IntDimIndexer)), ) object.__setattr__(self, "drop_axes", ()) def __iter__(self) -> Iterator[ChunkProjection]: for dim_projections in itertools.product(*self.dim_indexers): chunk_coords = tuple(p.dim_chunk_ix for p in dim_projections) chunk_selection = tuple(p.dim_chunk_sel for p in dim_projections) out_selection = tuple( p.dim_out_sel for p in dim_projections if p.dim_out_sel is not None ) is_complete_chunk = all(p.is_complete_chunk for p in dim_projections) yield ChunkProjection(chunk_coords, chunk_selection, out_selection, is_complete_chunk) @dataclass(frozen=True) class BoolArrayDimIndexer: dim_sel: npt.NDArray[np.bool_] dim_len: int dim_grid: DimensionGrid nchunks: int chunk_nitems: npt.NDArray[Any] chunk_nitems_cumsum: npt.NDArray[Any] nitems: int dim_chunk_ixs: npt.NDArray[np.intp] def __init__( self, dim_sel: npt.NDArray[np.bool_], dim_len: int, dim_grid: DimensionGrid, ) -> None: # check number of dimensions if not is_bool_array(dim_sel, 1): raise IndexError("Boolean arrays in an orthogonal selection must be 1-dimensional only") # check shape if dim_sel.shape[0] != dim_len: raise IndexError( f"Boolean array has the wrong length for dimension; expected {dim_len}, got {dim_sel.shape[0]}" ) g = dim_grid nchunks = g.nchunks # precompute number of selected items for each chunk chunk_nitems = np.zeros(nchunks, dtype="i8") for dim_chunk_ix in range(nchunks): dim_offset = g.chunk_offset(dim_chunk_ix) chunk_len = g.data_size(dim_chunk_ix) chunk_nitems[dim_chunk_ix] = np.count_nonzero( dim_sel[dim_offset : dim_offset + chunk_len] ) chunk_nitems_cumsum = np.cumsum(chunk_nitems) nitems = chunk_nitems_cumsum[-1] dim_chunk_ixs = np.nonzero(chunk_nitems)[0] # store attributes object.__setattr__(self, "dim_sel", dim_sel) object.__setattr__(self, "dim_len", dim_len) object.__setattr__(self, "dim_grid", dim_grid) object.__setattr__(self, "nchunks", nchunks) object.__setattr__(self, "chunk_nitems", chunk_nitems) object.__setattr__(self, "chunk_nitems_cumsum", chunk_nitems_cumsum) object.__setattr__(self, "nitems", nitems) object.__setattr__(self, "dim_chunk_ixs", dim_chunk_ixs) def __iter__(self) -> Iterator[ChunkDimProjection]: g = self.dim_grid # iterate over chunks with at least one item for dim_chunk_ix in self.dim_chunk_ixs: # find region in chunk dim_offset = g.chunk_offset(dim_chunk_ix) chunk_len = g.data_size(dim_chunk_ix) dim_chunk_sel = self.dim_sel[dim_offset : dim_offset + chunk_len] # pad out if boundary chunk (codec buffer may be larger than valid data region) codec_size = g.chunk_size(dim_chunk_ix) if dim_chunk_sel.shape[0] < codec_size: tmp = np.zeros(codec_size, dtype=bool) tmp[: dim_chunk_sel.shape[0]] = dim_chunk_sel dim_chunk_sel = tmp # find region in output if dim_chunk_ix == 0: start = 0 else: start = self.chunk_nitems_cumsum[dim_chunk_ix - 1] stop = self.chunk_nitems_cumsum[dim_chunk_ix] dim_out_sel = slice(start, stop) is_complete_chunk = False # TODO yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel, is_complete_chunk) class Order(Enum): """ Enum for indexing order. """ UNKNOWN = 0 INCREASING = 1 DECREASING = 2 UNORDERED = 3 @staticmethod def check(a: npt.NDArray[Any]) -> Order: # compare, don't subtract: np.diff wraps on unsigned dtypes diff_positive = a[1:] >= a[:-1] n_diff_positive = np.count_nonzero(diff_positive) all_increasing = n_diff_positive == len(diff_positive) any_increasing = n_diff_positive > 0 if all_increasing: order = Order.INCREASING elif any_increasing: order = Order.UNORDERED else: order = Order.DECREASING return order def wraparound_indices(x: npt.NDArray[Any], dim_len: int) -> npt.NDArray[Any]: """Normalize negative indices, copying only when normalization is needed.""" loc_neg = x < 0 if np.any(loc_neg): x = x.copy() x[loc_neg] += dim_len return x def boundscheck_indices(x: npt.NDArray[Any], dim_len: int) -> None: if np.any(x < 0) or np.any(x >= dim_len): msg = f"index out of bounds for dimension with length {dim_len}" raise BoundsCheckError(msg) def sorted_run_ends( a: npt.NDArray[Any], ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]: """Group a sorted 1-D integer array into runs of equal values. Returns `(values, run_ends)` where `values` holds the distinct values in order and `run_ends[i]` is the exclusive end offset of run `i` in `a`. Cost is O(len(a)), independent of the range of values — unlike a dense `np.bincount` histogram, which allocates O(max value) memory (see gh-4174). """ if a.size == 0: return np.empty(0, dtype=np.intp), np.empty(0, dtype=np.intp) run_starts = np.concatenate(([0], np.nonzero(np.diff(a))[0] + 1)) run_ends = np.append(run_starts[1:], a.size).astype(np.intp, copy=False) return a[run_starts].astype(np.intp, copy=False), run_ends @dataclass(frozen=True) class IntArrayDimIndexer: """Integer array selection against a single dimension.""" dim_len: int dim_grid: DimensionGrid nchunks: int nitems: int order: Order dim_sel: npt.NDArray[np.intp] dim_out_sel: npt.NDArray[np.intp] # Dense compatibility arrays are populated on first access, not during indexing. chunk_nitems: npt.NDArray[np.intp] = field(repr=False) dim_chunk_ixs: npt.NDArray[np.intp] chunk_nitems_cumsum: npt.NDArray[np.intp] = field(repr=False) # end offset of each occupied chunk's run of selected items, aligned with dim_chunk_ixs chunk_run_ends: npt.NDArray[np.intp] def __init__( self, dim_sel: npt.NDArray[np.intp], dim_len: int, dim_grid: DimensionGrid, wraparound: bool = True, boundscheck: bool = True, order: Order = Order.UNKNOWN, ) -> None: # ensure 1d array dim_sel = np.asanyarray(dim_sel) if not is_integer_array(dim_sel, 1): raise IndexError("integer arrays in an orthogonal selection must be 1-dimensional only") # Check unsigned values before narrowing: uint64 can wrap to a valid negative index. if boundscheck and dim_sel.dtype.kind == "u": boundscheck_indices(dim_sel, dim_len) # uint64 promotes to float against the signed chunk offset dim_sel = dim_sel.astype(np.intp, copy=False) nitems = len(dim_sel) g = dim_grid nchunks = g.nchunks # handle wraparound if wraparound: dim_sel = wraparound_indices(dim_sel, dim_len) # handle out of bounds if boundscheck: boundscheck_indices(dim_sel, dim_len) # determine which chunk is needed for each selection item # note: for dense integer selections, the division operation here is the # bottleneck dim_sel_chunk = g.indices_to_chunks(dim_sel) # determine order of indices if order == Order.UNKNOWN: order = Order.check(dim_sel) order = Order(order) if order == Order.INCREASING: dim_out_sel = None dim_sel_chunk_sorted = dim_sel_chunk elif order == Order.DECREASING: dim_sel = dim_sel[::-1] # TODO should be possible to do this without creating an arange dim_out_sel = np.arange(nitems - 1, -1, -1) dim_sel_chunk_sorted = dim_sel_chunk[::-1] else: # sort indices to group by chunk dim_out_sel = np.argsort(dim_sel_chunk) dim_sel = np.take(dim_sel, dim_out_sel) dim_sel_chunk_sorted = dim_sel_chunk[dim_out_sel] # the chunks to visit and, per occupied chunk, the end offset of its run of # selected items — O(nitems), never O(nchunks) dim_chunk_ixs, chunk_run_ends = sorted_run_ends(dim_sel_chunk_sorted) # store attributes object.__setattr__(self, "dim_len", dim_len) object.__setattr__(self, "dim_grid", dim_grid) object.__setattr__(self, "nchunks", nchunks) object.__setattr__(self, "nitems", nitems) object.__setattr__(self, "order", order) object.__setattr__(self, "dim_sel", dim_sel) object.__setattr__(self, "dim_out_sel", dim_out_sel) object.__setattr__(self, "dim_chunk_ixs", dim_chunk_ixs) object.__setattr__(self, "chunk_run_ends", chunk_run_ends) def __getattr__(self, name: str) -> npt.NDArray[np.intp]: if name not in ("chunk_nitems", "chunk_nitems_cumsum"): raise AttributeError(f"{type(self).__name__!r} object has no attribute {name!r}") dense = np.zeros(self.nchunks, dtype=np.intp) dense[self.dim_chunk_ixs] = np.diff(self.chunk_run_ends, prepend=0) if name == "chunk_nitems_cumsum": np.cumsum(dense, out=dense) object.__setattr__(self, name, dense) return dense def __iter__(self) -> Iterator[ChunkDimProjection]: g = self.dim_grid dense_cumsum = self.__dict__.get("chunk_nitems_cumsum") for i, dim_chunk_ix in enumerate(self.dim_chunk_ixs): dim_out_sel: slice | npt.NDArray[np.intp] # find region in output if dense_cumsum is None: start = 0 if i == 0 else self.chunk_run_ends[i - 1] stop = self.chunk_run_ends[i] else: start = 0 if dim_chunk_ix == 0 else dense_cumsum[dim_chunk_ix - 1] stop = dense_cumsum[dim_chunk_ix] if self.order == Order.INCREASING: dim_out_sel = slice(start, stop) else: dim_out_sel = self.dim_out_sel[start:stop] # find region in chunk dim_offset = g.chunk_offset(dim_chunk_ix) dim_chunk_sel = self.dim_sel[start:stop] - dim_offset is_complete_chunk = False # TODO yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel, is_complete_chunk) def slice_to_range(s: slice, length: int) -> range: return range(*s.indices(length)) def ix_(selection: Any, shape: tuple[int, ...]) -> npt.NDArray[np.intp]: """Convert an orthogonal selection to a numpy advanced (fancy) selection, like ``numpy.ix_`` but with support for slices and single ints.""" # normalisation selection = replace_ellipsis(selection, shape) # replace slice and int as these are not supported by numpy.ix_ selection = [ slice_to_range(dim_sel, dim_len) if isinstance(dim_sel, slice) else [dim_sel] if is_integer(dim_sel) else dim_sel for dim_sel, dim_len in zip(selection, shape, strict=True) ] # now get numpy to convert to a coordinate selection selection = np.ix_(*selection) return cast("npt.NDArray[np.intp]", selection) def oindex(a: npt.NDArray[Any], selection: Selection) -> npt.NDArray[Any]: """Implementation of orthogonal indexing with slices and ints.""" selection = replace_ellipsis(selection, a.shape) drop_axes = tuple(i for i, s in enumerate(selection) if is_integer(s)) selection = ix_(selection, a.shape) result = a[selection] if drop_axes: result = result.squeeze(axis=drop_axes) return result def oindex_set(a: npt.NDArray[Any], selection: Selection, value: Any) -> None: selection = replace_ellipsis(selection, a.shape) drop_axes = tuple(i for i, s in enumerate(selection) if is_integer(s)) selection = ix_(selection, a.shape) if not np.isscalar(value) and drop_axes: value = np.asanyarray(value) value_selection: list[Selector | None] = [slice(None)] * len(a.shape) for i in drop_axes: value_selection[i] = np.newaxis value = value[tuple(value_selection)] a[selection] = value @dataclass(frozen=True) class OrthogonalIndexer(Indexer): dim_indexers: list[IntDimIndexer | SliceDimIndexer | IntArrayDimIndexer | BoolArrayDimIndexer] dim_grids: tuple[DimensionGrid, ...] shape: tuple[int, ...] is_advanced: bool drop_axes: tuple[int, ...] def __init__(self, selection: Selection, shape: tuple[int, ...], chunk_grid: ChunkGrid) -> None: dim_grids = chunk_grid._dimensions # handle ellipsis selection = replace_ellipsis(selection, shape) # normalize list to array selection = replace_lists(selection) # setup per-dimension indexers dim_indexers: list[ IntDimIndexer | SliceDimIndexer | IntArrayDimIndexer | BoolArrayDimIndexer ] = [] for dim_sel, dim_len, dim_grid in zip(selection, shape, dim_grids, strict=True): dim_indexer: IntDimIndexer | SliceDimIndexer | IntArrayDimIndexer | BoolArrayDimIndexer if is_integer(dim_sel): dim_indexer = IntDimIndexer(dim_sel, dim_len, dim_grid) elif isinstance(dim_sel, slice): dim_indexer = SliceDimIndexer(dim_sel, dim_len, dim_grid) elif is_integer_array(dim_sel): dim_indexer = IntArrayDimIndexer(dim_sel, dim_len, dim_grid) elif is_bool_array(dim_sel): dim_indexer = BoolArrayDimIndexer(dim_sel, dim_len, dim_grid) else: raise IndexError( "unsupported selection item for orthogonal indexing; " "expected integer, slice, integer array or Boolean " f"array, got {type(dim_sel)!r}" ) dim_indexers.append(dim_indexer) shape = tuple(s.nitems for s in dim_indexers if not isinstance(s, IntDimIndexer)) is_advanced = not is_basic_selection(selection) if is_advanced: drop_axes = tuple( i for i, dim_indexer in enumerate(dim_indexers) if isinstance(dim_indexer, IntDimIndexer) ) else: drop_axes = () object.__setattr__(self, "dim_indexers", dim_indexers) object.__setattr__(self, "dim_grids", dim_grids) object.__setattr__(self, "shape", shape) object.__setattr__(self, "is_advanced", is_advanced) object.__setattr__(self, "drop_axes", drop_axes) def __iter__(self) -> Iterator[ChunkProjection]: for dim_projections in itertools.product(*self.dim_indexers): chunk_coords = tuple(p.dim_chunk_ix for p in dim_projections) chunk_selection: tuple[Selector, ...] | npt.NDArray[Any] = tuple( p.dim_chunk_sel for p in dim_projections ) out_selection: tuple[Selector, ...] | npt.NDArray[Any] = tuple( p.dim_out_sel for p in dim_projections if p.dim_out_sel is not None ) # handle advanced indexing arrays orthogonally if self.is_advanced: # NumPy can handle a single array-indexed dimension directly, # which preserves full slices and avoids an # unnecessary advanced-indexing copy. Integer-indexed # dimensions still need the ix_ path for downstream squeezing. # Example: we skip `ix_` for array[:, :, [1, 2, 3]] n_array_dims = sum(isinstance(sel, np.ndarray) for sel in chunk_selection) if n_array_dims > 1 or self.drop_axes: # N.B., numpy doesn't support orthogonal indexing directly # for multiple array-indexed dimensions, so we need to # convert the orthogonal selection into coordinate arrays. chunk_shape = tuple( g.chunk_size(p.dim_chunk_ix) for g, p in zip(self.dim_grids, dim_projections, strict=True) ) chunk_selection = ix_(chunk_selection, chunk_shape) # special case for non-monotonic indices if not is_basic_selection(out_selection): out_selection = ix_(out_selection, self.shape) is_complete_chunk = all(p.is_complete_chunk for p in dim_projections) yield ChunkProjection(chunk_coords, chunk_selection, out_selection, is_complete_chunk) @dataclass(frozen=True) class OIndex: array: AnyArray # TODO: develop Array generic and move zarr.Array[np.intp] | zarr.Array[np.bool_] to ArrayOfIntOrBool def __getitem__(self, selection: OrthogonalSelection | AnyArray) -> NDArrayLikeOrScalar: from zarr.core.array import Array # if input is a Zarr array, we materialize it now. if isinstance(selection, Array): selection = _zarr_array_to_int_or_bool_array(selection) fields, new_selection = pop_fields(selection) new_selection = ensure_tuple(new_selection) new_selection = replace_lists(new_selection) return self.array.get_orthogonal_selection( cast("OrthogonalSelection", new_selection), fields=fields ) def __setitem__(self, selection: OrthogonalSelection, value: npt.ArrayLike) -> None: fields, new_selection = pop_fields(selection) new_selection = ensure_tuple(new_selection) new_selection = replace_lists(new_selection) return self.array.set_orthogonal_selection( cast("OrthogonalSelection", new_selection), value, fields=fields ) @dataclass(frozen=True) class AsyncOIndex[T_ArrayMetadata: (ArrayV2Metadata, ArrayV3Metadata)]: array: AsyncArray[T_ArrayMetadata] async def getitem(self, selection: OrthogonalSelection | AnyArray) -> NDArrayLikeOrScalar: from zarr.core.array import Array # if input is a Zarr array, we materialize it now. if isinstance(selection, Array): selection = _zarr_array_to_int_or_bool_array(selection) fields, new_selection = pop_fields(selection) new_selection = ensure_tuple(new_selection) new_selection = replace_lists(new_selection) return await self.array.get_orthogonal_selection( cast(OrthogonalSelection, new_selection), fields=fields ) @dataclass(frozen=True) class BlockIndexer(Indexer): dim_indexers: list[SliceDimIndexer] shape: tuple[int, ...] drop_axes: tuple[int, ...] def __init__( self, selection: BasicSelection, shape: tuple[int, ...], chunk_grid: ChunkGrid ) -> None: dim_grids = chunk_grid._dimensions # handle ellipsis selection_normalized = replace_ellipsis(selection, shape) # normalize list to array selection_normalized = replace_lists(selection_normalized) # setup per-dimension indexers dim_indexers = [] for dim_sel, dim_len, dim_grid in zip(selection_normalized, shape, dim_grids, strict=True): dim_numchunks = dim_grid.nchunks if is_integer(dim_sel): if dim_sel < 0: dim_sel = dim_numchunks + dim_sel if dim_sel < 0 or dim_sel >= dim_numchunks: raise BoundsCheckError( f"block index out of bounds for dimension with {dim_numchunks} chunk(s)" ) start = dim_grid.chunk_offset(dim_sel) stop = start + dim_grid.chunk_size(dim_sel) slice_ = slice(start, stop) elif is_slice(dim_sel): start = dim_sel.start if dim_sel.start is not None else 0 stop = dim_sel.stop if dim_sel.stop is not None else dim_numchunks if dim_sel.step not in {1, None}: raise IndexError( "unsupported selection item for block indexing; " f"expected integer or slice with step=1, got {type(dim_sel)!r}" ) # Can't reuse wraparound_indices because it expects a numpy array # We have integers here. if start < 0: start = dim_numchunks + start if stop < 0: stop = dim_numchunks + stop start = dim_grid.chunk_offset(start) if start < dim_numchunks else dim_len stop = dim_grid.chunk_offset(stop) if stop < dim_numchunks else dim_len slice_ = slice(start, stop) else: raise IndexError( "unsupported selection item for block indexing; " f"expected integer or slice, got {type(dim_sel)!r}" ) dim_indexer = SliceDimIndexer(slice_, dim_len, dim_grid) dim_indexers.append(dim_indexer) if slice_.start >= dim_len or slice_.start < 0: msg = f"index out of bounds for dimension with length {dim_len}" raise BoundsCheckError(msg) shape = tuple(s.nitems for s in dim_indexers) object.__setattr__(self, "dim_indexers", dim_indexers) object.__setattr__(self, "shape", shape) object.__setattr__(self, "drop_axes", ()) def __iter__(self) -> Iterator[ChunkProjection]: for dim_projections in itertools.product(*self.dim_indexers): chunk_coords = tuple(p.dim_chunk_ix for p in dim_projections) chunk_selection = tuple(p.dim_chunk_sel for p in dim_projections) out_selection = tuple( p.dim_out_sel for p in dim_projections if p.dim_out_sel is not None ) is_complete_chunk = all(p.is_complete_chunk for p in dim_projections) yield ChunkProjection(chunk_coords, chunk_selection, out_selection, is_complete_chunk) @dataclass(frozen=True) class BlockIndex: array: AnyArray def __getitem__(self, selection: BasicSelection) -> NDArrayLikeOrScalar: fields, new_selection = pop_fields(selection) new_selection = ensure_tuple(new_selection) new_selection = replace_lists(new_selection) return self.array.get_block_selection(cast("BasicSelection", new_selection), fields=fields) def __setitem__(self, selection: BasicSelection, value: npt.ArrayLike) -> None: fields, new_selection = pop_fields(selection) new_selection = ensure_tuple(new_selection) new_selection = replace_lists(new_selection) return self.array.set_block_selection( cast("BasicSelection", new_selection), value, fields=fields ) def is_coordinate_selection( selection: SelectionNormalized, shape: tuple[int, ...] ) -> TypeGuard[CoordinateSelectionNormalized]: return ( isinstance(selection, tuple) and len(selection) == len(shape) and all(is_integer(dim_sel) or is_integer_array(dim_sel) for dim_sel in selection) ) def is_mask_selection(selection: Selection, shape: tuple[int, ...]) -> TypeGuard[MaskSelection]: return ( isinstance(selection, tuple) and len(selection) == 1 and is_bool_array(selection[0]) and selection[0].shape == shape ) @dataclass(frozen=True) class CoordinateIndexer(Indexer): sel_shape: tuple[int, ...] selection: CoordinateSelectionNormalized sel_sort: npt.NDArray[np.intp] | None # Exclude the lazy dense field from repr to keep inspection sparse. chunk_nitems_cumsum: npt.NDArray[np.intp] = field(repr=False) cdata_shape: tuple[int, ...] # end offset of each occupied chunk's run of selected points, aligned with chunk_rixs chunk_run_ends: npt.NDArray[np.intp] chunk_rixs: npt.NDArray[np.intp] chunk_mixs: tuple[npt.NDArray[np.intp], ...] shape: tuple[int, ...] dim_grids: tuple[DimensionGrid, ...] drop_axes: tuple[int, ...] def __init__( self, selection: CoordinateSelection, shape: tuple[int, ...], chunk_grid: ChunkGrid ) -> None: dim_grids = chunk_grid._dimensions cdata_shape: tuple[int, ...] if shape == (): cdata_shape = (1,) else: cdata_shape = tuple(g.nchunks for g in dim_grids) # some initial normalization selection_normalized = cast("CoordinateSelectionNormalized", ensure_tuple(selection)) selection_normalized = tuple( np.asarray([i], dtype=np.intp) if is_integer(i) else i for i in selection_normalized ) selection_normalized = cast( "CoordinateSelectionNormalized", replace_lists(selection_normalized) ) # validation if not is_coordinate_selection(selection_normalized, shape): raise IndexError( "invalid coordinate selection; expected one integer " "(coordinate) array per dimension of the target array, " f"got {selection!r}" ) # Check unsigned values before narrowing can turn an out-of-bounds value negative. for dim_sel, dim_len in zip(selection_normalized, shape, strict=True): if dim_sel.dtype.kind == "u": boundscheck_indices(dim_sel, dim_len) # keep indices integral: uint64 against a signed offset promotes to float selection_normalized = cast( "CoordinateSelectionNormalized", tuple(np.asarray(s, dtype=np.intp) for s in selection_normalized), ) # Optimization for a single sorted, in-bounds, 1-D integer coordinate array over a # regular (fixed-size) chunk grid. The general path below makes several full passes over # the flat selection. For sufficiently dense selections, locating the internal chunk # boundaries with searchsorted is cheaper. if len(selection_normalized) == 1: (coords,) = selection_normalized g0 = dim_grids[0] # coords is an integer ndarray here: is_coordinate_selection() validated above, and # the normalization turned ints/lists into arrays. Only the sorted-1D-over-regular-grid # shape is special-cased; everything else falls through to the general path below. if ( isinstance(g0, FixedDimension) and g0.size > 0 # guard the divide below and coords.ndim == 1 and coords.size > 0 and coords[0] >= 0 and coords[-1] < shape[0] and coords[0] <= coords[-1] ): size = g0.size first = int(coords[0]) // size last = int(coords[-1]) // size chunk_span = last - first + 1 # searchsorted does O(log n) work per chunk in the spanned range. Fall through # when directly processing the coordinates is expected to be cheaper. if ( chunk_span * coords.size.bit_length() < coords.size and bool((coords[:-1] <= coords[1:]).all()) # sorted -> grouped by chunk ): # Search only internal boundaries. Derive the first and last counts from the # selection bounds so that the boundary after the last chunk cannot overflow. if first == last: counts = np.array([coords.size], dtype=np.intp) else: edges = np.arange(first + 1, last + 1, dtype=coords.dtype) * size cuts = np.searchsorted(coords, edges) counts = np.diff(cuts, prepend=0, append=coords.size) occupied = np.nonzero(counts)[0] chunk_rixs = (first + occupied).astype(np.intp) chunk_run_ends = np.cumsum(counts[occupied]) object.__setattr__(self, "sel_shape", coords.shape) object.__setattr__(self, "selection", (coords,)) object.__setattr__(self, "sel_sort", None) object.__setattr__(self, "cdata_shape", cdata_shape) object.__setattr__(self, "chunk_run_ends", chunk_run_ends) object.__setattr__(self, "chunk_rixs", chunk_rixs) object.__setattr__(self, "chunk_mixs", (chunk_rixs,)) object.__setattr__(self, "dim_grids", dim_grids) object.__setattr__(self, "shape", coords.shape) object.__setattr__(self, "drop_axes", ()) return # Normalize each axis independently without modifying caller-owned index arrays. selection_normalized = tuple( wraparound_indices(dim_sel, dim_len) for dim_sel, dim_len in zip(selection_normalized, shape, strict=True) ) for dim_sel, dim_len in zip(selection_normalized, shape, strict=True): boundscheck_indices(dim_sel, dim_len) # compute chunk index for each point in the selection chunks_multi_index = tuple( g.indices_to_chunks(dim_sel) for (dim_sel, g) in zip(selection_normalized, dim_grids, strict=True) ) # broadcast selection - this will raise error if array dimensions don't match selection_broadcast = tuple(np.broadcast_arrays(*selection_normalized)) chunks_multi_index_broadcast = np.broadcast_arrays(*chunks_multi_index) # remember shape of selection, because we will flatten indices for processing sel_shape = selection_broadcast[0].shape or (1,) # flatten selection selection_broadcast = tuple(dim_sel.reshape(-1) for dim_sel in selection_broadcast) chunks_multi_index_broadcast = tuple( dim_chunks.reshape(-1) for dim_chunks in chunks_multi_index_broadcast ) # ravel chunk indices chunks_raveled_indices = np.ravel_multi_index( chunks_multi_index_broadcast, dims=cdata_shape ) # group points by chunk if np.any(np.diff(chunks_raveled_indices) < 0): # optimisation, only sort if needed sel_sort = np.argsort(chunks_raveled_indices) selection_broadcast = tuple(dim_sel[sel_sort] for dim_sel in selection_broadcast) chunks_raveled_indices = chunks_raveled_indices[sel_sort] else: sel_sort = None shape = selection_broadcast[0].shape or (1,) # the chunks to visit and, per occupied chunk, the end offset of its run of # selected points — O(npoints), never O(nchunks) chunk_rixs, chunk_run_ends = sorted_run_ends(chunks_raveled_indices) # unravel chunk indices chunk_mixs = np.unravel_index(chunk_rixs, cdata_shape) object.__setattr__(self, "sel_shape", sel_shape) object.__setattr__(self, "selection", selection_broadcast) object.__setattr__(self, "sel_sort", sel_sort) object.__setattr__(self, "cdata_shape", cdata_shape) object.__setattr__(self, "chunk_run_ends", chunk_run_ends) object.__setattr__(self, "chunk_rixs", chunk_rixs) object.__setattr__(self, "chunk_mixs", chunk_mixs) object.__setattr__(self, "dim_grids", dim_grids) object.__setattr__(self, "shape", shape) object.__setattr__(self, "drop_axes", ()) def __getattr__(self, name: str) -> npt.NDArray[np.intp]: if name != "chunk_nitems_cumsum": raise AttributeError(f"{type(self).__name__!r} object has no attribute {name!r}") dense = np.zeros(product(self.cdata_shape), dtype=np.intp) dense[self.chunk_rixs] = np.diff(self.chunk_run_ends, prepend=0) np.cumsum(dense, out=dense) object.__setattr__(self, name, dense) return dense def __iter__(self) -> Iterator[ChunkProjection]: dense_cumsum = self.__dict__.get("chunk_nitems_cumsum") # iterate over chunks for i in range(len(self.chunk_rixs)): chunk_coords = tuple(m[i] for m in self.chunk_mixs) if dense_cumsum is None: start = 0 if i == 0 else self.chunk_run_ends[i - 1] stop = self.chunk_run_ends[i] else: chunk_rix = self.chunk_rixs[i] start = 0 if chunk_rix == 0 else dense_cumsum[chunk_rix - 1] stop = dense_cumsum[chunk_rix] out_selection: slice | npt.NDArray[np.intp] if self.sel_sort is None: out_selection = slice(start, stop) else: out_selection = self.sel_sort[start:stop] chunk_offsets = tuple( g.chunk_offset(dim_chunk_ix) for dim_chunk_ix, g in zip(chunk_coords, self.dim_grids, strict=True) ) chunk_selection = tuple( dim_sel[start:stop] - dim_chunk_offset for (dim_sel, dim_chunk_offset) in zip(self.selection, chunk_offsets, strict=True) ) is_complete_chunk = False # TODO yield ChunkProjection(chunk_coords, chunk_selection, out_selection, is_complete_chunk) @dataclass(frozen=True) class MaskIndexer(CoordinateIndexer): def __init__( self, selection: MaskSelection, shape: tuple[int, ...], chunk_grid: ChunkGrid ) -> None: # some initial normalization selection_normalized = cast("tuple[MaskSelection]", ensure_tuple(selection)) selection_normalized = cast("tuple[MaskSelection]", replace_lists(selection_normalized)) # validation if not is_mask_selection(selection_normalized, shape): raise IndexError( "invalid mask selection; expected one Boolean (mask)" f"array with the same shape as the target array, got {selection_normalized!r}" ) # convert to indices selection_indices = np.nonzero(selection_normalized[0]) # delegate the rest to superclass super().__init__(selection_indices, shape, chunk_grid) @dataclass(frozen=True) class VIndex: array: AnyArray # TODO: develop Array generic and move zarr.Array[np.intp] | zarr.Array[np.bool_] to ArrayOfIntOrBool def __getitem__( self, selection: CoordinateSelection | MaskSelection | AnyArray ) -> NDArrayLikeOrScalar: from zarr.core.array import Array # if input is a Zarr array, we materialize it now. if isinstance(selection, Array): selection = _zarr_array_to_int_or_bool_array(selection) fields, new_selection = pop_fields(selection) new_selection = ensure_tuple(new_selection) new_selection = replace_lists(new_selection) if is_coordinate_selection(new_selection, self.array.shape): return self.array.get_coordinate_selection(new_selection, fields=fields) elif is_mask_selection(new_selection, self.array.shape): return self.array.get_mask_selection(new_selection, fields=fields) else: msg = ( "unsupported selection type for vectorized indexing; only " "coordinate selection (tuple of integer arrays) and mask selection " f"(single Boolean array) are supported; got {new_selection!r}" ) raise VindexInvalidSelectionError(msg) def __setitem__( self, selection: CoordinateSelection | MaskSelection, value: npt.ArrayLike ) -> None: fields, new_selection = pop_fields(selection) new_selection = ensure_tuple(new_selection) new_selection = replace_lists(new_selection) if is_coordinate_selection(new_selection, self.array.shape): self.array.set_coordinate_selection(new_selection, value, fields=fields) elif is_mask_selection(new_selection, self.array.shape): self.array.set_mask_selection(new_selection, value, fields=fields) else: msg = ( "unsupported selection type for vectorized indexing; only " "coordinate selection (tuple of integer arrays) and mask selection " f"(single Boolean array) are supported; got {new_selection!r}" ) raise VindexInvalidSelectionError(msg) @dataclass(frozen=True) class AsyncVIndex[T_ArrayMetadata: (ArrayV2Metadata, ArrayV3Metadata)]: array: AsyncArray[T_ArrayMetadata] # TODO: develop Array generic and move zarr.Array[np.intp] | zarr.Array[np.bool_] to ArrayOfIntOrBool async def getitem( self, selection: CoordinateSelection | MaskSelection | AnyArray ) -> NDArrayLikeOrScalar: # TODO deduplicate these internals with the sync version of getitem # TODO requires solving this circular sync issue: https://github.com/zarr-developers/zarr-python/pull/3083#discussion_r2230737448 from zarr.core.array import Array # if input is a Zarr array, we materialize it now. if isinstance(selection, Array): selection = _zarr_array_to_int_or_bool_array(selection) fields, new_selection = pop_fields(selection) new_selection = ensure_tuple(new_selection) new_selection = replace_lists(new_selection) if is_coordinate_selection(new_selection, self.array.shape): return await self.array.get_coordinate_selection(new_selection, fields=fields) elif is_mask_selection(new_selection, self.array.shape): return await self.array.get_mask_selection(new_selection, fields=fields) else: msg = ( "unsupported selection type for vectorized indexing; only " "coordinate selection (tuple of integer arrays) and mask selection " f"(single Boolean array) are supported; got {new_selection!r}" ) raise VindexInvalidSelectionError(msg) def check_fields(fields: Fields | None, dtype: np.dtype[Any]) -> np.dtype[Any]: # early out if fields is None: return dtype # check type if not isinstance(fields, str | list | tuple): raise IndexError( f"'fields' argument must be a string or list of strings; found {type(fields)!r}" ) if fields: if dtype.names is None: raise IndexError("invalid 'fields' argument, array does not have any fields") try: if isinstance(fields, str): # single field selection out_dtype = dtype[fields] else: # multiple field selection out_dtype = np.dtype([(f, dtype[f]) for f in fields]) except KeyError as e: raise IndexError(f"invalid 'fields' argument, field not found: {e!r}") from e else: return out_dtype else: return dtype def check_no_multi_fields(fields: Fields | None) -> Fields | None: if isinstance(fields, list): if len(fields) == 1: return fields[0] elif len(fields) > 1: raise IndexError("multiple fields are not supported for this operation") return fields def pop_fields(selection: SelectionWithFields) -> tuple[Fields | None, Selection]: if isinstance(selection, str): # single field selection return selection, () elif not isinstance(selection, tuple): # single selection item, no fields # leave selection as-is return None, cast("Selection", selection) else: # multiple items, split fields from selection items fields: Fields = [f for f in selection if isinstance(f, str)] fields = fields[0] if len(fields) == 1 else fields selection_tuple = tuple(s for s in selection if not isinstance(s, str)) selection = cast( "Selection", selection_tuple[0] if len(selection_tuple) == 1 else selection_tuple ) return fields, selection def make_slice_selection(selection: Any) -> list[slice]: ls: list[slice] = [] for dim_selection in selection: if is_integer(dim_selection): ls.append(slice(int(dim_selection), int(dim_selection) + 1, 1)) elif isinstance(dim_selection, np.ndarray): if len(dim_selection) == 1: ls.append(slice(int(dim_selection[0]), int(dim_selection[0]) + 1, 1)) else: raise ArrayIndexError else: ls.append(dim_selection) return ls def decode_morton(z: int, chunk_shape: tuple[int, ...]) -> tuple[int, ...]: # Inspired by compressed morton code as implemented in Neuroglancer # https://github.com/google/neuroglancer/blob/master/src/neuroglancer/datasource/precomputed/volume.md#compressed-morton-code bits = tuple((c - 1).bit_length() for c in chunk_shape) max_coords_bits = max(bits) input_bit = 0 input_value = z out = [0] * len(chunk_shape) for coord_bit in range(max_coords_bits): for dim in range(len(chunk_shape)): if coord_bit < bits[dim]: bit = (input_value >> input_bit) & 1 out[dim] |= bit << coord_bit input_bit += 1 return tuple(out) def decode_morton_vectorized( z: npt.NDArray[np.intp], chunk_shape: tuple[int, ...] ) -> npt.NDArray[np.intp]: """Vectorized Morton code decoding for multiple z values. Parameters ---------- z : ndarray 1D array of Morton codes to decode. chunk_shape : tuple of int Shape defining the coordinate space. Returns ------- ndarray 2D array of shape (len(z), len(chunk_shape)) containing decoded coordinates. """ n_dims = len(chunk_shape) bits = tuple((c - 1).bit_length() for c in chunk_shape) max_coords_bits = max(bits) if bits else 0 out = np.zeros((len(z), n_dims), dtype=np.intp) input_bit = 0 for coord_bit in range(max_coords_bits): for dim in range(n_dims): if coord_bit < bits[dim]: # Extract bit at position input_bit from all z values bit_values = (z >> input_bit) & 1 # Place bit at coord_bit position in dimension dim out[:, dim] |= bit_values << coord_bit input_bit += 1 return out @lru_cache(maxsize=16) def _morton_order(shape: tuple[int, ...]) -> npt.NDArray[np.intp]: n_total = product(shape) n_dims = len(shape) if n_total == 0: out = np.empty((0, n_dims), dtype=np.intp) out.flags.writeable = False return out # Ceiling hypercube: smallest power-of-2 hypercube whose Morton codes span # all valid coordinates in shape. (c-1).bit_length() gives the number # of bits needed to index c values (0 for singleton dims). n_z = 2**total_bits # is the size of this hypercube. total_bits = sum((c - 1).bit_length() for c in shape) n_z = 1 << total_bits if total_bits > 0 else 1 # Decode all Morton codes in the ceiling hypercube, then filter to valid coords. # This is fully vectorized. For shapes with n_z >> n_total (e.g. (33,33,33): # n_z=262144, n_total=35937), consider the argsort strategy below. order: npt.NDArray[np.intp] if n_z <= 4 * n_total: # Ceiling strategy: decode all n_z codes vectorized, filter in-bounds. # Works well when the overgeneration ratio n_z/n_total is small (≤4). z_values = np.arange(n_z, dtype=np.intp) all_coords = decode_morton_vectorized(z_values, shape) shape_arr = np.array(shape, dtype=np.intp) valid_mask = np.all(all_coords < shape_arr, axis=1) order = all_coords[valid_mask] else: # Argsort strategy: enumerate all n_total valid coordinates directly, # encode each to a Morton code, then sort by code. Avoids the 8x or # larger overgeneration penalty for near-miss shapes like (33,33,33). # Cost: O(n_total * bits) encode + O(n_total log n_total) sort, # vs O(n_z * bits) = O(8 * n_total * bits) for ceiling. grids = np.meshgrid(*[np.arange(c, dtype=np.intp) for c in shape], indexing="ij") all_coords = np.stack([g.ravel() for g in grids], axis=1) # Encode all coordinates to Morton codes (vectorized). bits_per_dim = tuple((c - 1).bit_length() for c in shape) max_coord_bits = max(bits_per_dim) z_codes = np.zeros(n_total, dtype=np.intp) output_bit = 0 for coord_bit in range(max_coord_bits): for dim in range(n_dims): if coord_bit < bits_per_dim[dim]: z_codes |= ((all_coords[:, dim] >> coord_bit) & 1) << output_bit output_bit += 1 sort_idx: npt.NDArray[np.intp] = np.argsort(z_codes, kind="stable") order = np.asarray(all_coords[sort_idx], dtype=np.intp) order.flags.writeable = False return order @lru_cache(maxsize=16) def morton_order_coords(shape: tuple[int, ...]) -> tuple[tuple[int, ...], ...]: # The grid coordinates in Morton (Z) order, as a cached sequence. The # coordinate set of a finite grid has a known length and is reused in full on # every shard write, so it is built once (vectorized, via `_morton_order`) and # cached per shape rather than recomputed. Indexable and `len`-able; iterate it # directly where an iterator is needed. # # `.tolist()` converts the whole array to native Python ints in one C-level # call; building the tuples row-by-row with `int(x)` is ~9x slower. return tuple(map(tuple, _morton_order(shape).tolist())) @lru_cache(maxsize=16) def _lexicographic_order(shape: tuple[int, ...]) -> npt.NDArray[np.intp]: # Lexicographic (C-order) coordinates, computed vectorized and cached so that # the sharding codec's per-shard chunk grid is not rebuilt on every call. # Equivalent to `np.array(list(np.ndindex(shape)))` but without the # Python-level iteration over every coordinate. n_dims = len(shape) if n_dims == 0: # A 0-d shard holds a single chunk addressed by the empty coordinate, so # the coordinate array has one row and zero columns. np.indices(()) cannot # express this, so build it directly. Matches list(np.ndindex(())) == [()]. order = np.empty((1, 0), dtype=np.intp) else: order = np.indices(shape, dtype=np.intp).reshape(n_dims, -1).T order.flags.writeable = False return order @lru_cache(maxsize=16) def lexicographic_order_coords(shape: tuple[int, ...]) -> tuple[tuple[int, ...], ...]: # The grid coordinates in lexicographic (row-major / C) order, as a cached # sequence. The coordinate set of a finite grid has a known length and is # reused in full on every shard write, so it is built once (vectorized, via # `_lexicographic_order`) and cached per shape. Indexable and `len`-able; # iterate it directly where an iterator is needed. # # `.tolist()` converts the whole array to native Python ints in one C-level # call; building the tuples row-by-row with `int(x)` is ~9x slower. return tuple(map(tuple, _lexicographic_order(shape).tolist())) @lru_cache(maxsize=16) def colexicographic_order_coords(shape: tuple[int, ...]) -> tuple[tuple[int, ...], ...]: # The grid coordinates in colexicographic (column-major / F) order, as a cached # sequence: the first axis varies fastest. Equivalent to reversing each axis, # taking lexicographic order, and reversing the coordinates back. Cached per # shape like its siblings so shard writes don't rebuild it. return tuple(c[::-1] for c in lexicographic_order_coords(shape[::-1])) def get_indexer( selection: SelectionWithFields, shape: tuple[int, ...], chunk_grid: ChunkGrid ) -> Indexer: _, pure_selection = pop_fields(selection) if is_pure_fancy_indexing(pure_selection, len(shape)): new_selection = ensure_tuple(selection) new_selection = replace_lists(new_selection) if is_coordinate_selection(new_selection, shape): return CoordinateIndexer(cast("CoordinateSelection", selection), shape, chunk_grid) elif is_mask_selection(new_selection, shape): return MaskIndexer(cast("MaskSelection", selection), shape, chunk_grid) else: msg = ( "unsupported selection type for vectorized indexing; only " "coordinate selection (tuple of integer arrays) and mask selection " f"(single Boolean array) are supported; got {new_selection!r}" ) raise VindexInvalidSelectionError(msg) elif is_pure_orthogonal_indexing(pure_selection, len(shape)): return OrthogonalIndexer(cast("OrthogonalSelection", selection), shape, chunk_grid) else: return BasicIndexer(cast("BasicSelection", selection), shape, chunk_grid) zarr-developers-zarr-python-a2f2415/src/zarr/core/json_parse.py000066400000000000000000000052531527014137600247040ustar00rootroot00000000000000"""Helpers for validating JSON-decoded metadata. Most JSON metadata validation is delegated to [`msgspec.convert`][msgspec.convert], which handles the type coercions Zarr needs (``Literal`` membership, ``int``/``bool`` strictness, list-to-tuple, ``TypedDict`` with ``NotRequired``). ``convert`` is a thin wrapper that translates [`msgspec.ValidationError`][msgspec.ValidationError] into the ``ValueError`` the rest of the codebase already raises. msgspec cannot handle two things in Zarr's metadata types: * the recursive ``JSON`` / ``JSONValue`` aliases, which it rejects at schema-build time, and * PEP 728 ``extra_items=`` extension fields, which it silently drops. User-defined attributes are left to the JSON reader and writer rather than recursively validated here. See https://github.com/zarr-developers/zarr-python/issues/3285. """ from __future__ import annotations from typing import Any, get_origin import msgspec __all__ = ["convert", "parse_field"] def _type_name(type_: Any) -> str: """Render ``type_`` for an error message. Parameterized types keep their arguments, so a ``Literal`` reports its members (``Literal[2, 3]``) rather than the bare origin name. ``__name__`` would drop them, which loses the most useful part of the message. """ if get_origin(type_) is not None: return str(type_).replace("typing.", "") return getattr(type_, "__name__", None) or str(type_).replace("typing.", "") def convert(value: object, type_: Any, *, strict: bool = True) -> Any: """Validate and coerce ``value`` against ``type_`` via [`msgspec.convert`][msgspec.convert]. On a mismatch msgspec raises [`msgspec.ValidationError`][msgspec.ValidationError]; this re-raises a plain, field-agnostic ``ValueError`` naming the expected type, so callers can add their own field context (see ``parse_field``). """ try: return msgspec.convert(value, type_, strict=strict) except msgspec.ValidationError as exc: raise ValueError(f"Expected instance of {_type_name(type_)}, got {value!r}.") from exc def parse_field( data: object, type_: Any, field: str, *, error: type[Exception] = ValueError ) -> Any: """Validate ``data`` for metadata field ``field`` against ``type_``. Wraps ``convert`` and, on failure, re-raises ``error`` with field context, chaining the underlying type error. This keeps the ``convert``-then-re-raise pattern in one place rather than repeating it in every per-field parser. """ try: return convert(data, type_) except ValueError as exc: raise error( f"Failed to parse input for {field!r}: expected {_type_name(type_)}, got {data!r}." ) from exc zarr-developers-zarr-python-a2f2415/src/zarr/core/metadata/000077500000000000000000000000001527014137600237425ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/core/metadata/__init__.py000066400000000000000000000006041527014137600260530ustar00rootroot00000000000000from .v2 import ArrayV2Metadata, ArrayV2MetadataDict from .v3 import ArrayMetadataJSON_V3, ArrayV3Metadata ArrayMetadata = ArrayV2Metadata | ArrayV3Metadata type ArrayMetadataDict = ArrayV2MetadataDict | ArrayMetadataJSON_V3 __all__ = [ "ArrayMetadata", "ArrayMetadataDict", "ArrayMetadataJSON_V3", "ArrayV2Metadata", "ArrayV2MetadataDict", "ArrayV3Metadata", ] zarr-developers-zarr-python-a2f2415/src/zarr/core/metadata/common.py000066400000000000000000000004051527014137600256030ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING if TYPE_CHECKING: from zarr.core.common import JSON def parse_attributes(data: dict[str, JSON] | None) -> dict[str, JSON]: if data is None: return {} return dict(data) zarr-developers-zarr-python-a2f2415/src/zarr/core/metadata/io.py000066400000000000000000000255401527014137600247310ustar00rootroot00000000000000from __future__ import annotations import asyncio from enum import Enum from itertools import zip_longest from typing import TYPE_CHECKING, Final, NamedTuple from zarr.abc.store import set_or_delete from zarr.core._json import buffer_to_json_object, json_equal from zarr.core.buffer.core import default_buffer_prototype from zarr.core.buffer.cpu import buffer_prototype as cpu_buffer_prototype from zarr.core.common import ZARR_JSON, ZARRAY_JSON, ZATTRS_JSON from zarr.core.metadata.repair import mark_repaired, repair_array_document from zarr.errors import ArrayNotFoundError, ContainsArrayError from zarr.storage._common import StorePath, ensure_no_existing_node if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Mapping from zarr.core.buffer import Buffer from zarr.core.common import JSON, ZarrFormat from zarr.core.group import GroupMetadata from zarr.core.metadata import ArrayMetadata class _Absent(Enum): ABSENT = "absent" ABSENT: Final = _Absent.ABSENT """Where a `DocumentChange` has no value: the document, member or element is not there.""" class DocumentChange(NamedTuple): """A JSON value that differs between a node's stored metadata documents and the documents its metadata would store.""" path: tuple[str | int, ...] """Where the value is: the document's key (e.g. `.zarray`), then the object members and array indices within it.""" stored: JSON | _Absent new: JSON | _Absent def diff_documents( stored: Mapping[str, JSON], new: Mapping[str, JSON] ) -> tuple[DocumentChange, ...]: """What differs between a node's stored metadata documents and the documents its metadata would store, each keyed by its store key (a document missing from `stored` is not stored). Empty if they are identical. Objects and arrays are compared member by member, so a change is the smallest value that differs; any other two values are identical only if their JSON encodings are, so `true` differs from `1` and `1.0` from `1`. """ return tuple(_diff((), stored, new)) def _diff( path: tuple[str | int, ...], stored: JSON | _Absent, new: JSON | _Absent ) -> Iterator[DocumentChange]: match stored, new: case dict(), dict(): for key in dict.fromkeys([*stored, *new]): yield from _diff((*path, key), stored.get(key, ABSENT), new.get(key, ABSENT)) case list(), list(): for index, pair in enumerate(zip_longest(stored, new, fillvalue=ABSENT)): yield from _diff((*path, index), *pair) case _ if stored is ABSENT or new is ABSENT or not json_equal(stored, new): yield DocumentChange(path, stored, new) def encode_documents( store_path: StorePath, metadata: ArrayMetadata | GroupMetadata ) -> dict[str, Buffer]: """The metadata documents `metadata` stores under `store_path`, by key (see `to_buffer_dict`). An operation that deletes or writes store content encodes its documents first, so metadata that cannot be stored fails with the store untouched; the error then names the node at `store_path`, as the warnings about stored documents do. """ from zarr.core.group import GroupMetadata try: return metadata.to_buffer_dict(default_buffer_prototype()) except ValueError as e: node = "Group" if isinstance(metadata, GroupMetadata) else "Array" e.add_note(f"{node} {str(store_path)!r}: nothing was stored.") raise async def store_documents(store_path: StorePath, documents: Mapping[str, Buffer]) -> None: """Store metadata documents encoded by `encode_documents` under `store_path`.""" await asyncio.gather( *(set_or_delete(store_path / key, value) for key, value in documents.items()) ) async def read_documents(store_path: StorePath, keys: Iterable[str]) -> dict[str, Buffer]: """The documents stored under `store_path` at `keys`, read concurrently, by key; a key with no document is left out.""" keys = tuple(keys) buffers = await asyncio.gather( *((store_path / key).get(prototype=cpu_buffer_prototype) for key in keys) ) return {key: buf for key, buf in zip(keys, buffers, strict=True) if buf is not None} async def upsert_metadata( store_path: StorePath, metadata: ArrayMetadata | GroupMetadata, stored: Mapping[str, Buffer] ) -> tuple[DocumentChange, ...]: """Store the documents of `metadata` under `store_path` that differ from `stored`, the documents `read_documents` read there, and return how they differed (see `diff_documents`): empty if nothing was stored. The documents are encoded before any is stored, so metadata that cannot be stored fails with the store untouched. """ documents = encode_documents(store_path, metadata) changes = diff_documents( {key: buffer_to_json_object(buf) for key, buf in stored.items() if key in documents}, {key: buffer_to_json_object(buf) for key, buf in documents.items()}, ) changed = {change.path[0] for change in changes} await store_documents(store_path, {k: v for k, v in documents.items() if k in changed}) return changes ARRAY_DOCUMENTS: Final[Mapping[ZarrFormat, tuple[str, ...]]] = { 2: (ZARRAY_JSON, ZATTRS_JSON), 3: (ZARR_JSON,), } """The store keys of the metadata documents of an array of each Zarr format.""" def parse_stored_array( documents: Mapping[str, Buffer], zarr_format: ZarrFormat, path: str | None = None ) -> ArrayMetadata: """The metadata of an array from its documents (by store key, see `ARRAY_DOCUMENTS`), read with the repairs but without their warnings (whoever asks has warned, or reads metadata built in code), and marked (see `mark_repaired`) if they had to be repaired. Raises `ArrayNotFoundError` if there is no array document among them. Only operations that store metadata read documents this way, so documents read as a rectilinear chunk grid require the rectilinear chunks flag, as storing it does; the error names the array at `path`.""" from zarr.core.array import ( _array_metadata_dict_v2, _array_metadata_dict_v3, parse_array_metadata, ) if zarr_format == 2 and ZARRAY_JSON in documents: stored = _array_metadata_dict_v2(documents[ZARRAY_JSON], documents.get(ZATTRS_JSON)) elif zarr_format == 3 and ZARR_JSON in documents: stored = _array_metadata_dict_v3(documents[ZARR_JSON]) else: raise ArrayNotFoundError(f"No Zarr format {zarr_format} array metadata document.") repaired, readings = repair_array_document(stored, zarr_format) metadata = parse_array_metadata(dict(repaired), path) return mark_repaired(metadata, stored, readings, None, warn=False) def _build_parents(store_path: StorePath, zarr_format: ZarrFormat) -> dict[str, GroupMetadata]: from zarr.core.group import GroupMetadata path = store_path.path if not path: return {} required_parts = path.split("/")[:-1] # the root group parents = {"": GroupMetadata(zarr_format=zarr_format)} for i, part in enumerate(required_parts): parent_path = "/".join(required_parts[:i] + [part]) parents[parent_path] = GroupMetadata(zarr_format=zarr_format) return parents async def save_metadata( store_path: StorePath, metadata: ArrayMetadata | GroupMetadata, ensure_parents: bool = False ) -> None: """Asynchronously save the array or group metadata. Parameters ---------- store_path : StorePath Location to save metadata. metadata : ArrayMetadata | GroupMetadata Metadata to save. ensure_parents : bool, optional Create any missing parent groups, and check no existing parents are arrays. Raises ------ ValueError """ to_save = encode_documents(store_path, metadata) await _write_metadata(store_path, to_save, metadata.zarr_format, ensure_parents=ensure_parents) async def save_new_metadata( store_path: StorePath, metadata: ArrayMetadata | GroupMetadata, *, overwrite: bool, ensure_parents: bool = False, ) -> None: """Save the metadata of a new array or group, replacing any existing node if requested. The metadata is encoded first (see `encode_documents`), so metadata that cannot be stored raises, naming the node, without deleting an existing node. Parameters ---------- store_path : StorePath Location of the new node. metadata : ArrayMetadata | GroupMetadata Metadata of the new node. overwrite : bool If true and the store supports deletes, delete everything under `store_path`. Otherwise, raise if a node already exists at `store_path`; a store that cannot delete keys cannot overwrite a node, so it raises the same error either way. ensure_parents : bool, optional Create any missing parent groups, and check no existing parents are arrays. """ to_save = encode_documents(store_path, metadata) if overwrite and store_path.store.supports_deletes: await store_path.delete_dir() else: await ensure_no_existing_node(store_path, zarr_format=metadata.zarr_format) await _write_metadata(store_path, to_save, metadata.zarr_format, ensure_parents=ensure_parents) async def _write_metadata( store_path: StorePath, to_save: dict[str, Buffer], zarr_format: ZarrFormat, *, ensure_parents: bool, ) -> None: set_awaitables = [set_or_delete(store_path / key, value) for key, value in to_save.items()] if ensure_parents: # To enable zarr.create(store, path="a/b/c"), we need to create all the intermediate groups. parents = _build_parents(store_path, zarr_format) ensure_array_awaitables = [] for parent_path, parent_metadata in parents.items(): parent_store_path = StorePath(store_path.store, parent_path) # Error if an array already exists at any parent location. Only groups can have child nodes. ensure_array_awaitables.append( ensure_no_existing_node( parent_store_path, parent_metadata.zarr_format, node_type="array" ) ) set_awaitables.extend( [ (parent_store_path / key).set_if_not_exists(value) for key, value in parent_metadata.to_buffer_dict( default_buffer_prototype() ).items() ] ) # Checks for parent arrays must happen first, before any metadata is modified try: await asyncio.gather(*ensure_array_awaitables) except ContainsArrayError as e: # clear awaitables to avoid RuntimeWarning: coroutine was never awaited for awaitable in set_awaitables: awaitable.close() raise ValueError( f"A parent of {store_path} is an array - only groups may have child nodes." ) from e await asyncio.gather(*set_awaitables) zarr-developers-zarr-python-a2f2415/src/zarr/core/metadata/repair.py000066400000000000000000000417761527014137600256150ustar00rootroot00000000000000"""Repairs that read invalid stored array metadata documents written by older software. This is the only place invalid metadata is read leniently. A repair maps a stored array metadata document (parsed JSON) to a valid one. `ArrayV2Metadata.from_dict` and `ArrayV3Metadata.from_dict` apply the repairs for their Zarr format, so every path that parses a stored document, including consolidated metadata, goes through them. A reading warns only where the user must act on it; a reading that gives what zarr read from the same document before these repairs existed is silent, so a document that opened without a warning still does. The warnings are given once per document, after the repaired document has passed the metadata constructor, so an invalid document raises its own error, not a warning about how it was read. Metadata read from a document whose repair moves chunks (a chunk size read as another size) is marked (see `mark_repaired`), silent or not, so the array stores the repair before it writes chunks under it. A repair that only respells a value zarr already read the same way (`true` as 1, `4.0` as 4) moves no chunks, so it is not marked. To read another kind of invalid document, add a repair to `ARRAY_REPAIRS`. """ from __future__ import annotations import copy import json import warnings from collections.abc import Callable, Iterable, Mapping, Sequence from itertools import chain, repeat from typing import TYPE_CHECKING, Final, NamedTuple, TypeGuard, cast from zarr.errors import ZarrUserWarning if TYPE_CHECKING: from zarr.core.common import JSON, ZarrFormat type ArrayDocument = Mapping[str, JSON] class Reading(NamedTuple): """How a repair read a stored document.""" moves_chunks: bool """Whether chunks written under the repaired document are not where a reader of the stored document looks for them, so the array must store the repair before it writes chunks.""" warning: str | None """What the user must act on, if anything.""" type Repair = Callable[[ArrayDocument], tuple[ArrayDocument, Reading] | None] """Returns `None` if the document needs no repair, else the repaired document and how it was read.""" RESAVE_HINT: Final = ( "To store valid metadata, open the array writable and call `array.update_attributes({})`; " "if a group holds consolidated metadata for the array, then also call " "`zarr.consolidate_metadata` on that group." ) """How to store the repair of a document whose array holds data.""" RECREATE_HINT: Final = ( "The array holds only its fill value, so recreate it with the chunk shape you want; " "nothing is lost: `zarr.from_array(array.store, name=array.path, data=array, " "chunks=, overwrite=True, write_data=False)` keeps its data type, fill " "value, attributes and codecs. To keep the array as it is instead, store valid metadata " "with `array.update_attributes({})`. Either way, open the array writable first, and if a " "group holds consolidated metadata for the array, then also call " "`zarr.consolidate_metadata` on that group." ) """How to act on a document whose array can hold no data: a stored chunk size of 0 is read as the smallest chunk size, which is a poor chunk shape for the data the array grows into, and a re-save would keep it.""" def mark_repaired[M]( metadata: M, stored: ArrayDocument, readings: Sequence[Reading], path: str | None, *, warn: bool = True, ) -> M: """Record that `metadata` was read from the document `stored`, which needed the repairs whose `readings` `repair_array_document` returned, if any. If a reading moves chunks, keep a copy of `stored` as `_stored_document` on `metadata`, so the array stores the repair before it writes chunks under it and consolidated metadata stores it as it was stored. If `warn`, warn once with the readings' warnings (each says what the user must act on and how), naming the array at `path` when the caller knows it.""" if any(reading.moves_chunks for reading in readings): object.__setattr__(metadata, "_stored_document", copy.deepcopy(stored)) messages = [reading.warning for reading in readings if reading.warning is not None] if warn and messages: subject = "" if path is None else f"Array {path!r}: " # The synchronous API parses metadata on zarr's IO thread, whose stack holds no # user code, so the warning points at the `from_dict` that read the document. warnings.warn(f"{subject}{' '.join(messages)}", ZarrUserWarning, stacklevel=2) return metadata def _is_int_list(value: object) -> TypeGuard[list[int]]: """Whether `value` is a JSON array of integers (JSON `false` and `true` count).""" return isinstance(value, list) and all(isinstance(v, int) for v in value) def _abbreviate(value: JSON, limit: int = 60) -> str: """`value` as JSON, cut to at most `limit` characters.""" text = json.dumps(value) return text if len(text) <= limit else f"{text[: limit - 3]}..." def _read_chunk_size( size: JSON, span: int | None, unit: int ) -> tuple[int | list[JSON], bool, str | None] | None: """Read one entry of a stored regular chunk shape as a chunk edge length, or as the chunk edge lengths of its axis. Returns the reading, whether it moves chunks (it is not the size a reader of the stored entry uses), and, where the user must act on how it was read, how it was read; `None` if the entry cannot be read, which leaves it for the metadata constructors to check. A JSON int >= 1 is kept, JSON `true` is read as 1, and 0 or JSON `false` is read as `unit`, the smallest chunk edge length the axis can have (1, or the inner chunk size of a shard), whatever the length `span` of the axis. That is what `chunks=-1` gives on an axis of length 0, where these sizes were written, and it does not depend on how far the axis has grown since. On an axis of positive length no chunk can have been stored under a chunk size of 0, so the array holds only its fill value. `span` is `None` where no stored 0 is known, as in the inner chunk shape of a sharding codec: 0 is then left as stored. A flat JSON list is kept as the chunk edge lengths of its axis, which only a rectilinear chunk grid can declare (see `_invalid_chunk_sizes_v3`); its edges are read as those of a stored rectilinear chunk grid (see `_invalid_edge_lengths_v3`). """ match size: case True: return 1, False, None case int() if size >= 1: return size, False, None case int() if size == 0 and span is not None: if span == 0: return unit, True, None return ( unit, True, ( "1, as no chunk can be stored under a chunk size of 0" if unit == 1 else f"{unit}, the inner chunk size, as no chunk can be stored under a " "chunk size of 0" ), ) case list() if not any(isinstance(edge, list) for edge in size): return size, False, None return None def _read_chunk_shape( stored: JSON, spans: Sequence[int | None], units: Iterable[int] = () ) -> tuple[list[int | list[JSON]], Reading | None] | None: """Read a stored regular chunk shape, entry by entry (see `_read_chunk_size`), for axes of lengths `spans` whose chunks are multiples of `units` (1 where not given). Returns the chunk shape and how it was read if any entry was read as another value (the warning says how the chunk shape was read and, as the array then holds only its fill value, recommends recreating it, see `RECREATE_HINT`), else `None`; `None` if it cannot be read. """ if not (isinstance(stored, list) and len(stored) == len(spans)): return None edges: list[int | list[JSON]] = [] changed = moves_chunks = False readings: list[str] = [] axes = zip(stored, spans, chain(units, repeat(1)), strict=False) for axis, (size, span, unit) in enumerate(axes): read = _read_chunk_size(size, span, unit) if read is None: return None edge, moved, how = read edges.append(edge) changed |= size is True or edge != size moves_chunks |= moved if how is not None: readings.append(f"{json.dumps(size)} in dimension {axis} as {how}") if not changed: return edges, None warning = ( f"The stored chunk shape {_abbreviate(stored)} is invalid: chunk sizes must be " f"integers of at least 1. It is read as {_abbreviate(edges)}, reading " f"{'; '.join(readings)}. {RECREATE_HINT}" if readings else None ) return edges, Reading(moves_chunks, warning) def _invalid_chunk_sizes_v2(doc: ArrayDocument) -> tuple[ArrayDocument, Reading] | None: shape = doc.get("shape") if not _is_int_list(shape): return None match _read_chunk_shape(doc.get("chunks"), shape): case chunks, Reading() as reading: return {**doc, "chunks": chunks}, reading return None def _read_codec(codec: JSON) -> JSON | None: """Read a stored codec: the inner chunk shape of a sharding codec is read as a chunk shape with no known axis lengths (see `_read_chunk_shape`), and so are those of the sharding codecs nested in its codecs. Returns `None` if nothing was read as another value. No stored inner chunk size is read as a different size, so the reading moves no chunks.""" match codec: case {"name": "sharding_indexed", "configuration": Mapping() as configuration}: repaired: dict[str, JSON] = {} stored = configuration.get("chunk_shape") match ( _read_chunk_shape(stored, [None] * len(stored)) if isinstance(stored, list) else None ): case chunk_shape, Reading(): repaired["chunk_shape"] = chunk_shape if isinstance(codecs := configuration.get("codecs"), list) and ( inner := _read_codecs(codecs) ): repaired["codecs"] = inner if not repaired: return None # The mapping pattern does not narrow `codec` for mypy. return { **cast("Mapping[str, JSON]", codec), "configuration": {**configuration, **repaired}, } return None def _read_codecs(codecs: list[JSON]) -> list[JSON] | None: """Read stored codecs (see `_read_codec`); `None` if nothing was read as another value.""" read = [_read_codec(codec) for codec in codecs] if all(codec is None for codec in read): return None return [stored if new is None else new for stored, new in zip(codecs, read, strict=True)] def _invalid_inner_chunk_sizes_v3(doc: ArrayDocument) -> tuple[ArrayDocument, Reading] | None: stored = doc.get("codecs") if not isinstance(stored, list) or (codecs := _read_codecs(stored)) is None: return None return {**doc, "codecs": codecs}, Reading(moves_chunks=False, warning=None) def _inner_chunk_shape(doc: ArrayDocument) -> list[int] | None: """The inner chunk shape of the sharding codec of a Zarr format 3 array document: `[]` if it has none, `None` if its inner chunk sizes are not all integers of at least 1 (the unit of its outer chunk shape is then unknown).""" match doc.get("codecs"): case list() as codecs: for codec in codecs: match codec: case {"name": "sharding_indexed", "configuration": {"chunk_shape": inner}}: return inner if _is_int_list(inner) and min(inner, default=1) >= 1 else None return [] def _invalid_chunk_sizes_v3(doc: ArrayDocument) -> tuple[ArrayDocument, Reading] | None: grid = doc.get("chunk_grid") shape = doc.get("shape") if not (isinstance(grid, Mapping) and grid.get("name") == "regular" and _is_int_list(shape)): return None if (units := _inner_chunk_shape(doc)) is None: return None configuration = grid.get("configuration") if not isinstance(configuration, Mapping): return None read = _read_chunk_shape(configuration.get("chunk_shape"), shape, units) if read is None: return None chunk_shape, reading = read edge_axes = [axis for axis, size in enumerate(chunk_shape) if isinstance(size, list)] if not edge_axes: if reading is None: return None repaired = {**configuration, "chunk_shape": chunk_shape} return {**doc, "chunk_grid": {**grid, "configuration": repaired}}, reading if len(edge_axes) == len(chunk_shape): # Only a mix of chunk sizes and edge lists was ever stored in a regular grid. return None # The user must act: re-saving this grid needs the rectilinear chunks flag. as_rectilinear = ( f"The stored chunk grid is named 'regular', but its chunk shape lists chunk edge " f"lengths in dimensions {edge_axes}, which only a rectilinear chunk grid can declare. " "It is read as that rectilinear chunk grid. Re-saving the metadata stores that " "rectilinear chunk grid, so each step that follows requires " f"`zarr.config.set({{'array.rectilinear_chunks': True}})`. {RESAVE_HINT}" ) rectilinear: JSON = { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": chunk_shape}, } warning = " ".join(filter(None, (reading and reading.warning, as_rectilinear))) # Only zarr 3.2.x reads the stored grid, so the array stores the rectilinear grid # before it writes chunks, for every other reader to find them. return {**doc, "chunk_grid": rectilinear}, Reading(moves_chunks=True, warning=warning) def _read_edge_length(edge: JSON) -> JSON: """Read one stored chunk edge length of a rectilinear chunk grid: JSON `true` is read as 1 and an integral JSON float of at least 1 (`4.0`) as the `int` it equals. Anything else is kept, for the metadata constructors to check.""" match edge: case True: return 1 case float() if edge.is_integer() and edge >= 1: return int(edge) return edge def _read_rectilinear_axis(spec: JSON) -> JSON: """Read the stored chunk edge lengths of one axis of a rectilinear chunk grid (see `_read_edge_length`): its explicit edges and the sizes of its run-length encoded `[size, count]` pairs. A bare chunk size and a repeat count are kept as stored: no writer stored them as `true` or as floats.""" match spec: case list(): return [_read_rectilinear_entry(entry) for entry in spec] return spec def _read_rectilinear_entry(entry: JSON) -> JSON: match entry: case [size, count]: return [_read_edge_length(size), count] return _read_edge_length(entry) def _respelled(read: JSON, stored: JSON) -> bool: """Whether reading `stored` as `read` gave a value of another type anywhere in it (`true` read as 1, `4.0` as 4). Compares types, not encodings, so a value that is not JSON (a NumPy integer in metadata built in code) is simply kept.""" if isinstance(stored, list) and isinstance(read, list): return any(_respelled(r, s) for r, s in zip(read, stored, strict=True)) return type(read) is not type(stored) def _invalid_edge_lengths_v3(doc: ArrayDocument) -> tuple[ArrayDocument, Reading] | None: grid = doc.get("chunk_grid") if not (isinstance(grid, Mapping) and grid.get("name") == "rectilinear"): return None configuration = grid.get("configuration") if not isinstance(configuration, Mapping): return None stored = configuration.get("chunk_shapes") if not isinstance(stored, list): return None read = [_read_rectilinear_axis(axis) for axis in stored] if not _respelled(read, stored): return None repaired = {**configuration, "chunk_shapes": read} # A stored edge is read as the value it equals, so the reading moves no chunks. return {**doc, "chunk_grid": {**grid, "configuration": repaired}}, Reading(False, None) ARRAY_REPAIRS: Final[Mapping[ZarrFormat, tuple[Repair, ...]]] = { 2: (_invalid_chunk_sizes_v2,), # The inner chunk shape is read first: it gives the unit of the outer chunk shape. # The rectilinear edge lengths are read last, after any repair that yields a # rectilinear chunk grid. 3: (_invalid_inner_chunk_sizes_v3, _invalid_chunk_sizes_v3, _invalid_edge_lengths_v3), } """The repairs of an array document of each Zarr format, applied in order.""" def repair_array_document( doc: ArrayDocument, zarr_format: ZarrFormat ) -> tuple[ArrayDocument, list[Reading]]: """Apply the repairs for `zarr_format` to a stored array metadata document. Returns the repaired document and the reading of each repair that changed it, for `mark_repaired` once the document has been validated. """ readings: list[Reading] = [] for repair in ARRAY_REPAIRS[zarr_format]: repaired = repair(doc) if repaired is not None: doc, reading = repaired readings.append(reading) return doc, readings zarr-developers-zarr-python-a2f2415/src/zarr/core/metadata/v2.py000066400000000000000000000342431527014137600246510ustar00rootroot00000000000000from __future__ import annotations import json import warnings from collections.abc import Iterable, Sequence from functools import cached_property from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypedDict, cast from zarr.abc.metadata import Metadata from zarr.abc.numcodec import Numcodec, _is_numcodec from zarr.core.dtype import get_data_type_from_json from zarr.core.dtype.common import OBJECT_CODEC_IDS, DTypeSpec_V2 from zarr.errors import ZarrUserWarning from zarr.registry import get_numcodec if TYPE_CHECKING: from typing import Self import numpy.typing as npt from zarr.core.buffer import Buffer, BufferPrototype from zarr.core.chunk_grids import ChunkGrid from zarr.core.dtype.wrapper import ( TBaseDType, TBaseScalar, ZDType, ) from zarr.core.metadata.repair import ArrayDocument from dataclasses import dataclass, field, fields, replace import numpy as np from zarr.core._json import json_to_buffer from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.chunk_key_encodings import parse_separator from zarr.core.common import ( JSON, ZARRAY_JSON, ZATTRS_JSON, MemoryOrder, ShapeLike, parse_shapelike, ) from zarr.core.config import config, parse_indexing_order from zarr.core.json_parse import parse_field from zarr.core.metadata.common import parse_attributes from zarr.core.metadata.repair import mark_repaired, repair_array_document class ArrayV2MetadataDict(TypedDict): """ A typed dictionary model for Zarr format 2 metadata. """ zarr_format: Literal[2] attributes: dict[str, JSON] # Union of acceptable types for v2 compressors type CompressorLikev2 = dict[str, JSON] | Numcodec | None @dataclass(frozen=True, kw_only=True) class ArrayV2Metadata(Metadata): shape: tuple[int, ...] chunks: tuple[int, ...] dtype: ZDType[TBaseDType, TBaseScalar] fill_value: int | float | str | bytes | None = None order: MemoryOrder = "C" filters: tuple[Numcodec, ...] | None = None dimension_separator: Literal[".", "/"] = "." compressor: Numcodec | None attributes: dict[str, JSON] = field(default_factory=dict) zarr_format: Literal[2] = field(init=False, default=2) _stored_document: ClassVar[ArrayDocument | None] = None """The stored document `from_dict` read this metadata from, if it had to repair it (set on the instance by `mark_repaired`): the store may still hold it.""" def __init__( self, *, shape: tuple[int, ...], dtype: ZDType[TBaseDType, TBaseScalar], chunks: tuple[int, ...], fill_value: Any, order: MemoryOrder, dimension_separator: Literal[".", "/"] = ".", compressor: CompressorLikev2 = None, filters: Iterable[Numcodec | dict[str, JSON]] | None = None, attributes: dict[str, JSON] | None = None, ) -> None: """ Metadata for a Zarr format 2 array. """ shape_parsed = parse_shapelike(shape) chunks_parsed = parse_chunks(chunks, shape_parsed) compressor_parsed = parse_compressor(compressor) order_parsed = parse_indexing_order(order) dimension_separator_parsed = parse_separator(dimension_separator) filters_parsed = parse_filters(filters) fill_value_parsed: TBaseScalar | None if fill_value is not None: fill_value_parsed = dtype.cast_scalar(fill_value) else: fill_value_parsed = fill_value attributes_parsed = parse_attributes(attributes) object.__setattr__(self, "shape", shape_parsed) object.__setattr__(self, "dtype", dtype) object.__setattr__(self, "chunks", chunks_parsed) object.__setattr__(self, "compressor", compressor_parsed) object.__setattr__(self, "order", order_parsed) object.__setattr__(self, "dimension_separator", dimension_separator_parsed) object.__setattr__(self, "filters", filters_parsed) object.__setattr__(self, "fill_value", fill_value_parsed) object.__setattr__(self, "attributes", attributes_parsed) @property def ndim(self) -> int: return len(self.shape) @cached_property def chunk_grid(self) -> ChunkGrid: """Backwards-compatible chunk grid property. !!! warning "Deprecated" Access the chunk grid via the array layer instead. This property will be removed in a future release. """ from zarr.core.chunk_grids import ChunkGrid warnings.warn( "ArrayV2Metadata.chunk_grid is deprecated. " "Use ChunkGrid.from_metadata(metadata) instead.", DeprecationWarning, stacklevel=2, ) return ChunkGrid.from_sizes(self.shape, tuple(self.chunks)) @property def shards(self) -> tuple[int, ...] | None: return None def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: zarray_dict = self.to_dict() zattrs_dict = zarray_dict.pop("attributes", {}) indent = config.get("json_indent") return { ZARRAY_JSON: json_to_buffer(zarray_dict, prototype=prototype, indent=indent), ZATTRS_JSON: json_to_buffer(zattrs_dict, prototype=prototype, indent=indent), } @classmethod def from_dict(cls, data: dict[str, Any], *, path: str | None = None) -> ArrayV2Metadata: """Read a stored `.zarray` document (with its attributes). An invalid document that `zarr.core.metadata.repair` can read is read as repaired; a reading the user must act on warns, naming the array at `path`.""" repaired, readings = repair_array_document(data, 2) # a new dict, because we are modifying it _data: dict[str, Any] = dict(repaired) # Check that the zarr_format attribute is correct. _ = parse_zarr_format(_data.pop("zarr_format")) # To resolve a numpy object dtype array, we need to search for an object codec, # which could be in filters or as a compressor. # we will reference a hard-coded collection of object codec ids for this search. _filters, _compressor = (_data.get("filters"), _data.get("compressor")) if _filters is not None: _filters = cast("tuple[dict[str, JSON], ...]", _filters) object_codec_id = get_object_codec_id(tuple(_filters) + (_compressor,)) else: object_codec_id = get_object_codec_id((_compressor,)) # we add a layer of indirection here around the dtype attribute of the array metadata # because we also need to know the object codec id, if any, to resolve the data type dtype_spec: DTypeSpec_V2 = { "name": _data["dtype"], "object_codec_id": object_codec_id, } dtype = get_data_type_from_json(dtype_spec, zarr_format=2) _data["dtype"] = dtype fill_value_encoded = _data.get("fill_value") if fill_value_encoded is not None: fill_value = dtype.from_json_scalar(fill_value_encoded, zarr_format=2) _data["fill_value"] = fill_value # zarr v2 allowed arbitrary keys here. # We don't want the ArrayV2Metadata constructor to fail just because someone put an # extra key in the metadata. expected = {x.name for x in fields(cls)} expected |= {"dtype", "chunks"} # check if `filters` is an empty sequence; if so use None instead and raise a warning filters = _data.get("filters") if ( isinstance(filters, Sequence) and not isinstance(filters, (str, bytes)) and len(filters) == 0 ): msg = ( "Found an empty list of filters in the array metadata document. " "This is contrary to the Zarr V2 specification, and will cause an error in the future. " "Use None (or Null in a JSON document) instead of an empty list of filters." ) warnings.warn(msg, ZarrUserWarning, stacklevel=1) _data["filters"] = None _data = {k: v for k, v in _data.items() if k in expected} return mark_repaired(cls(**_data), data, readings, path) def to_dict(self) -> dict[str, JSON]: zarray_dict = super().to_dict() if _is_numcodec(zarray_dict["compressor"]): codec_config = zarray_dict["compressor"].get_config() # Hotfix for https://github.com/zarr-developers/zarr-python/issues/2647 if codec_config["id"] == "zstd" and not codec_config.get("checksum", False): codec_config.pop("checksum") zarray_dict["compressor"] = codec_config if zarray_dict["filters"] is not None: raw_filters = zarray_dict["filters"] # TODO: remove this when we can stratically type the output JSON data structure # entirely if not isinstance(raw_filters, list | tuple): raise TypeError("Invalid type for filters. Expected a list or tuple.") new_filters = [] for f in raw_filters: if _is_numcodec(f): new_filters.append(f.get_config()) else: new_filters.append(f) zarray_dict["filters"] = new_filters # serialize the fill value after dtype-specific JSON encoding if self.fill_value is not None: fill_value = self.dtype.to_json_scalar(self.fill_value, zarr_format=2) zarray_dict["fill_value"] = fill_value # pull the "name" attribute out of the dtype spec returned by self.dtype.to_json zarray_dict["dtype"] = self.dtype.to_json(zarr_format=2)["name"] return zarray_dict def __eq__(self, other: object) -> bool: # The default dataclass __eq__ compares fields directly, which is wrong for a NaN # fill_value: NaN != NaN under IEEE 754. Comparing the JSON-serialized form instead # treats matching NaN (and inf) fill values as equal. See issue #2929. if not isinstance(other, ArrayV2Metadata): return NotImplemented return self.to_dict() == other.to_dict() def __hash__(self) -> int: # Hash the JSON-serialized form to stay consistent with __eq__: equal metadata # must hash equally, which a field-based hash violates for a NaN fill_value. return hash(json.dumps(self.to_dict(), sort_keys=True)) def get_chunk_spec( self, _chunk_coords: tuple[int, ...], array_config: ArrayConfig, prototype: BufferPrototype ) -> ArraySpec: return ArraySpec( shape=self.chunks, dtype=self.dtype, fill_value=self.fill_value, config=array_config, prototype=prototype, ) def encode_chunk_key(self, chunk_coords: tuple[int, ...]) -> str: chunk_identifier = self.dimension_separator.join(map(str, chunk_coords)) return "0" if chunk_identifier == "" else chunk_identifier def update_shape(self, shape: tuple[int, ...]) -> Self: return replace(self, shape=shape) def update_attributes(self, attributes: dict[str, JSON]) -> Self: return replace(self, attributes=attributes) def parse_dtype(data: npt.DTypeLike) -> np.dtype[Any]: if isinstance(data, list): # this is a valid _VoidDTypeLike check data = [tuple(d) for d in data] return np.dtype(data) def parse_zarr_format(data: object) -> Literal[2]: return cast("Literal[2]", parse_field(data, Literal[2], "zarr_format")) def parse_filters(data: object) -> tuple[Numcodec, ...] | None: """ Parse a potential tuple of filters """ out: list[Numcodec] = [] if data is None: return data if isinstance(data, Iterable): for idx, val in enumerate(data): if _is_numcodec(val): out.append(val) elif isinstance(val, dict): out.append(get_numcodec(val)) # type: ignore[arg-type] else: msg = f"Invalid filter at index {idx}. Expected a numcodecs.abc.Codec or a dict representation of numcodecs.abc.Codec. Got {type(val)} instead." raise TypeError(msg) if len(out) == 0: # Per the v2 spec, an empty tuple is not allowed -- use None to express "no filters" return None else: return tuple(out) # take a single codec instance and wrap it in a tuple if _is_numcodec(data): return (data,) msg = f"Invalid filters. Expected None, an iterable of numcodecs.abc.Codec or dict representations of numcodecs.abc.Codec. Got {type(data)} instead." raise TypeError(msg) def parse_compressor(data: object) -> Numcodec | None: """ Parse a potential compressor. """ if data is None or _is_numcodec(data): return data if isinstance(data, dict): return get_numcodec(data) # type: ignore[arg-type] msg = f"Invalid compressor. Expected None, a numcodecs.abc.Codec, or a dict representation of a numcodecs.abc.Codec. Got {type(data)} instead." raise ValueError(msg) def parse_chunks(chunks: ShapeLike, shape: tuple[int, ...]) -> tuple[int, ...]: """Check a chunk shape: one non-negative integer per array axis (see `parse_shapelike`). Stored chunk sizes of 0 are read by `zarr.core.metadata.repair`.""" chunks_parsed = parse_shapelike(chunks) if len(chunks_parsed) != len(shape): raise ValueError( f"The `shape` and `chunks` attributes must have the same length. " f"`chunks` has length {len(chunks_parsed)}, but `shape` has length {len(shape)}." ) return chunks_parsed def get_object_codec_id(maybe_object_codecs: Sequence[JSON]) -> str | None: """ Inspect a sequence of codecs / filters for an "object codec", i.e. a codec that can serialize object arrays to contiguous bytes. Zarr python maintains a hard-coded set of object codec ids. If any element from the input has an id that matches one of the hard-coded object codec ids, that id is returned immediately. """ object_codec_id = None for maybe_object_codec in maybe_object_codecs: if ( isinstance(maybe_object_codec, dict) and maybe_object_codec.get("id") in OBJECT_CODEC_IDS ): return cast("str", maybe_object_codec["id"]) return object_codec_id zarr-developers-zarr-python-a2f2415/src/zarr/core/metadata/v3.py000066400000000000000000000764351527014137600246630ustar00rootroot00000000000000from __future__ import annotations import json from collections.abc import Iterable, Mapping, Sequence from collections.abc import Set as AbstractSet from dataclasses import dataclass, field, replace from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NotRequired, TypeGuard, cast from typing_extensions import TypedDict from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec from zarr.abc.metadata import Metadata from zarr.core._json import json_to_buffer from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer.core import default_buffer_prototype from zarr.core.chunk_grids import FixedDimension, VaryingDimension from zarr.core.chunk_key_encodings import ( ChunkKeyEncoding, ChunkKeyEncodingLike, parse_chunk_key_encoding, ) from zarr.core.common import ( JSON, ZARR_JSON, DimensionNamesLike, NamedConfig, NamedRequiredConfig, NodeType, compress_rle, expand_rle, parse_chunk_edge, parse_chunk_shape, parse_named_configuration, parse_shapelike, validate_rectilinear_edges, validate_rectilinear_kind, ) from zarr.core.config import config from zarr.core.dtype import VariableLengthUTF8, ZDType, get_data_type_from_json from zarr.core.dtype.common import check_dtype_spec_v3 from zarr.core.json_parse import parse_field from zarr.core.metadata.common import parse_attributes from zarr.core.metadata.repair import ( mark_repaired, repair_array_document, ) from zarr.errors import MetadataValidationError, NodeTypeValidationError from zarr.registry import get_codec_class if TYPE_CHECKING: from typing import Self from zarr.codecs.sharding import ShardingCodec from zarr.core.buffer import Buffer, BufferPrototype from zarr.core.chunk_grids import ChunkGrid from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar from zarr.core.metadata.repair import ArrayDocument def parse_zarr_format(data: object) -> Literal[3]: return cast( "Literal[3]", parse_field(data, Literal[3], "zarr_format", error=MetadataValidationError) ) def parse_node_type_array(data: object) -> Literal["array"]: return cast( 'Literal["array"]', parse_field(data, Literal["array"], "node_type", error=NodeTypeValidationError), ) def parse_codecs(data: object) -> tuple[Codec, ...]: out: tuple[Codec, ...] = () if not isinstance(data, Iterable): raise TypeError(f"Expected iterable, got {type(data)}") for c in data: if isinstance( c, ArrayArrayCodec | ArrayBytesCodec | BytesBytesCodec ): # Can't use Codec here because of mypy limitation out += (c,) else: name_parsed, _ = parse_named_configuration(c, require_configuration=False) codec_cls = get_codec_class(name_parsed) try: out += (codec_cls.from_dict(c),) except KeyError as e: # A codec's `from_dict` may index its configuration directly, so a malformed # configuration surfaces as a KeyError. Convert it: a bare KeyError escaping # metadata parsing is swallowed by the array-then-group fallback in # `zarr.api.asynchronous.open`, which then reports an unrelated group error. # The KeyError may carry no arguments (`raise KeyError`), and it may come from # an internal lookup rather than the configuration mapping itself, so name the # key only when there is one and don't claim it was a missing configuration key. key_text = f" {e.args[0]!r}" if e.args else "" raise MetadataValidationError( f"KeyError{key_text} while parsing the configuration for codec {name_parsed!r}." ) from e return out def validate_array_bytes_codec(codecs: tuple[Codec, ...]) -> ArrayBytesCodec: # ensure that we have at least one ArrayBytesCodec abcs: list[ArrayBytesCodec] = [codec for codec in codecs if isinstance(codec, ArrayBytesCodec)] if len(abcs) == 0: raise ValueError("At least one ArrayBytesCodec is required.") elif len(abcs) > 1: raise ValueError("Only one ArrayBytesCodec is allowed.") return abcs[0] def validate_codecs(codecs: tuple[Codec, ...], dtype: ZDType[TBaseDType, TBaseScalar]) -> None: """Check that the codecs are valid for the given dtype""" from zarr.codecs.sharding import ShardingCodec abc = validate_array_bytes_codec(codecs) # Recursively resolve array-bytes codecs within sharding codecs while isinstance(abc, ShardingCodec): abc = validate_array_bytes_codec(abc.codecs) # we need to have special codecs if we are decoding vlen strings or bytestrings # TODO: use codec ID instead of class name codec_class_name = abc.__class__.__name__ # TODO: Fix typing here if isinstance(dtype, VariableLengthUTF8) and not codec_class_name == "VLenUTF8Codec": # type: ignore[unreachable] raise ValueError( f"For string dtype, ArrayBytesCodec must be `VLenUTF8Codec`, got `{codec_class_name}`." ) def parse_dimension_names(data: object) -> tuple[str | None, ...] | None: if data is None: return data elif isinstance(data, Iterable) and all(isinstance(x, type(None) | str) for x in data): return tuple(data) else: msg = f"Expected either None or an iterable of str, got {type(data)}" raise TypeError(msg) def parse_storage_transformers(data: object) -> tuple[dict[str, JSON], ...]: """ Parse storage_transformers. Zarr python cannot use storage transformers at this time, so this function doesn't attempt to validate them. """ if data is None: return () if isinstance(data, Iterable) and not isinstance(data, (str, bytes)): # Materialise once. The previous implementation called ``len(tuple(data))`` # and then returned ``data`` itself, which exhausted (and discarded) a # one-shot iterable and could return a value typed as a tuple that was not # actually a tuple. return tuple(data) raise TypeError( f"Invalid storage_transformers. Expected an iterable of dicts. Got {type(data)} instead." ) class AllowedExtraField(TypedDict, extra_items=JSON): # type: ignore[call-arg] """ This class models allowed extra fields in array or group metadata. They must have `must_understand` set to `False`, and may contain arbitrary additional JSON data. """ must_understand: Literal[False] def check_allowed_extra_field(data: object) -> TypeGuard[AllowedExtraField]: """ Check if the extra field is allowed according to the Zarr v3 spec. The object must be a mapping with a "must_understand" key set to `False`. """ return isinstance(data, Mapping) and data.get("must_understand") is False def parse_extra_fields( data: Mapping[str, object] | None, *, reserved_keys: AbstractSet[str], node_type: NodeType, ) -> dict[str, AllowedExtraField]: """ Check the extra fields of a Zarr V3 metadata document. Raises `ValueError` if a key collides with a key in `reserved_keys`, and `MetadataValidationError` if a value is not an object with `"must_understand": false`. """ if data is None: return {} conflict_keys = reserved_keys & set(data.keys()) if len(conflict_keys) > 0: msg = ( "Invalid extra fields. " "The following keys: " f"{sorted(conflict_keys)} " "are invalid because they collide with keys reserved for use by the " f"{node_type} metadata document." ) raise ValueError(msg) allowed_extra_fields: dict[str, AllowedExtraField] = {} invalid_extra_fields: list[str] = [] for key, val in data.items(): if check_allowed_extra_field(val): allowed_extra_fields[key] = val else: invalid_extra_fields.append(key) if len(invalid_extra_fields) > 0: msg = ( f"Got Zarr V3 {node_type} metadata with the following disallowed extra " f"fields: {sorted(invalid_extra_fields)}. " 'Extra fields are not allowed unless they are an object with a "must_understand" ' "key which is assigned the value `false`." ) raise MetadataValidationError(msg) return allowed_extra_fields # JSON type for a single dimension's rectilinear spec: # bare int (uniform shorthand), or list of ints / [value, count] RLE pairs. RectilinearDimSpecJSON = int | list[int | list[int]] class RegularChunkGridMetadataConfig(TypedDict): chunk_shape: Sequence[int] class RectilinearChunkGridMetadataConfig(TypedDict): kind: Literal["inline"] chunk_shapes: Sequence[RectilinearDimSpecJSON] RegularChunkGridMetadataJSON = NamedRequiredConfig[ Literal["regular"], RegularChunkGridMetadataConfig ] RectilinearChunkGridMetadataJSON = NamedRequiredConfig[ Literal["rectilinear"], RectilinearChunkGridMetadataConfig ] def _validate_chunk_shapes( chunk_shapes: Sequence[int | Sequence[int]], ) -> tuple[int | tuple[int, ...], ...]: """Validate per-dimension chunk specifications. Each element is either a bare ``int`` (regular step size, must be >= 1) or a sequence of explicit edge lengths (all must be >= 1, non-empty). """ result: list[int | tuple[int, ...]] = [] for dim_idx, dim_spec in enumerate(chunk_shapes): match dim_spec: case list() | tuple(): edges = tuple(parse_chunk_edge(edge, dim_idx) for edge in dim_spec) if not edges: raise ValueError(f"Dimension {dim_idx} has no chunk edges.") result.append(edges) case _: result.append(parse_chunk_edge(dim_spec, dim_idx)) return tuple(result) @dataclass(frozen=True, kw_only=True) class RegularChunkGridMetadata(Metadata): """Metadata-only description of a regular chunk grid. Stores just the chunk shape — no array extent, no runtime logic. This is what lives on ``ArrayV3Metadata.chunk_grid``. """ chunk_shape: tuple[int, ...] def __post_init__(self) -> None: chunk_shape_parsed = parse_chunk_shape(self.chunk_shape) object.__setattr__(self, "chunk_shape", chunk_shape_parsed) @property def ndim(self) -> int: return len(self.chunk_shape) def to_dict(self) -> RegularChunkGridMetadataJSON: # type: ignore[override] return { "name": "regular", "configuration": {"chunk_shape": self.chunk_shape}, } @classmethod def from_dict(cls, data: RegularChunkGridMetadataJSON) -> Self: # type: ignore[override] parse_named_configuration(data, "regular") # validate name configuration = data["configuration"] return cls(chunk_shape=parse_chunk_shape(configuration["chunk_shape"])) @dataclass(frozen=True, kw_only=True) class RectilinearChunkGridMetadata(Metadata): """Metadata-only description of a rectilinear chunk grid. Each element of ``chunk_shapes`` is either: - A bare ``int`` — a regular step size that repeats to cover the axis (the spec's single-integer shorthand). - A ``tuple[int, ...]`` — explicit per-chunk edge lengths (already expanded from any RLE encoding). This distinction matters for faithful round-tripping: a bare int serializes back as a bare int, while a single-element tuple serializes as a list. """ chunk_shapes: tuple[int | tuple[int, ...], ...] def __post_init__(self) -> None: object.__setattr__(self, "chunk_shapes", _validate_chunk_shapes(self.chunk_shapes)) @property def ndim(self) -> int: return len(self.chunk_shapes) def to_dict(self) -> RectilinearChunkGridMetadataJSON: # type: ignore[override] serialized_dims: list[RectilinearDimSpecJSON] = [] for dim_spec in self.chunk_shapes: if isinstance(dim_spec, int): # Bare int shorthand — serialize as-is serialized_dims.append(dim_spec) else: rle = compress_rle(dim_spec) # Use RLE only if it's actually shorter if len(rle) < len(dim_spec): serialized_dims.append(rle) else: serialized_dims.append(list(dim_spec)) return { "name": "rectilinear", "configuration": { "kind": "inline", "chunk_shapes": tuple(serialized_dims), }, } def update_shape( self, old_shape: tuple[int, ...], new_shape: tuple[int, ...] ) -> RectilinearChunkGridMetadata: """Return a new RectilinearChunkGridMetadata with edges adjusted for *new_shape*. - Bare-int dimensions stay as bare ints (they cover any extent). - Explicit-edge dimensions: if the new extent exceeds the sum of edges, a new chunk is appended to cover the additional extent. Otherwise edges are kept as-is (the spec allows trailing edges beyond the array extent). """ new_chunk_shapes: list[int | tuple[int, ...]] = [] for dim_spec, new_ext in zip(self.chunk_shapes, new_shape, strict=True): if isinstance(dim_spec, int): # Bare int covers any extent — no change needed new_chunk_shapes.append(dim_spec) else: edge_sum = sum(dim_spec) if new_ext > edge_sum: new_chunk_shapes.append((*dim_spec, new_ext - edge_sum)) else: new_chunk_shapes.append(dim_spec) return RectilinearChunkGridMetadata(chunk_shapes=tuple(new_chunk_shapes)) @classmethod def from_dict(cls, data: RectilinearChunkGridMetadataJSON) -> Self: # type: ignore[override] parse_named_configuration(data, "rectilinear") # validate name configuration = data["configuration"] validate_rectilinear_kind(configuration.get("kind")) raw_shapes = configuration["chunk_shapes"] parsed = [ tuple(expand_rle(dim_spec, axis)) if isinstance(dim_spec, list) else dim_spec for axis, dim_spec in enumerate(raw_shapes) ] return cls(chunk_shapes=tuple(parsed)) ChunkGridMetadata = RegularChunkGridMetadata | RectilinearChunkGridMetadata class RectilinearChunksDisabledError(ValueError): """Rectilinear chunk grids are used while the `array.rectilinear_chunks` flag is off.""" def _check_rectilinear_chunks_enabled() -> None: """Raise unless rectilinear chunks are enabled. The flag gates storing and reading array metadata documents that declare a rectilinear chunk grid; the chunk grid metadata classes themselves are not gated. """ if not config.get("array.rectilinear_chunks"): raise RectilinearChunksDisabledError( "Rectilinear chunk grids are experimental and disabled by default. " "Enable them with: zarr.config.set({'array.rectilinear_chunks': True}) " "or set the environment variable ZARR_ARRAY__RECTILINEAR_CHUNKS=True" ) def check_storable(metadata: ArrayV3Metadata) -> None: """Raise if `metadata` may not be stored: a rectilinear chunk grid requires the rectilinear chunks flag. `zarr.core.metadata.io.encode_documents` names the node in the error. Every serialization of array metadata for a store calls this, before the store is touched: `ArrayV3Metadata.to_buffer_dict` and, for the arrays in a group's consolidated metadata, `GroupMetadata.to_buffer_dict`. """ if isinstance(metadata.chunk_grid, RectilinearChunkGridMetadata): _check_rectilinear_chunks_enabled() def create_chunk_grid_metadata( chunks: ChunkGrid, ) -> ChunkGridMetadata: """Construct a chunk grid metadata object from a normalized `ChunkGrid`. Regular grids produce a `RegularChunkGridMetadata`. Rectilinear grids produce a `RectilinearChunkGridMetadata`. Parameters ---------- chunks : ChunkGrid Normalized chunk grid, as returned by `normalize_chunks_nd` or `guess_chunks`. See Also -------- parse_chunk_grid : Deserialize a chunk grid from stored JSON metadata. """ if chunks.is_regular: return RegularChunkGridMetadata(chunk_shape=chunks.chunk_shape) # Uniform dimensions stay bare ints — the rectilinear grid spec treats # a bare int as a step size repeating to cover the axis. chunk_shapes: list[int | tuple[int, ...]] = [] for dim in chunks.dimensions: if isinstance(dim, FixedDimension): chunk_shapes.append(dim.size) elif isinstance(dim, VaryingDimension): chunk_shapes.append(dim.edges) else: raise TypeError(f"Unknown dimension grid type: {type(dim)}") return RectilinearChunkGridMetadata(chunk_shapes=tuple(chunk_shapes)) def parse_chunk_grid( data: dict[str, JSON] | ChunkGridMetadata | NamedConfig[str, Any], ) -> ChunkGridMetadata: """Deserialize a chunk grid from stored JSON metadata or pass through an existing instance. See Also -------- create_chunk_grid_metadata : Construct a chunk grid from user-facing input. """ if isinstance(data, (RegularChunkGridMetadata, RectilinearChunkGridMetadata)): return data name, _ = parse_named_configuration(data) if name == "regular": return RegularChunkGridMetadata.from_dict(data) # type: ignore[arg-type] if name == "rectilinear": return RectilinearChunkGridMetadata.from_dict(data) # type: ignore[arg-type] raise ValueError(f"Unknown chunk grid name: {name!r}") class ArrayMetadataJSON_V3(TypedDict, extra_items=AllowedExtraField): # type: ignore[call-arg] """ A typed dictionary model for zarr v3 array metadata. Extra keys are permitted if they conform to ``AllowedExtraField`` (i.e. they are mappings with ``must_understand: false``). """ zarr_format: Literal[3] node_type: Literal["array"] data_type: str | NamedConfig[str, Mapping[str, JSON]] shape: tuple[int, ...] chunk_grid: str | NamedConfig[str, Mapping[str, JSON]] chunk_key_encoding: str | NamedConfig[str, Mapping[str, JSON]] fill_value: JSON codecs: tuple[str | NamedConfig[str, Mapping[str, JSON]], ...] attributes: NotRequired[Mapping[str, JSON]] storage_transformers: NotRequired[tuple[str | NamedConfig[str, Mapping[str, JSON]], ...]] dimension_names: NotRequired[tuple[str | None, ...]] """ The names of the fields of the array metadata document defined in the zarr V3 spec. """ ARRAY_METADATA_KEYS: Final[set[str]] = { "zarr_format", "node_type", "data_type", "shape", "chunk_grid", "chunk_key_encoding", "fill_value", "codecs", "attributes", "storage_transformers", "dimension_names", } @dataclass(frozen=True, kw_only=True) class ArrayV3Metadata(Metadata): shape: tuple[int, ...] data_type: ZDType[TBaseDType, TBaseScalar] chunk_grid: ChunkGridMetadata chunk_key_encoding: ChunkKeyEncoding fill_value: Any codecs: tuple[Codec, ...] attributes: dict[str, Any] = field(default_factory=dict) dimension_names: tuple[str | None, ...] | None = None zarr_format: Literal[3] = field(default=3, init=False) node_type: Literal["array"] = field(default="array", init=False) storage_transformers: tuple[dict[str, JSON], ...] extra_fields: dict[str, AllowedExtraField] _stored_document: ClassVar[ArrayDocument | None] = None """The stored document `from_dict` read this metadata from, if it had to repair it (set on the instance by `mark_repaired`): the store may still hold it.""" def __init__( self, *, shape: Iterable[int], data_type: ZDType[TBaseDType, TBaseScalar], chunk_grid: dict[str, JSON] | ChunkGridMetadata | NamedConfig[str, Any], chunk_key_encoding: ChunkKeyEncodingLike, fill_value: object, codecs: Iterable[Codec | dict[str, JSON] | NamedConfig[str, Any] | str], attributes: dict[str, JSON] | None, dimension_names: DimensionNamesLike, storage_transformers: Iterable[dict[str, JSON]] | None = None, extra_fields: Mapping[str, AllowedExtraField] | None = None, ) -> None: """ Because the class is a frozen dataclass, we set attributes using object.__setattr__ """ shape_parsed = parse_shapelike(shape) chunk_grid_parsed = parse_chunk_grid(chunk_grid) chunk_key_encoding_parsed = parse_chunk_key_encoding(chunk_key_encoding) dimension_names_parsed = parse_dimension_names(dimension_names) # Note: relying on a type method is numpy-specific fill_value_parsed = data_type.cast_scalar(fill_value) attributes_parsed = parse_attributes(attributes) codecs_parsed_partial = parse_codecs(codecs) storage_transformers_parsed = parse_storage_transformers(storage_transformers) extra_fields_parsed = parse_extra_fields( extra_fields, reserved_keys=ARRAY_METADATA_KEYS, node_type="array" ) array_spec = ArraySpec( shape=shape_parsed, dtype=data_type, fill_value=fill_value_parsed, config=ArrayConfig.from_dict({}), # TODO: config is not needed here. prototype=default_buffer_prototype(), # TODO: prototype is not needed here. ) # Thread the spec through evolution: each codec must be evolved against # the spec it will actually see at run-time, not the original array spec. # Earlier array->array codecs may transform the dtype (e.g. cast_value), # so the spec passed to later codecs must reflect those transformations. # Per-codec validate() must run before resolve_metadata(), since the # latter may rely on invariants the former checks (e.g. cast_value # rejects complex source dtypes that would otherwise crash _do_cast). evolved: list[Codec] = [] spec = array_spec for c in codecs_parsed_partial: evolved_codec = c.evolve_from_array_spec(spec) evolved_codec.validate(shape=spec.shape, dtype=spec.dtype, chunk_grid=chunk_grid_parsed) evolved.append(evolved_codec) spec = evolved_codec.resolve_metadata(spec) codecs_parsed = tuple(evolved) validate_codecs(codecs_parsed_partial, data_type) object.__setattr__(self, "shape", shape_parsed) object.__setattr__(self, "data_type", data_type) object.__setattr__(self, "chunk_grid", chunk_grid_parsed) object.__setattr__(self, "chunk_key_encoding", chunk_key_encoding_parsed) object.__setattr__(self, "codecs", codecs_parsed) object.__setattr__(self, "dimension_names", dimension_names_parsed) object.__setattr__(self, "fill_value", fill_value_parsed) object.__setattr__(self, "attributes", attributes_parsed) object.__setattr__(self, "storage_transformers", storage_transformers_parsed) object.__setattr__(self, "extra_fields", extra_fields_parsed) self._validate_metadata() def _validate_metadata(self) -> None: if len(self.shape) != self.chunk_grid.ndim: raise ValueError("`chunk_grid` and `shape` need to have the same number of dimensions.") if isinstance(self.chunk_grid, RectilinearChunkGridMetadata): validate_rectilinear_edges(self.chunk_grid.chunk_shapes, self.shape) if self.dimension_names is not None and len(self.shape) != len(self.dimension_names): raise ValueError( "`dimension_names` and `shape` need to have the same number of dimensions." ) if self.fill_value is None: raise ValueError("`fill_value` is required.") for codec in self.codecs: codec.validate(shape=self.shape, dtype=self.data_type, chunk_grid=self.chunk_grid) @property def ndim(self) -> int: return len(self.shape) @property def dtype(self) -> ZDType[TBaseDType, TBaseScalar]: return self.data_type # TODO: move these properties to the Array class. # They require knowledge of codecs (ShardingCodec) and don't belong on a metadata DTO. @property def sharding_codec(self) -> ShardingCodec | None: """The array's sharding codec, or None if the array is not sharded.""" from zarr.codecs.sharding import ShardingCodec if len(self.codecs) == 1 and isinstance(self.codecs[0], ShardingCodec): return self.codecs[0] return None @property def chunks(self) -> tuple[int, ...]: if (sharding_codec := self.sharding_codec) is not None: # Inner chunks are always regular, whatever the shape of the outer # (shard) grid. return sharding_codec.chunk_shape if not isinstance(self.chunk_grid, RegularChunkGridMetadata): msg = ( "The `chunks` attribute is only defined for arrays using regular chunk grids. " "This array has a rectilinear chunk grid. Use `read_chunk_sizes` for general access." ) raise NotImplementedError(msg) return self.chunk_grid.chunk_shape @property def shards(self) -> tuple[int, ...] | None: if self.sharding_codec is not None: if not isinstance(self.chunk_grid, RegularChunkGridMetadata): msg = ( "The `shards` attribute is only defined for arrays using regular chunk grids. " "This array has a rectilinear chunk grid. Use `write_chunk_sizes` for general access." ) raise NotImplementedError(msg) return self.chunk_grid.chunk_shape return None @property def inner_codecs(self) -> tuple[Codec, ...]: if (sharding_codec := self.sharding_codec) is not None: return sharding_codec.codecs return self.codecs def encode_chunk_key(self, chunk_coords: tuple[int, ...]) -> str: return self.chunk_key_encoding.encode_chunk_key(chunk_coords) def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: check_storable(self) indent = config.get("json_indent") return {ZARR_JSON: json_to_buffer(self.to_dict(), prototype=prototype, indent=indent)} @classmethod def from_dict(cls, data: dict[str, JSON], *, path: str | None = None) -> Self: """Read a stored `zarr.json` array document. An invalid document that `zarr.core.metadata.repair` can read is read as repaired; a reading the user must act on warns, and a document the rectilinear chunks flag refuses raises, naming the array at `path`.""" # The flag gates what the document declares, so it is checked before repairs. chunk_grid = data.get("chunk_grid") if isinstance(chunk_grid, Mapping) and chunk_grid.get("name") == "rectilinear": try: _check_rectilinear_chunks_enabled() except ValueError as e: if path is not None: e.add_note(f"Array {path!r}: nothing was read.") raise repaired, readings = repair_array_document(data, 3) # a new dict, because we are modifying it _data = dict(repaired) # check that the zarr_format attribute is correct _ = parse_zarr_format(_data.pop("zarr_format")) # check that the node_type attribute is correct _ = parse_node_type_array(_data.pop("node_type")) data_type_json = _data.pop("data_type") if not check_dtype_spec_v3(data_type_json): raise ValueError(f"Invalid data_type: {data_type_json!r}") data_type = get_data_type_from_json(data_type_json, zarr_format=3) # check that the fill value is consistent with the data type try: fill = _data.pop("fill_value") fill_value_parsed = data_type.from_json_scalar(fill, zarr_format=3) except ValueError as e: raise TypeError(f"Invalid fill_value: {fill!r}") from e # check if there are extra keys allowed_extra_fields = parse_extra_fields( {k: v for k, v in _data.items() if k not in ARRAY_METADATA_KEYS}, reserved_keys=ARRAY_METADATA_KEYS, node_type="array", ) # TODO: replace this with a real type check! _data_typed = cast(ArrayMetadataJSON_V3, _data) metadata = cls( shape=_data_typed["shape"], chunk_grid=_data_typed["chunk_grid"], # type: ignore[arg-type] chunk_key_encoding=_data_typed["chunk_key_encoding"], # type: ignore[arg-type] codecs=_data_typed["codecs"], # Attribute values are arbitrary JSON, so they have no field-specific # schema to validate. `__init__` checks the outer dict via `parse_attributes`. attributes=_data_typed.get("attributes", {}), # type: ignore[arg-type] dimension_names=_data_typed.get("dimension_names", None), fill_value=fill_value_parsed, data_type=data_type, extra_fields=allowed_extra_fields, storage_transformers=_data_typed.get("storage_transformers", ()), # type: ignore[arg-type] ) return mark_repaired(metadata, data, readings, path) def to_dict(self) -> dict[str, JSON]: out_dict = super().to_dict() extra_fields = out_dict.pop("extra_fields") out_dict = out_dict | extra_fields # type: ignore[operator] out_dict["chunk_grid"] = self.chunk_grid.to_dict() out_dict["fill_value"] = self.data_type.to_json_scalar( self.fill_value, zarr_format=self.zarr_format ) if not isinstance(out_dict, dict): raise TypeError(f"Expected dict. Got {type(out_dict)}.") # if `dimension_names` is `None`, we do not include it in # the metadata document if out_dict["dimension_names"] is None: out_dict.pop("dimension_names") # TODO: replace the `to_dict` / `from_dict` on the `Metadata`` class with # to_json, from_json, and have ZDType inherit from `Metadata` # until then, we have this hack here, which relies on the fact that to_dict will pass through # any non-`Metadata` fields as-is. dtype_meta = out_dict["data_type"] if isinstance(dtype_meta, ZDType): out_dict["data_type"] = dtype_meta.to_json(zarr_format=3) # type: ignore[unreachable] return out_dict def __eq__(self, other: object) -> bool: # The default dataclass __eq__ compares fields directly, which is wrong for a NaN # fill_value: NaN != NaN under IEEE 754. Comparing the JSON-serialized form instead # treats matching NaN (and inf) fill values as equal. See issue #2929. if not isinstance(other, ArrayV3Metadata): return NotImplemented return self.to_dict() == other.to_dict() def __hash__(self) -> int: # Hash the JSON-serialized form to stay consistent with __eq__: equal metadata # must hash equally, which a field-based hash violates for a NaN fill_value. return hash(json.dumps(self.to_dict(), sort_keys=True)) def update_shape(self, shape: tuple[int, ...]) -> Self: chunk_grid = self.chunk_grid if isinstance(chunk_grid, RectilinearChunkGridMetadata): chunk_grid = chunk_grid.update_shape(self.shape, shape) return replace(self, shape=shape, chunk_grid=chunk_grid) def update_attributes(self, attributes: dict[str, JSON]) -> Self: return replace(self, attributes=attributes) zarr-developers-zarr-python-a2f2415/src/zarr/core/sync.py000066400000000000000000000163321527014137600235150ustar00rootroot00000000000000from __future__ import annotations import asyncio import atexit import logging import os import threading from concurrent.futures import ThreadPoolExecutor, wait from typing import TYPE_CHECKING from zarr.core.config import config if TYPE_CHECKING: from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine from typing import Any logger = logging.getLogger(__name__) # From https://github.com/fsspec/filesystem_spec/blob/master/fsspec/asyn.py iothread: list[threading.Thread | None] = [None] # dedicated IO thread loop: list[asyncio.AbstractEventLoop | None] = [ None ] # global event loop for any non-async instance _lock: threading.Lock | None = None # global lock placeholder _executor: ThreadPoolExecutor | None = None # global executor placeholder class SyncError(Exception): pass def _get_lock() -> threading.Lock: """Allocate or return a threading lock. The lock is allocated on first use to allow setting one lock per forked process. """ global _lock if not _lock: _lock = threading.Lock() return _lock def _get_executor() -> ThreadPoolExecutor: """Return Zarr Thread Pool Executor The executor is allocated on first use. """ global _executor if not _executor: max_workers = config.get("threading.max_workers", None) logger.debug("Creating Zarr ThreadPoolExecutor with max_workers=%s", max_workers) _executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="zarr_pool") _get_loop().set_default_executor(_executor) return _executor def cleanup_resources() -> None: global _executor if _executor: _executor.shutdown(wait=True, cancel_futures=True) _executor = None if loop[0] is not None: with _get_lock(): # Stop the event loop safely loop[0].call_soon_threadsafe(loop[0].stop) # Stop loop from another thread if iothread[0] is not None: iothread[0].join(timeout=0.2) # Add a timeout to avoid hanging if iothread[0].is_alive(): logger.warning( "Thread did not finish cleanly; forcefully closing the event loop." ) # Forcefully close the event loop to release resources loop[0].close() # dereference the loop and iothread loop[0] = None iothread[0] = None atexit.register(cleanup_resources) def reset_resources_after_fork() -> None: """ Ensure that global resources are reset after a fork. Without this function, forked processes will retain invalid references to the parent process's resources. """ # `loop` and `iothread` are mutated in place rather than rebound, so only # `_executor` needs the global declaration. global _executor # These lines are excluded from coverage because this function only runs in a child process, # which is not observed by the test coverage instrumentation. Despite the apparent lack of # test coverage, this function should be adequately tested by any test that uses Zarr IO with # multiprocessing. loop[0] = None # pragma: no cover iothread[0] = None # pragma: no cover _executor = None # pragma: no cover # this is only available on certain operating systems if hasattr(os, "register_at_fork"): os.register_at_fork(after_in_child=reset_resources_after_fork) async def _runner[T](coro: Coroutine[Any, Any, T]) -> T | BaseException: """ Await a coroutine and return the result of running it. If awaiting the coroutine raises an exception, the exception will be returned. """ try: return await coro except Exception as ex: # noqa: BLE001 -- the caller re-raises the returned exception return ex def sync[T]( coro: Coroutine[Any, Any, T], loop: asyncio.AbstractEventLoop | None = None, timeout: float | None = None, ) -> T: """ Make loop run coroutine until it returns. Runs in other thread """ if loop is None: # NB: if the loop is not running *yet*, it is OK to submit work # and we will wait for it loop = _get_loop() if _executor is None and config.get("threading.max_workers", None) is not None: # trigger executor creation and attach to loop _ = _get_executor() if not isinstance(loop, asyncio.AbstractEventLoop): raise TypeError(f"loop cannot be of type {type(loop)}") if loop.is_closed(): raise RuntimeError("Loop is not running") try: loop0 = asyncio.events.get_running_loop() if loop0 is loop: raise SyncError("Calling sync() from within a running loop") except RuntimeError: pass future = asyncio.run_coroutine_threadsafe(_runner(coro), loop) finished, unfinished = wait([future], return_when=asyncio.ALL_COMPLETED, timeout=timeout) if len(unfinished) > 0: raise TimeoutError(f"Coroutine {coro} failed to finish within {timeout} s") return_result = next(iter(finished)).result() if isinstance(return_result, BaseException): raise return_result else: return return_result def _get_loop() -> asyncio.AbstractEventLoop: """Create or return the default fsspec IO loop The loop will be running on a separate thread. """ current = loop[0] if current is None: with _get_lock(): # repeat the check just in case the loop got filled between the # previous two calls from another thread current = loop[0] if current is None: logger.debug("Creating Zarr event loop") current = asyncio.new_event_loop() loop[0] = current thread = threading.Thread(target=current.run_forever, name="zarr_io") thread.daemon = True iothread[0] = thread thread.start() return current async def _collect_aiterator[T](data: AsyncIterator[T]) -> tuple[T, ...]: """ Collect an entire async iterator into a tuple """ result = [x async for x in data] return tuple(result) def collect_aiterator[T](data: AsyncIterator[T]) -> tuple[T, ...]: """ Synchronously collect an entire async iterator into a tuple. """ return sync(_collect_aiterator(data)) class SyncMixin: def _sync[T](self, coroutine: Coroutine[Any, Any, T]) -> T: # TODO: refactor this to take *args and **kwargs and pass those to the method # this should allow us to better type the sync wrapper return sync( coroutine, timeout=config.get("async.timeout"), ) def _sync_iter[T](self, async_iterator: AsyncIterator[T]) -> list[T]: async def iter_to_list() -> list[T]: return [item async for item in async_iterator] return self._sync(iter_to_list()) async def _with_semaphore[T]( func: Callable[[], Awaitable[T]], semaphore: asyncio.Semaphore | None = None ) -> T: """ Await the result of invoking the no-argument-callable ``func`` within the context manager provided by a Semaphore, if one is provided. Otherwise, just await the result of invoking ``func``. """ if semaphore is None: return await func() async with semaphore: return await func() zarr-developers-zarr-python-a2f2415/src/zarr/core/sync_group.py000066400000000000000000000137001527014137600247250ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING from zarr.core.group import Group, GroupMetadata, _parse_async_node from zarr.core.group import create_hierarchy as create_hierarchy_async from zarr.core.group import create_nodes as create_nodes_async from zarr.core.group import create_rooted_hierarchy as create_rooted_hierarchy_async from zarr.core.group import get_node as get_node_async from zarr.core.sync import _collect_aiterator, sync if TYPE_CHECKING: from collections.abc import Iterator from zarr.abc.store import Store from zarr.core.common import ZarrFormat from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.types import AnyArray def create_nodes( *, store: Store, nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] ) -> Iterator[tuple[str, Group | AnyArray]]: """Create a collection of arrays and / or groups concurrently. Note: no attempt is made to validate that these arrays and / or groups collectively form a valid Zarr hierarchy. It is the responsibility of the caller of this function to ensure that the ``nodes`` parameter satisfies any correctness constraints. Parameters ---------- store : Store The storage backend to use. nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, and the values are the metadata of the nodes. The metadata must be either an instance of GroupMetadata, ArrayV3Metadata or ArrayV2Metadata. Yields ------ Group | Array The created nodes. """ coro = create_nodes_async(store=store, nodes=nodes) for key, value in sync(_collect_aiterator(coro)): yield key, _parse_async_node(value) def create_hierarchy( *, store: Store, nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], overwrite: bool = False, ) -> Iterator[tuple[str, Group | AnyArray]]: """ Create a complete zarr hierarchy from a collection of metadata objects. This function will parse its input to ensure that the hierarchy is complete. Any implicit groups will be inserted as needed. For example, an input like ```{'a/b': GroupMetadata}``` will be parsed to ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}``` After input parsing, this function then creates all the nodes in the hierarchy concurrently. Arrays and Groups are yielded in the order they are created. This order is not stable and should not be relied on. Parameters ---------- store : Store The storage backend to use. nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, relative to the root of the ``Store``. The root of the store can be specified with the empty string ``''``. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that all values must have the same ``zarr_format`` -- it is an error to mix zarr versions in the same hierarchy. Leading "/" characters from keys will be removed. overwrite : bool Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is raised instead of overwriting an existing array or group. This function will not erase an existing group unless that group is explicitly named in ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a group already exists at path ``a``, then this function will leave the group at ``a`` as-is. Yields ------ tuple[str, Group | Array] This function yields (path, node) pairs, in the order the nodes were created. Examples -------- ```python from zarr import create_hierarchy from zarr.storage import MemoryStore from zarr.core.group import GroupMetadata store = MemoryStore() nodes = {'a': GroupMetadata(attributes={'name': 'leaf'})} nodes_created = dict(create_hierarchy(store=store, nodes=nodes)) print(nodes) # {'a': GroupMetadata(attributes={'name': 'leaf'}, zarr_format=3, consolidated_metadata=None, node_type='group', extra_fields={})} ``` """ coro = create_hierarchy_async(store=store, nodes=nodes, overwrite=overwrite) for key, value in sync(_collect_aiterator(coro)): yield key, _parse_async_node(value) def create_rooted_hierarchy( *, store: Store, nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], overwrite: bool = False, ) -> Group | AnyArray: """ Create a Zarr hierarchy with a root, and return the root node, which could be a ``Group`` or ``Array`` instance. Parameters ---------- store : Store The storage backend to use. nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, and the values are the metadata of the nodes. The metadata must be either an instance of GroupMetadata, ArrayV3Metadata or ArrayV2Metadata. overwrite : bool Whether to overwrite existing nodes. Default is ``False``. Returns ------- Group | Array """ async_node = sync(create_rooted_hierarchy_async(store=store, nodes=nodes, overwrite=overwrite)) return _parse_async_node(async_node) def get_node(store: Store, path: str, zarr_format: ZarrFormat) -> AnyArray | Group: """ Get an Array or Group from a path in a Store. Parameters ---------- store : Store The store-like object to read from. path : str The path to the node to read. zarr_format : {2, 3} The zarr format of the node to read. Returns ------- Array | Group """ return _parse_async_node(sync(get_node_async(store=store, path=path, zarr_format=zarr_format))) zarr-developers-zarr-python-a2f2415/src/zarr/dtype.py000066400000000000000000000050341527014137600227330ustar00rootroot00000000000000from zarr.core.dtype import ( Bool, Complex64, Complex128, DateTime64, DateTime64JSON_V2, DateTime64JSON_V3, FixedLengthUTF32, FixedLengthUTF32JSON_V2, FixedLengthUTF32JSON_V3, Float16, Float32, Float64, Int8, Int16, Int32, Int64, NullTerminatedBytes, NullterminatedBytesJSON_V2, NullTerminatedBytesJSON_V3, RawBytes, RawBytesJSON_V2, RawBytesJSON_V3, Struct, StructJSON_V3, Structured, StructuredJSON_V2, StructuredJSON_V3, TimeDelta64, TimeDelta64JSON_V2, TimeDelta64JSON_V3, UInt8, UInt16, UInt32, UInt64, VariableLengthBytes, VariableLengthBytesJSON_V2, VariableLengthUTF8, VariableLengthUTF8JSON_V2, ZDType, data_type_registry, # Import for backwards compatibility, but not included in __all__ # so it doesn't show up in the docs parse_data_type, # noqa: F401 parse_dtype, ) from zarr.core.dtype.common import ( DTypeSpec_V2, check_dtype_spec_no_object_codec_v2, check_dtype_spec_v2, ) __all__ = [ "Bool", "Complex64", "Complex128", "DTypeSpec_V2", "DateTime64", "DateTime64JSON_V2", "DateTime64JSON_V3", "FixedLengthUTF32", "FixedLengthUTF32JSON_V2", "FixedLengthUTF32JSON_V3", "Float16", "Float32", "Float64", "Int8", "Int16", "Int32", "Int64", "NullTerminatedBytes", "NullTerminatedBytesJSON_V3", "NullterminatedBytesJSON_V2", "RawBytes", "RawBytesJSON_V2", "RawBytesJSON_V3", "Struct", "StructJSON_V3", "Structured", "StructuredJSON_V2", "StructuredJSON_V3", "TimeDelta64", "TimeDelta64JSON_V2", "TimeDelta64JSON_V3", "UInt8", "UInt16", "UInt32", "UInt64", "VariableLengthBytes", "VariableLengthBytesJSON_V2", "VariableLengthUTF8", "VariableLengthUTF8JSON_V2", "ZDType", "check_dtype_spec_no_object_codec_v2", "check_dtype_spec_v2", "data_type_registry", "parse_dtype", ] def __getattr__(name: str) -> object: if name == "DataTypeValidationError": import warnings from zarr.errors import DataTypeValidationError, ZarrDeprecationWarning warnings.warn( "Importing DataTypeValidationError from zarr.dtype is deprecated. " "Use zarr.errors.DataTypeValidationError instead.", ZarrDeprecationWarning, stacklevel=2, ) return DataTypeValidationError raise AttributeError(f"module {__name__!r} has no attribute {name!r}") zarr-developers-zarr-python-a2f2415/src/zarr/errors.py000066400000000000000000000103411527014137600231170ustar00rootroot00000000000000__all__ = [ "ArrayIndexError", "ArrayNotFoundError", "BaseZarrError", "BoundsCheckError", "ChunkNotFoundError", "ContainsArrayAndGroupError", "ContainsArrayError", "ContainsGroupError", "DataTypeValidationError", "GroupNotFoundError", "MetadataValidationError", "NegativeStepError", "NodeTypeValidationError", "PickleCodecDisabledError", "UnknownCodecError", "UnstableSpecificationWarning", "VindexInvalidSelectionError", "ZarrDeprecationWarning", "ZarrFutureWarning", "ZarrRuntimeWarning", ] class BaseZarrError(ValueError): """ Base error which all zarr errors are sub-classed from. """ _msg: str = "{}" def __init__(self, *args: object) -> None: """ If a single argument is passed, treat it as a pre-formatted message. If multiple arguments are passed, they are used as arguments for a template string class variable. This behavior is deprecated. """ if len(args) == 1: super().__init__(args[0]) else: super().__init__(self._msg.format(*args)) class NodeNotFoundError(BaseZarrError, FileNotFoundError): """ Raised when a node (array or group) is not found at a certain path. """ class ArrayNotFoundError(NodeNotFoundError): """ Raised when an array isn't found at a certain path. """ _msg = "No array found in store {!r} at path {!r}" class GroupNotFoundError(NodeNotFoundError): """ Raised when a group isn't found at a certain path. """ _msg = "No group found in store {!r} at path {!r}" class ContainsGroupError(BaseZarrError): """Raised when a group already exists at a certain path.""" _msg = "A group exists in store {!r} at path {!r}." class ContainsArrayError(BaseZarrError): """Raised when an array already exists at a certain path.""" _msg = "An array exists in store {!r} at path {!r}." class ContainsArrayAndGroupError(BaseZarrError): """Raised when both array and group metadata are found at the same path.""" _msg = ( "Array and group metadata documents (.zarray and .zgroup) were both found in store " "{!r} at path {!r}. " "Only one of these files may be present in a given directory / prefix. " "Remove the .zarray file, or the .zgroup file, or both." ) class DataTypeValidationError(ValueError): ... class MetadataValidationError(BaseZarrError): """Raised when the Zarr metadata is invalid in some way""" _msg = "Invalid value for '{}'. Expected '{}'. Got '{}'." class UnknownCodecError(BaseZarrError): """ Raised when an unknown codec was used. """ class PickleCodecDisabledError(BaseZarrError): """ Raised when reading data encoded with the pickle codec without opting in. Unpickling can execute arbitrary code, so decoding with the pickle codec is disabled unless the ``array.allow_unsafe_pickle_decode`` configuration option is set to ``True``. """ class NodeTypeValidationError(MetadataValidationError): """ Specialized exception when the node_type of the metadata document is incorrect. This can be raised when the value is invalid or unexpected given the context, for example an 'array' node when we expected a 'group'. """ class ZarrFutureWarning(FutureWarning): """ A warning intended for end users raised to indicate deprecated features. """ class UnstableSpecificationWarning(ZarrFutureWarning): """ A warning raised to indicate that a feature is outside the Zarr specification. """ class ZarrDeprecationWarning(DeprecationWarning): """ A warning raised to indicate that a feature will be removed in a future release. """ class ZarrUserWarning(UserWarning): """ A warning raised to report problems with user code. """ class ZarrRuntimeWarning(RuntimeWarning): """ A warning for dubious runtime behavior. """ class VindexInvalidSelectionError(IndexError): ... class NegativeStepError(IndexError): ... class BoundsCheckError(IndexError): ... class ArrayIndexError(IndexError): ... class ChunkNotFoundError(BaseZarrError): """ Raised when a chunk that was expected to exist in storage was not retrieved successfully. """ zarr-developers-zarr-python-a2f2415/src/zarr/experimental/000077500000000000000000000000001527014137600237275ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/experimental/__init__.py000066400000000000000000000002671527014137600260450ustar00rootroot00000000000000"""The experimental module is a site for exporting new or experimental Zarr features.""" from zarr.core.chunk_grids import ChunkGrid, ChunkSpec __all__ = ["ChunkGrid", "ChunkSpec"] zarr-developers-zarr-python-a2f2415/src/zarr/experimental/cache_store.py000066400000000000000000000427051527014137600265700ustar00rootroot00000000000000from __future__ import annotations import asyncio import logging import time from collections import OrderedDict from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Literal, Self from zarr.abc.store import ByteRequest, Store from zarr.storage._wrapper import WrapperStore logger = logging.getLogger(__name__) if TYPE_CHECKING: from zarr.core.buffer.core import Buffer, BufferPrototype # A cache entry identifier. Plain ``str`` for full-key entries that live in # the Store-backed cache; ``(str, ByteRequest)`` for byte-range entries that # live in the in-memory range cache. _CacheEntryKey = str | tuple[str, ByteRequest] @dataclass(slots=True) class _CacheState: cache_order: OrderedDict[_CacheEntryKey, None] = field(default_factory=OrderedDict) current_size: int = 0 key_sizes: dict[_CacheEntryKey, int] = field(default_factory=dict) lock: asyncio.Lock = field(default_factory=asyncio.Lock) hits: int = 0 misses: int = 0 evictions: int = 0 key_insert_times: dict[_CacheEntryKey, float] = field(default_factory=dict) range_cache: dict[str, dict[ByteRequest, Buffer]] = field(default_factory=dict) class CacheStore(WrapperStore[Store]): """ A dual-store caching implementation for Zarr stores. This cache wraps any Store implementation and uses a separate Store instance as the cache backend. This provides persistent caching capabilities with time-based expiration, size-based eviction, and flexible cache storage options. Full-key reads are cached in the Store-backed cache. Byte-range reads are cached in a separate in-memory dictionary so that partial reads never pollute the filesystem (or other persistent backend). Both caches share the same ``max_size`` budget and LRU eviction policy. Parameters ---------- store : Store The underlying store to wrap with caching cache_store : Store The store to use for caching (can be any Store implementation that supports deletes) max_age_seconds : int or "infinity", optional Maximum age of cached entries in seconds. The string "infinity" means entries never expire. Default is "infinity". max_size : int | None, optional Maximum size of the cache in bytes. When exceeded, least recently used items are evicted. None means unlimited size. Default is None. Note: Individual values larger than max_size will not be cached. cache_set_data : bool, optional Whether to cache data when it's written to the store. Default is True. Examples -------- ```python import zarr from zarr.storage import MemoryStore from zarr.experimental.cache_store import CacheStore # Create a cached store source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore( store=source_store, cache_store=cache_store, max_age_seconds=60, max_size=1024*1024 ) # Use it like any other store array = zarr.create(shape=(100,), store=cached_store) array[:] = 42 ``` """ _cache: Store max_age_seconds: int | Literal["infinity"] max_size: int | None cache_set_data: bool _state: _CacheState def __init__( self, store: Store, *, cache_store: Store, max_age_seconds: int | str = "infinity", max_size: int | None = None, cache_set_data: bool = True, ) -> None: super().__init__(store) if not cache_store.supports_deletes: msg = ( f"The provided cache store {cache_store} does not support deletes. " "The cache_store must support deletes for CacheStore to function properly." ) raise ValueError(msg) self._cache = cache_store # Validate and set max_age_seconds if isinstance(max_age_seconds, str): if max_age_seconds != "infinity": raise ValueError("max_age_seconds string value must be 'infinity'") self.max_age_seconds = "infinity" else: self.max_age_seconds = max_age_seconds self.max_size = max_size self.cache_set_data = cache_set_data self._state = _CacheState() def _with_store(self, store: Store) -> Self: # Cannot support this operation because it would share a cache, but have a new store # So cache keys would conflict raise NotImplementedError("CacheStore does not support this operation.") def with_read_only(self, read_only: bool = False) -> Self: # Create a new cache store that shares the same cache and mutable state store = type(self)( store=self._store.with_read_only(read_only), cache_store=self._cache, max_age_seconds=self.max_age_seconds, max_size=self.max_size, cache_set_data=self.cache_set_data, ) store._state = self._state return store def _is_key_fresh(self, entry_key: _CacheEntryKey) -> bool: """Check if a cached entry is still fresh based on max_age_seconds. Uses monotonic time for accurate elapsed time measurement. """ if self.max_age_seconds == "infinity": return True now = time.monotonic() elapsed = now - self._state.key_insert_times.get(entry_key, 0) return elapsed < self.max_age_seconds async def _accommodate_value(self, value_size: int) -> None: """Ensure there is enough space in the cache for a new value. Must be called while holding self._state.lock. """ if self.max_size is None: return # Remove least recently used items until we have enough space while self._state.current_size + value_size > self.max_size and self._state.cache_order: # Get the least recently used key (first in OrderedDict) lru_key = next(iter(self._state.cache_order)) await self._evict_key(lru_key) async def _evict_key(self, entry_key: _CacheEntryKey) -> None: """Evict a cache entry. Must be called while holding self._state.lock. For ``str`` keys the entry is deleted from the Store-backed cache. For ``(str, ByteRequest)`` keys the entry is removed from the in-memory range cache. """ key_size = self._state.key_sizes.get(entry_key, 0) if isinstance(entry_key, str): await self._cache.delete(entry_key) else: base_key, byte_range = entry_key per_key = self._state.range_cache.get(base_key) if per_key is not None: per_key.pop(byte_range, None) if not per_key: del self._state.range_cache[base_key] self._state.cache_order.pop(entry_key, None) self._state.key_insert_times.pop(entry_key, None) self._state.key_sizes.pop(entry_key, None) self._state.current_size = max(0, self._state.current_size - key_size) self._state.evictions += 1 async def _track_entry(self, entry_key: _CacheEntryKey, value: Buffer) -> bool: """Register *entry_key* in the shared size / LRU tracking. Returns ``True`` if the entry was tracked, ``False`` if the value exceeds ``max_size`` and was skipped. Callers should roll back any data they already stored when this returns ``False``. This method holds the lock for the entire operation to ensure atomicity. """ value_size = len(value) # Check if value exceeds max size if self.max_size is not None and value_size > self.max_size: return False async with self._state.lock: # If key already exists, subtract old size first if entry_key in self._state.key_sizes: old_size = self._state.key_sizes[entry_key] self._state.current_size -= old_size # Make room for the new value await self._accommodate_value(value_size) # Update tracking atomically self._state.cache_order[entry_key] = None self._state.current_size += value_size self._state.key_sizes[entry_key] = value_size self._state.key_insert_times[entry_key] = time.monotonic() return True async def _update_access_order(self, entry_key: _CacheEntryKey) -> None: """Update the access order for LRU tracking.""" if entry_key in self._state.cache_order: async with self._state.lock: self._state.cache_order.move_to_end(entry_key) def _remove_from_tracking(self, entry_key: _CacheEntryKey) -> None: """Remove an entry from all tracking structures. Must be called while holding self._state.lock. """ self._state.cache_order.pop(entry_key, None) self._state.key_insert_times.pop(entry_key, None) self._state.key_sizes.pop(entry_key, None) def _invalidate_range_entries(self, key: str) -> None: """Remove all byte-range entries for *key* from the range cache and tracking. Must be called while holding self._state.lock. """ per_key = self._state.range_cache.pop(key, None) if per_key is not None: for byte_range in per_key: entry_key: _CacheEntryKey = (key, byte_range) entry_size = self._state.key_sizes.pop(entry_key, 0) self._state.cache_order.pop(entry_key, None) self._state.key_insert_times.pop(entry_key, None) self._state.current_size = max(0, self._state.current_size - entry_size) # ------------------------------------------------------------------ # get helpers # ------------------------------------------------------------------ async def _cache_miss( self, key: str, byte_range: ByteRequest | None, result: Buffer | None ) -> None: """Handle a cache miss by storing or cleaning up after a source-store fetch.""" if result is None: if byte_range is None: await self._cache.delete(key) async with self._state.lock: self._remove_from_tracking(key) else: entry_key: _CacheEntryKey = (key, byte_range) async with self._state.lock: per_key = self._state.range_cache.get(key) if per_key is not None: per_key.pop(byte_range, None) if not per_key: del self._state.range_cache[key] self._remove_from_tracking(entry_key) else: if byte_range is None: await self._cache.set(key, result) await self._track_entry(key, result) else: entry_key = (key, byte_range) self._state.range_cache.setdefault(key, {})[byte_range] = result tracked = await self._track_entry(entry_key, result) if not tracked: # Value too large for the cache — roll back the insertion per_key = self._state.range_cache.get(key) if per_key is not None: per_key.pop(byte_range, None) if not per_key: del self._state.range_cache[key] async def _get_try_cache( self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: """Try to get data from cache first, falling back to source store.""" if byte_range is None: # Full-key read — use Store-backed cache maybe_cached = await self._cache.get(key, prototype) if maybe_cached is not None: self._state.hits += 1 await self._update_access_order(key) return maybe_cached else: # Byte-range read — use in-memory range cache entry_key: _CacheEntryKey = (key, byte_range) per_key = self._state.range_cache.get(key) if per_key is not None: cached_buf = per_key.get(byte_range) if cached_buf is not None: self._state.hits += 1 await self._update_access_order(entry_key) return cached_buf # Cache miss — fetch from source store self._state.misses += 1 result = await super().get(key, prototype, byte_range) await self._cache_miss(key, byte_range, result) return result async def _get_no_cache( self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: """Get data directly from source store and update cache.""" self._state.misses += 1 result = await super().get(key, prototype, byte_range) await self._cache_miss(key, byte_range, result) return result @property def _supports_sync_io(self) -> bool: # The caching logic lives only in the async get/set/delete overrides; # the sync methods inherited from `WrapperStore` delegate straight to # the source store, so a sync-capable consumer (the fused codec # pipeline) would write and delete around the cache, leaving stale # entries that later async reads serve as current data. Opt out of # sync IO until the sync surface is cache-aware. return False async def get( self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None, ) -> Buffer | None: """ Retrieve data from the store, using cache when appropriate. Parameters ---------- key : str The key to retrieve prototype : BufferPrototype Buffer prototype for creating the result buffer byte_range : ByteRequest, optional Byte range to retrieve Returns ------- Buffer | None The retrieved data, or None if not found """ entry_key: _CacheEntryKey = (key, byte_range) if byte_range is not None else key if not self._is_key_fresh(entry_key): return await self._get_no_cache(key, prototype, byte_range) else: return await self._get_try_cache(key, prototype, byte_range) async def set(self, key: str, value: Buffer) -> None: """ Store data in the underlying store and optionally in cache. Parameters ---------- key : str The key to store under value : Buffer The data to store """ await super().set(key, value) # Invalidate all cached byte-range entries (source data changed) async with self._state.lock: self._invalidate_range_entries(key) if self.cache_set_data: await self._cache.set(key, value) await self._track_entry(key, value) else: await self._cache.delete(key) async with self._state.lock: self._remove_from_tracking(key) async def delete(self, key: str) -> None: """ Delete data from both the underlying store and cache. Parameters ---------- key : str The key to delete """ await super().delete(key) # Invalidate all cached byte-range entries async with self._state.lock: self._invalidate_range_entries(key) await self._cache.delete(key) async with self._state.lock: self._remove_from_tracking(key) def cache_info(self) -> dict[str, Any]: """Return information about the cache state.""" return { "cache_store_type": type(self._cache).__name__, "max_age_seconds": "infinity" if self.max_age_seconds == "infinity" else self.max_age_seconds, "max_size": self.max_size, "current_size": self._state.current_size, "cache_set_data": self.cache_set_data, "tracked_keys": len(self._state.key_insert_times), "cached_keys": len(self._state.cache_order), } def cache_stats(self) -> dict[str, Any]: """Return cache performance statistics.""" total_requests = self._state.hits + self._state.misses hit_rate = self._state.hits / total_requests if total_requests > 0 else 0.0 return { "hits": self._state.hits, "misses": self._state.misses, "evictions": self._state.evictions, "total_requests": total_requests, "hit_rate": hit_rate, } async def clear_cache(self) -> None: """Clear all cached data and tracking information.""" # Clear the cache store if it supports clear if hasattr(self._cache, "clear"): await self._cache.clear() # Reset tracking async with self._state.lock: self._state.key_insert_times.clear() self._state.cache_order.clear() self._state.key_sizes.clear() self._state.range_cache.clear() self._state.current_size = 0 def __repr__(self) -> str: """Return string representation of the cache store.""" return ( f"{self.__class__.__name__}(" f"store={self._store!r}, " f"cache_store={self._cache!r}, " f"max_age_seconds={self.max_age_seconds}, " f"max_size={self.max_size}, " f"current_size={self._state.current_size}, " f"cached_keys={len(self._state.cache_order)})" ) zarr-developers-zarr-python-a2f2415/src/zarr/metadata/000077500000000000000000000000001527014137600230125ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/metadata/__init__.py000066400000000000000000000000001527014137600251110ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/metadata/migrate_v3.py000066400000000000000000000245041527014137600254310ustar00rootroot00000000000000import asyncio import logging from typing import cast import numcodecs.abc import zarr from zarr import Group from zarr.abc.codec import ArrayArrayCodec, BytesBytesCodec, Codec from zarr.abc.store import Store from zarr.codecs.blosc import BloscCodec, BloscShuffle from zarr.codecs.bytes import BytesCodec from zarr.codecs.gzip import GzipCodec from zarr.codecs.transpose import TransposeCodec from zarr.codecs.zstd import ZstdCodec from zarr.core.buffer.core import default_buffer_prototype from zarr.core.chunk_key_encodings import V2ChunkKeyEncoding from zarr.core.common import ( ZARR_JSON, ZARRAY_JSON, ZATTRS_JSON, ZGROUP_JSON, ZMETADATA_V2_JSON, ZarrFormat, ) from zarr.core.dtype.common import HasEndianness from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType from zarr.core.group import GroupMetadata from zarr.core.metadata.v2 import ArrayV2Metadata from zarr.core.metadata.v3 import ArrayV3Metadata, RegularChunkGridMetadata from zarr.core.sync import sync from zarr.errors import UnknownCodecError from zarr.registry import get_codec_class from zarr.storage import StorePath from zarr.types import AnyArray _logger = logging.getLogger(__name__) def migrate_v2_to_v3( *, input_store: Store, output_store: Store | None = None, dry_run: bool = False, ) -> None: """Migrate all v2 metadata in a Zarr store to v3. This will create a zarr.json file at each level of a Zarr hierarchy (for every group / array). v2 files (.zarray, .zattrs etc.) will be left as-is. Parameters ---------- input_store : Store Input Zarr to migrate. output_store : Store, optional Output location to write v3 metadata (no array data will be copied). If not provided, v3 metadata will be written to input_store. dry_run : bool, optional Enable a 'dry run' - files that would be created are logged, but no files are created or changed. """ zarr_v2 = zarr.open(store=input_store, mode="r+") if output_store is not None: # w- access to not allow overwrite of existing data output_path = sync(StorePath.open(output_store, path="", mode="w-")) else: output_path = zarr_v2.store_path migrate_to_v3(zarr_v2, output_path, dry_run=dry_run) def migrate_to_v3(zarr_v2: AnyArray | Group, output_path: StorePath, dry_run: bool = False) -> None: """Migrate all v2 metadata in a Zarr array/group to v3. Note - if a group is provided, then all arrays / groups within this group will also be converted. A zarr.json file will be created for each level and written to output_path, with any v2 files (.zarray, .zattrs etc.) left as-is. Parameters ---------- zarr_v2 : Array | Group An array or group with zarr_format = 2 output_path : StorePath The store path to write generated v3 metadata to. dry_run : bool, optional Enable a 'dry run' - files that would be created are logged, but no files are created or changed. """ if not zarr_v2.metadata.zarr_format == 2: raise TypeError("Only arrays / groups with zarr v2 metadata can be converted") if isinstance(zarr_v2.metadata, GroupMetadata): _convert_group(zarr_v2, output_path, dry_run) else: _convert_array(zarr_v2, output_path, dry_run) async def remove_metadata( store: Store, zarr_format: ZarrFormat, force: bool = False, dry_run: bool = False, ) -> None: """Remove all v2 (.zarray, .zattrs, .zgroup, .zmetadata) or v3 (zarr.json) metadata files from the given Zarr. Note - this will remove metadata files at all levels of the hierarchy (every group and array). Parameters ---------- store : Store Zarr to remove metadata from. zarr_format : ZarrFormat Which format's metadata to remove - 2 or 3. force : bool, optional When False, metadata can only be removed if a valid alternative exists e.g. deletion of v2 metadata will only be allowed when v3 metadata is also present. When True, metadata can be removed when there is no alternative. dry_run : bool, optional Enable a 'dry run' - files that would be deleted are logged, but no files are removed or changed. """ if not store.supports_deletes: raise ValueError("Store must support deletes to remove metadata") store_path = await StorePath.open(store, path="", mode="r+") metadata_files_all = { 2: [ZARRAY_JSON, ZATTRS_JSON, ZGROUP_JSON, ZMETADATA_V2_JSON], 3: [ZARR_JSON], } if zarr_format == 2: alternative_metadata = 3 else: alternative_metadata = 2 awaitables = [] async for file_path in store.list(): parent_path, _, file_name = file_path.rpartition("/") if file_name not in metadata_files_all[zarr_format]: continue if force or await _metadata_exists( cast(ZarrFormat, alternative_metadata), store_path / parent_path ): _logger.info("Deleting metadata at %s", store_path / file_path) if not dry_run: awaitables.append((store_path / file_path).delete()) else: raise ValueError( f"Cannot remove v{zarr_format} metadata at {store_path / file_path} - no v{alternative_metadata} " "metadata exists. To delete anyway, use the 'force' option." ) await asyncio.gather(*awaitables) def _convert_group(zarr_v2: Group, output_path: StorePath, dry_run: bool) -> None: if zarr_v2.metadata.consolidated_metadata is not None: raise NotImplementedError("Migration of consolidated metadata isn't supported.") # process members of the group for key in zarr_v2: migrate_to_v3(zarr_v2[key], output_path=output_path / key, dry_run=dry_run) # write group's converted metadata group_metadata_v3 = GroupMetadata( attributes=zarr_v2.metadata.attributes, zarr_format=3, consolidated_metadata=None ) sync(_save_v3_metadata(group_metadata_v3, output_path, dry_run=dry_run)) def _convert_array(zarr_v2: AnyArray, output_path: StorePath, dry_run: bool) -> None: array_metadata_v3 = _convert_array_metadata(cast(ArrayV2Metadata, zarr_v2.metadata)) sync(_save_v3_metadata(array_metadata_v3, output_path, dry_run=dry_run)) async def _metadata_exists(zarr_format: ZarrFormat, store_path: StorePath) -> bool: metadata_files_required = {2: [ZARRAY_JSON, ZGROUP_JSON], 3: [ZARR_JSON]} for metadata_file in metadata_files_required[zarr_format]: if await (store_path / metadata_file).exists(): return True return False def _convert_array_metadata(metadata_v2: ArrayV2Metadata) -> ArrayV3Metadata: chunk_key_encoding = V2ChunkKeyEncoding(separator=metadata_v2.dimension_separator) codecs: list[Codec] = [] # array-array codecs if metadata_v2.order == "F": # F is equivalent to order: n-1, ... 1, 0 codecs.append(TransposeCodec(order=list(range(len(metadata_v2.shape) - 1, -1, -1)))) if metadata_v2.filters is not None: codecs.extend(_convert_filters(metadata_v2.filters)) # array-bytes codecs if not isinstance(metadata_v2.dtype, HasEndianness): codecs.append(BytesCodec(endian=None)) else: codecs.append(BytesCodec(endian=metadata_v2.dtype.endianness)) # bytes-bytes codecs if metadata_v2.compressor is not None: bytes_bytes_codec = _convert_compressor(metadata_v2.compressor, metadata_v2.dtype) codecs.append(bytes_bytes_codec) return ArrayV3Metadata( shape=metadata_v2.shape, data_type=metadata_v2.dtype, chunk_grid=RegularChunkGridMetadata(chunk_shape=metadata_v2.chunks), chunk_key_encoding=chunk_key_encoding, fill_value=metadata_v2.fill_value, codecs=codecs, attributes=metadata_v2.attributes, dimension_names=None, storage_transformers=None, ) def _convert_filters(filters: tuple[numcodecs.abc.Codec, ...]) -> list[ArrayArrayCodec]: filters_codecs = [_find_numcodecs_zarr3(filter) for filter in filters] for codec in filters_codecs: if not isinstance(codec, ArrayArrayCodec): raise TypeError(f"Filter {type(codec)} is not an ArrayArrayCodec") return cast(list[ArrayArrayCodec], filters_codecs) def _convert_compressor( compressor: numcodecs.abc.Codec, dtype: ZDType[TBaseDType, TBaseScalar] ) -> BytesBytesCodec: match compressor.codec_id: case "blosc": return BloscCodec( typesize=dtype.to_native_dtype().itemsize, cname=compressor.cname, clevel=compressor.clevel, shuffle=BloscShuffle.from_int(compressor.shuffle), blocksize=compressor.blocksize, ) case "zstd": return ZstdCodec( level=compressor.level, checksum=compressor.checksum, ) case "gzip": return GzipCodec(level=compressor.level) case _: # If possible, find matching zarr.codecs.numcodecs codec compressor_codec = _find_numcodecs_zarr3(compressor) if not isinstance(compressor_codec, BytesBytesCodec): raise TypeError(f"Compressor {type(compressor_codec)} is not a BytesBytesCodec") return compressor_codec def _find_numcodecs_zarr3(numcodecs_codec: numcodecs.abc.Codec) -> Codec: """Find matching zarr.codecs.numcodecs codec (if it exists)""" numcodec_name = f"numcodecs.{numcodecs_codec.codec_id}" numcodec_dict = { "name": numcodec_name, "configuration": numcodecs_codec.get_config(), } try: codec_v3 = get_codec_class(numcodec_name) except UnknownCodecError as exc: raise ValueError( f"Couldn't find corresponding zarr.codecs.numcodecs codec for {numcodecs_codec.codec_id}" ) from exc return codec_v3.from_dict(numcodec_dict) async def _save_v3_metadata( metadata_v3: ArrayV3Metadata | GroupMetadata, output_path: StorePath, dry_run: bool = False ) -> None: zarr_json_path = output_path / ZARR_JSON if await zarr_json_path.exists(): raise ValueError(f"{ZARR_JSON} already exists at {zarr_json_path}") _logger.info("Saving metadata to %s", zarr_json_path) to_save = metadata_v3.to_buffer_dict(default_buffer_prototype()) if not dry_run: await zarr_json_path.set_if_not_exists(to_save[ZARR_JSON]) zarr-developers-zarr-python-a2f2415/src/zarr/py.typed000066400000000000000000000000001527014137600227170ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/registry.py000066400000000000000000000535271527014137600234700ustar00rootroot00000000000000from __future__ import annotations import warnings from collections import defaultdict from collections.abc import Mapping from importlib.metadata import entry_points as get_entry_points from typing import TYPE_CHECKING, Any from zarr.core.config import BadConfigError, config from zarr.core.dtype import data_type_registry from zarr.errors import UnknownCodecError, ZarrUserWarning if TYPE_CHECKING: from importlib.metadata import EntryPoint from zarr.abc.codec import ( ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec, CodecJSON_V2, CodecPipeline, ) from zarr.abc.numcodec import Numcodec from zarr.core.buffer import Buffer, NDBuffer from zarr.core.chunk_key_encodings import ChunkKeyEncoding from zarr.core.common import JSON, ZarrFormat __all__ = [ "Registry", "get_buffer_class", "get_chunk_key_encoding_class", "get_codec_class", "get_ndbuffer_class", "get_pipeline_class", "register_buffer", "register_chunk_key_encoding", "register_codec", "register_ndbuffer", "register_pipeline", ] _ZARR_CODEC_DOCS_URL = "https://zarr.readthedocs.io/en/stable/user-guide/extending/#custom-codecs" _NUMCODECS_CODEC_DOCS_URL = ( "https://numcodecs.readthedocs.io/en/stable/registry.html#numcodecs.registry.register_codec" ) # Codecs zarr-python does not implement, mapped to the names of Python packages that do. # These tables exist purely to make the "no implementation for this codec" error actionable; # nothing here affects which codecs zarr can actually read or write. Values are what you would # pass to `pip install`. Only add an entry you have verified against the package's declared # entry points, and only for a package that is actually published. # # The two Zarr formats resolve codecs through different registries, so they get different # tables: a name can mean one thing as a Zarr format 3 codec name and another as a Zarr # format 2 codec id. For example, `imagecodecs_*` names are registered by # `imagecodecs-zarr` and `virtual-tiff` under `zarr.codecs`, and by # `imagecodecs-numcodecs` under `numcodecs.codecs`. # Zarr format 3 codec names (entry point group "zarr.codecs"). _CODEC_PACKAGES: dict[str, tuple[str, ...]] = { "gribberish": ("gribberish",), # Verified against imagecodecs-zarr 2026.8.16's published zarr.codecs entry points: # https://pypi.org/project/imagecodecs-zarr/2026.8.16/ # virtual-tiff also provides 13 of these names, plus jpeg8 and jetraw. # Keep exact names: neither package provides every possible imagecodecs_* name. "imagecodecs_aec": ("imagecodecs-zarr",), "imagecodecs_apng": ("imagecodecs-zarr",), "imagecodecs_avif": ("imagecodecs-zarr",), "imagecodecs_b2nd": ("imagecodecs-zarr",), "imagecodecs_bfloat16": ("imagecodecs-zarr",), "imagecodecs_bitorder": ("imagecodecs-zarr",), "imagecodecs_bitshuffle": ("imagecodecs-zarr",), "imagecodecs_blosc": ("imagecodecs-zarr",), "imagecodecs_blosc2": ("imagecodecs-zarr",), "imagecodecs_bmp": ("imagecodecs-zarr",), "imagecodecs_brotli": ("imagecodecs-zarr",), "imagecodecs_byteshuffle": ("imagecodecs-zarr",), "imagecodecs_bz2": ("imagecodecs-zarr",), "imagecodecs_ccittfax3": ("imagecodecs-zarr",), "imagecodecs_ccittfax4": ("imagecodecs-zarr",), "imagecodecs_ccittrle": ("imagecodecs-zarr",), "imagecodecs_checksum": ("imagecodecs-zarr",), "imagecodecs_chunked": ("imagecodecs-zarr",), "imagecodecs_cms": ("imagecodecs-zarr",), "imagecodecs_dds": ("imagecodecs-zarr",), "imagecodecs_deflate": ("imagecodecs-zarr", "virtual-tiff"), "imagecodecs_delta": ("imagecodecs-zarr", "virtual-tiff"), "imagecodecs_dicomrle": ("imagecodecs-zarr",), "imagecodecs_eer": ("imagecodecs-zarr",), "imagecodecs_exr": ("imagecodecs-zarr",), "imagecodecs_float24": ("imagecodecs-zarr",), "imagecodecs_floatpred": ("imagecodecs-zarr", "virtual-tiff"), "imagecodecs_gif": ("imagecodecs-zarr",), "imagecodecs_hcomp": ("imagecodecs-zarr",), "imagecodecs_heif": ("imagecodecs-zarr",), "imagecodecs_htj2k": ("imagecodecs-zarr",), "imagecodecs_isal": ("imagecodecs-zarr",), "imagecodecs_jetraw": ("virtual-tiff",), "imagecodecs_jpeg": ("imagecodecs-zarr", "virtual-tiff"), "imagecodecs_jpeg2k": ("imagecodecs-zarr", "virtual-tiff"), "imagecodecs_jpeg8": ("virtual-tiff",), "imagecodecs_jpegls": ("imagecodecs-zarr",), "imagecodecs_jpegxl": ("imagecodecs-zarr", "virtual-tiff"), "imagecodecs_jpegxr": ("imagecodecs-zarr", "virtual-tiff"), "imagecodecs_jpegxs": ("imagecodecs-zarr",), "imagecodecs_lerc": ("imagecodecs-zarr", "virtual-tiff"), "imagecodecs_ljpeg": ("imagecodecs-zarr",), "imagecodecs_lz4": ("imagecodecs-zarr",), "imagecodecs_lz4f": ("imagecodecs-zarr",), "imagecodecs_lz4h5": ("imagecodecs-zarr",), "imagecodecs_lzf": ("imagecodecs-zarr",), "imagecodecs_lzfse": ("imagecodecs-zarr",), "imagecodecs_lzham": ("imagecodecs-zarr",), "imagecodecs_lzma": ("imagecodecs-zarr",), "imagecodecs_lzo": ("imagecodecs-zarr",), "imagecodecs_lzw": ("imagecodecs-zarr", "virtual-tiff"), "imagecodecs_meshopt": ("imagecodecs-zarr",), "imagecodecs_openzl": ("imagecodecs-zarr",), "imagecodecs_packbits": ("imagecodecs-zarr", "virtual-tiff"), "imagecodecs_packints": ("imagecodecs-zarr",), "imagecodecs_pcodec": ("imagecodecs-zarr",), "imagecodecs_pcx": ("imagecodecs-zarr",), "imagecodecs_pglz": ("imagecodecs-zarr",), "imagecodecs_pixarlog": ("imagecodecs-zarr",), "imagecodecs_plio": ("imagecodecs-zarr",), "imagecodecs_png": ("imagecodecs-zarr", "virtual-tiff"), "imagecodecs_qoi": ("imagecodecs-zarr",), "imagecodecs_quantize": ("imagecodecs-zarr",), "imagecodecs_rcomp": ("imagecodecs-zarr",), "imagecodecs_rgbe": ("imagecodecs-zarr",), "imagecodecs_snappy": ("imagecodecs-zarr",), "imagecodecs_sperr": ("imagecodecs-zarr",), "imagecodecs_spng": ("imagecodecs-zarr",), "imagecodecs_sz3": ("imagecodecs-zarr",), "imagecodecs_szip": ("imagecodecs-zarr",), "imagecodecs_tga": ("imagecodecs-zarr",), "imagecodecs_tiff": ("imagecodecs-zarr",), "imagecodecs_ultrahdr": ("imagecodecs-zarr",), "imagecodecs_wavpack": ("imagecodecs-zarr",), "imagecodecs_webp": ("imagecodecs-zarr", "virtual-tiff"), "imagecodecs_wic": ("imagecodecs-zarr",), "imagecodecs_xor": ("imagecodecs-zarr",), "imagecodecs_zfp": ("imagecodecs-zarr",), "imagecodecs_zlib": ("imagecodecs-zarr",), "imagecodecs_zlibng": ("imagecodecs-zarr",), "imagecodecs_zopfli": ("imagecodecs-zarr",), "imagecodecs_zstd": ("imagecodecs-zarr", "virtual-tiff"), "imagecodecs_zstd1": ("imagecodecs-zarr",), "n5_default": ("zarr-n5",), } # As `_CODEC_PACKAGES`, but each key is matched against the start of the codec name. Packages # that provide many codecs namespace them behind a shared prefix, so one entry covers them all. _CODEC_PACKAGE_PREFIXES: dict[str, tuple[str, ...]] = { "any-numcodecs.": ("zarr-any-numcodecs",), "omfiles.": ("omfiles",), "virtual_tiff.": ("virtual-tiff",), } # Zarr format 2 codec ids (entry point group "numcodecs.codecs"). `numcodecs` itself gates # several of its own codecs behind optional dependencies, so the package to install for those # is an extra of numcodecs rather than a third-party distribution. _NUMCODEC_PACKAGES: dict[str, tuple[str, ...]] = { "FITSAscii": ("kerchunk",), "FITSVarBintable": ("kerchunk",), "crc32c": ("numcodecs[crc32c]",), "fill_hdf_strings": ("kerchunk",), "grib": ("kerchunk",), "msgpack2": ("numcodecs[msgpack]",), "pcodec": ("numcodecs[pcodec]",), "rawgrib": ("gribscan",), "record_member": ("kerchunk",), "vc-delta3d": ("vc-delta3d",), "wavpack": ("wavpack-numcodecs",), "zfpy": ("numcodecs[zfpy]",), } # As `_NUMCODEC_PACKAGES`, but matched against the start of the codec id. _NUMCODEC_PACKAGE_PREFIXES: dict[str, tuple[str, ...]] = { "gribscan.": ("gribscan",), "imagecodecs_": ("imagecodecs-numcodecs",), } def _packages_for_codec(name: str, *, zarr_format: ZarrFormat) -> tuple[str, ...]: """ Names of Python packages known to provide an implementation of the codec ``name``. Returns an empty tuple if we don't know of any. Parameters ---------- name : str The codec name (Zarr format 3) or codec id (Zarr format 2) we failed to resolve. zarr_format : ZarrFormat Which registry the codec was looked up in. """ if zarr_format == 2: exact, prefixes = _NUMCODEC_PACKAGES, _NUMCODEC_PACKAGE_PREFIXES else: exact, prefixes = _CODEC_PACKAGES, _CODEC_PACKAGE_PREFIXES if name in exact: return exact[name] for prefix, packages in prefixes.items(): if name.startswith(prefix): return packages return () def _missing_codec_message(name: str, *, zarr_format: ZarrFormat) -> str: """ Build the error message raised when no implementation of the codec ``name`` is available. Parameters ---------- name : str The codec name (Zarr format 3) or codec id (Zarr format 2) we failed to resolve. zarr_format : ZarrFormat Which registry the codec was looked up in. Zarr format 2 codecs are resolved through numcodecs, so that case points at the numcodecs registry rather than at zarr's. """ if zarr_format == 2: docs_url, registry = _NUMCODECS_CODEC_DOCS_URL, "numcodecs" else: docs_url, registry = _ZARR_CODEC_DOCS_URL, "zarr" msg = ( f"An implementation for codec {name!r} is not available. Register one explicitly " f"using the codec registry (see {docs_url}), or install a Python package that " f"registers a codec implementation with {registry}." ) packages = _packages_for_codec(name, zarr_format=zarr_format) if packages: msg += f" Known packages supporting this codec: {', '.join(packages)}." return msg class Registry[T](dict[str, type[T]]): def __init__(self) -> None: super().__init__() self.lazy_load_list: list[EntryPoint] = [] def lazy_load(self, use_entrypoint_name: bool = False) -> None: for e in self.lazy_load_list: self.register(e.load(), qualname=e.name if use_entrypoint_name else None) self.lazy_load_list.clear() def register(self, cls: type[T], qualname: str | None = None) -> None: if qualname is None: qualname = fully_qualified_name(cls) self[qualname] = cls _codec_registries: dict[str, Registry[Codec]] = defaultdict(Registry) _pipeline_registry: Registry[CodecPipeline] = Registry() _buffer_registry: Registry[Buffer] = Registry() _ndbuffer_registry: Registry[NDBuffer] = Registry() _chunk_key_encoding_registry: Registry[ChunkKeyEncoding] = Registry() """ The registry module is responsible for managing implementations of codecs, pipelines, buffers, ndbuffers, and chunk key encodings and collecting them from entrypoints. The implementation used is determined by the config. The registry module is also responsible for managing dtypes. """ def _collect_entrypoints() -> list[Registry[Any]]: """ Collects codecs, pipelines, dtypes, buffers and ndbuffers from entrypoints. Entry points can either be single items or groups of items. Allowed syntax for entry_points.txt is e.g. [zarr.codecs] gzip = package:EntrypointGzipCodec1 [zarr.codecs.gzip] some_name = package:EntrypointGzipCodec2 another = package:EntrypointGzipCodec3 [zarr] buffer = package:TestBuffer1 [zarr.buffer] xyz = package:TestBuffer2 abc = package:TestBuffer3 ... """ entry_points = get_entry_points() _buffer_registry.lazy_load_list.extend(entry_points.select(group="zarr.buffer")) _buffer_registry.lazy_load_list.extend(entry_points.select(group="zarr", name="buffer")) _ndbuffer_registry.lazy_load_list.extend(entry_points.select(group="zarr.ndbuffer")) _ndbuffer_registry.lazy_load_list.extend(entry_points.select(group="zarr", name="ndbuffer")) data_type_registry._lazy_load_list.extend(entry_points.select(group="zarr.data_type")) data_type_registry._lazy_load_list.extend(entry_points.select(group="zarr", name="data_type")) _chunk_key_encoding_registry.lazy_load_list.extend( entry_points.select(group="zarr.chunk_key_encoding") ) _chunk_key_encoding_registry.lazy_load_list.extend( entry_points.select(group="zarr", name="chunk_key_encoding") ) _pipeline_registry.lazy_load_list.extend(entry_points.select(group="zarr.codec_pipeline")) _pipeline_registry.lazy_load_list.extend( entry_points.select(group="zarr", name="codec_pipeline") ) for e in entry_points.select(group="zarr.codecs"): _codec_registries[e.name].lazy_load_list.append(e) for group in entry_points.groups: if group.startswith("zarr.codecs."): codec_name = group.split(".")[2] _codec_registries[codec_name].lazy_load_list.extend(entry_points.select(group=group)) return [ *_codec_registries.values(), _pipeline_registry, _buffer_registry, _ndbuffer_registry, _chunk_key_encoding_registry, ] def _reload_config() -> None: config.refresh() def fully_qualified_name(cls: type) -> str: module = cls.__module__ return f"{module}.{cls.__qualname__}" def register_codec(key: str, codec_cls: type[Codec], *, qualname: str | None = None) -> None: if key not in _codec_registries: _codec_registries[key] = Registry() _codec_registries[key].register(codec_cls, qualname=qualname) def register_pipeline(pipe_cls: type[CodecPipeline]) -> None: _pipeline_registry.register(pipe_cls) def register_ndbuffer(cls: type[NDBuffer], qualname: str | None = None) -> None: _ndbuffer_registry.register(cls, qualname) def register_buffer(cls: type[Buffer], qualname: str | None = None) -> None: _buffer_registry.register(cls, qualname) def register_chunk_key_encoding(key: str, cls: type) -> None: _chunk_key_encoding_registry.register(cls, key) def get_codec_class(key: str, reload_config: bool = False) -> type[Codec]: if reload_config: _reload_config() if key in _codec_registries: # logger.debug("Auto loading codec '%s' from entrypoint", codec_id) _codec_registries[key].lazy_load() codec_classes = _codec_registries[key] if not codec_classes: raise UnknownCodecError(_missing_codec_message(key, zarr_format=3)) config_entry = config.get("codecs", {}).get(key) if config_entry is None: if len(codec_classes) == 1: return next(iter(codec_classes.values())) warnings.warn( f"Codec '{key}' not configured in config. Selecting any implementation.", stacklevel=2, category=ZarrUserWarning, ) return list(codec_classes.values())[-1] selected_codec_cls = codec_classes.get(config_entry) if selected_codec_cls is None: # Not UnknownCodecError: the codec is known, the implementation named in the config is # not registered. That is a configuration problem, which is what the sibling getters in # this module raise BadConfigError for. raise BadConfigError( f"Codec {key!r} is configured to use the implementation {config_entry!r}, which is " f"not registered. Registered implementations of this codec: " f"{sorted(codec_classes)}." ) return selected_codec_cls def _resolve_codec(data: dict[str, JSON]) -> Codec: """ Get a codec instance from a dict representation of that codec. """ # TODO: narrow the type of the input to only those dicts that map on to codec class instances. return get_codec_class(data["name"]).from_dict(data) # type: ignore[arg-type] def _parse_bytes_bytes_codec(data: dict[str, JSON] | Codec) -> BytesBytesCodec: """ Normalize the input to a `BytesBytesCodec` instance. If the input is already a `BytesBytesCodec`, it is returned as is. If the input is a dict, it is converted to a `BytesBytesCodec` instance via the `_resolve_codec` function. """ from zarr.abc.codec import BytesBytesCodec if isinstance(data, dict): result = _resolve_codec(data) if not isinstance(result, BytesBytesCodec): msg = f"Expected a dict representation of a BytesBytesCodec; got a dict representation of a {type(result)} instead." raise TypeError(msg) else: if not isinstance(data, BytesBytesCodec): raise TypeError(f"Expected a BytesBytesCodec. Got {type(data)} instead.") result = data return result def _parse_array_bytes_codec(data: dict[str, JSON] | Codec) -> ArrayBytesCodec: """ Normalize the input to a `ArrayBytesCodec` instance. If the input is already a `ArrayBytesCodec`, it is returned as is. If the input is a dict, it is converted to a `ArrayBytesCodec` instance via the `_resolve_codec` function. """ from zarr.abc.codec import ArrayBytesCodec if isinstance(data, dict): result = _resolve_codec(data) if not isinstance(result, ArrayBytesCodec): msg = f"Expected a dict representation of an ArrayBytesCodec; got a dict representation of a {type(result)} instead." raise TypeError(msg) else: if not isinstance(data, ArrayBytesCodec): raise TypeError(f"Expected an ArrayBytesCodec. Got {type(data)} instead.") result = data return result def _parse_array_array_codec(data: dict[str, JSON] | Codec) -> ArrayArrayCodec: """ Normalize the input to a `ArrayArrayCodec` instance. If the input is already a `ArrayArrayCodec`, it is returned as is. If the input is a dict, it is converted to a `ArrayArrayCodec` instance via the `_resolve_codec` function. """ from zarr.abc.codec import ArrayArrayCodec if isinstance(data, dict): result = _resolve_codec(data) if not isinstance(result, ArrayArrayCodec): msg = f"Expected a dict representation of an ArrayArrayCodec; got a dict representation of a {type(result)} instead." raise TypeError(msg) else: if not isinstance(data, ArrayArrayCodec): raise TypeError(f"Expected an ArrayArrayCodec. Got {type(data)} instead.") result = data return result def get_pipeline_class(reload_config: bool = False) -> type[CodecPipeline]: if reload_config: _reload_config() _pipeline_registry.lazy_load() path = config.get("codec_pipeline.path") pipeline_class = _pipeline_registry.get(path) if pipeline_class: return pipeline_class raise BadConfigError( f"Pipeline class '{path}' not found in registered pipelines: {list(_pipeline_registry)}." ) def get_buffer_class(reload_config: bool = False) -> type[Buffer]: if reload_config: _reload_config() _buffer_registry.lazy_load() path = config.get("buffer") buffer_class = _buffer_registry.get(path) if buffer_class: return buffer_class raise BadConfigError( f"Buffer class '{path}' not found in registered buffers: {list(_buffer_registry)}." ) def get_ndbuffer_class(reload_config: bool = False) -> type[NDBuffer]: if reload_config: _reload_config() _ndbuffer_registry.lazy_load() path = config.get("ndbuffer") ndbuffer_class = _ndbuffer_registry.get(path) if ndbuffer_class: return ndbuffer_class raise BadConfigError( f"NDBuffer class '{path}' not found in registered buffers: {list(_ndbuffer_registry)}." ) def get_chunk_key_encoding_class(key: str) -> type[ChunkKeyEncoding]: _chunk_key_encoding_registry.lazy_load(use_entrypoint_name=True) if key not in _chunk_key_encoding_registry: raise KeyError( f"Chunk key encoding '{key}' not found in registered chunk key encodings: {list(_chunk_key_encoding_registry)}." ) return _chunk_key_encoding_registry[key] _collect_entrypoints() def get_numcodec(data: CodecJSON_V2[str]) -> Numcodec: """ Resolve a numcodec codec from the numcodecs registry. This requires the Numcodecs package to be installed. Parameters ---------- data : CodecJSON_V2 The JSON metadata for the codec. Returns ------- codec : Numcodec Raises ------ UnknownCodecError If ``data`` carries a string ``"id"`` that is not registered with numcodecs. Any other failure, including a registered codec rejecting its configuration and a ``data`` that is not a mapping, propagates from numcodecs unchanged. Examples -------- ```python from zarr.registry import get_numcodec codec = get_numcodec({'id': 'zlib', 'level': 1}) codec # Zlib(level=1) ``` """ from numcodecs.registry import codec_registry, entries, get_codec # Check whether numcodecs can resolve the id *before* handing off, rather than catching what # `get_codec` raises. Catching cannot tell "this id is unregistered" from "a registered codec # rejected its configuration" or from "a wrapper codec failed to resolve an inner codec", and # relabelling either of those with this id would attach a package hint that is simply wrong. # This mirrors the two lookups `get_codec` performs (it then tests the result for # truthiness rather than membership, which only differs for a falsy registry value). # Widened to `object` deliberately: `data` is annotated as a TypedDict, but this is a public # function and callers pass whatever they like. numcodecs coerces with `dict(config)` and # raises for anything that is not a mapping, which is the behaviour to preserve. raw: object = data codec_id = raw.get("id") if isinstance(raw, Mapping) else None if isinstance(codec_id, str) and codec_id not in codec_registry and codec_id not in entries: raise UnknownCodecError(_missing_codec_message(codec_id, zarr_format=2)) return get_codec(data) # type: ignore[no-any-return] zarr-developers-zarr-python-a2f2415/src/zarr/storage/000077500000000000000000000000001527014137600226765ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/storage/__init__.py000066400000000000000000000032721527014137600250130ustar00rootroot00000000000000import importlib import sys import warnings from collections.abc import Callable from types import ModuleType from typing import Any from zarr.errors import ZarrDeprecationWarning from zarr.storage._common import StoreLike, StorePath from zarr.storage._fsspec import FsspecStore from zarr.storage._local import LocalStore from zarr.storage._logging import LoggingStore from zarr.storage._memory import GpuMemoryStore, ManagedMemoryStore, MemoryStore from zarr.storage._obstore import ObjectStore from zarr.storage._wrapper import WrapperStore from zarr.storage._zip import ZipStore __all__ = [ "FsspecStore", "GpuMemoryStore", "LocalStore", "LoggingStore", "ManagedMemoryStore", "MemoryStore", "ObjectStore", "StoreLike", "StorePath", "WrapperStore", "ZipStore", ] class VerboseModule(ModuleType): def __setattr__(self, attr: str, value: Any) -> None: if attr == "default_compressor": warnings.warn( "setting zarr.storage.default_compressor is deprecated, use " "zarr.config to configure array.v2_default_compressor " "e.g. config.set({'codecs.zstd':'numcodecs.Zstd', 'array.v2_default_compressor.numeric': 'zstd'})", ZarrDeprecationWarning, stacklevel=1, ) else: super().__setattr__(attr, value) def __reduce__(self) -> tuple[Callable[[str], ModuleType], tuple[str]]: # Serializers such as cloudpickle dispatch on the exact type, so they have no # reduce for this subclass. Pickle the module by name instead. return importlib.import_module, (self.__name__,) sys.modules[__name__].__class__ = VerboseModule zarr-developers-zarr-python-a2f2415/src/zarr/storage/_common.py000066400000000000000000000560131527014137600247040ustar00rootroot00000000000000from __future__ import annotations import importlib.util from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, Self from zarr.abc.store import ( ByteRequest, Store, SupportsDeleteSync, SupportsGetSync, SupportsSetSync, ) from zarr.core._json import buffer_to_json_object, get_json from zarr.core.buffer import Buffer, default_buffer_prototype from zarr.core.common import ( ANY_ACCESS_MODE, ZARR_JSON, ZARRAY_JSON, ZGROUP_JSON, AccessModeLiteral, ZarrFormat, ) from zarr.errors import ContainsArrayAndGroupError, ContainsArrayError, ContainsGroupError from zarr.storage._local import LocalStore from zarr.storage._memory import ManagedMemoryStore, MemoryStore from zarr.storage._utils import UPath, _join_paths, normalize_path, parse_store_url _has_fsspec = importlib.util.find_spec("fsspec") if _has_fsspec: from fsspec.mapping import FSMap else: FSMap = None if TYPE_CHECKING: from zarr.core.buffer import BufferPrototype from zarr.core.common import JSON class StorePath: """ Path-like interface for a Store. Parameters ---------- store : Store The store to use. path : str The path within the store. """ store: Store path: str def __init__(self, store: Store, path: str = "") -> None: self.store = store self.path = normalize_path(path) @property def read_only(self) -> bool: return self.store.read_only @classmethod async def _create_open_instance(cls, store: Store, path: str) -> Self: """Helper to create and return a StorePath instance.""" await store._ensure_open() return cls(store, path) @classmethod async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = None) -> Self: """ Open StorePath based on the provided mode. * If the mode is None, return an opened version of the store with no changes. * If the mode is 'r+', 'w-', 'w', or 'a' and the store is read-only, raise a ValueError. * If the mode is 'r' and the store is not read-only, return a copy of the store with read_only set to True. * If the mode is 'w-' and the store is not read-only and the StorePath contains keys, raise a FileExistsError. No mode modifies the store. In particular, mode 'w' does not delete the keys nested within the StorePath: the node created at the path replaces them, once that node is valid (see `zarr.core.metadata.io.save_new_metadata`). Parameters ---------- mode : AccessModeLiteral The mode to use when initializing the store path. The accepted values are: - `'r'`: read only (must exist) - `'r+'`: read/write (must exist) - `'a'`: read/write (create if doesn't exist) - `'w'`: read/write (the node created at the path overwrites what exists) - `'w-'`: read/write (create if doesn't exist). Raises ------ FileExistsError If the mode is 'w-' and the store path already exists. ValueError If the mode is not "r" and the store is read-only, or """ # fastpath if mode is None if mode is None: return await cls._create_open_instance(store, path) if mode not in ANY_ACCESS_MODE: raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") if store.read_only: # Don't allow write operations on a read-only store if mode != "r": raise ValueError( f"Store is read-only but mode is {mode!r}. Create a writable store or use 'r' mode." ) self = await cls._create_open_instance(store, path) elif mode == "r": # Create read-only copy for read mode on writable store try: read_only_store = store.with_read_only(True) except NotImplementedError as e: raise ValueError( "Store is not read-only but mode is 'r'. Unable to create a read-only copy of the store. " "Please use a read-only store or a storage class that implements .with_read_only()." ) from e self = await cls._create_open_instance(read_only_store, path) else: # writable store and writable mode self = await cls._create_open_instance(store, path) if mode == "w-" and not await self.is_empty(): raise FileExistsError( f"Cannot create '{path}' with mode 'w-' because it already contains data. " f"Use mode 'w' to overwrite or 'a' to append." ) return self async def get( self, prototype: BufferPrototype | None = None, byte_range: ByteRequest | None = None, ) -> Buffer | None: """ Read bytes from the store. Parameters ---------- prototype : BufferPrototype, optional The buffer prototype to use when reading the bytes. byte_range : ByteRequest, optional The range of bytes to read. Returns ------- buffer : Buffer or None The read bytes, or None if the key does not exist. """ if prototype is None: prototype = default_buffer_prototype() return await self.store.get(self.path, prototype=prototype, byte_range=byte_range) async def get_json(self, *, byte_range: ByteRequest | None = None) -> JSON | None: """ Read and parse the JSON document at this path, or None if it is absent. Parameters ---------- byte_range : ByteRequest, optional If given, read only this portion of the value. Note that a partial read of a JSON document may not be valid JSON. Returns ------- JSON or None The parsed JSON value, or None if this path does not exist. """ return await get_json(self.store, self.path, byte_range=byte_range) async def set(self, value: Buffer) -> None: """ Write bytes to the store. Parameters ---------- value : Buffer The buffer to write. """ await self.store.set(self.path, value) async def delete(self) -> None: """ Delete the key from the store. Raises ------ NotImplementedError If the store does not support deletion. """ await self.store.delete(self.path) async def delete_dir(self) -> None: """ Delete all keys with the given prefix from the store. """ await self.store.delete_dir(self.path) async def set_if_not_exists(self, default: Buffer) -> None: """ Store a key to `value` if the key is not already present. Parameters ---------- default : Buffer The buffer to store if the key is not already present. """ await self.store.set_if_not_exists(self.path, default) async def exists(self) -> bool: """ Check if the key exists in the store. Returns ------- bool True if the key exists in the store, False otherwise. """ return await self.store.exists(self.path) async def is_empty(self) -> bool: """ Check if any keys exist in the store with the given prefix. Returns ------- bool True if no keys exist in the store with the given prefix, False otherwise. """ return await self.store.is_empty(self.path) # ------------------------------------------------------------------- # Synchronous IO delegation # ------------------------------------------------------------------- def get_sync( self, *, prototype: BufferPrototype | None = None, byte_range: ByteRequest | None = None, ) -> Buffer | None: """Synchronous read — delegates to `self.store.get_sync(self.path, ...)`.""" if not isinstance(self.store, SupportsGetSync): raise TypeError(f"Store {type(self.store).__name__} does not support synchronous get.") if prototype is None: prototype = default_buffer_prototype() return self.store.get_sync(self.path, prototype=prototype, byte_range=byte_range) def set_sync(self, value: Buffer) -> None: """Synchronous write — delegates to `self.store.set_sync(self.path, value)`.""" if not isinstance(self.store, SupportsSetSync): raise TypeError(f"Store {type(self.store).__name__} does not support synchronous set.") self.store.set_sync(self.path, value) def delete_sync(self) -> None: """Synchronous delete — delegates to `self.store.delete_sync(self.path)`.""" if not isinstance(self.store, SupportsDeleteSync): raise TypeError( f"Store {type(self.store).__name__} does not support synchronous delete." ) self.store.delete_sync(self.path) def __truediv__(self, other: str) -> StorePath: """Combine this store path with another path""" return self.__class__(self.store, _join_paths([self.path, other])) def __str__(self) -> str: return _join_paths([str(self.store), self.path]) def __repr__(self) -> str: return f"StorePath({self.store.__class__.__name__}, '{self}')" def __eq__(self, other: object) -> bool: """ Check if two StorePath objects are equal. Returns ------- bool True if the two objects are equal, False otherwise. Notes ----- Two StorePath objects are considered equal if their stores are equal and their paths are equal. """ try: return self.store == other.store and self.path == other.path # type: ignore[attr-defined, no-any-return] except AttributeError: return False type StoreLike = Store | StorePath | FSMap | Path | UPath | str | dict[str, Buffer] async def make_store( store_like: StoreLike | None, *, mode: AccessModeLiteral | None = None, storage_options: dict[str, Any] | None = None, ) -> Store: """ Convert a `StoreLike` object into a Store object. `StoreLike` objects are converted to `Store` as follows: - `Store` or `StorePath` = `Store` object. - `Path` or `str` = `LocalStore` object. - `str` that starts with a protocol = `FsspecStore` object. - `dict[str, Buffer]` = `MemoryStore` object. - `None` = `MemoryStore` object. - `FSMap` = `FsspecStore` object. - `UPath` = `FsspecStore` object, or `LocalStore` for a local `UPath`. Parameters ---------- store_like : StoreLike | None The `StoreLike` object to convert to a `Store` object. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. mode : StoreAccessMode | None, optional The mode to use when creating the `Store` object. If None, the default mode is 'r'. storage_options : dict[str, Any] | None, optional The storage options to use when creating the `RemoteStore` object. If None, the default storage options are used. Returns ------- Store The converted Store object. Raises ------ TypeError If the StoreLike object is not one of the supported types, or if storage_options is provided but not used. """ from zarr.storage._fsspec import FsspecStore # circular import # Parse URL early so we can reuse the result for both validation and routing parsed = parse_store_url(store_like) if isinstance(store_like, str) else None # Check if storage_options is valid for this store_like if storage_options is not None: is_fsspec_uri = parsed is not None and parsed.scheme not in ("", "memory", "file") if not is_fsspec_uri: raise TypeError( "'storage_options' was provided but unused. " "'storage_options' is only used when the store is passed as an FSSpec URI string.", ) if mode is not None and mode not in ANY_ACCESS_MODE: raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}") _read_only = mode == "r" if isinstance(store_like, StorePath): # Get underlying store return store_like.store elif isinstance(store_like, Store): # Already a Store return store_like elif isinstance(store_like, dict): # Already a dictionary that can be a MemoryStore # # We deliberate only consider dict[str, Buffer] here, and not arbitrary mutable mappings. # By only allowing dictionaries, which are in-memory, we know that MemoryStore appropriate. return await MemoryStore.open(store_dict=store_like, read_only=_read_only) elif store_like is None: # Create a new in-memory store return await make_store({}, mode=mode, storage_options=storage_options) elif isinstance(store_like, UPath): # This must be checked before Path: in universal-pathlib < 0.3 every UPath, including # remote ones like S3Path, subclasses pathlib.Path, and would otherwise be misrouted to a # LocalStore. Local UPaths get a LocalStore so that UPath("/data") and Path("/data") agree, # mirroring how the equivalent strings are routed below. if store_like.protocol in ("", "file"): return await make_store( Path(store_like.path), mode=mode, storage_options=storage_options ) return FsspecStore.from_upath(store_like, read_only=_read_only) elif isinstance(store_like, Path): # Create a new LocalStore return await LocalStore.open(root=store_like, mode=mode, read_only=_read_only) elif isinstance(store_like, str) and parsed is not None: if parsed.scheme == "memory" and not _has_fsspec: # Create or get a ManagedMemoryStore return ManagedMemoryStore(name=parsed.name, path=parsed.path, read_only=_read_only) elif parsed.scheme == "file" or not parsed.scheme: # Local filesystem path — use parsed.path to strip the file:// scheme return await make_store(Path(parsed.path), mode=mode, storage_options=storage_options) else: # Assume fsspec can handle it (s3://, gs://, http://, etc.) return FsspecStore.from_url( store_like, storage_options=storage_options, read_only=_read_only ) elif _has_fsspec and isinstance(store_like, FSMap): return FsspecStore.from_mapper(store_like, read_only=_read_only) else: raise TypeError(f"Unsupported type for store_like: '{type(store_like).__name__}'") async def make_store_path( store_like: StoreLike | None, *, path: str | None = "", mode: AccessModeLiteral | None = None, storage_options: dict[str, Any] | None = None, ) -> StorePath: """ Convert a `StoreLike` object into a StorePath object. This function takes a `StoreLike` object and returns a `StorePath` object. See `make_store` for details of which `Store` is used for each type of `store_like` object. Parameters ---------- store_like : StoreLike or None, default=None The `StoreLike` object to convert to a `StorePath` object. See the [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. path : str | None, optional The path to use when creating the `StorePath` object. If None, the default path is the empty string. mode : StoreAccessMode | None, optional The mode to use when creating the `StorePath` object. If None, the default mode is 'r'. storage_options : dict[str, Any] | None, optional The storage options to use when creating the `RemoteStore` object. If None, the default storage options are used. Returns ------- StorePath The converted StorePath object. Raises ------ TypeError If the StoreLike object is not one of the supported types, or if storage_options is provided but not used. ValueError If path is provided for a store that does not support it. See Also -------- make_store """ path_normalized = normalize_path(path) if isinstance(store_like, StorePath): # Already a StorePath if storage_options: raise TypeError( "'storage_options' was provided but unused. " "'storage_options' is only used when the store is passed as an FSSpec URI string.", ) return store_like / path_normalized elif _has_fsspec and isinstance(store_like, FSMap) and path: raise ValueError( "'path' was provided but is not used for FSMap store_like objects. Specify the path when creating the FSMap instance instead." ) else: store = await make_store(store_like, mode=mode, storage_options=storage_options) return await StorePath.open(store, path=path_normalized, mode=mode) async def ensure_no_existing_node( store_path: StorePath, zarr_format: ZarrFormat, node_type: Literal["array", "group"] | None = None, ) -> None: """ Check if a store_path is safe for array / group creation. Returns `None` or raises an exception. Parameters ---------- store_path : StorePath The storage location to check. zarr_format : ZarrFormat The Zarr format to check. node_type : str | None, optional Raise an error if an "array", or "group" exists. By default (when None), raises an error for either. Raises ------ ContainsArrayError, ContainsGroupError, ContainsArrayAndGroupError """ if zarr_format == 2: extant_node = await _contains_node_v2(store_path) elif zarr_format == 3: extant_node = await _contains_node_v3(store_path) match extant_node: case "array": if node_type != "group": msg = f"An array exists in store {store_path.store!r} at path {store_path.path!r}." raise ContainsArrayError(msg) case "group": if node_type != "array": msg = f"A group exists in store {store_path.store!r} at path {store_path.path!r}." raise ContainsGroupError(msg) case "nothing": return case _: msg = f"Invalid value for extant_node: {extant_node}" # type: ignore[unreachable] raise ValueError(msg) async def _contains_node_v3(store_path: StorePath) -> Literal["array", "group", "nothing"]: """ Check if a store_path contains nothing, an array, or a group. This function returns the string "array", "group", or "nothing" to denote containing an array, a group, or nothing. Parameters ---------- store_path : StorePath The location in storage to check. Returns ------- Literal["array", "group", "nothing"] A string representing the zarr node found at store_path. """ result: Literal["array", "group", "nothing"] = "nothing" extant_meta_bytes = await (store_path / ZARR_JSON).get() # if no metadata document could be loaded, then we just return "nothing" if extant_meta_bytes is not None: try: extant_meta_json = buffer_to_json_object(extant_meta_bytes) # avoid constructing a full metadata document here in the name of speed. if extant_meta_json["node_type"] == "array": result = "array" elif extant_meta_json["node_type"] == "group": result = "group" except (KeyError, TypeError, ValueError): # any of these errors is consistent with no array or group present. `ValueError` # covers both malformed JSON (`json.JSONDecodeError`) and non-UTF-8 bytes # (`UnicodeDecodeError`), each a `ValueError` subclass. pass return result async def _contains_node_v2(store_path: StorePath) -> Literal["array", "group", "nothing"]: """ Check if a store_path contains nothing, an array, a group, or both. If both an array and a group are detected, a `ContainsArrayAndGroup` exception is raised. Otherwise, this function returns the string "array", "group", or "nothing" to denote containing an array, a group, or nothing. Parameters ---------- store_path : StorePath The location in storage to check. Returns ------- Literal["array", "group", "nothing"] A string representing the zarr node found at store_path. """ _array = await contains_array(store_path=store_path, zarr_format=2) _group = await contains_group(store_path=store_path, zarr_format=2) if _array and _group: msg = ( "Array and group metadata documents (.zarray and .zgroup) were both found in store " f"{store_path.store!r} at path {store_path.path!r}. " "Only one of these files may be present in a given directory / prefix. " "Remove the .zarray file, or the .zgroup file, or both." ) raise ContainsArrayAndGroupError(msg) elif _array: return "array" elif _group: return "group" else: return "nothing" async def contains_array(store_path: StorePath, zarr_format: ZarrFormat) -> bool: """ Check if an array exists at a given StorePath. Parameters ---------- store_path : StorePath The StorePath to check for an existing group. zarr_format : The zarr format to check for. Returns ------- bool True if the StorePath contains a group, False otherwise. """ if zarr_format == 3: extant_meta_bytes = await (store_path / ZARR_JSON).get() if extant_meta_bytes is None: return False else: try: extant_meta_json = buffer_to_json_object(extant_meta_bytes) # we avoid constructing a full metadata document here in the name of speed. if extant_meta_json["node_type"] == "array": return True except (ValueError, KeyError, TypeError): return False elif zarr_format == 2: return await (store_path / ZARRAY_JSON).exists() msg = f"Invalid zarr_format provided. Got {zarr_format}, expected 2 or 3" raise ValueError(msg) async def contains_group(store_path: StorePath, zarr_format: ZarrFormat) -> bool: """ Check if a group exists at a given StorePath. Parameters ---------- store_path : StorePath The StorePath to check for an existing group. zarr_format : The zarr format to check for. Returns ------- bool True if the StorePath contains a group, False otherwise """ if zarr_format == 3: return (await _contains_node_v3(store_path)) == "group" elif zarr_format == 2: return await (store_path / ZGROUP_JSON).exists() msg = f"Invalid zarr_format provided. Got {zarr_format}, expected 2 or 3" # type: ignore[unreachable] raise ValueError(msg) zarr-developers-zarr-python-a2f2415/src/zarr/storage/_fsspec.py000066400000000000000000000407351527014137600247030ustar00rootroot00000000000000from __future__ import annotations import posixpath import warnings from contextlib import suppress from typing import TYPE_CHECKING, Any from packaging.version import parse as parse_version from zarr.abc.store import ( ByteRequest, OffsetByteRequest, RangeByteRequest, Store, SuffixByteRequest, ) from zarr.core.buffer import Buffer from zarr.errors import ZarrUserWarning from zarr.storage._utils import _dereference_path if TYPE_CHECKING: from collections.abc import AsyncIterator, Iterable from fsspec import AbstractFileSystem from fsspec.asyn import AsyncFileSystem from fsspec.mapping import FSMap from zarr.core.buffer import BufferPrototype ALLOWED_EXCEPTIONS: tuple[type[Exception], ...] = ( FileNotFoundError, IsADirectoryError, NotADirectoryError, ) def _make_async(fs: AbstractFileSystem) -> AsyncFileSystem: """Convert a sync FSSpec filesystem to an async FFSpec filesystem If the filesystem class supports async operations, a new async instance is created from the existing instance. If the filesystem class does not support async operations, the existing instance is wrapped with AsyncFileSystemWrapper. """ import fsspec fsspec_version = parse_version(fsspec.__version__) if fs.async_impl and fs.asynchronous: # Already an async instance of an async filesystem, nothing to do return fs if fs.async_impl: # Convert sync instance of an async fs to an async instance. Reuse the original # constructor arguments rather than round-tripping through JSON, since storage # options may hold objects that are not JSON-serializable (e.g. credentials). return type(fs)(*fs.storage_args, **{**fs.storage_options, "asynchronous": True}) if fsspec_version < parse_version("2024.12.0"): raise ImportError( f"The filesystem '{fs}' is synchronous, and the required " "AsyncFileSystemWrapper is not available. Upgrade fsspec to version " "2024.12.0 or later to enable this functionality." ) from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper return AsyncFileSystemWrapper(fs, asynchronous=True) class FsspecStore(Store): """ Store for remote data based on FSSpec. Parameters ---------- fs : AsyncFileSystem The Async FSSpec filesystem to use with this store. read_only : bool Whether the store is read-only path : str The root path of the store. This should be a relative path and must not include the filesystem scheme. allowed_exceptions : tuple[type[Exception], ...] When fetching data, these cases will be deemed to correspond to missing keys. Attributes ---------- fs allowed_exceptions supports_writes supports_deletes supports_listing Raises ------ TypeError If the Filesystem does not support async operations. ValueError If the path argument includes a scheme. Warns ----- ZarrUserWarning If the file system (fs) was not created with `asynchronous=True`. Notes ----- Closing the store does not close the underlying filesystem or its network session. fsspec caches and shares filesystem instances across callers, so the store cannot know whether it is the only user, and closing a shared session would break other stores. The filesystem's lifecycle belongs to whoever created it; use fsspec's own tools (e.g. `clear_instance_cache`) to release it. See Also -------- FsspecStore.from_upath FsspecStore.from_url """ # based on FSSpec supports_writes: bool = True supports_deletes: bool = True supports_listing: bool = True fs: AsyncFileSystem allowed_exceptions: tuple[type[Exception], ...] path: str def __init__( self, fs: AsyncFileSystem, read_only: bool = False, path: str = "/", allowed_exceptions: tuple[type[Exception], ...] = ALLOWED_EXCEPTIONS, ) -> None: super().__init__(read_only=read_only) self.fs = fs self.path = path self.allowed_exceptions = allowed_exceptions if not self.fs.async_impl: raise TypeError("Filesystem needs to support async operations.") if not self.fs.asynchronous: warnings.warn( f"fs ({fs}) was not created with `asynchronous=True`, this may lead to surprising behavior", category=ZarrUserWarning, stacklevel=2, ) @classmethod def from_upath( cls, upath: Any, read_only: bool = False, allowed_exceptions: tuple[type[Exception], ...] = ALLOWED_EXCEPTIONS, ) -> FsspecStore: """ Create an FsspecStore from a upath object. Parameters ---------- upath : UPath The upath to the root of the store. read_only : bool Whether the store is read-only, defaults to False. allowed_exceptions : tuple, optional The exceptions that are allowed to be raised when accessing the store. Defaults to ALLOWED_EXCEPTIONS. Returns ------- FsspecStore """ # A UPath hands back a filesystem in whatever mode it was constructed with, which is # synchronous unless the caller passed asynchronous=True. Route it through _make_async so # that sync-mode instances of async filesystems are re-created in async mode, and # genuinely synchronous filesystems are wrapped. return cls( fs=_make_async(upath.fs), path=upath.path.rstrip("/"), read_only=read_only, allowed_exceptions=allowed_exceptions, ) @classmethod def from_mapper( cls, fs_map: FSMap, read_only: bool = False, allowed_exceptions: tuple[type[Exception], ...] = ALLOWED_EXCEPTIONS, ) -> FsspecStore: """ Create an FsspecStore from an FSMap object. Parameters ---------- fs_map : FSMap Fsspec mutable mapping object. read_only : bool Whether the store is read-only, defaults to False. allowed_exceptions : tuple, optional The exceptions that are allowed to be raised when accessing the store. Defaults to ALLOWED_EXCEPTIONS. Returns ------- FsspecStore """ fs = _make_async(fs_map.fs) return cls( fs=fs, path=fs_map.root, read_only=read_only, allowed_exceptions=allowed_exceptions, ) @classmethod def from_url( cls, url: str, storage_options: dict[str, Any] | None = None, read_only: bool = False, allowed_exceptions: tuple[type[Exception], ...] = ALLOWED_EXCEPTIONS, ) -> FsspecStore: """ Create an FsspecStore from a URL. The type of store is determined from the URL scheme. Parameters ---------- url : str The URL to the root of the store. storage_options : dict, optional The options to pass to fsspec when creating the filesystem. read_only : bool Whether the store is read-only, defaults to False. allowed_exceptions : tuple, optional The exceptions that are allowed to be raised when accessing the store. Defaults to ALLOWED_EXCEPTIONS. Returns ------- FsspecStore """ try: from fsspec import url_to_fs except ImportError: # before fsspec==2024.3.1 from fsspec.core import url_to_fs opts = storage_options or {} opts = {"asynchronous": True, **opts} fs, path = url_to_fs(url, **opts) if not fs.async_impl: fs = _make_async(fs) return cls(fs=fs, path=path, read_only=read_only, allowed_exceptions=allowed_exceptions) def with_read_only(self, read_only: bool = False) -> FsspecStore: # docstring inherited return type(self)( fs=self.fs, path=self.path, allowed_exceptions=self.allowed_exceptions, read_only=read_only, ) async def clear(self) -> None: # docstring inherited try: for subpath in await self.fs._find(self.path, withdirs=True): if subpath != self.path: await self.fs._rm(subpath, recursive=True) except FileNotFoundError: pass def __repr__(self) -> str: return f"" def __eq__(self, other: object) -> bool: return ( isinstance(other, type(self)) and self.path == other.path and self.read_only == other.read_only and self.fs == other.fs ) async def get( self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited if not self._is_open: await self._open() path = _dereference_path(self.path, key) try: if byte_range is None: value = prototype.buffer.from_bytes(await self.fs._cat_file(path)) elif isinstance(byte_range, RangeByteRequest): value = prototype.buffer.from_bytes( await self.fs._cat_file( path, start=byte_range.start, end=byte_range.end, ) ) elif isinstance(byte_range, OffsetByteRequest): value = prototype.buffer.from_bytes( await self.fs._cat_file(path, start=byte_range.offset, end=None) ) elif isinstance(byte_range, SuffixByteRequest): value = prototype.buffer.from_bytes( await self.fs._cat_file(path, start=-byte_range.suffix, end=None) ) else: raise ValueError(f"Unexpected byte_range, got {byte_range}.") except self.allowed_exceptions: return None except OSError as e: if "not satisfiable" in str(e): # this is an s3-specific condition we probably don't want to leak return prototype.buffer.from_bytes(b"") raise else: return value async def set( self, key: str, value: Buffer, byte_range: tuple[int, int] | None = None, ) -> None: # docstring inherited if not self._is_open: await self._open() self._check_writable() if not isinstance(value, Buffer): raise TypeError( f"FsspecStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." ) path = _dereference_path(self.path, key) # write data if byte_range: raise NotImplementedError await self.fs._pipe_file(path, value.to_bytes()) async def delete(self, key: str) -> None: # docstring inherited self._check_writable() path = _dereference_path(self.path, key) try: await self.fs._rm(path) except FileNotFoundError: pass except self.allowed_exceptions: pass async def delete_dir(self, prefix: str) -> None: # docstring inherited if not self.supports_deletes: raise NotImplementedError( "This method is only available for stores that support deletes." ) self._check_writable() path_to_delete = _dereference_path(self.path, prefix) with suppress(*self.allowed_exceptions): await self.fs._rm(path_to_delete, recursive=True) async def exists(self, key: str) -> bool: # docstring inherited path = _dereference_path(self.path, key) exists: bool = await self.fs._exists(path) return exists async def get_partial_values( self, prototype: BufferPrototype, key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: # docstring inherited # Materialise first: key_ranges may be a one-shot iterable, so a bare # truthiness check (e.g. `if key_ranges`) would be unreliable for an # empty generator. _cat_ranges also expects lists of paths/starts/stops. key_ranges = list(key_ranges) if not key_ranges: return [] paths: list[str] = [] starts: list[int | None] = [] stops: list[int | None] = [] for key, byte_range in key_ranges: paths.append(_dereference_path(self.path, key)) if byte_range is None: starts.append(None) stops.append(None) elif isinstance(byte_range, RangeByteRequest): starts.append(byte_range.start) stops.append(byte_range.end) elif isinstance(byte_range, OffsetByteRequest): starts.append(byte_range.offset) stops.append(None) elif isinstance(byte_range, SuffixByteRequest): starts.append(-byte_range.suffix) stops.append(None) else: raise ValueError(f"Unexpected byte_range, got {byte_range}.") # TODO: expectations for exceptions or missing keys? res = await self.fs._cat_ranges(paths, starts, stops, on_error="return") # the following is an s3-specific condition we probably don't want to leak res = [b"" if (isinstance(r, OSError) and "not satisfiable" in str(r)) else r for r in res] for r in res: if isinstance(r, Exception) and not isinstance(r, self.allowed_exceptions): raise r return [None if isinstance(r, Exception) else prototype.buffer.from_bytes(r) for r in res] async def list(self) -> AsyncIterator[str]: # docstring inherited allfiles = await self.fs._find(self.path, detail=False, withdirs=False) for onefile in (a.removeprefix(f"{self.path}/") for a in allfiles): yield onefile async def list_dir(self, prefix: str) -> AsyncIterator[str]: # docstring inherited # A store path of "" or "/" is the root of the filesystem. Joining it with a # leading "/" gives "/a" or "//a", which ReferenceFileSystem does not find, # since it keys entries by bare names like "a/zarr.json". root = self.path.rstrip("/") prefix = f"{root}/{prefix.rstrip('/')}" if root else prefix.rstrip("/") try: allfiles = await self.fs._ls(prefix, detail=False) except FileNotFoundError: return seen: set[str] = set() for onefile in allfiles: name = onefile.replace(f"{prefix}/", "").removeprefix(self.path).removeprefix("/") if onefile.startswith(("http://", "https://")): # An HTTP listing is scraped from the links on an HTML page. In-page anchors # and queries name the page itself, not an object, and relative links such # as "./a" need normalizing. if "#" in name or "?" in name: continue name = posixpath.normpath(name) # Only direct children, each once: an HTTP listing marks directories with a # trailing "/", and a link back to the listed directory reduces to "" or ".". name = name.rstrip("/") if name in ("", ".", "..") or "/" in name or name in seen: continue seen.add(name) yield name async def list_prefix(self, prefix: str) -> AsyncIterator[str]: # docstring inherited # Same root handling as list_dir. root = self.path.rstrip("/") for onefile in await self.fs._find( f"{root}/{prefix}" if root else prefix, detail=False, maxdepth=None, withdirs=False ): yield onefile.removeprefix(f"{root}/") if root else onefile.removeprefix("/") async def getsize(self, key: str) -> int: path = _dereference_path(self.path, key) info = await self.fs._info(path) size = info.get("size") if size is None: # Not all filesystems support size. Fall back to reading the entire object return await super().getsize(key) else: # fsspec doesn't have typing. We'll need to assume or verify this is true return int(size) zarr-developers-zarr-python-a2f2415/src/zarr/storage/_local.py000066400000000000000000000350651527014137600245120ustar00rootroot00000000000000from __future__ import annotations import asyncio import contextlib import io import os import shutil import sys import time import uuid from pathlib import Path from typing import TYPE_CHECKING, BinaryIO, Literal, Self from zarr.abc.store import ( ByteRequest, OffsetByteRequest, RangeByteRequest, Store, SuffixByteRequest, ) from zarr.core.buffer import Buffer from zarr.core.buffer.core import default_buffer_prototype from zarr.core.common import AccessModeLiteral, concurrent_map if TYPE_CHECKING: from collections.abc import AsyncIterator, Callable, Iterable, Iterator from zarr.core.buffer import BufferPrototype def _get(path: Path, prototype: BufferPrototype, byte_range: ByteRequest | None) -> Buffer: if byte_range is None: return prototype.buffer.from_bytes(path.read_bytes()) with path.open("rb") as f: size = f.seek(0, io.SEEK_END) if isinstance(byte_range, RangeByteRequest): f.seek(byte_range.start) return prototype.buffer.from_bytes(f.read(byte_range.end - f.tell())) elif isinstance(byte_range, OffsetByteRequest): f.seek(byte_range.offset) elif isinstance(byte_range, SuffixByteRequest): f.seek(max(0, size - byte_range.suffix)) else: raise TypeError(f"Unexpected byte_range, got {byte_range}.") return prototype.buffer.from_bytes(f.read()) if sys.platform == "win32": # Per the os.rename docs: # On Windows, if dst exists a FileExistsError is always raised. _safe_move = os.rename else: # On Unix, os.rename silently replace files, so instead we use os.link like # atomicwrites: # https://github.com/untitaker/python-atomicwrites/blob/1.4.1/atomicwrites/__init__.py#L59-L60 # This also raises FileExistsError if dst exists. def _safe_move(src: Path, dst: Path) -> None: os.link(src, dst) os.unlink(src) # Windows error codes that MoveFileEx reports when the destination could not be # superseded. Replacing a name that was itself replaced moments earlier # intermittently fails this way with no second process and no open handle # involved, and a retry clears it in well under a millisecond. Zarr v2 hit the # same thing and fixed it in #698. # # ERROR_ACCESS_DENIED is also what Windows reports for conditions that will not # clear (the destination is a directory, is read-only, or is ACL-denied). Those # are retried too and surface the same error after the bounded delay below. # # Nothing else is retried. In particular the FileExistsError that the exclusive # path relies on to report an existing node is ERROR_ALREADY_EXISTS (183), so it # still propagates on the first attempt. _TRANSIENT_WINERRORS = frozenset( { 5, # ERROR_ACCESS_DENIED 32, # ERROR_SHARING_VIOLATION } ) # Delay before each retry. The sequence sums to well under a second so that a # genuine failure still surfaces promptly; in practice most transient failures # clear on the first retry. _RETRY_DELAYS = (0.001, 0.005, 0.02, 0.05, 0.2) def _move_with_retry(tmp_path: Path, path: Path, move: Callable[[Path, Path], object]) -> None: """Run `move(tmp_path, path)`, retrying while the destination is busy. This is a single attempt on every platform but Windows, without needing to test for one: only `winerror` values are ever retried, and off Windows an `OSError` does not carry one. """ for delay in _RETRY_DELAYS: try: move(tmp_path, path) except OSError as e: if getattr(e, "winerror", None) not in _TRANSIENT_WINERRORS: raise time.sleep(delay) else: return move(tmp_path, path) @contextlib.contextmanager def _atomic_write( path: Path, mode: Literal["r+b", "wb"], exclusive: bool = False, ) -> Iterator[BinaryIO]: tmp_path = path.with_suffix(f".{uuid.uuid4().hex}.partial") try: with tmp_path.open(mode) as f: yield f if exclusive: _move_with_retry(tmp_path, path, _safe_move) else: _move_with_retry(tmp_path, path, Path.replace) except Exception: tmp_path.unlink(missing_ok=True) raise def _put(path: Path, value: Buffer, exclusive: bool = False) -> int: path.parent.mkdir(parents=True, exist_ok=True) # write takes any object supporting the buffer protocol view = value.as_buffer_like() with _atomic_write(path, "wb", exclusive=exclusive) as f: return f.write(view) # The helpers below do the blocking filesystem work behind LocalStore's async methods. # Each async method runs exactly one of them via `asyncio.to_thread` so that the event # loop is never stalled on disk I/O; the synchronous methods call them directly. def _ensure_root(root: Path, *, create: bool) -> None: if create: root.mkdir(parents=True, exist_ok=True) if not root.exists(): raise FileNotFoundError(f"{root} does not exist") def _clear(root: Path) -> None: shutil.rmtree(root) root.mkdir() def _delete(path: Path) -> None: if path.is_dir(): shutil.rmtree(path) else: path.unlink(missing_ok=True) def _delete_dir(path: Path, prefix: str) -> None: if path.is_dir(): shutil.rmtree(path) elif path.is_file(): raise ValueError(f"delete_dir was passed a {prefix=!r} that is a file") # A non-existent directory is a no-op; test_group:test_create_creates_parents relies on it. def _exists(path: Path) -> bool: return path.is_file() def _getsize(path: Path) -> int: return path.stat().st_size def _list_files(root: Path, prefix: str) -> list[str]: """Keys (paths relative to `root`, POSIX style) of every file under `root / prefix`.""" to_strip = root.as_posix() + "/" return [p.as_posix().removeprefix(to_strip) for p in (root / prefix).rglob("*") if p.is_file()] def _list_dir(base: Path) -> list[str]: try: return [p.name for p in base.iterdir()] except (FileNotFoundError, NotADirectoryError): return [] def _move(src: Path, dest_root: Path) -> None: dest_root.parent.mkdir(parents=True, exist_ok=True) if dest_root.exists(): raise FileExistsError(f"Destination root {dest_root} already exists.") shutil.move(src, dest_root) class LocalStore(Store): """ Store for the local file system. Parameters ---------- root : str or Path Directory to use as root of store. read_only : bool Whether the store is read-only Attributes ---------- supports_writes supports_deletes supports_listing root """ supports_writes: bool = True supports_deletes: bool = True supports_listing: bool = True root: Path def __init__(self, root: Path | str, *, read_only: bool = False) -> None: super().__init__(read_only=read_only) if isinstance(root, str): root = Path(root) if not isinstance(root, Path): raise TypeError( f"'root' must be a string or Path instance. Got an instance of {type(root)} instead." ) self.root = root def with_read_only(self, read_only: bool = False) -> Self: # docstring inherited return type(self)( root=self.root, read_only=read_only, ) @classmethod async def open( cls, root: Path | str, *, read_only: bool = False, mode: AccessModeLiteral | None = None ) -> Self: """ Create and open the store. Parameters ---------- root : str or Path Directory to use as root of store. read_only : bool Whether the store is read-only mode : Mode in which to create the store. This only affects opening the store, and the final read-only state of the store is controlled through the read_only parameter. Returns ------- Store The opened store instance. """ # If mode = 'r+', want to open in read only mode (fail if exists), # but return a writeable store if mode is not None: read_only_creation = mode in ["r", "r+"] else: read_only_creation = read_only store = cls(root, read_only=read_only_creation) await store._open() # Set read_only state store = store.with_read_only(read_only) await store._open() return store async def _open(self, *, mode: AccessModeLiteral | None = None) -> None: await asyncio.to_thread(_ensure_root, self.root, create=not self.read_only) return await super()._open() async def _ensure_open(self) -> None: # docstring inherited if not self._is_open: # Concurrent lazy opens (every `set` of a `set_many`, say) can all get here # before any of them finishes, so each one verifies the root. That is # idempotent, and so is setting the flag, so the calls cannot conflict. # Going through `self._open()` instead would make every call after the first # fail, because `Store._open` raises on a store that is already open. # As in `_ensure_open_sync`, a subclass's `_open` override is not run here. await asyncio.to_thread(_ensure_root, self.root, create=not self.read_only) self._is_open = True async def clear(self) -> None: # docstring inherited self._check_writable() await asyncio.to_thread(_clear, self.root) def __str__(self) -> str: return f"file://{self.root.as_posix()}" def __repr__(self) -> str: return f"LocalStore('{self}')" def __eq__(self, other: object) -> bool: return isinstance(other, type(self)) and self.root == other.root # ------------------------------------------------------------------- # Synchronous store methods # ------------------------------------------------------------------- def _ensure_open_sync(self) -> None: if not self._is_open: _ensure_root(self.root, create=not self.read_only) self._is_open = True def get_sync( self, key: str, *, prototype: BufferPrototype | None = None, byte_range: ByteRequest | None = None, ) -> Buffer | None: if prototype is None: prototype = default_buffer_prototype() self._ensure_open_sync() path = self.root / key try: return _get(path, prototype, byte_range) except (FileNotFoundError, IsADirectoryError, NotADirectoryError): return None def set_sync(self, key: str, value: Buffer) -> None: self._ensure_open_sync() self._check_writable() if not isinstance(value, Buffer): raise TypeError( f"LocalStore.set(): `value` must be a Buffer instance. " f"Got an instance of {type(value)} instead." ) path = self.root / key _put(path, value) def delete_sync(self, key: str) -> None: self._ensure_open_sync() self._check_writable() _delete(self.root / key) async def get( self, key: str, prototype: BufferPrototype | None = None, byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited if prototype is None: prototype = default_buffer_prototype() await self._ensure_open() path = self.root / key try: return await asyncio.to_thread(_get, path, prototype, byte_range) except (FileNotFoundError, IsADirectoryError, NotADirectoryError): return None async def get_partial_values( self, prototype: BufferPrototype, key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: # docstring inherited args = [] for key, byte_range in key_ranges: path = self.root / key args.append((_get, path, prototype, byte_range)) return await concurrent_map(args, asyncio.to_thread, limit=None) # TODO: fix limit async def set(self, key: str, value: Buffer) -> None: # docstring inherited return await self._set(key, value) async def set_if_not_exists(self, key: str, value: Buffer) -> None: # docstring inherited try: return await self._set(key, value, exclusive=True) except FileExistsError: pass async def _set(self, key: str, value: Buffer, exclusive: bool = False) -> None: await self._ensure_open() self._check_writable() if not isinstance(value, Buffer): raise TypeError( f"LocalStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." ) path = self.root / key await asyncio.to_thread(_put, path, value, exclusive=exclusive) async def delete(self, key: str) -> None: """ Remove a key from the store. Parameters ---------- key : str Notes ----- If ``key`` is a directory within this store, the entire directory at ``store.root / key`` is deleted. """ # docstring inherited self._check_writable() await asyncio.to_thread(_delete, self.root / key) async def delete_dir(self, prefix: str) -> None: # docstring inherited self._check_writable() await asyncio.to_thread(_delete_dir, self.root / prefix, prefix) async def exists(self, key: str) -> bool: # docstring inherited return await asyncio.to_thread(_exists, self.root / key) async def list(self) -> AsyncIterator[str]: # docstring inherited for key in await asyncio.to_thread(_list_files, self.root, ""): yield key async def list_prefix(self, prefix: str) -> AsyncIterator[str]: # docstring inherited for key in await asyncio.to_thread(_list_files, self.root, prefix.rstrip("/")): yield key async def list_dir(self, prefix: str) -> AsyncIterator[str]: # docstring inherited for name in await asyncio.to_thread(_list_dir, self.root / prefix): yield name async def move(self, dest_root: Path | str) -> None: """ Move the store to another path. The old root directory is deleted. """ if isinstance(dest_root, str): dest_root = Path(dest_root) await asyncio.to_thread(_move, self.root, dest_root) self.root = dest_root async def getsize(self, key: str) -> int: # docstring inherited return await asyncio.to_thread(_getsize, self.root / key) zarr-developers-zarr-python-a2f2415/src/zarr/storage/_logging.py000066400000000000000000000177341527014137600250510ustar00rootroot00000000000000from __future__ import annotations import inspect import logging import sys import time from collections import defaultdict from contextlib import contextmanager from typing import TYPE_CHECKING, Any, Self from zarr.abc.store import Store from zarr.storage._wrapper import WrapperStore if TYPE_CHECKING: from collections.abc import AsyncGenerator, Generator, Iterable from zarr.abc.store import ByteRequest from zarr.core.buffer import Buffer, BufferPrototype counter: defaultdict[str, int] class LoggingStore[T_Store: Store](WrapperStore[T_Store]): """ Store that logs all calls to another wrapped store. Parameters ---------- store : Store Store to wrap log_level : str Log level log_handler : logging.Handler Log handler Attributes ---------- counter : dict Counter of number of times each method has been called """ counter: defaultdict[str, int] def __init__( self, store: T_Store, log_level: str = "DEBUG", log_handler: logging.Handler | None = None, ) -> None: super().__init__(store) self.counter = defaultdict(int) self.log_level = log_level self.log_handler = log_handler self._configure_logger(log_level, log_handler) def _configure_logger( self, log_level: str = "DEBUG", log_handler: logging.Handler | None = None ) -> None: self.log_level = log_level self.logger = logging.getLogger(f"LoggingStore({self._store})") self.logger.setLevel(log_level) if not self.logger.hasHandlers(): if not log_handler: log_handler = self._default_handler() # Add handler to logger self.logger.addHandler(log_handler) def _default_handler(self) -> logging.Handler: """Define a default log handler""" handler = logging.StreamHandler(stream=sys.stdout) handler.setLevel(self.log_level) handler.setFormatter( logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") ) return handler def _with_store(self, store: T_Store) -> Self: return type(self)(store=store, log_level=self.log_level, log_handler=self.log_handler) @contextmanager def log(self, hint: Any = "") -> Generator[None, None, None]: """Context manager to log method calls Each call to the wrapped store is logged to the configured logger and added to the counter dict. """ method = inspect.stack()[2].function op = f"{type(self._store).__name__}.{method}" if hint: op = f"{op}({hint})" self.logger.info(" Calling %s", op) start_time = time.time() try: self.counter[method] += 1 yield finally: end_time = time.time() self.logger.info("Finished %s [%.2f s]", op, end_time - start_time) @classmethod async def open(cls: type[Self], store_cls: type[T_Store], *args: Any, **kwargs: Any) -> Self: log_level = kwargs.pop("log_level", "DEBUG") log_handler = kwargs.pop("log_handler", None) store = store_cls(*args, **kwargs) await store._open() return cls(store=store, log_level=log_level, log_handler=log_handler) @property def supports_writes(self) -> bool: with self.log(): return self._store.supports_writes @property def supports_deletes(self) -> bool: with self.log(): return self._store.supports_deletes @property def supports_listing(self) -> bool: with self.log(): return self._store.supports_listing @property def read_only(self) -> bool: with self.log(): return self._store.read_only @property def _is_open(self) -> bool: with self.log(): return self._store._is_open @_is_open.setter def _is_open(self, value: bool) -> None: raise NotImplementedError("LoggingStore must be opened via the `_open` method") async def _open(self) -> None: with self.log(): return await self._store._open() async def _ensure_open(self) -> None: with self.log(): return await self._store._ensure_open() async def is_empty(self, prefix: str = "") -> bool: # docstring inherited with self.log(): return await self._store.is_empty(prefix=prefix) async def clear(self) -> None: # docstring inherited with self.log(): return await self._store.clear() def __str__(self) -> str: return f"logging-{self._store}" def __repr__(self) -> str: return f"LoggingStore({self._store.__class__.__name__}, '{self._store}')" def __eq__(self, other: object) -> bool: with self.log(other): return type(self) is type(other) and self._store.__eq__(other._store) async def get( self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited with self.log(key): return await self._store.get(key=key, prototype=prototype, byte_range=byte_range) async def get_partial_values( self, prototype: BufferPrototype, key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: # docstring inherited key_ranges = list(key_ranges) keys = ",".join([k[0] for k in key_ranges]) with self.log(keys): return await self._store.get_partial_values(prototype=prototype, key_ranges=key_ranges) async def exists(self, key: str) -> bool: # docstring inherited with self.log(key): return await self._store.exists(key) async def set(self, key: str, value: Buffer) -> None: # docstring inherited with self.log(key): return await self._store.set(key=key, value=value) async def set_if_not_exists(self, key: str, value: Buffer) -> None: # docstring inherited with self.log(key): return await self._store.set_if_not_exists(key=key, value=value) async def delete(self, key: str) -> None: # docstring inherited with self.log(key): return await self._store.delete(key=key) def get_sync( self, key: str, *, prototype: BufferPrototype | None = None, byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited with self.log(key): return super().get_sync(key, prototype=prototype, byte_range=byte_range) def set_sync(self, key: str, value: Buffer) -> None: # docstring inherited with self.log(key): return super().set_sync(key, value) def delete_sync(self, key: str) -> None: # docstring inherited with self.log(key): return super().delete_sync(key) async def list(self) -> AsyncGenerator[str, None]: # docstring inherited with self.log(): async for key in self._store.list(): yield key async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: # docstring inherited with self.log(prefix): async for key in self._store.list_prefix(prefix=prefix): yield key async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: # docstring inherited with self.log(prefix): async for key in self._store.list_dir(prefix=prefix): yield key async def delete_dir(self, prefix: str) -> None: # docstring inherited with self.log(prefix): await self._store.delete_dir(prefix=prefix) async def getsize(self, key: str) -> int: with self.log(key): return await self._store.getsize(key) async def getsize_prefix(self, prefix: str) -> int: with self.log(prefix): return await self._store.getsize_prefix(prefix) zarr-developers-zarr-python-a2f2415/src/zarr/storage/_memory.py000066400000000000000000000575531527014137600247360ustar00rootroot00000000000000from __future__ import annotations import os import threading import weakref from logging import getLogger from typing import TYPE_CHECKING, Any, Self from zarr.abc.store import ByteRequest, Store from zarr.core.buffer import Buffer, gpu from zarr.core.buffer.core import default_buffer_prototype from zarr.core.common import concurrent_map from zarr.storage._utils import ( _join_paths, _normalize_byte_range_index, normalize_path, parse_store_url, ) if TYPE_CHECKING: from collections.abc import AsyncIterator, Iterable, MutableMapping from zarr.core.buffer import BufferPrototype logger = getLogger(__name__) def _copy_buffer(value: Buffer) -> Buffer: """Copy `value` so the store does not retain the caller's memory. Encoding a chunk can hand the store a zero-copy view of the user's array (an uncompressed write is the common case), and unlike stores that serialize on write, this one keeps whatever it is given alive in a dict. Without this copy a later mutation of the user's array would rewrite chunks already committed to the store. """ return type(value).from_array_like(value.as_array_like().copy()) class MemoryStore(Store): """ Store for local memory. Parameters ---------- store_dict : dict Initial data read_only : bool Whether the store is read-only Attributes ---------- supports_writes supports_deletes supports_listing Notes ----- Writes copy the buffer they are given, so the store never aliases the caller's memory. Buffers passed via `store_dict` are the caller's responsibility and are stored as-is. """ supports_writes: bool = True supports_deletes: bool = True supports_listing: bool = True _store_dict: MutableMapping[str, Buffer] def __init__( self, store_dict: MutableMapping[str, Buffer] | None = None, *, read_only: bool = False, ) -> None: super().__init__(read_only=read_only) if store_dict is None: store_dict = {} self._store_dict = store_dict def with_read_only(self, read_only: bool = False) -> MemoryStore: # docstring inherited return type(self)( store_dict=self._store_dict, read_only=read_only, ) async def clear(self) -> None: # docstring inherited self._store_dict.clear() def __str__(self) -> str: return f"memory://{id(self._store_dict)}" def __repr__(self) -> str: return f"MemoryStore('{self}')" def __eq__(self, other: object) -> bool: return ( isinstance(other, type(self)) and self._store_dict == other._store_dict and self.read_only == other.read_only ) # ------------------------------------------------------------------- # Synchronous store methods # ------------------------------------------------------------------- def get_sync( self, key: str, *, prototype: BufferPrototype | None = None, byte_range: ByteRequest | None = None, ) -> Buffer | None: if prototype is None: prototype = default_buffer_prototype() if not self._is_open: self._is_open = True try: value = self._store_dict[key] start, stop = _normalize_byte_range_index(value, byte_range) return prototype.buffer.from_buffer(value[start:stop]) except KeyError: return None def set_sync(self, key: str, value: Buffer) -> None: self._check_writable() if not self._is_open: self._is_open = True if not isinstance(value, Buffer): raise TypeError( f"MemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." ) self._store_dict[key] = _copy_buffer(value) def delete_sync(self, key: str) -> None: self._check_writable() if not self._is_open: self._is_open = True try: del self._store_dict[key] except KeyError: logger.debug("Key %s does not exist.", key) async def get( self, key: str, prototype: BufferPrototype | None = None, byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited if prototype is None: prototype = default_buffer_prototype() if not self._is_open: await self._open() try: value = self._store_dict[key] start, stop = _normalize_byte_range_index(value, byte_range) return prototype.buffer.from_buffer(value[start:stop]) except KeyError: return None async def get_partial_values( self, prototype: BufferPrototype, key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: # docstring inherited # All the key-ranges arguments goes with the same prototype async def _get(key: str, byte_range: ByteRequest | None) -> Buffer | None: return await self.get(key, prototype=prototype, byte_range=byte_range) return await concurrent_map(key_ranges, _get, limit=None) async def exists(self, key: str) -> bool: # docstring inherited return key in self._store_dict async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None = None) -> None: # docstring inherited self._check_writable() await self._ensure_open() if not isinstance(value, Buffer): raise TypeError( f"MemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." ) if byte_range is not None: buf = self._store_dict[key] buf[byte_range[0] : byte_range[1]] = value self._store_dict[key] = buf else: self._store_dict[key] = _copy_buffer(value) async def set_if_not_exists(self, key: str, value: Buffer) -> None: # docstring inherited self._check_writable() await self._ensure_open() self._store_dict.setdefault(key, _copy_buffer(value)) async def delete(self, key: str) -> None: # docstring inherited self._check_writable() try: del self._store_dict[key] except KeyError: logger.debug("Key %s does not exist.", key) async def list(self) -> AsyncIterator[str]: # docstring inherited for key in self._store_dict: yield key async def list_prefix(self, prefix: str) -> AsyncIterator[str]: # docstring inherited # note: we materialize all dict keys into a list here so we can mutate the dict in-place (e.g. in delete_prefix) for key in list(self._store_dict): if key.startswith(prefix): yield key async def list_dir(self, prefix: str) -> AsyncIterator[str]: # docstring inherited prefix = prefix.rstrip("/") if prefix == "": keys_unique = {k.split("/")[0] for k in self._store_dict} else: # Our dictionary doesn't contain directory markers, but we want to include # a pseudo directory when there's a nested item and we're listing an # intermediate level. keys_unique = { key.removeprefix(f"{prefix}/").split("/")[0] for key in self._store_dict if key.startswith(f"{prefix}/") and key not in {prefix, f"{prefix}/"} } for key in keys_unique: yield key class GpuMemoryStore(MemoryStore): """ Store for GPU memory. Stores every chunk in GPU memory irrespective of the original location. The dictionary of buffers to initialize this memory store with *must* be GPU Buffers. Writing data to this store through ``.set`` will move the buffer to the GPU if necessary. Parameters ---------- store_dict : MutableMapping, optional A mutable mapping with string keys and [zarr.core.buffer.gpu.Buffer][] values. read_only : bool Whether to open the store in read-only mode. """ _store_dict: MutableMapping[str, gpu.Buffer] # type: ignore[assignment] def __init__( self, store_dict: MutableMapping[str, gpu.Buffer] | None = None, *, read_only: bool = False, ) -> None: super().__init__(store_dict=store_dict, read_only=read_only) # type: ignore[arg-type] def __str__(self) -> str: return f"gpumemory://{id(self._store_dict)}" def __repr__(self) -> str: return f"GpuMemoryStore('{self}')" @classmethod def from_dict(cls, store_dict: MutableMapping[str, Buffer]) -> Self: """ Create a GpuMemoryStore from a dictionary of buffers at any location. The dictionary backing the newly created ``GpuMemoryStore`` will not be the same as ``store_dict``. Parameters ---------- store_dict : mapping A mapping of strings keys to arbitrary Buffers. The buffer data will be moved into a [`gpu.Buffer`][zarr.core.buffer.gpu.Buffer]. Returns ------- GpuMemoryStore """ gpu_store_dict = {k: gpu.Buffer.from_buffer(v) for k, v in store_dict.items()} return cls(gpu_store_dict) async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None = None) -> None: # docstring inherited self._check_writable() if not isinstance(value, Buffer): raise TypeError( f"GpuMemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." ) # Convert to gpu.Buffer gpu_value = value if isinstance(value, gpu.Buffer) else gpu.Buffer.from_buffer(value) await super().set(key, gpu_value, byte_range=byte_range) def set_sync(self, key: str, value: Buffer) -> None: # docstring inherited self._check_writable() if not isinstance(value, Buffer): raise TypeError( f"GpuMemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." ) # Convert to gpu.Buffer, mirroring `set` above: every value in this store's # backing dict must be a gpu.Buffer, regardless of which API wrote it. gpu_value = value if isinstance(value, gpu.Buffer) else gpu.Buffer.from_buffer(value) super().set_sync(key, gpu_value) # ----------------------------------------------------------------------------- # ManagedMemoryStore and its registry # ----------------------------------------------------------------------------- # ManagedMemoryStore owns the lifecycle of its backing dict, enabling proper # weakref-based tracking. This allows memory:// URLs to be resolved back to # the store's dict within the same process. class _ManagedStoreDict(dict[str, Buffer]): """ A dict subclass that supports weak references. Regular dicts don't support weakrefs, but we need to track managed store dicts in a WeakValueDictionary so they can be garbage collected when no longer referenced. This subclass adds the necessary __weakref__ slot. """ __slots__ = ("__weakref__",) class _ManagedStoreDictRegistry: """ Registry for managed store dicts. This registry is the source of truth for managed store dicts. It creates new dicts, tracks them via weak references, and looks them up by name. """ def __init__(self) -> None: self._registry: weakref.WeakValueDictionary[str, _ManagedStoreDict] = ( weakref.WeakValueDictionary() ) self._counter = 0 self._lock = threading.Lock() def _generate_name(self) -> str: """Generate a unique name for a store. Must be called while holding `self._lock`. """ name = str(self._counter) self._counter += 1 return name def get_or_create(self, name: str | None = None) -> tuple[_ManagedStoreDict, str]: """ Get an existing managed dict by name, or create a new one. Thread-safe: uses a lock to prevent TOCTOU races between checking for an existing entry and inserting a new one. Parameters ---------- name : str | None The name for the store. If None, a unique name is auto-generated. If a store with this name already exists, returns the existing store. Names cannot contain '/' characters. Returns ------- tuple[_ManagedStoreDict, str] The store dict and its name. Raises ------ ValueError If the name contains '/' characters. """ with self._lock: if name is None: name = self._generate_name() elif "/" in name: raise ValueError( f"Store name cannot contain '/': {name!r}. " "Use the 'path' parameter to specify a path within the store." ) existing = self._registry.get(name) if existing is not None: return existing, name store_dict = _ManagedStoreDict() self._registry[name] = store_dict return store_dict, name def get(self, name: str) -> _ManagedStoreDict | None: """ Look up a managed store dict by name. Parameters ---------- name : str The name of the store. Returns ------- _ManagedStoreDict | None The store dict if found, None otherwise. """ return self._registry.get(name) _managed_store_dict_registry = _ManagedStoreDictRegistry() class ManagedMemoryStore(MemoryStore): """ A memory store that owns and manages the lifecycle of its backing dict. Unlike ``MemoryStore`` which accepts any ``MutableMapping``, this store creates and owns its backing dict internally. This enables proper lifecycle management and allows the store to be looked up by its ``memory://`` URL within the same process. Parameters ---------- name : str | None The name for this store, used in the ``memory://`` URL. If None, a unique name is auto-generated. If a store with this name already exists, the new store will share the same backing dict. path : str The root path for this store. All keys will be prefixed with this path. read_only : bool Whether the store is read-only. Attributes ---------- name : str The name of this store. path : str The root path of this store. Notes ----- The backing dict is tracked via weak references and will be garbage collected when no ``ManagedMemoryStore`` instances reference it. URLs pointing to a garbage-collected store will fail to resolve. See Also -------- MemoryStore : A memory store that accepts any MutableMapping. Examples -------- >>> store = ManagedMemoryStore(name="my-data") >>> str(store) 'memory://my-data' >>> # Later, resolve the URL back to the store's dict >>> store2 = ManagedMemoryStore.from_url("memory://my-data") >>> store2._store_dict is store._store_dict True >>> # Create a store with a path prefix >>> store3 = ManagedMemoryStore.from_url("memory://my-data/subdir") >>> store3.path 'subdir' """ _store_dict: _ManagedStoreDict _name: str path: str def __init__(self, name: str | None = None, *, path: str = "", read_only: bool = False) -> None: # Skip MemoryStore.__init__ and call Store.__init__ directly # because we manage _store_dict via the registry, not via a user-supplied # MutableMapping. If MemoryStore.__init__ ever adds logic beyond setting # _store_dict, that logic must be replicated here. Store.__init__(self, read_only=read_only) # Get or create a managed dict from the registry self._store_dict, self._name = _managed_store_dict_registry.get_or_create(name) self.path = normalize_path(path) def __str__(self) -> str: return _join_paths([f"memory://{self._name}", self.path]) def __repr__(self) -> str: return f"ManagedMemoryStore('{self}')" def __eq__(self, other: object) -> bool: return ( isinstance(other, type(self)) and self._store_dict is other._store_dict and self.path == other.path and self.read_only == other.read_only ) @property def name(self) -> str: """The name of this store, used in the memory:// URL.""" return self._name @classmethod def _from_managed_dict( cls, managed_dict: _ManagedStoreDict, name: str, *, path: str = "", read_only: bool = False, ) -> ManagedMemoryStore: """Internal: create a store from an existing managed dict.""" store = object.__new__(cls) Store.__init__(store, read_only=read_only) store._store_dict = managed_dict store._name = name store.path = normalize_path(path) return store def with_read_only(self, read_only: bool = False) -> ManagedMemoryStore: # docstring inherited return type(self)._from_managed_dict( self._store_dict, self._name, path=self.path, read_only=read_only ) @classmethod def from_url(cls, url: str, *, read_only: bool = False) -> ManagedMemoryStore: """ Create a ManagedMemoryStore from a memory:// URL. This looks up the backing dict in the process-wide registry and creates a new store instance that shares the same dict. Parameters ---------- url : str A URL like "memory://my-store" or "memory://my-store/path/to/data" identifying the store and optional path prefix. read_only : bool Whether the new store should be read-only. Returns ------- ManagedMemoryStore A store sharing the same backing dict as the original. Raises ------ ValueError If the URL is not a valid memory:// URL or the store has been garbage collected. """ parsed = parse_store_url(url) if parsed.scheme != "memory": raise ValueError( f"Expected a 'memory://' URL, got scheme {parsed.scheme!r} in '{url}'." ) name = parsed.name or "" managed_dict = _managed_store_dict_registry.get(name) if managed_dict is None: raise ValueError( f"Memory store not found for URL '{url}'. " "The store may have been garbage collected." ) return cls._from_managed_dict(managed_dict, name, path=parsed.path, read_only=read_only) # Override MemoryStore methods to use path prefix and check process def get_sync( self, key: str, *, prototype: BufferPrototype | None = None, byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited return super().get_sync( _join_paths([self.path, key]), prototype=prototype, byte_range=byte_range ) def set_sync(self, key: str, value: Buffer) -> None: # docstring inherited super().set_sync(_join_paths([self.path, key]), value) def delete_sync(self, key: str) -> None: # docstring inherited super().delete_sync(_join_paths([self.path, key])) async def get( self, key: str, prototype: BufferPrototype | None = None, byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited return await super().get( _join_paths([self.path, key]), prototype=prototype, byte_range=byte_range ) # get_partial_values is intentionally NOT overridden here: MemoryStore.get_partial_values # dispatches per-key through `self.get`, which already resolves to the override above. # Re-prefixing the keys here as well would apply `self.path` twice. async def exists(self, key: str) -> bool: # docstring inherited return await super().exists(_join_paths([self.path, key])) async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None = None) -> None: # docstring inherited return await super().set(_join_paths([self.path, key]), value, byte_range=byte_range) async def set_if_not_exists(self, key: str, value: Buffer) -> None: # docstring inherited return await super().set_if_not_exists(_join_paths([self.path, key]), value) async def delete(self, key: str) -> None: # docstring inherited return await super().delete(_join_paths([self.path, key])) async def list(self) -> AsyncIterator[str]: # docstring inherited prefix = f"{self.path}/" if self.path else "" async for key in super().list(): if key.startswith(prefix): yield key.removeprefix(prefix) async def list_prefix(self, prefix: str) -> AsyncIterator[str]: # docstring inherited # Manual concatenation instead of _join_paths because we need "path/" # as the prefix when prefix is empty (to list all keys under self.path) full_prefix = f"{self.path}/{prefix}" if self.path else prefix path_prefix = f"{self.path}/" if self.path else "" async for key in super().list_prefix(full_prefix): yield key.removeprefix(path_prefix) async def list_dir(self, prefix: str) -> AsyncIterator[str]: # docstring inherited full_prefix = _join_paths([self.path, prefix]) async for key in super().list_dir(full_prefix): yield key def __reduce__( self, ) -> tuple[type[ManagedMemoryStore], tuple[str | None], dict[str, Any]]: """ Support pickling of ManagedMemoryStore. On unpickle, the store will reconnect to an existing store with the same name if one exists in the registry, or create a new empty store otherwise. Note that the backing dict data is NOT serialized - only the store's identity (name, path, read_only) is preserved. If the original store has been garbage collected, the unpickled store will have an empty dict. The current process ID is preserved so that cross-process unpickling can be detected and will raise an error at unpickle time. """ return ( self.__class__, (self._name,), { "path": self.path, "read_only": self.read_only, "created_pid": os.getpid(), }, ) def __setstate__(self, state: dict[str, Any]) -> None: """Restore state after unpickling. The pickle protocol calls ``cls(name)`` (via ``__reduce__``'s args) then ``__setstate__(state)``. ``__init__`` already set up ``_store_dict`` and ``_name`` from the registry — we just restore path and read_only here. """ # Check for cross-process usage first, before mutating state created_pid = state.get("created_pid") if created_pid is not None and created_pid != os.getpid(): raise RuntimeError( f"ManagedMemoryStore '{self._name}' was created in process {created_pid} " f"but is being unpickled in process {os.getpid()}. " "ManagedMemoryStore instances cannot be shared across processes because " "their backing dict is not serialized. Use a persistent store (e.g., " "LocalStore, ZipStore) for cross-process data sharing." ) self.path = normalize_path(state.get("path", "")) # Use the Store-level _read_only attribute directly because # Store.__init__ was already called by __init__ during unpickling self._read_only = state.get("read_only", False) zarr-developers-zarr-python-a2f2415/src/zarr/storage/_obstore.py000066400000000000000000000422241527014137600250700ustar00rootroot00000000000000from __future__ import annotations import asyncio import contextlib import pickle from collections import defaultdict from itertools import chain from operator import itemgetter from typing import TYPE_CHECKING, Self, TypedDict from zarr.abc.store import ( ByteRequest, OffsetByteRequest, RangeByteRequest, Store, SuffixByteRequest, ) from zarr.core.common import concurrent_map from zarr.core.config import config from zarr.storage._utils import _relativize_path if TYPE_CHECKING: from collections.abc import AsyncGenerator, Coroutine, Iterable, Sequence from typing import Any from obstore import ListResult, ListStream, ObjectMeta, OffsetRange, SuffixRange from obstore.store import ObjectStore as _UpstreamObjectStore from zarr.core.buffer import Buffer, BufferPrototype __all__ = ["ObjectStore"] _ALLOWED_EXCEPTIONS: tuple[type[Exception], ...] = ( FileNotFoundError, IsADirectoryError, NotADirectoryError, ) class ObjectStore[T_Store: "_UpstreamObjectStore"](Store): """ Store that uses obstore for fast read/write from AWS, GCP, Azure. Parameters ---------- store : obstore.store.ObjectStore An obstore store instance that is set up with the proper credentials. read_only : bool Whether to open the store in read-only mode. Warnings -------- ObjectStore is experimental and subject to API changes without notice. Please raise an issue with any comments/concerns about the store. """ store: T_Store """The underlying obstore instance.""" def __eq__(self, value: object) -> bool: if not isinstance(value, ObjectStore): return False if not self.read_only == value.read_only: return False return self.store == value.store # type: ignore[no-any-return] def __init__(self, store: T_Store, *, read_only: bool = False) -> None: if not store.__class__.__module__.startswith("obstore"): raise TypeError(f"expected ObjectStore class, got {store!r}") super().__init__(read_only=read_only) self.store = store def with_read_only(self, read_only: bool = False) -> Self: # docstring inherited return type(self)( store=self.store, read_only=read_only, ) def __str__(self) -> str: return f"object_store://{self.store}" def __repr__(self) -> str: return f"{type(self).__name__}({self})" def __getstate__(self) -> dict[Any, Any]: state = self.__dict__.copy() state["store"] = pickle.dumps(self.store) return state def __setstate__(self, state: dict[Any, Any]) -> None: state["store"] = pickle.loads(state["store"]) self.__dict__.update(state) async def get( self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: # docstring inherited import obstore as obs try: if byte_range is None: resp = await obs.get_async(self.store, key) return prototype.buffer.from_bytes(await resp.bytes_async()) # type: ignore[arg-type] elif isinstance(byte_range, RangeByteRequest): bytes = await obs.get_range_async( self.store, key, start=byte_range.start, end=byte_range.end ) return prototype.buffer.from_bytes(bytes) # type: ignore[arg-type] elif isinstance(byte_range, OffsetByteRequest): resp = await obs.get_async( self.store, key, options={"range": {"offset": byte_range.offset}} ) return prototype.buffer.from_bytes(await resp.bytes_async()) # type: ignore[arg-type] elif isinstance(byte_range, SuffixByteRequest): # some object stores (Azure) don't support suffix requests. In this # case, our workaround is to first get the length of the object and then # manually request the byte range at the end. try: resp = await obs.get_async( self.store, key, options={"range": {"suffix": byte_range.suffix}} ) return prototype.buffer.from_bytes(await resp.bytes_async()) # type: ignore[arg-type] except obs.exceptions.NotSupportedError: head_resp = await obs.head_async(self.store, key) file_size = head_resp["size"] suffix_len = byte_range.suffix buffer = await obs.get_range_async( self.store, key, start=file_size - suffix_len, length=suffix_len, ) return prototype.buffer.from_bytes(buffer) # type: ignore[arg-type] else: raise ValueError(f"Unexpected byte_range, got {byte_range}") except _ALLOWED_EXCEPTIONS: return None async def get_partial_values( self, prototype: BufferPrototype, key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: # docstring inherited return await _get_partial_values(self.store, prototype=prototype, key_ranges=key_ranges) async def exists(self, key: str) -> bool: # docstring inherited import obstore as obs try: await obs.head_async(self.store, key) except FileNotFoundError: return False else: return True @property def supports_writes(self) -> bool: # docstring inherited return True async def set(self, key: str, value: Buffer) -> None: # docstring inherited import obstore as obs self._check_writable() buf = value.as_buffer_like() await obs.put_async(self.store, key, buf) async def set_if_not_exists(self, key: str, value: Buffer) -> None: # docstring inherited import obstore as obs self._check_writable() buf = value.as_buffer_like() with contextlib.suppress(obs.exceptions.AlreadyExistsError): await obs.put_async(self.store, key, buf, mode="create") @property def supports_deletes(self) -> bool: # docstring inherited return True async def delete(self, key: str) -> None: # docstring inherited import obstore as obs self._check_writable() # Some obstore stores such as local filesystems, GCP and Azure raise an error # when deleting a non-existent key, while others such as S3 and in-memory do # not. We suppress the error to make the behavior consistent across all obstore # stores. This is also in line with the behavior of the other Zarr store adapters. with contextlib.suppress(FileNotFoundError): await obs.delete_async(self.store, key) async def delete_dir(self, prefix: str) -> None: # docstring inherited import obstore as obs self._check_writable() if prefix != "" and not prefix.endswith("/"): prefix += "/" metas = await obs.list(self.store, prefix).collect_async() keys = [(m["path"],) for m in metas] await concurrent_map(keys, self.delete, limit=config.get("async.concurrency")) @property def supports_listing(self) -> bool: # docstring inherited return True async def _list(self, prefix: str | None = None) -> AsyncGenerator[ObjectMeta, None]: import obstore as obs objects: ListStream[Sequence[ObjectMeta]] = obs.list(self.store, prefix=prefix) async for batch in objects: for item in batch: yield item def list(self) -> AsyncGenerator[str, None]: # docstring inherited return (obj["path"] async for obj in self._list()) def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: # docstring inherited return (obj["path"] async for obj in self._list(prefix)) def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: # docstring inherited import obstore as obs coroutine = obs.list_with_delimiter_async(self.store, prefix=prefix) return _transform_list_dir(coroutine, prefix) async def getsize(self, key: str) -> int: # docstring inherited import obstore as obs resp = await obs.head_async(self.store, key) return resp["size"] async def getsize_prefix(self, prefix: str) -> int: # docstring inherited sizes = [obj["size"] async for obj in self._list(prefix=prefix)] return sum(sizes) async def _transform_list_dir( list_result_coroutine: Coroutine[Any, Any, ListResult[Sequence[ObjectMeta]]], prefix: str ) -> AsyncGenerator[str, None]: """ Transform the result of list_with_delimiter into an async generator of paths. """ list_result = await list_result_coroutine # We assume that the underlying object-store implementation correctly handles the # prefix, so we don't double-check that the returned results actually start with the # given prefix. prefix = prefix.rstrip("/") for path in chain( list_result["common_prefixes"], map(itemgetter("path"), list_result["objects"]) ): if prefix != "" and path == prefix: continue relpath = _relativize_path(path=path, prefix=prefix) if relpath: yield relpath class _BoundedRequest(TypedDict): """Range request with a known start and end byte. These requests can be multiplexed natively on the Rust side with `obstore.get_ranges_async`. """ original_request_index: int """The positional index in the original key_ranges input""" start: int """Start byte offset.""" end: int """End byte offset.""" class _OtherRequest(TypedDict): """Offset or suffix range requests. These requests cannot be concurrent on the Rust side, and each need their own call to `obstore.get_async`, passing in the `range` parameter. """ original_request_index: int """The positional index in the original key_ranges input""" path: str """The path to request from.""" range: OffsetRange | None # Note: suffix requests are handled separately because some object stores (Azure) # don't support them """The range request type.""" class _SuffixRequest(TypedDict): """Offset or suffix range requests. These requests cannot be concurrent on the Rust side, and each need their own call to `obstore.get_async`, passing in the `range` parameter. """ original_request_index: int """The positional index in the original key_ranges input""" path: str """The path to request from.""" range: SuffixRange """The suffix range.""" class _Response(TypedDict): """A response buffer associated with the original index that it should be restored to.""" original_request_index: int """The positional index in the original key_ranges input""" buffer: Buffer """The buffer returned from obstore's range request.""" async def _make_bounded_requests( store: _UpstreamObjectStore, path: str, requests: list[_BoundedRequest], prototype: BufferPrototype, semaphore: asyncio.Semaphore, ) -> list[_Response]: """Make all bounded requests for a specific file. `obstore.get_ranges_async` allows for making concurrent requests for multiple ranges within a single file, and will e.g. merge concurrent requests. This only uses one single Python coroutine. """ import obstore as obs starts = [r["start"] for r in requests] ends = [r["end"] for r in requests] async with semaphore: responses = await obs.get_ranges_async(store, path=path, starts=starts, ends=ends) buffer_responses: list[_Response] = [] for request, response in zip(requests, responses, strict=True): buffer_responses.append( { "original_request_index": request["original_request_index"], "buffer": prototype.buffer.from_bytes(response), # type: ignore[arg-type] } ) return buffer_responses async def _make_other_request( store: _UpstreamObjectStore, request: _OtherRequest, prototype: BufferPrototype, semaphore: asyncio.Semaphore, ) -> list[_Response]: """Make offset or full-file requests. We return a `list[_Response]` for symmetry with `_make_bounded_requests` so that all futures can be gathered together. """ import obstore as obs async with semaphore: if request["range"] is None: resp = await obs.get_async(store, request["path"]) else: resp = await obs.get_async(store, request["path"], options={"range": request["range"]}) buffer = await resp.bytes_async() return [ { "original_request_index": request["original_request_index"], "buffer": prototype.buffer.from_bytes(buffer), # type: ignore[arg-type] } ] async def _make_suffix_request( store: _UpstreamObjectStore, request: _SuffixRequest, prototype: BufferPrototype, semaphore: asyncio.Semaphore, ) -> list[_Response]: """Make suffix requests. This is separated out from `_make_other_request` because some object stores (Azure) don't support suffix requests. In this case, our workaround is to first get the length of the object and then manually request the byte range at the end. We return a `list[_Response]` for symmetry with `_make_bounded_requests` so that all futures can be gathered together. """ import obstore as obs async with semaphore: try: resp = await obs.get_async(store, request["path"], options={"range": request["range"]}) buffer = await resp.bytes_async() except obs.exceptions.NotSupportedError: head_resp = await obs.head_async(store, request["path"]) file_size = head_resp["size"] suffix_len = request["range"]["suffix"] buffer = await obs.get_range_async( store, request["path"], start=file_size - suffix_len, length=suffix_len, ) return [ { "original_request_index": request["original_request_index"], "buffer": prototype.buffer.from_bytes(buffer), # type: ignore[arg-type] } ] async def _get_partial_values( store: _UpstreamObjectStore, prototype: BufferPrototype, key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: """Make multiple range requests. ObjectStore has a `get_ranges` method that will additionally merge nearby ranges, but it's _per_ file. So we need to split these key_ranges into **per-file** key ranges, and then reassemble the results in the original order. We separate into different requests: - One call to `obstore.get_ranges_async` **per target file** - One call to `obstore.get_async` for each other request. """ key_ranges = list(key_ranges) per_file_bounded_requests: dict[str, list[_BoundedRequest]] = defaultdict(list) other_requests: list[_OtherRequest] = [] suffix_requests: list[_SuffixRequest] = [] for idx, (path, byte_range) in enumerate(key_ranges): if byte_range is None: other_requests.append( { "original_request_index": idx, "path": path, "range": None, } ) elif isinstance(byte_range, RangeByteRequest): per_file_bounded_requests[path].append( {"original_request_index": idx, "start": byte_range.start, "end": byte_range.end} ) elif isinstance(byte_range, OffsetByteRequest): other_requests.append( { "original_request_index": idx, "path": path, "range": {"offset": byte_range.offset}, } ) elif isinstance(byte_range, SuffixByteRequest): suffix_requests.append( { "original_request_index": idx, "path": path, "range": {"suffix": byte_range.suffix}, } ) else: raise ValueError(f"Unsupported range input: {byte_range}") semaphore = asyncio.Semaphore(config.get("async.concurrency")) futs: list[Coroutine[Any, Any, list[_Response]]] = [] for path, bounded_ranges in per_file_bounded_requests.items(): futs.append( _make_bounded_requests(store, path, bounded_ranges, prototype, semaphore=semaphore) ) for request in other_requests: futs.append(_make_other_request(store, request, prototype, semaphore=semaphore)) # noqa: PERF401 for suffix_request in suffix_requests: futs.append(_make_suffix_request(store, suffix_request, prototype, semaphore=semaphore)) # noqa: PERF401 buffers: list[Buffer | None] = [None] * len(key_ranges) for responses in await asyncio.gather(*futs): for resp in responses: buffers[resp["original_request_index"]] = resp["buffer"] return buffers zarr-developers-zarr-python-a2f2415/src/zarr/storage/_utils.py000066400000000000000000000230661527014137600245560ustar00rootroot00000000000000from __future__ import annotations import importlib import re from pathlib import Path, PureWindowsPath from urllib.parse import urlparse if importlib.util.find_spec("upath"): # Re-exported for zarr.storage._common, which needs it to recognize UPath store_like values. # The redundant-looking alias is the explicit re-export mypy requires under strict mode. from upath.core import UPath as UPath # noqa: PLC0414 else: class UPath: # type: ignore[no-redef] pass import sys from typing import TYPE_CHECKING, NamedTuple from zarr.abc.store import OffsetByteRequest, RangeByteRequest, SuffixByteRequest if TYPE_CHECKING: from collections.abc import Iterable, Mapping from zarr.abc.store import ByteRequest from zarr.core.buffer import Buffer class ParsedStoreUrl(NamedTuple): """ Parsed components of a store URL. Attributes ---------- scheme : str The URL scheme (e.g., "memory", "file", "s3"). Empty string for local paths. name : str | None The store name/host component. For memory:// URLs this is the store name. None if empty. path : str The path component within the store. raw : str The original URL string. """ scheme: str name: str | None path: str raw: str def parse_store_url(url: str) -> ParsedStoreUrl: """ Parse a store URL into its components. Parameters ---------- url : str A URL like "memory://store-name/path" or "s3://bucket/key" or a local path. Returns ------- ParsedStoreUrl Named tuple with scheme, name, path, and raw URL. Examples -------- >>> parse_store_url("memory://mystore") ParsedStoreUrl(scheme='memory', name='mystore', path='', raw='memory://mystore') >>> parse_store_url("memory://mystore/path/to/data") ParsedStoreUrl(scheme='memory', name='mystore', path='path/to/data', raw='memory://mystore/path/to/data') >>> parse_store_url("s3://bucket/key") ParsedStoreUrl(scheme='s3', name='bucket', path='key', raw='s3://bucket/key') >>> parse_store_url("/local/path") ParsedStoreUrl(scheme='', name=None, path='/local/path', raw='/local/path') Note that ``memory://name/path`` and ``memory:///path`` are different: the first has ``name="name"`` and ``path="path"``, while the second has ``name=None`` and ``path="/path"`` (no host component between ``//`` and ``/``). """ # On Windows, bare paths like "C:\foo" or "C:/foo" cause urlparse to # misinterpret the drive letter as a URL scheme. Detect this early and # return a local-path result without going through urlparse. if sys.platform == "win32" and PureWindowsPath(url).drive: return ParsedStoreUrl(scheme="", name=None, path=url, raw=url) parsed = urlparse(url) # netloc is the "host" part (store name for memory://, bucket for s3://, etc.) name = parsed.netloc or None # For URLs with a scheme and netloc (like memory://store/path or s3://bucket/key), # strip the leading slash from the path component. # For local paths (no scheme), preserve the path as-is. if parsed.scheme and parsed.netloc: path = parsed.path.lstrip("/") else: path = parsed.path return ParsedStoreUrl(scheme=parsed.scheme, name=name, path=path, raw=url) def normalize_path(path: str | bytes | Path | None) -> str: if path is None: result = "" elif isinstance(path, bytes): result = str(path, "ascii") # handle pathlib.Path elif isinstance(path, Path | UPath): result = str(path) elif isinstance(path, str): result = path else: raise TypeError(f'Object {path} has an invalid type for "path": {type(path).__name__}') # convert backslash to forward slash result = result.replace("\\", "/") # remove leading and trailing slashes result = result.strip("/") # collapse any repeated slashes pat = re.compile(r"//+") result = pat.sub("/", result) # disallow path segments with just '.' or '..' segments = result.split("/") if any(s in {".", ".."} for s in segments): raise ValueError( f"The path {path!r} is invalid because its string representation contains '.' or '..' segments." ) return result def _normalize_byte_range_index(data: Buffer, byte_range: ByteRequest | None) -> tuple[int, int]: """ Convert a ByteRequest into an explicit start and stop """ if byte_range is None: start = 0 stop = len(data) + 1 elif isinstance(byte_range, RangeByteRequest): start = byte_range.start stop = byte_range.end elif isinstance(byte_range, OffsetByteRequest): start = byte_range.offset stop = len(data) + 1 elif isinstance(byte_range, SuffixByteRequest): start = max(0, len(data) - byte_range.suffix) stop = len(data) + 1 else: raise ValueError(f"Unexpected byte_range, got {byte_range}.") return (start, stop) def _join_paths(paths: Iterable[str]) -> str: """ Filter out instances of '' and join the remaining strings with '/'. Parameters ---------- paths : Iterable[str] Returns ------- str Examples -------- ```python from zarr.storage._utils import _join_paths _join_paths(["", "a", "b"]) # 'a/b' _join_paths(["a", "b", "c"]) # 'a/b/c' ``` """ return "/".join(filter(lambda v: v != "", paths)) def _dereference_path(root: str, path: str) -> str: """ Combine a store-side root with a key into a single fully-qualified path. Unlike `_join_paths`, this is purpose-built for the case where `root` is an opaque backend-side prefix that may use `"/"` as a sentinel for "root of the filesystem" (notably for fsspec's `ReferenceFileSystem`). A trailing `"/"` is stripped from `root` before joining; if `root` is then empty, the bare `path` is returned so that joining `"/"` with `"key"` yields `"key"` rather than `"//key"`. A trailing `"/"` on the result is also stripped. Leading slashes on `root` are preserved -- a backend-side path like `"/home/foo/data.zarr"` is an absolute filesystem path for `LocalFileSystem` and must not lose its leading separator. Parameters ---------- root : str The backend-side root of a store. May be `""`, `"/"`, an absolute filesystem path, or a backend-specific prefix. path : str The key within the store, typically a zarr key like `"zarr.json"` or `"a/b/c/zarr.json"`. Returns ------- str `root` and `path` joined by a single `"/"`, with the `"/"` sentinel collapsed and trailing slashes removed. Examples -------- ```python from zarr.storage._utils import _dereference_path _dereference_path("/", "zarr.json") # 'zarr.json' _dereference_path("", "zarr.json") # 'zarr.json' _dereference_path("/home/foo", "zarr.json") # '/home/foo/zarr.json' _dereference_path("/home/foo/", "zarr.json") # '/home/foo/zarr.json' _dereference_path("bucket/p", "zarr.json") # 'bucket/p/zarr.json' ``` """ root = root.rstrip("/") path = f"{root}/{path}" if root else path return path.rstrip("/") def _relativize_path(*, path: str, prefix: str) -> str: """ Make a "/"-delimited path relative to some prefix. If the prefix is '', then the path is returned as-is. Otherwise, the prefix is removed from the path as well as the separator string "/". If ``prefix`` is not the empty string and ``path`` does not start with ``prefix`` followed by a "/" character, then an error is raised. This function assumes that the prefix does not end with "/". Parameters ---------- path : str The path to make relative to the prefix. prefix : str The prefix to make the path relative to. Returns ------- str Examples -------- ```python from zarr.storage._utils import _relativize_path _relativize_path(path="a/b", prefix="") # 'a/b' _relativize_path(path="a/b/c", prefix="a/b") # 'c' ``` """ if prefix == "": return path else: _prefix = f"{prefix}/" if not path.startswith(_prefix): raise ValueError(f"The first component of {path} does not start with {prefix}.") return path.removeprefix(_prefix) def _normalize_paths(paths: Iterable[str]) -> tuple[str, ...]: """ Normalize the input paths according to the normalization scheme used for zarr node paths. If any two paths normalize to the same value, raise a ValueError. """ path_map: dict[str, str] = {} for path in paths: parsed = normalize_path(path) if parsed in path_map: msg = ( f"After normalization, the value '{path}' collides with '{path_map[parsed]}'. " f"Both '{path}' and '{path_map[parsed]}' normalize to the same value: '{parsed}'. " f"You should use either '{path}' or '{path_map[parsed]}', but not both." ) raise ValueError(msg) path_map[parsed] = path return tuple(path_map.keys()) def _normalize_path_keys[T](data: Mapping[str, T]) -> dict[str, T]: """ Normalize the keys of the input dict according to the normalization scheme used for zarr node paths. If any two keys in the input normalize to the same value, raise a ValueError. Returns a dict where the keys are the elements of the input and the values are the normalized form of each key. """ parsed_keys = _normalize_paths(data.keys()) return dict(zip(parsed_keys, data.values(), strict=True)) zarr-developers-zarr-python-a2f2415/src/zarr/storage/_wrapper.py000066400000000000000000000165711527014137600251010ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING, cast if TYPE_CHECKING: from collections.abc import AsyncGenerator, AsyncIterator, Iterable, Sequence from types import TracebackType from typing import Any, Self from zarr.abc.buffer import Buffer from zarr.abc.store import ByteRequest from zarr.core.buffer import BufferPrototype from zarr.abc.store import ( Store, SupportsDeleteSync, SupportsGetSync, SupportsSetSync, _store_supports_sync_io, ) class WrapperStore[T_Store: Store](Store): """ Store that wraps an existing Store. By default all of the store methods are delegated to the wrapped store instance, which is accessible via the ``._store`` attribute of this class. Use this class to modify or extend the behavior of the other store classes. """ _store: T_Store def __init__(self, store: T_Store) -> None: self._store = store def _with_store(self, store: T_Store) -> Self: """ Constructs a new instance of the wrapper store with the same details but a new store. """ return type(self)(store=store) @classmethod async def open(cls: type[Self], store_cls: type[T_Store], *args: Any, **kwargs: Any) -> Self: store = store_cls(*args, **kwargs) await store._open() return cls(store=store) def with_read_only(self, read_only: bool = False) -> Self: return self._with_store(cast(T_Store, self._store.with_read_only(read_only))) def __enter__(self) -> Self: return self._with_store(self._store.__enter__()) def __exit__( self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None, ) -> None: return self._store.__exit__(exc_type, exc_value, traceback) async def _open(self) -> None: await self._store._open() async def _ensure_open(self) -> None: await self._store._ensure_open() async def is_empty(self, prefix: str) -> bool: return await self._store.is_empty(prefix) @property def _is_open(self) -> bool: return self._store._is_open @_is_open.setter def _is_open(self, value: bool) -> None: raise NotImplementedError("WrapperStore must be opened via the `_open` method") async def clear(self) -> None: return await self._store.clear() @property def read_only(self) -> bool: return self._store.read_only def _check_writable(self) -> None: return self._store._check_writable() def __eq__(self, value: object) -> bool: return type(self) is type(value) and self._store.__eq__(value._store) def __str__(self) -> str: return f"wrapping-{self._store}" def __repr__(self) -> str: return f"WrapperStore({self._store.__class__.__name__}, '{self._store}')" async def get( self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: return await self._store.get(key, prototype, byte_range) async def get_partial_values( self, prototype: BufferPrototype, key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: return await self._store.get_partial_values(prototype, key_ranges) async def get_ranges( self, key: str, byte_ranges: Sequence[ByteRequest | None], *, prototype: BufferPrototype, max_concurrency: int | None = None, max_gap_bytes: int | None = None, max_coalesced_bytes: int | None = None, ) -> AsyncIterator[Sequence[tuple[int, Buffer | None]]]: """Forward `get_ranges` to the wrapped store. Default values for the coalescing kwargs are not declared here; the wrapped store decides them. `None` means "don't override the wrapped store's default". """ kwargs: dict[str, int] = {} if max_concurrency is not None: kwargs["max_concurrency"] = max_concurrency if max_gap_bytes is not None: kwargs["max_gap_bytes"] = max_gap_bytes if max_coalesced_bytes is not None: kwargs["max_coalesced_bytes"] = max_coalesced_bytes async for group in self._store.get_ranges(key, byte_ranges, prototype=prototype, **kwargs): yield group async def exists(self, key: str) -> bool: return await self._store.exists(key) async def set(self, key: str, value: Buffer) -> None: await self._store.set(key, value) async def set_if_not_exists(self, key: str, value: Buffer) -> None: return await self._store.set_if_not_exists(key, value) async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None: await self._store._set_many(values) @property def supports_writes(self) -> bool: return self._store.supports_writes @property def supports_deletes(self) -> bool: return self._store.supports_deletes @property def _supports_sync_io(self) -> bool: # The delegating `*_sync` methods below make every wrapper structurally # satisfy `SupportsSyncStore`; whether they can actually run depends on # the wrapped store, so forward its capability (see # `zarr.abc.store._store_supports_sync_io`). return _store_supports_sync_io(self._store) def get_sync( self, key: str, *, prototype: BufferPrototype | None = None, byte_range: ByteRequest | None = None, ) -> Buffer | None: """Forward `get_sync` to the wrapped store.""" if not isinstance(self._store, SupportsGetSync): raise TypeError(f"Store {type(self._store).__name__} does not support synchronous get.") return self._store.get_sync(key, prototype=prototype, byte_range=byte_range) # type: ignore[unreachable] def set_sync(self, key: str, value: Buffer) -> None: """Forward `set_sync` to the wrapped store.""" if not isinstance(self._store, SupportsSetSync): raise TypeError(f"Store {type(self._store).__name__} does not support synchronous set.") self._store.set_sync(key, value) # type: ignore[unreachable] def delete_sync(self, key: str) -> None: """Forward `delete_sync` to the wrapped store.""" if not isinstance(self._store, SupportsDeleteSync): raise TypeError( f"Store {type(self._store).__name__} does not support synchronous delete." ) self._store.delete_sync(key) # type: ignore[unreachable] async def delete(self, key: str) -> None: await self._store.delete(key) @property def supports_listing(self) -> bool: return self._store.supports_listing def list(self) -> AsyncIterator[str]: return self._store.list() def list_prefix(self, prefix: str) -> AsyncIterator[str]: return self._store.list_prefix(prefix) def list_dir(self, prefix: str) -> AsyncIterator[str]: return self._store.list_dir(prefix) async def delete_dir(self, prefix: str) -> None: return await self._store.delete_dir(prefix) def close(self) -> None: self._store.close() async def _get_many( self, requests: Iterable[tuple[str, BufferPrototype, ByteRequest | None]] ) -> AsyncGenerator[tuple[str, Buffer | None], None]: async for req in self._store._get_many(requests): yield req zarr-developers-zarr-python-a2f2415/src/zarr/storage/_zip.py000066400000000000000000000435021527014137600242150ustar00rootroot00000000000000from __future__ import annotations import io import os import shutil import threading import time import zipfile from pathlib import Path from typing import IO, TYPE_CHECKING, Any, Literal from zarr.abc.store import ( ByteRequest, OffsetByteRequest, RangeByteRequest, Store, SuffixByteRequest, ) from zarr.core.buffer import Buffer, BufferPrototype if TYPE_CHECKING: from collections.abc import AsyncIterator, Iterable ZipStoreAccessModeLiteral = Literal["r", "w", "a", "x"] class _RawReaderAdapter(io.RawIOBase): """ Adapt a minimal seekable reader to the `io` interface `zipfile` needs. Some file-like objects (e.g. `obstore.ReadableFile`) implement `read`/`seek`/`tell` but are not `io.IOBase` instances, and their `read` may return a buffer-protocol object rather than `bytes`. Wrapping in this adapter plus `io.BufferedReader` yields real `bytes`. Reads are clamped to the bytes remaining before EOF: some readers (obstore < 0.6) raise on short reads rather than returning fewer bytes. The size is cached, which is safe because the adapter is only used for read-only access. """ def __init__(self, fileobj: IO[bytes]) -> None: self._fileobj = fileobj self._size: int | None = None def _get_size(self) -> int: if self._size is None: pos = self._fileobj.tell() self._size = self._fileobj.seek(0, os.SEEK_END) self._fileobj.seek(pos) return self._size def readable(self) -> bool: return True def seekable(self) -> bool: return True def seek(self, pos: int, whence: int = 0) -> int: return self._fileobj.seek(pos, whence) def tell(self) -> int: return self._fileobj.tell() def readinto(self, b: Any) -> int: n_requested = min(len(b), self._get_size() - self._fileobj.tell()) if n_requested <= 0: return 0 data = self._fileobj.read(n_requested) n = len(data) b[:n] = memoryview(data) return n def _is_unfinished_archive(path: Path) -> bool: """Whether the file at `path` exists but is not a zip archive.""" try: with path.open("rb") as f: # is_zipfile swallows OSError, so open the file here to let a # directory or a permission error surface as itself return not zipfile.is_zipfile(f) except FileNotFoundError: return False class ZipStore(Store): """ Store using a ZIP file. Parameters ---------- path : str, Path, or IO[bytes] Location of file, or an open binary file object. A file object must support `read`, `seek`, and `tell`; objects that are not `io.IOBase` instances (e.g. an `obstore` reader) are adapted automatically but can only be used for reading (`mode="r"`). The file object must stay open for the lifetime of the store, and operations that require a filesystem location (`clear`, `move`, pickling) are not supported. Using the store again after `close()` reopens the archive, which requires a file object that is readable and seekable; otherwise it raises `io.UnsupportedOperation`. mode : str, optional One of 'r' to read an existing file, 'w' to truncate and write a new file, 'a' to append to an existing file, or 'x' to exclusively create and write a new file. 'w' and 'x' apply to the first open only; the store reopens its archive with 'a' after `close()`, `move()`, or unpickling, so the entries it already wrote are kept. An unpickled store opens its archive on first use. If `close()` raises, the archive may be incomplete, and every later use that would reopen it raises `RuntimeError` instead; for a writable store backed by a path, `clear()` replaces the archive and makes the store usable again. compression : int, optional Compression method to use when writing to the archive. allowZip64 : bool, optional If True (the default) will create ZIP files that use the ZIP64 extensions when the zipfile is larger than 2 GiB. If False will raise an exception when the ZIP file would require ZIP64 extensions. Attributes ---------- allowed_exceptions supports_writes supports_deletes supports_listing path compression allowZip64 """ supports_writes: bool = True supports_deletes: bool = False supports_listing: bool = True path: Path | None compression: int allowZip64: bool _zf: zipfile.ZipFile _lock: threading.RLock _fileobj: IO[bytes] | None def __init__( self, path: Path | str | IO[bytes], *, mode: ZipStoreAccessModeLiteral = "r", read_only: bool | None = None, compression: int = zipfile.ZIP_STORED, allowZip64: bool = True, ) -> None: if read_only is None: read_only = mode == "r" super().__init__(read_only=read_only) if isinstance(path, str): path = Path(path) if isinstance(path, Path): self.path = path # root? self._fileobj = None else: self.path = None if not isinstance(path, io.IOBase): if not all( callable(getattr(path, attr, None)) for attr in ("read", "seek", "tell") ): raise TypeError( f"expected a path or an open binary file object supporting " f"read/seek/tell, got {type(path).__name__}" ) if mode != "r": raise TypeError( f"a file object that is not an io.IOBase instance can only be " f"opened for reading (mode='r', got mode={mode!r})" ) # e.g. an obstore ReadableFile: readable and seekable, but # not an io object and reads may not return bytes path = io.BufferedReader(_RawReaderAdapter(path)) self._fileobj = path self._zmode = mode self.compression = compression self.allowZip64 = allowZip64 self._lock = threading.RLock() self._was_opened = False self._close_failed = False def _sync_open(self) -> None: if self._is_open: raise ValueError("store is already open") if self._close_failed: # the central directory may be missing; appending to such a file # makes zipfile start a new archive and drop the earlier entries raise RuntimeError( f"closing the archive of {self!r} failed, so it may be incomplete; " "the store will not reopen it" ) if ( self.path is not None and self._was_opened and self._zmode == "a" and _is_unfinished_archive(self.path) ): # the file lacks a central directory, e.g. because another copy of # this store still has it open for writing; appending would start # a new archive and drop the entries already written raise zipfile.BadZipFile( f"{self!r} cannot reopen {self.path}: it is not a zip archive. It " "may be incomplete, for example because another store still has it " "open for writing." ) if ( self.path is None and self._fileobj is not None and self._was_opened and not (self._fileobj.readable() and self._fileobj.seekable()) ): # reopening appends, which needs to read the archive back; on a # write-only file zipfile would start a new archive and drop the # earlier entries raise io.UnsupportedOperation( "a ZipStore backed by a file object that is not readable and " "seekable cannot be used again after close(), because the " "archive it wrote cannot be read back" ) self._zf = zipfile.ZipFile( self.path if self.path is not None else self._fileobj, # type: ignore[arg-type] mode=self._zmode, compression=self.compression, allowZip64=self.allowZip64, ) # "w" truncates and "x" refuses an existing file. Both apply only to the # first open: reopening after close(), move(), or unpickling must keep # the entries already written. if self._zmode in ("w", "x"): self._zmode = "a" self._was_opened = True self._is_open = True def _zipfile(self) -> zipfile.ZipFile: """Return the archive, opening it on first use.""" with self._lock: if not self._is_open: self._sync_open() return self._zf async def _open(self) -> None: with self._lock: self._sync_open() async def _ensure_open(self) -> None: # the base class checks _is_open outside the lock self._zipfile() def __getstate__(self) -> dict[str, Any]: if self.path is None: # A path-backed store pickles its path and reopens the file on # unpickling; an open file object cannot be serialized that way. raise TypeError( "cannot pickle a ZipStore backed by a file-like object; " "construct the store from a path instead" ) # We need a copy to not modify the state of the original store. The # lock keeps it from catching _sync_open between opening with "w" and # switching to "a", which would make unpickling truncate the archive. with self._lock: state = self.__dict__.copy() for attr in ["_zf", "_lock"]: state.pop(attr, None) return state def __setstate__(self, state: dict[str, Any]) -> None: self.__dict__ = state self.__dict__.setdefault("_close_failed", False) self.__dict__.setdefault("_was_opened", False) self._lock = threading.RLock() # open on first use: a copy that is never used, like those dask makes # while building a graph, must not open the file, because zipfile # writes a central directory when it closes an archive it could not read self._is_open = False def close(self) -> None: # docstring inherited # hold the lock until the archive is closed: a thread that reopened it # before its central directory was written would lose the entries with self._lock: if not self._is_open: return try: self._zf.close() except BaseException: self._close_failed = True raise finally: super().close() async def clear(self) -> None: # docstring inherited with self._lock: self._check_writable() if self.path is None: raise NotImplementedError( "clear() is not supported for a ZipStore backed by a file-like object" ) if self._zmode == "x": # "x" lasts only until the first open; opening now keeps it from # deleting a file this store never claimed self._zipfile() self.close() # the file may be damaged or gone; clear() replaces it either way self.path.unlink(missing_ok=True) # replacing the file cannot drop entries, so clear() is the one way # to recover a store whose close() failed self._close_failed = False # if this open fails the store stays closed, and the next use # creates the archive again self._zmode = "w" self._sync_open() def __str__(self) -> str: if self.path is None: return f"zip://{self._fileobj!r}" return f"zip://{self.path}" def __repr__(self) -> str: return f"ZipStore('{self}')" def __eq__(self, other: object) -> bool: return ( isinstance(other, type(self)) and self.path == other.path and self._fileobj is other._fileobj ) def _get( self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited try: with self._zipfile().open(key) as f: # will raise KeyError if byte_range is None: return prototype.buffer.from_bytes(f.read()) elif isinstance(byte_range, RangeByteRequest): f.seek(byte_range.start) return prototype.buffer.from_bytes(f.read(byte_range.end - f.tell())) size = f.seek(0, os.SEEK_END) if isinstance(byte_range, OffsetByteRequest): f.seek(byte_range.offset) elif isinstance(byte_range, SuffixByteRequest): f.seek(max(0, size - byte_range.suffix)) else: raise TypeError(f"Unexpected byte_range, got {byte_range}.") return prototype.buffer.from_bytes(f.read()) except KeyError: return None async def get( self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None, ) -> Buffer | None: # docstring inherited with self._lock: return self._get(key, prototype=prototype, byte_range=byte_range) async def get_partial_values( self, prototype: BufferPrototype, key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: # docstring inherited out = [] with self._lock: for key, byte_range in key_ranges: out.append(self._get(key, prototype=prototype, byte_range=byte_range)) return out def _set(self, key: str, value: Buffer) -> None: # generally, this should be called inside a lock keyinfo = zipfile.ZipInfo(filename=key, date_time=time.localtime(time.time())[:6]) keyinfo.compress_type = self.compression if keyinfo.filename[-1] == os.sep: keyinfo.external_attr = 0o40775 << 16 # drwxrwxr-x keyinfo.external_attr |= 0x10 # MS-DOS directory flag else: keyinfo.external_attr = 0o644 << 16 # ?rw-r--r-- self._zipfile().writestr(keyinfo, value.to_bytes()) async def set(self, key: str, value: Buffer) -> None: # docstring inherited self._check_writable() if not isinstance(value, Buffer): raise TypeError( f"ZipStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." ) with self._lock: self._set(key, value) async def set_if_not_exists(self, key: str, value: Buffer) -> None: self._check_writable() with self._lock: members = self._zipfile().namelist() if key not in members: self._set(key, value) async def delete_dir(self, prefix: str) -> None: # only raise NotImplementedError if any keys are found self._check_writable() if prefix != "" and not prefix.endswith("/"): prefix += "/" async for _ in self.list_prefix(prefix): raise NotImplementedError async def delete(self, key: str) -> None: # docstring inherited # we choose to only raise NotImplementedError here if the key exists # this allows the array/group APIs to avoid the overhead of existence checks self._check_writable() if await self.exists(key): raise NotImplementedError async def exists(self, key: str) -> bool: # docstring inherited with self._lock: try: self._zipfile().getinfo(key) except KeyError: return False else: return True async def list(self) -> AsyncIterator[str]: # docstring inherited # namelist() is a copy; holding the lock across yield would block # other threads for as long as the caller iterates for key in self._zipfile().namelist(): yield key async def list_prefix(self, prefix: str) -> AsyncIterator[str]: # docstring inherited async for key in self.list(): if key.startswith(prefix): yield key async def list_dir(self, prefix: str) -> AsyncIterator[str]: # docstring inherited prefix = prefix.rstrip("/") keys = self._zipfile().namelist() seen = set() if prefix == "": keys_unique = {k.split("/")[0] for k in keys} for key in keys_unique: if key not in seen: seen.add(key) yield key else: for key in keys: if key.startswith(f"{prefix}/") and key.strip("/") != prefix: k = key.removeprefix(f"{prefix}/").split("/")[0] if k not in seen: seen.add(k) yield k async def move(self, path: Path | str) -> None: """ Move the store to another path. """ if self.path is None: raise NotImplementedError( "move() is not supported for a ZipStore backed by a file-like object" ) if isinstance(path, str): path = Path(path) # hold the lock so that no thread reopens the old path mid-move with self._lock: # opening first keeps mode "x" from moving a file it may not claim self._zipfile() self.close() os.makedirs(path.parent, exist_ok=True) shutil.move(self.path, path) self.path = path self._sync_open() zarr-developers-zarr-python-a2f2415/src/zarr/testing/000077500000000000000000000000001527014137600227075ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/src/zarr/testing/__init__.py000066400000000000000000000016721527014137600250260ustar00rootroot00000000000000import importlib.util import warnings from typing import TYPE_CHECKING from zarr.errors import ZarrUserWarning if importlib.util.find_spec("pytest") is not None: from zarr.testing.store import StoreTests from zarr.testing.utils import assert_bytes_equal else: warnings.warn( "pytest not installed, skipping test suite", category=ZarrUserWarning, stacklevel=2 ) if TYPE_CHECKING: import pytest def pytest_configure(config: "pytest.Config") -> None: # The tests in zarr.testing are intended to be run by downstream projects. # To allow those downstream projects to run with `--strict-markers`, we need # to register an entry point with pytest11 and register our "plugin" with it, # which just registers the markers used in zarr.testing config.addinivalue_line("markers", "gpu: mark a test as requiring CuPy and GPU") # TODO: import public buffer tests? __all__ = ["StoreTests", "assert_bytes_equal"] zarr-developers-zarr-python-a2f2415/src/zarr/testing/buffer.py000066400000000000000000000063251527014137600245400ustar00rootroot00000000000000# mypy: ignore-errors from __future__ import annotations from typing import TYPE_CHECKING, Any, Literal import numpy as np import numpy.typing as npt from zarr.core.buffer import Buffer, BufferPrototype, cpu from zarr.storage import MemoryStore if TYPE_CHECKING: from collections.abc import Iterable from typing import Self from zarr.abc.store import ByteRequest __all__ = [ "NDBufferUsingTestNDArrayLike", "StoreExpectingTestBuffer", "TestBuffer", ] class TestNDArrayLike(np.ndarray): """An example of an ndarray-like class""" __test__ = False class TestBuffer(cpu.Buffer): """Example of a custom Buffer that handles ArrayLike""" __test__ = False class NDBufferUsingTestNDArrayLike(cpu.NDBuffer): """Example of a custom NDBuffer that handles MyNDArrayLike""" @classmethod def create( cls, *, shape: Iterable[int], dtype: npt.DTypeLike, order: Literal["C", "F"] = "C", fill_value: Any | None = None, ) -> Self: """Overwrite `NDBuffer.create` to create a TestNDArrayLike instance""" ret = cls(TestNDArrayLike(shape=shape, dtype=dtype, order=order)) if fill_value is not None: ret.fill(fill_value) return ret @classmethod def empty( cls, shape: tuple[int, ...], dtype: npt.DTypeLike, order: Literal["C", "F"] = "C", ) -> Self: return super(cpu.NDBuffer, cls).empty(shape=shape, dtype=dtype, order=order) class StoreExpectingTestBuffer(MemoryStore): """Example of a custom Store that expect MyBuffer for all its non-metadata We assume that keys containing "json" is metadata """ async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None = None) -> None: if "json" not in key: assert isinstance(value, TestBuffer) await super().set(key, value, byte_range) def set_sync(self, key: str, value: Buffer) -> None: # Synchronous counterpart of `set`, used by FusedCodecPipeline. Mirror the # same buffer-type guard so the invariant holds whichever pipeline writes. if "json" not in key: assert isinstance(value, TestBuffer) super().set_sync(key, value) async def get( self, key: str, prototype: BufferPrototype, byte_range: tuple[int, int | None] | None = None, ) -> Buffer | None: if "json" not in key: assert prototype.buffer is TestBuffer ret = await super().get(key=key, prototype=prototype, byte_range=byte_range) if ret is not None: assert isinstance(ret, prototype.buffer) return ret def get_sync( self, key: str, *, prototype: BufferPrototype | None = None, byte_range: ByteRequest | None = None, ) -> Buffer | None: # Synchronous counterpart of `get`, used by FusedCodecPipeline. if "json" not in key and prototype is not None: assert prototype.buffer is TestBuffer ret = super().get_sync(key=key, prototype=prototype, byte_range=byte_range) if ret is not None and prototype is not None: assert isinstance(ret, prototype.buffer) return ret zarr-developers-zarr-python-a2f2415/src/zarr/testing/stateful.py000066400000000000000000000635611527014137600251230ustar00rootroot00000000000000import builtins import functools from collections.abc import Callable, Iterable from typing import Any, cast import hypothesis.extra.numpy as npst import hypothesis.strategies as st import numpy as np from hypothesis import assume, note from hypothesis.stateful import ( RuleBasedStateMachine, initialize, invariant, precondition, rule, ) from hypothesis.strategies import DataObject import zarr from zarr import Array from zarr.abc.store import ( OffsetByteRequest, RangeByteRequest, Store, SuffixByteRequest, ) from zarr.codecs.bytes import BytesCodec from zarr.core.buffer import Buffer, BufferPrototype, cpu, default_buffer_prototype from zarr.core.sync import SyncMixin from zarr.storage import LocalStore, MemoryStore from zarr.testing.strategies import ( arrays as zarr_arrays, ) from zarr.testing.strategies import ( basic_indices, chunk_paths, key_ranges, node_names, orthogonal_indices, ) from zarr.testing.strategies import keys as zarr_keys MAX_BINARY_SIZE = 100 def with_frequency[F: Callable[..., Any]](frequency: float) -> Callable[[F], F]: """This needs to be deterministic for hypothesis replaying""" def decorator(func: F) -> F: counter_attr = f"__{func.__name__}_counter" @functools.wraps(func) def wrapper(*args: Any, **kwargs: Any) -> Any: return func(*args, **kwargs) @precondition def frequency_check(f: Any) -> Any: if not hasattr(f, counter_attr): setattr(f, counter_attr, 0) current_count = getattr(f, counter_attr) + 1 setattr(f, counter_attr, current_count) return (current_count * frequency) % 1.0 >= (1.0 - frequency) return cast(F, frequency_check(wrapper)) return decorator def split_prefix_name(path: str) -> tuple[str, str]: split = path.rsplit("/", maxsplit=1) if len(split) > 1: prefix, name = split else: prefix = "" (name,) = split return prefix, name class ZarrHierarchyStateMachine(SyncMixin, RuleBasedStateMachine): """ This state machine models operations that modify a zarr store's hierarchy. That is, user actions that modify arrays/groups as well as list operations. It is intended to be used by external stores, and compares their results to a MemoryStore that is assumed to be perfect. """ def __init__(self, store: Store) -> None: super().__init__() self.store = store self.model = MemoryStore() zarr.group(store=self.model) # Track state of the hierarchy, these should contain fully qualified paths self.all_groups: set[str] = set() self.all_arrays: set[str] = set() @initialize() def init_store(self) -> None: # This lets us reuse the fixture provided store. self._sync(self.store.clear()) zarr.group(store=self.store) def can_add(self, path: str) -> bool: return path not in self.all_groups and path not in self.all_arrays # -------------------- store operations ----------------------- @rule(name=node_names, data=st.data()) def add_group(self, name: str, data: DataObject) -> None: # Handle possible case-insensitive file systems (e.g. MacOS) if isinstance(self.store, LocalStore): name = name.lower() if self.all_groups: parent = data.draw(st.sampled_from(sorted(self.all_groups)), label="Group parent") else: parent = "" path = f"{parent}/{name}".lstrip("/") assume(self.can_add(path)) note(f"Adding group: path='{path}'") self.all_groups.add(path) zarr.group(store=self.store, path=path) zarr.group(store=self.model, path=path) @rule(data=st.data(), name=node_names) def add_array(self, data: DataObject, name: str) -> None: # Handle possible case-insensitive file systems (e.g. MacOS) if isinstance(self.store, LocalStore): name = name.lower() if self.all_groups: parent = data.draw(st.sampled_from(sorted(self.all_groups)), label="Array parent") else: parent = "" # TODO: support creating deeper paths # TODO: support overwriting potentially by just skipping `self.can_add` path = f"{parent}/{name}".lstrip("/") assume(self.can_add(path)) # Generate array on the model store using the arrays strategy a = data.draw( zarr_arrays( stores=st.just(self.model), paths=st.just(parent), array_names=st.just(name), zarr_formats=st.just(3), compressors=st.just(BytesCodec()), open_mode="a", ), label="generated array", ) note(f"Adding array: path='{path}' shape={a.shape} chunks={a.metadata.chunk_grid}") # Recreate the same array, including its data, in the store under test. zarr.from_array( self.store, data=a, name=path, ) self.all_arrays.add(path) @rule() @with_frequency(0.25) def clear(self) -> None: note("clearing") import zarr self._sync(self.store.clear()) self._sync(self.model.clear()) assert self._sync(self.store.is_empty("/")) assert self._sync(self.model.is_empty("/")) self.all_groups.clear() self.all_arrays.clear() zarr.group(store=self.store) zarr.group(store=self.model) # TODO: MemoryStore is broken? # assert not self._sync(self.store.is_empty("/")) # assert not self._sync(self.model.is_empty("/")) def draw_directory(self, data: DataObject) -> str: group_st = st.sampled_from(sorted(self.all_groups)) if self.all_groups else st.nothing() array_st = st.sampled_from(sorted(self.all_arrays)) if self.all_arrays else st.nothing() array_or_group = data.draw(st.one_of(group_st, array_st)) if data.draw(st.booleans()) and array_or_group in self.all_arrays: arr = zarr.open_array(path=array_or_group, store=self.model) path = data.draw( st.one_of( st.sampled_from([array_or_group]), chunk_paths(ndim=arr.ndim, numblocks=arr.cdata_shape).map( lambda x: f"{array_or_group}/c/" ), ) ) else: path = array_or_group return path @precondition(lambda self: bool(self.all_groups)) @rule(data=st.data()) def check_list_dir(self, data: DataObject) -> None: path = self.draw_directory(data) note(f"list_dir for {path=!r}") # Consider .list_dir("path/to/array") for an array with a single chunk. # The MemoryStore model will return `"c", "zarr.json"` only if the chunk exists # If that chunk was deleted, then `"c"` is not returned. # LocalStore will not have this behaviour :/ # There are similar consistency issues with delete_dir("/path/to/array/c/0/0") assume(not isinstance(self.store, LocalStore)) model_ls = sorted(self._sync_iter(self.model.list_dir(path))) store_ls = sorted(self._sync_iter(self.store.list_dir(path))) assert model_ls == store_ls, (model_ls, store_ls) @precondition(lambda self: bool(self.all_arrays)) @rule(data=st.data()) def delete_chunk(self, data: DataObject) -> None: array = data.draw(st.sampled_from(sorted(self.all_arrays))) arr = zarr.open_array(path=array, store=self.model) chunk_path = data.draw(chunk_paths(ndim=arr.ndim, numblocks=arr.cdata_shape, subset=False)) path = f"{array}/c/{chunk_path}" note(f"deleting chunk {path=!r}") self._sync(self.model.delete(path)) self._sync(self.store.delete(path)) @precondition(lambda self: bool(self.all_arrays)) @rule(data=st.data()) def check_array(self, data: DataObject) -> None: path = data.draw(st.sampled_from(sorted(self.all_arrays))) actual = zarr.open_array(self.store, path=path)[:] expected = zarr.open_array(self.model, path=path)[:] np.testing.assert_equal(actual, expected) @precondition(lambda self: bool(self.all_arrays)) @rule(data=st.data()) def overwrite_array_basic_indexing(self, data: DataObject) -> None: array = data.draw(st.sampled_from(sorted(self.all_arrays))) model_array = zarr.open_array(path=array, store=self.model) store_array = zarr.open_array(path=array, store=self.store) slicer = data.draw(basic_indices(shape=model_array.shape)) note(f"overwriting array with basic indexer: {slicer=}") new_data = data.draw( npst.arrays(shape=np.shape(model_array[slicer]), dtype=model_array.dtype) ) model_array[slicer] = new_data store_array[slicer] = new_data @precondition(lambda self: bool(self.all_arrays)) @rule(data=st.data()) def overwrite_array_orthogonal_indexing(self, data: DataObject) -> None: array = data.draw(st.sampled_from(sorted(self.all_arrays))) model_array = zarr.open_array(path=array, store=self.model) store_array = zarr.open_array(path=array, store=self.store) indexer, _ = data.draw(orthogonal_indices(shape=model_array.shape)) note(f"overwriting array orthogonal {indexer=}") new_data = data.draw( npst.arrays(shape=model_array.oindex[indexer].shape, dtype=model_array.dtype) # type: ignore[union-attr] ) model_array.oindex[indexer] = new_data store_array.oindex[indexer] = new_data @precondition(lambda self: bool(self.all_arrays)) @rule(data=st.data()) def resize_array(self, data: DataObject) -> None: array = data.draw(st.sampled_from(sorted(self.all_arrays))) model_array = zarr.open_array(path=array, store=self.model) store_array = zarr.open_array(path=array, store=self.store) ndim = model_array.ndim new_shape = tuple( 0 if oldsize == 0 else newsize for newsize, oldsize in zip( data.draw(npst.array_shapes(max_dims=ndim, min_dims=ndim, min_side=0)), model_array.shape, strict=True, ) ) note(f"resizing array from {model_array.shape} to {new_shape}") model_array.resize(new_shape) store_array.resize(new_shape) @precondition(lambda self: bool(self.all_arrays) or bool(self.all_groups)) @rule(data=st.data()) def delete_dir(self, data: DataObject) -> None: path = self.draw_directory(data) note(f"delete_dir with {path=!r}") self._sync(self.model.delete_dir(path)) self._sync(self.store.delete_dir(path)) matches = set() for node in self.all_groups | self.all_arrays: if node == path or node.startswith(path + "/"): matches.add(node) self.all_groups = self.all_groups - matches self.all_arrays = self.all_arrays - matches # @precondition(lambda self: bool(self.all_groups)) # @precondition(lambda self: bool(self.all_arrays)) # @rule(data=st.data()) # def move_array(self, data): # array_path = data.draw(st.sampled_from(self.all_arrays), label="Array move source") # to_group = data.draw(st.sampled_from(self.all_groups), label="Array move destination") # # fixme renaming to self? # array_name = os.path.basename(array_path) # assume(self.model.can_add(to_group, array_name)) # new_path = f"{to_group}/{array_name}".lstrip("/") # note(f"moving array '{array_path}' -> '{new_path}'") # self.model.rename(array_path, new_path) # self.repo.store.rename(array_path, new_path) # @precondition(lambda self: len(self.all_groups) >= 2) # @rule(data=st.data()) # def move_group(self, data): # from_group = data.draw(st.sampled_from(self.all_groups), label="Group move source") # to_group = data.draw(st.sampled_from(self.all_groups), label="Group move destination") # assume(not to_group.startswith(from_group)) # from_group_name = os.path.basename(from_group) # assume(self.model.can_add(to_group, from_group_name)) # # fixme renaming to self? # new_path = f"{to_group}/{from_group_name}".lstrip("/") # note(f"moving group '{from_group}' -> '{new_path}'") # self.model.rename(from_group, new_path) # self.repo.store.rename(from_group, new_path) @precondition(lambda self: self.store.supports_deletes) @precondition(lambda self: len(self.all_arrays) >= 1) @rule(data=st.data()) def delete_array_using_del(self, data: DataObject) -> None: array_path = data.draw( st.sampled_from(sorted(self.all_arrays)), label="Array deletion target" ) prefix, array_name = split_prefix_name(array_path) note(f"Deleting array '{array_path}' ({prefix=!r}, {array_name=!r}) using del") for store in [self.model, self.store]: group = zarr.open_group(path=prefix, store=store) group[array_name] # check that it exists del group[array_name] self.all_arrays.remove(array_path) @precondition(lambda self: self.store.supports_deletes) @precondition(lambda self: bool(self.all_groups)) @rule(data=st.data()) def delete_group_using_del(self, data: DataObject) -> None: group_path = data.draw( st.sampled_from(sorted(self.all_groups)), label="Group deletion target", ) prefix, group_name = split_prefix_name(group_path) note(f"Deleting group '{group_path=!r}', {prefix=!r}, {group_name=!r} using delete") members = zarr.open_group(store=self.model, path=group_path).members(max_depth=None) for _, obj in members: if isinstance(obj, Array): self.all_arrays.remove(obj.path) else: self.all_groups.remove(obj.path) for store in [self.store, self.model]: group = zarr.open_group(store=store, path=prefix) group[group_name] # check that it exists del group[group_name] self.all_groups.remove(group_path) # # --------------- assertions ----------------- # def check_group_arrays(self, group): # # note(f"Checking arrays of '{group}'") # g1 = self.model.get_group(group) # g2 = zarr.open_group(path=group, mode="r", store=self.repo.store) # model_arrays = sorted(g1.arrays(), key=itemgetter(0)) # our_arrays = sorted(g2.arrays(), key=itemgetter(0)) # for (n1, a1), (n2, a2) in zip_longest(model_arrays, our_arrays): # assert n1 == n2 # assert_array_equal(a1, a2) # def check_subgroups(self, group_path): # g1 = self.model.get_group(group_path) # g2 = zarr.open_group(path=group_path, mode="r", store=self.repo.store) # g1_children = [name for (name, _) in g1.groups()] # g2_children = [name for (name, _) in g2.groups()] # # note(f"Checking {len(g1_children)} subgroups of group '{group_path}'") # assert g1_children == g2_children # def check_list_prefix_from_group(self, group): # prefix = f"meta/root/{group}" # model_list = sorted(self.model.list_prefix(prefix)) # al_list = sorted(self.repo.store.list_prefix(prefix)) # # note(f"Checking {len(model_list)} keys under '{prefix}'") # assert model_list == al_list # prefix = f"data/root/{group}" # model_list = sorted(self.model.list_prefix(prefix)) # al_list = sorted(self.repo.store.list_prefix(prefix)) # # note(f"Checking {len(model_list)} keys under '{prefix}'") # assert model_list == al_list # @precondition(lambda self: self.model.is_persistent_session()) # @rule(data=st.data()) # def check_group_path(self, data): # t0 = time.time() # group = data.draw(st.sampled_from(self.all_groups)) # self.check_list_prefix_from_group(group) # self.check_subgroups(group) # self.check_group_arrays(group) # t1 = time.time() # note(f"Checks took {t1 - t0} sec.") @invariant() def check_list_prefix_from_root(self) -> None: model_list = self._sync_iter(self.model.list_prefix("")) store_list = self._sync_iter(self.store.list_prefix("")) note(f"Checking {len(model_list)} expected keys vs {len(store_list)} actual keys") assert sorted(model_list) == sorted(store_list), ( sorted(model_list), sorted(store_list), ) # check that our internal state matches that of the store and model assert all(f"{path}/zarr.json" in model_list for path in self.all_groups | self.all_arrays) assert all(f"{path}/zarr.json" in store_list for path in self.all_groups | self.all_arrays) class SyncStoreWrapper(zarr.core.sync.SyncMixin): def __init__(self, store: Store) -> None: """Synchronous Store wrapper This class holds synchronous methods that map to async methods of Store classes. The synchronous wrapper is needed because hypothesis' stateful testing infra does not support asyncio so we redefine sync versions of the Store API. https://github.com/HypothesisWorks/hypothesis/issues/3712#issuecomment-1668999041 """ self.store = store @property def read_only(self) -> bool: return self.store.read_only def set(self, key: str, data_buffer: Buffer) -> None: return self._sync(self.store.set(key, data_buffer)) def list(self) -> builtins.list[str]: return self._sync_iter(self.store.list()) def get(self, key: str, prototype: BufferPrototype) -> Buffer | None: return self._sync(self.store.get(key, prototype=prototype)) def get_partial_values( self, key_ranges: Iterable[Any], prototype: BufferPrototype ) -> builtins.list[Buffer | None]: return self._sync(self.store.get_partial_values(prototype=prototype, key_ranges=key_ranges)) def delete(self, path: str) -> None: return self._sync(self.store.delete(path)) def is_empty(self, prefix: str) -> bool: return self._sync(self.store.is_empty(prefix=prefix)) def clear(self) -> None: return self._sync(self.store.clear()) def exists(self, key: str) -> bool: return self._sync(self.store.exists(key)) def getsize_prefix(self, prefix: str) -> int: return self._sync(self.store.getsize_prefix(prefix)) def list_dir(self, prefix: str) -> None: raise NotImplementedError def list_prefix(self, prefix: str) -> None: raise NotImplementedError @property def supports_listing(self) -> bool: return self.store.supports_listing @property def supports_writes(self) -> bool: return self.store.supports_writes @property def supports_deletes(self) -> bool: return self.store.supports_deletes class ZarrStoreStateMachine(RuleBasedStateMachine): """ " Zarr store state machine This is a subclass of a Hypothesis RuleBasedStateMachine. It is testing a framework to ensure that the state of a Zarr store matches an expected state after a set of random operations. It contains a store (currently, a Zarr MemoryStore) and a model, a simplified version of a zarr store (in this case, a dict). It also contains rules which represent actions that can be applied to a zarr store. Rules apply an action to both the store and the model, and invariants assert that the state of the model is equal to the state of the store. Hypothesis then generates sequences of rules, running invariants after each rule. It raises an error if a sequence produces discontinuity between state of the model and state of the store (ie. an invariant is violated). https://hypothesis.readthedocs.io/en/latest/stateful.html """ def __init__(self, store: Store) -> None: super().__init__() self.model: dict[str, Buffer] = {} self.store = SyncStoreWrapper(store) self.prototype = default_buffer_prototype() @initialize() def init_store(self) -> None: self.store.clear() @rule(key=zarr_keys(), data=st.binary(min_size=0, max_size=MAX_BINARY_SIZE)) def set(self, key: str, data: bytes) -> None: note(f"(set) Setting {key!r} with {data!r}") assert not self.store.read_only data_buf = cpu.Buffer.from_bytes(data) self.store.set(key, data_buf) self.model[key] = data_buf @precondition(lambda self: len(self.model.keys()) > 0) @rule(key=zarr_keys(), data=st.data()) def get(self, key: str, data: DataObject) -> None: key = data.draw( st.sampled_from(sorted(self.model.keys())) ) # hypothesis wants to sample from sorted list note("(get)") store_value = self.store.get(key, self.prototype) # to bytes here necessary because data_buf set to model in set() assert self.model[key] == store_value @rule(key=zarr_keys(), data=st.data()) def get_invalid_zarr_keys(self, key: str, data: DataObject) -> None: note("(get_invalid)") assume(key not in self.model) assert self.store.get(key, self.prototype) is None @precondition(lambda self: len(self.model.keys()) > 0) @rule(data=st.data()) def get_partial_values(self, data: DataObject) -> None: key_range = data.draw( key_ranges(keys=st.sampled_from(sorted(self.model.keys())), max_size=MAX_BINARY_SIZE) ) note(f"(get partial) {key_range=}") # Pass a one-shot generator rather than a list: stores (and wrappers such # as LoggingStore) must not exhaust the iterable before using it. obs_maybe = self.store.get_partial_values((kr for kr in key_range), self.prototype) observed = [] for obs in obs_maybe: assert obs is not None observed.append(obs.to_bytes()) model_vals_ls = [] for key, byte_range in key_range: # Independently model each ByteRequest variant (do NOT reuse the # store's _normalize_byte_range_index helper, so this stays an # independent oracle). Bounds may exceed the value length. value = self.model[key] n = len(value) if byte_range is None: expected = value[:] elif isinstance(byte_range, RangeByteRequest): expected = value[byte_range.start : byte_range.end] elif isinstance(byte_range, OffsetByteRequest): expected = value[byte_range.offset :] elif isinstance(byte_range, SuffixByteRequest): # "last suffix bytes"; suffix > n means the whole value. expected = value[max(0, n - byte_range.suffix) :] else: raise AssertionError(f"unexpected byte_range {byte_range!r}") model_vals_ls.append(expected) assert all( obs == exp.to_bytes() for obs, exp in zip(observed, model_vals_ls, strict=True) ), ( observed, model_vals_ls, ) @precondition(lambda self: self.store.supports_deletes) @precondition(lambda self: len(self.model.keys()) > 0) @rule(data=st.data()) def delete(self, data: DataObject) -> None: key = data.draw(st.sampled_from(sorted(self.model.keys()))) note(f"(delete) Deleting {key=}") self.store.delete(key) del self.model[key] @rule() def clear(self) -> None: assert not self.store.read_only note("(clear)") self.store.clear() self.model.clear() assert self.store.is_empty("") assert len(self.model.keys()) == len(list(self.store.list())) == 0 @rule() # Local store can be non-empty when there are subdirectories but no files @precondition(lambda self: not isinstance(self.store.store, LocalStore)) def is_empty(self) -> None: note("(is_empty)") # make sure they either both are or both aren't empty (same state) assert self.store.is_empty("") == (not self.model) @rule(key=zarr_keys()) def exists(self, key: str) -> None: note("(exists)") assert self.store.exists(key) == (key in self.model) @precondition(lambda self: len(self.model.keys()) > 0) @rule(data=st.data()) def getsize_prefix(self, data: DataObject) -> None: # Measure the size under the first path segment of some existing key. # getsize_prefix(node) must count only keys under the directory "node/", # not sibling keys that merely share the string prefix (e.g. measuring # "a" must not include a sibling key "ab/..."). key = data.draw(st.sampled_from(sorted(self.model.keys()))) node = key.split("/")[0] note(f"(getsize_prefix) {node=}") observed = self.store.getsize_prefix(node) expected = sum(len(value) for k, value in self.model.items() if k.startswith(node + "/")) assert observed == expected, (observed, expected, node) @invariant() def check_paths_equal(self) -> None: note("Checking that paths are equal") paths = sorted(self.store.list()) assert sorted(self.model.keys()) == paths @invariant() def check_vals_equal(self) -> None: note("Checking values equal") for key, val in self.model.items(): store_item = self.store.get(key, self.prototype) assert val == store_item @invariant() def check_num_zarr_keys_equal(self) -> None: note("check num zarr_keys equal") assert len(self.model) == len(list(self.store.list())) @invariant() def check_zarr_keys(self) -> None: keys = list(self.store.list()) if not keys: assert self.store.is_empty("") is True else: assert self.store.is_empty("") is False for key in keys: assert self.store.exists(key) is True note("checking keys / exists / empty") zarr-developers-zarr-python-a2f2415/src/zarr/testing/store.py000066400000000000000000001013471527014137600244230ustar00rootroot00000000000000from __future__ import annotations import asyncio import pickle import time from abc import abstractmethod from typing import TYPE_CHECKING, Self import numpy as np from zarr.storage import WrapperStore if TYPE_CHECKING: from collections.abc import AsyncIterator, Iterable, Sequence from typing import Any from zarr.core.buffer.core import BufferPrototype import pytest from zarr.abc.store import ( ByteRequest, OffsetByteRequest, RangeByteRequest, Store, SuffixByteRequest, SupportsDeleteSync, SupportsGetSync, SupportsSetSync, ) from zarr.core.buffer import Buffer, default_buffer_prototype from zarr.core.sync import _collect_aiterator, sync from zarr.storage._utils import _normalize_byte_range_index from zarr.testing.utils import assert_bytes_equal __all__ = ["StoreTests"] class StoreTests[S: Store, B: Buffer]: store_cls: type[S] buffer_cls: type[B] @staticmethod def _require_get_sync(store: S) -> SupportsGetSync: """Skip unless *store* implements [`SupportsGetSync`][zarr.abc.store.SupportsGetSync].""" if not isinstance(store, SupportsGetSync): pytest.skip("store does not implement SupportsGetSync") return store # type: ignore[unreachable] @staticmethod def _require_set_sync(store: S) -> SupportsSetSync: """Skip unless *store* implements [`SupportsSetSync`][zarr.abc.store.SupportsSetSync].""" if not isinstance(store, SupportsSetSync): pytest.skip("store does not implement SupportsSetSync") return store # type: ignore[unreachable] @staticmethod def _require_delete_sync(store: S) -> SupportsDeleteSync: """Skip unless *store* implements [`SupportsDeleteSync`][zarr.abc.store.SupportsDeleteSync].""" if not isinstance(store, SupportsDeleteSync): pytest.skip("store does not implement SupportsDeleteSync") return store # type: ignore[unreachable] @abstractmethod async def set(self, store: S, key: str, value: Buffer) -> None: """ Insert a value into a storage backend, with a specific key. This should not use any store methods. Bypassing the store methods allows them to be tested. """ ... @abstractmethod async def get(self, store: S, key: str) -> Buffer: """ Retrieve a value from a storage backend, by key. This should not use any store methods. Bypassing the store methods allows them to be tested. """ ... @abstractmethod @pytest.fixture def store_kwargs(self, *args: Any, **kwargs: Any) -> dict[str, Any]: """Kwargs for instantiating a store""" ... @abstractmethod def test_store_repr(self, store: S) -> None: ... @abstractmethod def test_store_supports_writes(self, store: S) -> None: ... def test_store_supports_partial_writes(self, store: S) -> None: assert not store.supports_partial_writes @abstractmethod def test_store_supports_listing(self, store: S) -> None: ... @pytest.fixture def open_kwargs(self, store_kwargs: dict[str, Any]) -> dict[str, Any]: return store_kwargs @pytest.fixture async def store(self, open_kwargs: dict[str, Any]) -> Store: return await self.store_cls.open(**open_kwargs) @pytest.fixture async def store_not_open(self, store_kwargs: dict[str, Any]) -> Store: return self.store_cls(**store_kwargs) def test_store_type(self, store: S) -> None: assert isinstance(store, Store) assert isinstance(store, self.store_cls) def test_store_eq(self, store: S, store_kwargs: dict[str, Any]) -> None: # check self equality assert store == store # noqa: PLR0124 -- self-equality is the property under test # check store equality with same inputs # asserting this is important for being able to compare (de)serialized stores store2 = self.store_cls(**store_kwargs) assert store == store2 async def test_serializable_store(self, store: S) -> None: new_store: S = pickle.loads(pickle.dumps(store)) assert new_store == store assert new_store.read_only == store.read_only # quickly roundtrip data to a key to test that new store works data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") key = "foo" await store.set(key, data_buf) observed = await store.get(key, prototype=default_buffer_prototype()) assert_bytes_equal(observed, data_buf) def test_store_read_only(self, store: S) -> None: assert not store.read_only with pytest.raises(AttributeError): store.read_only = False # type: ignore[misc] @pytest.mark.parametrize("read_only", [True, False]) async def test_store_open_read_only(self, open_kwargs: dict[str, Any], read_only: bool) -> None: open_kwargs["read_only"] = read_only store = await self.store_cls.open(**open_kwargs) assert store._is_open assert store.read_only == read_only async def test_store_context_manager(self, open_kwargs: dict[str, Any]) -> None: # Test that the context manager closes the store with await self.store_cls.open(**open_kwargs) as store: assert store._is_open # Test trying to open an already open store with pytest.raises(ValueError, match="store is already open"): await store._open() assert not store._is_open async def test_read_only_store_raises(self, open_kwargs: dict[str, Any]) -> None: kwargs = {**open_kwargs, "read_only": True} store = await self.store_cls.open(**kwargs) assert store.read_only # set with pytest.raises( ValueError, match="store was opened in read-only mode and does not support writing" ): await store.set("foo", self.buffer_cls.from_bytes(b"bar")) # delete with pytest.raises( ValueError, match="store was opened in read-only mode and does not support writing" ): await store.delete("foo") async def test_with_read_only_store(self, open_kwargs: dict[str, Any]) -> None: kwargs = {**open_kwargs, "read_only": True} store = await self.store_cls.open(**kwargs) assert store.read_only # Test that you cannot write to a read-only store with pytest.raises( ValueError, match="store was opened in read-only mode and does not support writing" ): await store.set("foo", self.buffer_cls.from_bytes(b"bar")) # Check if the store implements with_read_only try: writer = store.with_read_only(read_only=False) except NotImplementedError: # Test that stores that do not implement with_read_only raise NotImplementedError with the correct message with pytest.raises( NotImplementedError, match=f"with_read_only is not implemented for the {type(store)} store type.", ): store.with_read_only(read_only=False) return # Test that you can write to a new store copy assert not writer._is_open assert not writer.read_only await writer.set("foo", self.buffer_cls.from_bytes(b"bar")) await writer.delete("foo") # Test that you cannot write to the original store assert store.read_only with pytest.raises( ValueError, match="store was opened in read-only mode and does not support writing" ): await store.set("foo", self.buffer_cls.from_bytes(b"bar")) with pytest.raises( ValueError, match="store was opened in read-only mode and does not support writing" ): await store.delete("foo") # Test that you cannot write to a read-only store copy reader = store.with_read_only(read_only=True) assert reader.read_only with pytest.raises( ValueError, match="store was opened in read-only mode and does not support writing" ): await reader.set("foo", self.buffer_cls.from_bytes(b"bar")) with pytest.raises( ValueError, match="store was opened in read-only mode and does not support writing" ): await reader.delete("foo") @pytest.mark.parametrize("key", ["c/0", "foo/c/0.0", "foo/0/0"]) @pytest.mark.parametrize( ("data", "byte_range"), [ (b"\x01\x02\x03\x04", None), (b"\x01\x02\x03\x04", RangeByteRequest(1, 4)), (b"\x01\x02\x03\x04", OffsetByteRequest(1)), (b"\x01\x02\x03\x04", SuffixByteRequest(1)), (b"", None), ], ) async def test_get(self, store: S, key: str, data: bytes, byte_range: ByteRequest) -> None: """ Ensure that data can be read from the store using the store.get method. """ data_buf = self.buffer_cls.from_bytes(data) await self.set(store, key, data_buf) observed = await store.get(key, prototype=default_buffer_prototype(), byte_range=byte_range) start, stop = _normalize_byte_range_index(data_buf, byte_range=byte_range) expected = data_buf[start:stop] assert_bytes_equal(observed, expected) async def test_get_not_open(self, store_not_open: S) -> None: """ Ensure that data can be read from the store that isn't yet open using the store.get method. """ assert not store_not_open._is_open data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") key = "c/0" await self.set(store_not_open, key, data_buf) observed = await store_not_open.get(key, prototype=default_buffer_prototype()) assert_bytes_equal(observed, data_buf) async def test_get_raises(self, store: S) -> None: """ Ensure that a ValueError is raise for invalid byte range syntax """ data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") await self.set(store, "c/0", data_buf) with pytest.raises((ValueError, TypeError), match=r"Unexpected byte_range, got.*"): await store.get("c/0", prototype=default_buffer_prototype(), byte_range=(0, 2)) # type: ignore[arg-type] async def test_get_many(self, store: S) -> None: """ Ensure that multiple keys can be retrieved at once with the _get_many method. """ keys = tuple(map(str, range(10))) values = tuple(f"{k}".encode() for k in keys) for k, v in zip(keys, values, strict=False): await self.set(store, k, self.buffer_cls.from_bytes(v)) observed_buffers = await _collect_aiterator( store._get_many( zip( keys, (default_buffer_prototype(),) * len(keys), (None,) * len(keys), strict=False, ) ) ) observed_kvs = sorted(((k, b.to_bytes()) for k, b in observed_buffers)) # type: ignore[union-attr] expected_kvs = sorted(((k, b) for k, b in zip(keys, values, strict=False))) assert observed_kvs == expected_kvs @pytest.mark.parametrize("key", ["c/0", "foo/c/0.0", "foo/0/0"]) @pytest.mark.parametrize("data", [b"\x01\x02\x03\x04", b""]) async def test_getsize(self, store: S, key: str, data: bytes) -> None: """ Test the result of store.getsize(). """ data_buf = self.buffer_cls.from_bytes(data) expected = len(data_buf) await self.set(store, key, data_buf) observed = await store.getsize(key) assert observed == expected async def test_getsize_prefix(self, store: S) -> None: """ Test the result of store.getsize_prefix(). Includes a sibling key ("cc/0") that shares the string prefix "c" but belongs to a different directory: getsize_prefix("c") must not count it, i.e. the prefix is matched as a directory ("c/...") not a raw substring. """ data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") keys = ["c/0/0", "c/0/1", "c/1/0", "c/1/1"] # Sibling directory sharing the "c" string prefix; must be excluded. sibling_keys = ["cc/0"] keys_values = [(k, data_buf) for k in keys + sibling_keys] await store._set_many(keys_values) expected = len(data_buf) * len(keys) observed = await store.getsize_prefix("c") assert observed == expected async def test_getsize_raises(self, store: S) -> None: """ Test that getsize() raise a FileNotFoundError if the key doesn't exist. """ with pytest.raises(FileNotFoundError): await store.getsize("c/1000") @pytest.mark.parametrize("key", ["zarr.json", "c/0", "foo/c/0.0", "foo/0/0"]) @pytest.mark.parametrize("data", [b"\x01\x02\x03\x04", b""]) async def test_set(self, store: S, key: str, data: bytes) -> None: """ Ensure that data can be written to the store using the store.set method. """ assert not store.read_only data_buf = self.buffer_cls.from_bytes(data) await store.set(key, data_buf) observed = await self.get(store, key) assert_bytes_equal(observed, data_buf) async def test_set_not_open(self, store_not_open: S) -> None: """ Ensure that data can be written to the store that's not yet open using the store.set method. """ assert not store_not_open._is_open data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") key = "c/0" await store_not_open.set(key, data_buf) observed = await self.get(store_not_open, key) assert_bytes_equal(observed, data_buf) async def test_set_many(self, store: S) -> None: """ Test that a dict of key : value pairs can be inserted into the store via the `_set_many` method. """ keys = ["zarr.json", "c/0", "foo/c/0.0", "foo/0/0"] data_buf = [self.buffer_cls.from_bytes(k.encode()) for k in keys] store_dict = dict(zip(keys, data_buf, strict=True)) await store._set_many(store_dict.items()) for k, v in store_dict.items(): assert (await self.get(store, k)).to_bytes() == v.to_bytes() @pytest.mark.parametrize( "key_ranges", [ [], [("zarr.json", RangeByteRequest(0, 2))], [("c/0", RangeByteRequest(0, 2)), ("zarr.json", None)], [ ("c/0/0", RangeByteRequest(0, 2)), ("c/0/1", SuffixByteRequest(2)), ("c/0/2", OffsetByteRequest(2)), ], ], ) async def test_get_partial_values( self, store: S, key_ranges: list[tuple[str, ByteRequest]] ) -> None: # put all of the data for key, _ in key_ranges: await self.set(store, key, self.buffer_cls.from_bytes(bytes(key, encoding="utf-8"))) # read back just part of it. Pass key_ranges as a one-shot generator # (a valid Iterable per the method signature) to ensure stores and # wrappers do not exhaust the iterable before handing it to the backend. observed_maybe = await store.get_partial_values( prototype=default_buffer_prototype(), key_ranges=(kr for kr in key_ranges), ) # One result must be returned per requested key range. Checking this # explicitly guards against a store/wrapper exhausting the key_ranges # iterable early and silently returning fewer (or no) results. assert len(observed_maybe) == len(key_ranges) observed: list[Buffer] = [] expected: list[Buffer] = [] for obs in observed_maybe: assert obs is not None observed.append(obs) for key, byte_range in key_ranges: result = await store.get( key, prototype=default_buffer_prototype(), byte_range=byte_range ) assert result is not None expected.append(result) assert all( obs.to_bytes() == exp.to_bytes() for obs, exp in zip(observed, expected, strict=True) ) async def test_exists(self, store: S) -> None: assert not await store.exists("foo") await store.set("foo/zarr.json", self.buffer_cls.from_bytes(b"bar")) assert await store.exists("foo/zarr.json") async def test_delete(self, store: S) -> None: if not store.supports_deletes: pytest.skip("store does not support deletes") await store.set("foo/zarr.json", self.buffer_cls.from_bytes(b"bar")) assert await store.exists("foo/zarr.json") await store.delete("foo/zarr.json") assert not await store.exists("foo/zarr.json") async def test_delete_dir(self, store: S) -> None: if not store.supports_deletes: pytest.skip("store does not support deletes") await store.set("zarr.json", self.buffer_cls.from_bytes(b"root")) await store.set("foo-bar/zarr.json", self.buffer_cls.from_bytes(b"root")) await store.set("foo/zarr.json", self.buffer_cls.from_bytes(b"bar")) await store.set("foo/c/0", self.buffer_cls.from_bytes(b"chunk")) await store.delete_dir("foo") assert await store.exists("zarr.json") assert await store.exists("foo-bar/zarr.json") assert not await store.exists("foo/zarr.json") assert not await store.exists("foo/c/0") async def test_delete_nonexistent_key_does_not_raise(self, store: S) -> None: if not store.supports_deletes: pytest.skip("store does not support deletes") await store.delete("nonexistent_key") async def test_is_empty(self, store: S) -> None: assert await store.is_empty("") await self.set( store, "foo/bar", self.buffer_cls.from_bytes(bytes("something", encoding="utf-8")) ) assert not await store.is_empty("") assert await store.is_empty("fo") assert not await store.is_empty("foo/") assert not await store.is_empty("foo") assert await store.is_empty("spam/") async def test_clear(self, store: S) -> None: await self.set( store, "key", self.buffer_cls.from_bytes(bytes("something", encoding="utf-8")) ) await store.clear() assert await store.is_empty("") async def test_list(self, store: S) -> None: assert await _collect_aiterator(store.list()) == () prefix = "foo" data = self.buffer_cls.from_bytes(b"") store_dict = { f"{prefix}/zarr.json": data, **{f"{prefix}/c/{idx}": data for idx in range(10)}, } await store._set_many(store_dict.items()) expected_sorted = sorted(store_dict.keys()) observed = await _collect_aiterator(store.list()) observed_sorted = sorted(observed) assert observed_sorted == expected_sorted async def test_list_prefix(self, store: S) -> None: """ Test that the `list_prefix` method works as intended. Given a prefix, it should return all the keys in storage that start with this prefix. """ prefixes = ("", "a/", "a/b/", "a/b/c/") data = self.buffer_cls.from_bytes(b"") fname = "zarr.json" store_dict = {p + fname: data for p in prefixes} await store._set_many(store_dict.items()) for prefix in prefixes: observed = tuple(sorted(await _collect_aiterator(store.list_prefix(prefix)))) expected: tuple[str, ...] = () for key in store_dict: if key.startswith(prefix): expected += (key,) expected = tuple(sorted(expected)) assert observed == expected async def test_list_empty_path(self, store: S) -> None: """ Verify that list and list_prefix work correctly when path is an empty string, i.e. no unwanted replacement occurs. """ data = self.buffer_cls.from_bytes(b"") store_dict = { "foo/bar/zarr.json": data, "foo/bar/c/1": data, "foo/baz/c/0": data, } await store._set_many(store_dict.items()) # Test list() observed_list = await _collect_aiterator(store.list()) observed_list_sorted = sorted(observed_list) expected_list_sorted = sorted(store_dict.keys()) assert observed_list_sorted == expected_list_sorted # Test list_prefix() with an empty prefix observed_prefix_empty = await _collect_aiterator(store.list_prefix("")) observed_prefix_empty_sorted = sorted(observed_prefix_empty) expected_prefix_empty_sorted = sorted(store_dict.keys()) assert observed_prefix_empty_sorted == expected_prefix_empty_sorted # Test list_prefix() with a non-empty prefix observed_prefix = await _collect_aiterator(store.list_prefix("foo/bar/")) observed_prefix_sorted = sorted(observed_prefix) expected_prefix_sorted = sorted(k for k in store_dict if k.startswith("foo/bar/")) assert observed_prefix_sorted == expected_prefix_sorted async def test_list_dir(self, store: S) -> None: roots_and_keys: list[tuple[str, dict[str, Buffer]]] = [ ( "foo", { "foo/zarr.json": self.buffer_cls.from_bytes(b"bar"), "foo/c/1": self.buffer_cls.from_bytes(b"\x01"), }, ), ( "foo/bar", { "foo/bar/foobar_first_child": self.buffer_cls.from_bytes(b"1"), "foo/bar/foobar_second_child/zarr.json": self.buffer_cls.from_bytes(b"2"), }, ), ] assert await _collect_aiterator(store.list_dir("")) == () for root, store_dict in roots_and_keys: assert await _collect_aiterator(store.list_dir(root)) == () await store._set_many(store_dict.items()) keys_observed = await _collect_aiterator(store.list_dir(root)) keys_expected = {k.removeprefix(f"{root}/").split("/")[0] for k in store_dict} assert sorted(keys_observed) == sorted(keys_expected) keys_observed = await _collect_aiterator(store.list_dir(f"{root}/")) assert sorted(keys_expected) == sorted(keys_observed) async def test_set_if_not_exists(self, store: S) -> None: key = "k" data_buf = self.buffer_cls.from_bytes(b"0000") await self.set(store, key, data_buf) new = self.buffer_cls.from_bytes(b"1111") await store.set_if_not_exists("k", new) # no error result = await store.get(key, default_buffer_prototype()) assert result == data_buf await store.set_if_not_exists("k2", new) # no error result = await store.get("k2", default_buffer_prototype()) assert result == new # ------------------------------------------------------------------- # Synchronous store methods (SupportsSyncStore protocol) # ------------------------------------------------------------------- def test_get_sync(self, store: S) -> None: getter = self._require_get_sync(store) data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") key = "sync_get" sync(self.set(store, key, data_buf)) result = getter.get_sync(key) assert result is not None assert_bytes_equal(result, data_buf) def test_get_sync_missing(self, store: S) -> None: getter = self._require_get_sync(store) result = getter.get_sync("nonexistent") assert result is None def test_set_sync(self, store: S) -> None: setter = self._require_set_sync(store) data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") key = "sync_set" setter.set_sync(key, data_buf) result = sync(self.get(store, key)) assert_bytes_equal(result, data_buf) def test_delete_sync(self, store: S) -> None: setter = self._require_set_sync(store) deleter = self._require_delete_sync(store) getter = self._require_get_sync(store) if not store.supports_deletes: pytest.skip("store does not support deletes") data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") key = "sync_delete" setter.set_sync(key, data_buf) deleter.delete_sync(key) result = getter.get_sync(key) assert result is None def test_delete_sync_missing(self, store: S) -> None: deleter = self._require_delete_sync(store) if not store.supports_deletes: pytest.skip("store does not support deletes") # should not raise deleter.delete_sync("nonexistent_sync") # ------------------------------------------------------------------- # Sync/async parity laws # ------------------------------------------------------------------- # A store's sync and async methods must observe the same key the same # way. This is stronger than the individual test_get_sync/test_set_sync/ # test_delete_sync tests above: those write and read back through the # *same* API (sync-only or, via `self.set`/`self.get`, bypassing the # store entirely), so a sync method that skips logic the async method # applies (e.g. a path prefix) can still pass them. These laws write # through one API and observe through the other. @pytest.mark.parametrize("direction", ["set_async_get_sync", "set_sync_get_async"]) async def test_sync_async_set_get_parity(self, store: S, direction: str) -> None: setter = self._require_set_sync(store) getter = self._require_get_sync(store) data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") key = "parity_set_get" if direction == "set_async_get_sync": await store.set(key, data_buf) result = getter.get_sync(key) else: setter.set_sync(key, data_buf) result = await store.get(key, prototype=default_buffer_prototype()) assert result is not None assert_bytes_equal(result, data_buf) async def test_delete_sync_visible_to_async_get(self, store: S) -> None: deleter = self._require_delete_sync(store) if not store.supports_deletes: pytest.skip("store does not support deletes") data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") key = "parity_delete" await store.set(key, data_buf) deleter.delete_sync(key) result = await store.get(key, prototype=default_buffer_prototype()) assert result is None @pytest.mark.parametrize( "byte_range", [ None, RangeByteRequest(1, 4), OffsetByteRequest(1), SuffixByteRequest(1), RangeByteRequest(10, 20), ], ids=["none", "range", "offset", "suffix", "range-past-eof"], ) async def test_get_sync_byte_range_parity( self, store: S, byte_range: ByteRequest | None ) -> None: getter = self._require_get_sync(store) data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") key = "parity_byte_range" await store.set(key, data_buf) sync_result = getter.get_sync(key, byte_range=byte_range) async_result = await store.get( key, prototype=default_buffer_prototype(), byte_range=byte_range ) if async_result is None: assert sync_result is None else: assert sync_result is not None assert_bytes_equal(sync_result, async_result) class LatencyStore(WrapperStore[Store]): """ A wrapper class that takes any store class in its constructor and adds latency to the `set` and `get` methods. This can be used for performance testing. """ _get_latency: float | tuple[float, float] _set_latency: float | tuple[float, float] def __init__( self, store: Store, *, get_latency: float | tuple[float, float] = 0, set_latency: float | tuple[float, float] = 0, ) -> None: super().__init__(store) self._get_latency = get_latency if isinstance(get_latency, tuple) else float(get_latency) self._set_latency = set_latency if isinstance(set_latency, tuple) else float(set_latency) @property def get_latency(self) -> float: if isinstance(self._get_latency, float): return self._get_latency return max(0.0, np.random.normal(loc=self._get_latency[0], scale=self._get_latency[1])) @property def set_latency(self) -> float: if isinstance(self._set_latency, float): return self._set_latency return max(0.0, np.random.normal(loc=self._set_latency[0], scale=self._set_latency[1])) def _with_store(self, store: Store) -> Self: # Pass the raw latency config, not the sampled `get_latency`/`set_latency` # properties — sampling would freeze a `(loc, scale)` distribution into # one fixed float on derived stores (e.g. via `with_read_only`). return type(self)(store, get_latency=self._get_latency, set_latency=self._set_latency) async def set(self, key: str, value: Buffer) -> None: """ Add latency to the ``set`` method. Calls ``asyncio.sleep(self.set_latency)`` before invoking the wrapped ``set`` method. Parameters ---------- key : str The key to set value : Buffer The value to set Returns ------- None """ await asyncio.sleep(self.set_latency) await self._store.set(key, value) async def get( self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: """ Add latency to the ``get`` method. Calls ``asyncio.sleep(self.get_latency)`` before invoking the wrapped ``get`` method. Parameters ---------- key : str The key to get prototype : BufferPrototype The BufferPrototype to use. byte_range : ByteRequest, optional An optional byte range. Returns ------- buffer : Buffer or None """ await asyncio.sleep(self.get_latency) return await self._store.get(key, prototype=prototype, byte_range=byte_range) def get_sync( self, key: str, *, prototype: BufferPrototype | None = None, byte_range: ByteRequest | None = None, ) -> Buffer | None: """Add latency to `get_sync`. Sleeps `self.get_latency` on the calling thread (the sync path runs on worker threads, not the event loop) before delegating to the wrapped store. """ time.sleep(self.get_latency) return super().get_sync(key, prototype=prototype, byte_range=byte_range) def set_sync(self, key: str, value: Buffer) -> None: """Add latency to `set_sync`. Sleeps `self.set_latency` on the calling thread (the sync path runs on worker threads, not the event loop) before delegating to the wrapped store. """ time.sleep(self.set_latency) super().set_sync(key, value) async def get_ranges( self, key: str, byte_ranges: Sequence[ByteRequest | None], *, prototype: BufferPrototype, max_concurrency: int | None = None, max_gap_bytes: int | None = None, max_coalesced_bytes: int | None = None, ) -> AsyncIterator[Sequence[tuple[int, Buffer | None]]]: """Byte-range reads built on `self.get`, so each fetch pays latency. Routes through the coalescing `Store.get_ranges` default instead of the `WrapperStore` delegation, which would bypass this wrapper's `get` and therefore the synthetic latency. `None` for a coalescing kwarg means "use the `Store` default". """ kwargs: dict[str, int] = {} if max_concurrency is not None: kwargs["max_concurrency"] = max_concurrency if max_gap_bytes is not None: kwargs["max_gap_bytes"] = max_gap_bytes if max_coalesced_bytes is not None: kwargs["max_coalesced_bytes"] = max_coalesced_bytes async for group in Store.get_ranges(self, key, byte_ranges, prototype=prototype, **kwargs): yield group async def get_partial_values( self, prototype: BufferPrototype, key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: """Partial-value reads built on `self.get`, so each fetch pays latency. Issues one `self.get` per `(key, byte_range)` pair instead of the `WrapperStore` delegation, which would bypass this wrapper's `get` and therefore the synthetic latency. """ return list( await asyncio.gather( *( self.get(key, prototype=prototype, byte_range=byte_range) for key, byte_range in key_ranges ) ) ) zarr-developers-zarr-python-a2f2415/src/zarr/testing/strategies.py000066400000000000000000001355501527014137600254440ustar00rootroot00000000000000import dataclasses import itertools import math import sys from collections.abc import Callable, Mapping, Sequence from typing import Any, Literal import hypothesis.extra.numpy as npst import hypothesis.strategies as st import numpy as np import numpy.typing as npt from hypothesis import event from hypothesis.strategies import SearchStrategy import zarr from zarr.abc.codec import Codec from zarr.abc.store import ( ByteRequest, OffsetByteRequest, RangeByteRequest, Store, SuffixByteRequest, ) from zarr.codecs.bytes import BytesCodec from zarr.codecs.crc32c_ import Crc32cCodec from zarr.codecs.sharding import SUBCHUNK_WRITE_ORDER, ShardingCodec, SubchunkWriteOrder from zarr.codecs.zstd import ZstdCodec from zarr.core.array import Array, CompressorsLike, SerializerLike from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding from zarr.core.common import JSON, AccessModeLiteral, ZarrFormat from zarr.core.dtype import data_type_registry, get_data_type_from_native_dtype from zarr.core.dtype.common import HasItemSize from zarr.core.dtype.npy.common import DATETIME_UNIT from zarr.core.dtype.npy.structured import Struct from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.core.metadata.v3 import ( RectilinearChunkGridMetadata, RectilinearDimSpecJSON, RegularChunkGridMetadata, ) from zarr.core.sync import sync from zarr.storage import MemoryStore, StoreLike from zarr.storage._utils import _join_paths, normalize_path from zarr.types import AnyArray TrueOrFalse = Literal[True, False] # Copied from Xarray _attr_keys = st.text(st.characters(), min_size=1) _attr_values = st.recursive( st.none() | st.booleans() | st.text(st.characters(), max_size=5), lambda children: st.lists(children) | st.dictionaries(_attr_keys, children), max_leaves=3, ) @st.composite def keys(draw: st.DrawFn, *, max_num_nodes: int | None = None) -> str: return draw(st.lists(node_names, min_size=1, max_size=max_num_nodes).map("/".join)) @st.composite def paths(draw: st.DrawFn, *, max_num_nodes: int | None = None) -> str: return draw(st.just("/") | keys(max_num_nodes=max_num_nodes)) def dtypes() -> st.SearchStrategy[np.dtype[Any]]: return ( npst.boolean_dtypes() | npst.integer_dtypes(endianness="=") | npst.unsigned_integer_dtypes(endianness="=") | npst.floating_dtypes(endianness="=") | npst.complex_number_dtypes(endianness="=") | npst.byte_string_dtypes(endianness="=") | npst.unicode_string_dtypes(endianness="=") | npst.datetime64_dtypes(endianness="=") | npst.timedelta64_dtypes(endianness="=") ) _field_names = st.text( alphabet=st.characters(min_codepoint=97, max_codepoint=122), min_size=1, max_size=4 ) _field_titles = st.text( alphabet=st.characters(min_codepoint=65, max_codepoint=90), min_size=1, max_size=4 ) def _leaf_zdtypes(cls: type[ZDType[TBaseDType, TBaseScalar]]) -> SearchStrategy[ZDType[Any, Any]]: """ A strategy for instances of a single non-struct `ZDType` class, drawing each constructor parameter the class declares from its valid range. """ params = {f.name for f in dataclasses.fields(cls)} kwargs: dict[str, SearchStrategy[Any]] = {} if "endianness" in params: kwargs["endianness"] = st.sampled_from(["little", "big"]) if "length" in params: kwargs["length"] = st.integers(min_value=1, max_value=16) if "unit" in params: # The constructor normalizes the microsecond alias; all units accept a scale. kwargs["unit"] = st.sampled_from(DATETIME_UNIT) kwargs["scale_factor"] = st.integers(min_value=1, max_value=2**31 - 1) return st.builds(cls, **kwargs) def _struct_zdtypes( children: SearchStrategy[ZDType[Any, Any]], ) -> SearchStrategy[ZDType[Any, Any]]: """A strategy for `Struct` instances whose field data types are drawn from `children`.""" @st.composite def _draw(draw: st.DrawFn) -> ZDType[Any, Any]: num_fields = draw(st.integers(min_value=1, max_value=4)) # suffix with the index so that names are unique without filtering names = [f"{draw(_field_names)}{i}" for i in range(num_fields)] return Struct(fields=tuple((name, draw(children)) for name in names)) return _draw() def zdtypes(*, max_leaves: int = 6) -> SearchStrategy[ZDType[Any, Any]]: """ Generate instances of the built-in registered `ZDType` classes, including nested `Struct`. Struct fields are restricted to fixed-size data types, as required by the V3 `struct` extension. This strategy samples bounded lengths and normalized datetime units/scales; it does not cover every valid instance or arbitrary third-party dtype constructors. """ leaf_classes = [cls for cls in data_type_registry.contents.values() if cls is not Struct] leaves = st.one_of([_leaf_zdtypes(cls) for cls in leaf_classes]) fixed_size_leaves = st.one_of( [_leaf_zdtypes(cls) for cls in leaf_classes if issubclass(cls, HasItemSize)] ) structs = st.recursive(fixed_size_leaves, _struct_zdtypes, max_leaves=max_leaves).filter( lambda dt: isinstance(dt, Struct) ) return leaves | structs @st.composite def structured_dtypes( draw: st.DrawFn, *, allow_extended: bool = False, max_depth: int = 3 ) -> np.dtype[np.void]: """ A strategy for native NumPy structured dtypes, flat or nested. With `allow_extended=False` (the default), generate packed fields without titles or subarray shapes. With `allow_extended=True`, also generate field titles, subarray fields, and `align=True` layouts, independently. The current native dtype conversion rejects titles and subarray fields and accepts padding with a warning. These are implementation behaviors, not restrictions imposed by the V2 format. """ fixed_size_leaves = st.one_of( [ _leaf_zdtypes(cls) for cls in data_type_registry.contents.values() if cls is not Struct and issubclass(cls, HasItemSize) ] ) def build(depth: int) -> np.dtype[np.void]: num_fields = draw(st.integers(min_value=1, max_value=4)) # suffix with the index so that names and titles are unique without filtering; titles # draw from a different alphabet so they never collide with names either names = [f"{draw(_field_names)}{i}" for i in range(num_fields)] titles = [f"{draw(_field_titles)}{i}" for i in range(num_fields)] specs: list[tuple[Any, Any]] = [] for name, title in zip(names, titles, strict=True): field_dtype: Any if depth < max_depth and draw(st.booleans()): field_dtype = build(depth + 1) else: field_dtype = draw(fixed_size_leaves).to_native_dtype() key: Any = name if allow_extended and draw(st.booleans()): key = (title, name) if allow_extended and draw(st.booleans()): field_dtype = (field_dtype, draw(npst.array_shapes(max_dims=2, max_side=3))) specs.append((key, field_dtype)) align = allow_extended and draw(st.booleans()) return np.dtype(specs, align=align) return build(0) def v3_dtypes() -> st.SearchStrategy[np.dtype[Any]]: return dtypes() def v2_dtypes() -> st.SearchStrategy[np.dtype[Any]]: return dtypes() def safe_unicode_for_dtype(dtype: np.dtype[np.str_]) -> st.SearchStrategy[str]: """Generate UTF-8-safe text constrained to max_len of dtype.""" # account for utf-32 encoding (i.e. 4 bytes/character) max_len = max(1, dtype.itemsize // 4) return st.text( alphabet=st.characters( exclude_categories=["Cs"], # Avoid *technically allowed* surrogates min_codepoint=32, ), min_size=1, max_size=max_len, ) def clear_store(x: Store) -> Store: sync(x.clear()) return x # From https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#node-names # 1. must not be the empty string ("") # 2. must not include the character "/" # 3. must not be a string composed only of period characters, e.g. "." or ".." # 4. must not start with the reserved prefix "__" zarr_key_chars = st.sampled_from( ".-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" ) node_names = ( st.text(zarr_key_chars, min_size=1) .filter(lambda t: t not in (".", "..") and not t.startswith("__")) .filter(lambda name: name.lower() != "zarr.json") ) short_node_names = ( st.text(zarr_key_chars, max_size=3, min_size=1) .filter(lambda t: t not in (".", "..") and not t.startswith("__")) .filter(lambda name: name.lower() != "zarr.json") ) array_names = node_names attrs: st.SearchStrategy[Mapping[str, JSON] | None] = st.none() | st.dictionaries( _attr_keys, _attr_values ) # st.builds will only call a new store constructor for different keyword arguments # i.e. stores.examples() will always return the same object per Store class. # So we map a clear to reset the store. stores = st.builds(MemoryStore, st.just({})).map(clear_store) compressors = st.sampled_from([None, "default"]) zarr_formats: st.SearchStrategy[ZarrFormat] = st.sampled_from([3, 2]) # We de-prioritize arrays having dim sizes 0, 1, 2 array_shapes = npst.array_shapes(max_dims=4, min_side=3, max_side=5) | npst.array_shapes( max_dims=4, min_side=0 ) @st.composite def dimension_names(draw: st.DrawFn, *, ndim: int | None = None) -> list[str | None] | None: simple_text = st.text(zarr_key_chars, min_size=0) return draw(st.none() | st.lists(st.none() | simple_text, min_size=ndim, max_size=ndim)) # type: ignore[arg-type] subchunk_write_orders: st.SearchStrategy[SubchunkWriteOrder] = st.sampled_from(SUBCHUNK_WRITE_ORDER) # Inner codec chains for a ShardingCodec. We MUST sample the uncompressed, # single-BytesCodec configuration (no Zstd) — that is the only configuration in # which the FusedCodecPipeline's vectorized whole-shard "bulk decode" fast path # engages, so it is the only one that can exercise (and regress-guard) that path # against arbitrary indexing. Freezing the inner codecs to [BytesCodec, ZstdCodec] # silently disables the fast path under every property test. sharding_inner_codecs: st.SearchStrategy[list[BytesCodec | ZstdCodec]] = st.sampled_from( [ [BytesCodec()], [BytesCodec(), ZstdCodec()], ] ) @st.composite def array_metadata( draw: st.DrawFn, *, array_shapes: Callable[..., st.SearchStrategy[tuple[int, ...]]] = npst.array_shapes, zarr_formats: st.SearchStrategy[ZarrFormat] = zarr_formats, attributes: SearchStrategy[Mapping[str, JSON] | None] = attrs, ) -> ArrayV2Metadata | ArrayV3Metadata: zarr_format = draw(zarr_formats) # separator = draw(st.sampled_from(['/', '\\'])) shape = draw(array_shapes()) ndim = len(shape) np_dtype = draw(dtypes()) dtype = get_data_type_from_native_dtype(np_dtype) fill_value = draw(npst.from_dtype(np_dtype)) if zarr_format == 2: chunk_shape = draw(array_shapes(min_dims=ndim, max_dims=ndim, min_side=1)) return ArrayV2Metadata( shape=shape, chunks=chunk_shape, dtype=dtype, fill_value=fill_value, order=draw(st.sampled_from(["C", "F"])), attributes=draw(attributes), # type: ignore[arg-type] dimension_separator=draw(st.sampled_from([".", "/"])), filters=None, compressor=None, ) else: chunk_grid = draw(chunk_grids(shape=shape)) return ArrayV3Metadata( shape=shape, data_type=dtype, chunk_grid=chunk_grid, fill_value=fill_value, attributes=draw(attributes), # type: ignore[arg-type] dimension_names=draw(dimension_names(ndim=ndim)), chunk_key_encoding=DefaultChunkKeyEncoding(separator="/"), # FIXME codecs=[BytesCodec()], storage_transformers=(), ) @st.composite def numpy_arrays( draw: st.DrawFn, *, shapes: st.SearchStrategy[tuple[int, ...]] = array_shapes, dtype: np.dtype[Any] | None = None, ) -> npt.NDArray[Any]: """ Generate numpy arrays that can be saved in the provided Zarr format. """ if dtype is None: dtype = draw(dtypes()) if np.issubdtype(dtype, np.str_): safe_unicode_strings = safe_unicode_for_dtype(dtype) return draw(npst.arrays(dtype=dtype, shape=shapes, elements=safe_unicode_strings)) return draw(npst.arrays(dtype=dtype, shape=shapes)) @st.composite def chunk_shapes(draw: st.DrawFn, *, shape: tuple[int, ...]) -> tuple[int, ...]: # We want this strategy to shrink towards arrays with smaller number of chunks # 1. st.integers() shrinks towards smaller values. So we use that to generate number of chunks numchunks = draw( st.tuples( *[ st.integers(min_value=0 if size == 0 else 1, max_value=max(size, 1)) for size in shape ] ) ) # 2. and now generate the chunks tuple # Chunk sizes must be >= 1 per spec; for zero-extent dimensions use 1. chunks = tuple( max(1, size // nchunks) if nchunks > 0 else 1 for size, nchunks in zip(shape, numchunks, strict=True) ) for c in chunks: event("chunk size", c) if any((c != 0 and s % c != 0) for s, c in zip(shape, chunks, strict=True)): event("smaller last chunk") return chunks @st.composite def shard_shapes( draw: st.DrawFn, *, shape: tuple[int, ...], chunk_shape: tuple[int, ...] ) -> tuple[int, ...]: # We want this strategy to shrink towards arrays with smaller number of shards # shards must be an integral number of chunks assert all(c != 0 for c in chunk_shape) numchunks = tuple(s // c for s, c in zip(shape, chunk_shape, strict=True)) multiples = tuple(draw(st.integers(min_value=1, max_value=nc)) for nc in numchunks) return tuple(m * c for m, c in zip(multiples, chunk_shape, strict=True)) @st.composite def _sharding_codecs( draw: st.DrawFn, *, chunk_shape: tuple[int, ...], codecs: Sequence[Codec] | None = None, ) -> ShardingCodec: """A ``ShardingCodec`` over ``chunk_shape`` with a drawn subchunk write order. The inner codec chain is drawn from ``sharding_inner_codecs`` unless ``codecs`` is given, which lets a caller nest another ``ShardingCodec`` inside. """ subchunk_write_order = draw(subchunk_write_orders) inner_codecs: Sequence[Codec] = ( draw(sharding_inner_codecs, label="sharding inner codecs") if codecs is None else codecs ) return ShardingCodec( subchunk_write_order=subchunk_write_order, codecs=inner_codecs, index_codecs=[BytesCodec(), Crc32cCodec()], chunk_shape=chunk_shape, ) @st.composite def np_array_and_chunks( draw: st.DrawFn, *, arrays: st.SearchStrategy[npt.NDArray[Any]] = numpy_arrays(), # noqa: B008 ) -> tuple[np.ndarray[Any, Any], tuple[int, ...]]: """A hypothesis strategy to generate small sized random arrays. Returns: a tuple of the array and a suitable random chunking for it. """ array = draw(arrays) return (array, draw(chunk_shapes(shape=array.shape))) @st.composite def arrays( draw: st.DrawFn, *, shapes: st.SearchStrategy[tuple[int, ...]] = array_shapes, compressors: st.SearchStrategy = compressors, stores: st.SearchStrategy[StoreLike] = stores, paths: st.SearchStrategy[str] = paths(), # noqa: B008 array_names: st.SearchStrategy = array_names, arrays: st.SearchStrategy | None = None, attrs: st.SearchStrategy = attrs, zarr_formats: st.SearchStrategy = zarr_formats, subchunk_write_orders: SearchStrategy[SubchunkWriteOrder] = subchunk_write_orders, open_mode: AccessModeLiteral = "w", ) -> AnyArray: store = draw(stores, label="store") path = draw(paths, label="array parent") name = draw(array_names, label="array name") attributes = draw(attrs, label="attributes") zarr_format = draw(zarr_formats, label="zarr format") if arrays is None: arrays = numpy_arrays(shapes=shapes) nparray = draw(arrays, label="array data") dim_names: list[str | None] | None = None serializer: SerializerLike = "auto" compressors_unsearched: CompressorsLike = "auto" # For v3 arrays, optionally use RectilinearChunkGridMetadata chunk_grid_meta: RegularChunkGridMetadata | RectilinearChunkGridMetadata | None = None # test that None works too. fill_value = draw(st.one_of([st.none(), npst.from_dtype(nparray.dtype)])) # compressor = draw(compressors) expected_attrs = {} if attributes is None else attributes array_path = _join_paths([path, name]) root = zarr.open_group(store, mode=open_mode, zarr_format=zarr_format) chunks_param: tuple[int, ...] | list[int | list[int]] | RectilinearChunkGridMetadata shard_shape = None dim_names = None if zarr_format == 3: chunk_grid_meta = draw(st.none() | chunk_grids(shape=nparray.shape), label="chunk grid") dim_names = draw(dimension_names(ndim=nparray.ndim), label="dimension names") if isinstance(chunk_grid_meta, RectilinearChunkGridMetadata): # A rectilinear grid is passed either as the metadata object or in # the list form of `chunks=`, drawn from its own strategy. A 0-d # array has no dimension to hold an edge list. if nparray.ndim > 0 and draw(st.booleans(), label="chunks as lists"): event("rectilinear chunks= as lists") chunks_param = draw(_rectilinear_chunks(shape=nparray.shape), label="chunks") chunk_grid_meta = RectilinearChunkGridMetadata( chunk_shapes=tuple( dim if isinstance(dim, int) else tuple(dim) for dim in chunks_param ) ) else: event("rectilinear chunks= as metadata") chunks_param = chunk_grid_meta elif isinstance(chunk_grid_meta, RegularChunkGridMetadata): chunks_param = chunk_grid_meta.chunk_shape else: chunks_param = draw(chunk_shapes(shape=nparray.shape), label="chunk shape") # Any chunk that fits the array can be sharded: shard_shapes draws a # whole number of chunks per axis, one inner chunk included. if all(s >= c >= 1 for s, c in zip(nparray.shape, chunks_param, strict=True)): shard_shape = draw( st.none() | shard_shapes(shape=nparray.shape, chunk_shape=chunks_param), label="shard shape", ) event("sharded" if shard_shape is not None else "unsharded") if shard_shape is not None: serializer = draw(_sharding_codecs(chunk_shape=chunks_param)) compressors_unsearched = None else: chunks_param = draw(chunk_shapes(shape=nparray.shape), label="chunk shape") a = root.create_array( array_path, shape=nparray.shape, chunks=chunks_param, shards=shard_shape, dtype=nparray.dtype, attributes=attributes, compressors=compressors_unsearched, # FIXME fill_value=fill_value, dimension_names=dim_names, serializer=serializer, ) assert isinstance(a, Array) if a.metadata.zarr_format == 3: assert a.fill_value is not None assert a.name is not None assert a.path == normalize_path(array_path) assert a.name == f"/{a.path}" assert isinstance(root[array_path], Array) assert nparray.shape == a.shape # Verify chunks — for rectilinear grids, .chunks raises if zarr_format == 3: assert shard_shape == a.shards if isinstance(a.metadata.chunk_grid, RegularChunkGridMetadata): assert a.metadata.chunk_grid.chunk_shape == ( a.shards if shard_shape is not None else a.chunks ) assert shard_shape == a.shards else: # The stored grid is exactly the declared one: bare ints stay bare # ints, edge lists keep their edges. assert a.metadata.chunk_grid == chunk_grid_meta assert shard_shape is None assert a.basename == name, (a.basename, name) assert dict(a.attrs) == expected_attrs a[:] = nparray return a @st.composite def simple_arrays( draw: st.DrawFn, *, shapes: st.SearchStrategy[tuple[int, ...]] = array_shapes, ) -> Any: return draw( arrays( shapes=shapes, paths=paths(max_num_nodes=2), array_names=short_node_names, attrs=st.none(), compressors=st.sampled_from([None, "default"]), ) ) def chunks_param_from_rectilinear( meta: RectilinearChunkGridMetadata, ) -> list[int | list[int]]: """Convert rectilinear chunk grid metadata into a `chunks=` argument. Explicit edge tuples become lists. Bare ints — the spec's step-size shorthand meaning "repeat to cover the axis" — pass through unchanged; wrapping one in a single-element list would instead declare exactly one chunk, which fails normalization whenever the axis needs more than one. """ return [list(dim) if isinstance(dim, tuple) else dim for dim in meta.chunk_shapes] # The most chunks a drawn rectilinear grid declares along one axis, and over # the array, all axes together. Indexing tests visit every chunk, so the # product is what costs time. _RECTILINEAR_MAX_CHUNKS_PER_DIM = 20 _RECTILINEAR_CHUNK_BUDGET = 400 def _max_chunks_per_dim(ndim: int) -> int: """At most `_RECTILINEAR_MAX_CHUNKS_PER_DIM` chunks per dimension, and at most `_RECTILINEAR_CHUNK_BUDGET` in total over `ndim` dimensions.""" return max( k for k in range(1, _RECTILINEAR_MAX_CHUNKS_PER_DIM + 1) if k**ndim <= _RECTILINEAR_CHUNK_BUDGET ) @st.composite def rectilinear_dim_edges( draw: st.DrawFn, *, extent: int, max_chunks: int = _RECTILINEAR_MAX_CHUNKS_PER_DIM ) -> list[int]: """Explicit chunk edge lengths summing exactly to `extent`. A zero `extent` has no chunks for edges to cover, and no non-empty list of positive edges sums to 0; its edges are the chunks the axis grows into on `append` or `resize`, so any non-empty list of positive edges is drawn. Two modes: "uneven" cuts the extent at random dividers; "uniform" repeats one size with an optional remainder, optionally shuffled so equal edges are not all adjacent. At most `max_chunks` chunks keeps property tests fast. """ assert extent >= 0 if extent == 0: event("rectilinear edges: zero extent") return draw(st.lists(st.integers(min_value=1, max_value=10), min_size=1, max_size=5)) if draw(st.booleans(), label="uneven edges"): nchunks = draw(st.integers(min_value=1, max_value=min(extent, max_chunks))) # Draw distinct dividers by index into the unused positions: no # rejection, unlike `st.lists(..., unique=True)`. positions = list(range(1, extent)) dividers = sorted( positions.pop(draw(st.integers(min_value=0, max_value=len(positions) - 1))) for _ in range(nchunks - 1) ) return [b - a for a, b in zip([0, *dividers], [*dividers, extent], strict=True)] size = draw(st.integers(min_value=math.ceil(extent / max_chunks), max_value=extent)) edges = [size] * (extent // size) if extent % size: edges.append(extent % size) if draw(st.booleans(), label="shuffle uniform edges"): return list(draw(st.permutations(edges))) return edges def _rectilinear_step(draw: st.DrawFn, *, extent: int, max_chunks: int) -> int: """A bare-int chunk size for one dimension: a step that repeats to cover the extent, with the last chunk possibly smaller. A step larger than the extent (one overhanging chunk) is allowed, as for a regular grid.""" step = draw(st.integers(min_value=max(1, math.ceil(extent / max_chunks)), max_value=extent + 3)) if step > extent: event("rectilinear step: larger than extent") return step @st.composite def rectilinear_chunks(draw: st.DrawFn, *, shape: tuple[int, ...]) -> list[list[int]]: """A `chunks=` specification declaring a rectilinear grid over `shape`, as an explicit edge list per dimension summing to the extent (any edges, for a zero extent). A 0-d `shape` gives `[]`. To also draw bare-int steps, see `_rectilinear_chunks`. """ max_chunks = _max_chunks_per_dim(len(shape)) return [draw(rectilinear_dim_edges(extent=e, max_chunks=max_chunks)) for e in shape] @st.composite def _rectilinear_chunks(draw: st.DrawFn, *, shape: tuple[int, ...]) -> list[int | list[int]]: """A `chunks=` specification declaring a rectilinear grid over `shape`. Each dimension is either a bare int (a step size; the last chunk may be smaller) or an explicit edge list summing to the extent (any edges, for a zero extent). At least one dimension is an edge list, since bare ints alone declare a regular grid. Run-length encoding is not part of the `chunks=` syntax; it belongs to stored metadata, see `rectilinear_chunk_shape_declarations`. `shape` must have at least one dimension: a 0-d array has no dimension to hold an edge list, so it cannot have a rectilinear grid. """ assert shape, "a rectilinear grid needs at least one dimension" max_chunks = _max_chunks_per_dim(len(shape)) forced_list = draw(st.integers(min_value=0, max_value=len(shape) - 1)) chunks: list[int | list[int]] = [] for i, extent in enumerate(shape): if i != forced_list and draw(st.booleans(), label="bare int"): chunks.append(_rectilinear_step(draw, extent=extent, max_chunks=max_chunks)) else: chunks.append(draw(rectilinear_dim_edges(extent=extent, max_chunks=max_chunks))) return chunks def _rle_encode(draw: st.DrawFn, edges: list[int]) -> list[int | list[int]]: """Run-length encode `edges` as the spec allows: a mix of bare ints and `[size, count]` pairs. Either the canonical form (each run as one pair, a run of one as a bare int) or an arbitrary grouping, which may split a run across pairs and use `count == 1`.""" canonical = draw(st.booleans(), label="canonical rle") if not canonical: event("rectilinear rle: arbitrary grouping") encoded: list[int | list[int]] = [] i = 0 while i < len(edges): run = 1 while i + run < len(edges) and edges[i + run] == edges[i]: run += 1 if canonical: count = run bare = run == 1 else: count = draw(st.integers(min_value=1, max_value=run)) bare = count == 1 and draw(st.booleans(), label="bare edge") encoded.append(edges[i] if bare else [edges[i], count]) i += count return encoded def _rectilinear_chunk_shapes( draw: st.DrawFn, shape: tuple[int, ...] ) -> tuple[int | tuple[int, ...], ...]: """The `chunk_shapes` of a stored rectilinear grid over `shape`: per dimension a bare-int step, or explicit edges. Edges may sum beyond the extent, which the spec allows and a shrinking resize produces.""" max_chunks = _max_chunks_per_dim(len(shape)) chunk_shapes: list[int | tuple[int, ...]] = [] for extent in shape: if draw(st.booleans(), label="bare int"): chunk_shapes.append(_rectilinear_step(draw, extent=extent, max_chunks=max_chunks)) continue edges = draw(rectilinear_dim_edges(extent=extent, max_chunks=max_chunks)) if extent > 0 and draw(st.booleans(), label="overhang"): event("rectilinear edges: overhang") if draw(st.booleans(), label="trailing edge"): edges = [*edges, draw(st.integers(min_value=1, max_value=5))] else: edges[-1] += draw(st.integers(min_value=1, max_value=5)) chunk_shapes.append(tuple(edges)) return tuple(chunk_shapes) @st.composite def rectilinear_chunk_shape_declarations( draw: st.DrawFn, *, shape: tuple[int, ...] ) -> tuple[list[RectilinearDimSpecJSON], tuple[int | tuple[int, ...], ...]]: """The `chunk_shapes` of a stored rectilinear chunk grid, with its meaning. Samples the whole declaration space of the spec. Per dimension: a bare int step, or an edge list written in full or run-length encoded (canonically, or with arbitrary grouping). Edge lists may sum beyond the extent. Returns `(declaration, chunk_shapes)`: the JSON value to store, and the `chunk_shapes` that parsing it must produce. """ chunk_shapes = _rectilinear_chunk_shapes(draw, shape) declaration: list[RectilinearDimSpecJSON] = [ dim if isinstance(dim, int) else list(dim) if draw(st.booleans(), label="write edges in full") else _rle_encode(draw, list(dim)) for dim in chunk_shapes ] return declaration, chunk_shapes @st.composite def rectilinear_chunk_grids( draw: st.DrawFn, *, shape: tuple[int, ...] ) -> RectilinearChunkGridMetadata: """A `RectilinearChunkGridMetadata` over `shape`, per dimension a bare-int step or explicit edges, which may sum beyond the extent.""" return RectilinearChunkGridMetadata(chunk_shapes=_rectilinear_chunk_shapes(draw, shape)) @st.composite def chunk_grids( draw: st.DrawFn, *, shape: tuple[int, ...] ) -> RegularChunkGridMetadata | RectilinearChunkGridMetadata: """Generate either a RegularChunkGridMetadata or RectilinearChunkGridMetadata. This strategy depends on the global state of the config having rectilinear chunk grids enabled or not. This means that it may be a possible source of a hypothesis FlakyStrategy error due dependence on global state. However, in practice this seems unlikely to happen. This allows property tests to exercise both chunk grid types. """ if zarr.config.get("array.rectilinear_chunks") and draw(st.booleans()): event("using RectilinearChunkGridMetadata") return draw(rectilinear_chunk_grids(shape=shape)) else: event("using RegularChunkGridMetadata") return RegularChunkGridMetadata(chunk_shape=draw(chunk_shapes(shape=shape))) # Rectilinear arrays need min_side >= 1 so every dimension has at least one element _rectilinear_shapes = npst.array_shapes(max_dims=3, min_side=1, max_side=20) @st.composite def rectilinear_arrays( draw: st.DrawFn, *, shapes: st.SearchStrategy[tuple[int, ...]] = _rectilinear_shapes, ) -> Any: """Generate a zarr v3 array with rectilinear (variable) chunk grid.""" shape = draw(shapes) chunk_shapes = draw(_rectilinear_chunks(shape=shape)) np_dtype = draw(dtypes()) nparray = draw(numpy_arrays(shapes=st.just(shape), dtype=np_dtype)) fill_value = draw(st.one_of([st.none(), npst.from_dtype(np_dtype)])) dim_names = draw(dimension_names(ndim=len(shape))) store = MemoryStore() with zarr.config.set({"array.rectilinear_chunks": True}): a = zarr.create_array( store=store, shape=shape, chunks=chunk_shapes, dtype=np_dtype, fill_value=fill_value, dimension_names=dim_names, ) a[:] = nparray return a # Sharded arrays need min_side >= 1: a shard must hold at least one chunk on every axis. _sharded_shapes = npst.array_shapes(max_dims=4, min_side=1, max_side=8) @st.composite def sharded_arrays( draw: st.DrawFn, *, shapes: st.SearchStrategy[tuple[int, ...]] = _sharded_shapes, nested: bool | None = None, ) -> Any: """Generate a zarr v3 array whose chunks are grouped into shards. ``arrays`` shards only a small fraction of its draws (a v3 array with a regular chunk grid, every axis larger than a chunk that is itself larger than 1, and then only half the time), so a property test that must exercise the sharding codec should draw from this strategy directly. Every draw is sharded: the chunk shape and the shard shape (an integral number of chunks per axis, possibly a single chunk) are drawn from ``shapes``, and the codec's subchunk write order and inner codec chain are drawn as in ``arrays``. ``shapes`` must generate shapes with at least one element on every axis. ``nested`` selects one level of recursive sharding: the drawn chunks are grouped into inner shards, which are themselves grouped into the shards stored in the array, so the outer ``ShardingCodec`` wraps an inner one with its own subchunk write order. ``None`` (the default) draws it, so half the examples nest. For a nested array ``Array.chunks`` is the inner shard shape (the outer codec's chunk shape); the innermost chunk shape is the inner codec's ``chunk_shape``. """ shape = draw(shapes) chunk_shape = draw(chunk_shapes(shape=shape), label="chunk shape") serializer = draw(_sharding_codecs(chunk_shape=chunk_shape)) nest = draw(st.booleans(), label="nested sharding") if nested is None else nested if nest: # Each level's shard is an integral number of the level below's chunks. codec_chunk_shape = draw( shard_shapes(shape=shape, chunk_shape=chunk_shape), label="inner shard shape" ) serializer = draw(_sharding_codecs(chunk_shape=codec_chunk_shape, codecs=[serializer])) else: codec_chunk_shape = chunk_shape shard_shape = draw( shard_shapes(shape=shape, chunk_shape=codec_chunk_shape), label="shard shape" ) event("nested sharding" if nest else "single-level sharding") nparray = draw(numpy_arrays(shapes=st.just(shape)), label="array data") fill_value = draw(st.one_of([st.none(), npst.from_dtype(nparray.dtype)])) dim_names = draw(dimension_names(ndim=len(shape)), label="dimension names") # The shard is the array's chunk grid and the drawn codec is its serializer. # Passing ``shards=`` instead would make ``create_array`` wrap the codec in a # second ``ShardingCodec`` of the same chunk shape, hiding the drawn write # order behind a default outer one. a = zarr.create_array( store=MemoryStore(), shape=shape, chunks=shard_shape, dtype=nparray.dtype, fill_value=fill_value, dimension_names=dim_names, serializer=serializer, filters=None, compressors=None, ) assert a.shards == shard_shape assert a.chunks == codec_chunk_shape assert isinstance(a.metadata, ArrayV3Metadata) (codec,) = a.metadata.codecs assert isinstance(codec, ShardingCodec) assert codec.subchunk_write_order == serializer.subchunk_write_order a[:] = nparray return a def is_negative_slice(idx: Any) -> bool: return isinstance(idx, slice) and idx.step is not None and idx.step < 0 @st.composite def end_slices(draw: st.DrawFn, *, shape: tuple[int, ...]) -> Any: """ A strategy that slices ranges that include the last chunk. This is intended to stress-test handling of a possibly smaller last chunk. """ slicers = [] for size in shape: start = draw(st.integers(min_value=size // 2, max_value=size - 1)) length = draw(st.integers(min_value=0, max_value=size - start)) slicers.append(slice(start, start + length)) event("drawing end slice") return tuple(slicers) @st.composite def basic_indices( draw: st.DrawFn, *, shape: tuple[int, ...], min_dims: int = 0, max_dims: int | None = None, allow_newaxis: TrueOrFalse = False, allow_ellipsis: TrueOrFalse = True, ) -> Any: """Basic indices without unsupported negative slices.""" strategy = npst.basic_indices( shape=shape, min_dims=min_dims, max_dims=max_dims, allow_newaxis=allow_newaxis, allow_ellipsis=allow_ellipsis, ).filter( lambda idxr: ( not ( is_negative_slice(idxr) or (isinstance(idxr, tuple) and any(is_negative_slice(idx) for idx in idxr)) ) ) ) if math.prod(shape) >= 3: strategy = end_slices(shape=shape) | strategy return draw(strategy) @st.composite def orthogonal_indices( draw: st.DrawFn, *, shape: tuple[int, ...] ) -> tuple[tuple[int | slice | np.ndarray[Any, Any], ...], tuple[np.ndarray[Any, Any], ...]]: """ Strategy that returns (1) a tuple of per-axis selectors (integer array, slice, or bare integer) for orthogonal indexing of Zarr arrays. (2) a tuple of broadcast integer arrays that index a numpy array to the same result. A bare integer drops its axis, as ``oindex`` does, so it is given as a 0-d array and does not contribute a result dimension. """ zindexer: list[int | slice | np.ndarray[Any, Any]] = [] kept: list[tuple[int, np.ndarray[Any, Any]]] = [] npindexer: dict[int, np.ndarray[Any, Any]] = {} for axis, size in enumerate(shape): if size != 0: strategy = ( npst.integer_array_indices( shape=(size,), result_shape=npst.array_shapes(min_side=1, max_side=size, max_dims=1), ) | basic_indices(min_dims=1, shape=(size,), allow_ellipsis=False) # basic_indices(min_dims=1) never yields a bare integer, so draw # one explicitly: it is the only selector that drops an axis. | st.integers(min_value=-size, max_value=size - 1) ) else: strategy = basic_indices(min_dims=1, shape=(size,), allow_ellipsis=False) val = draw( strategy # bare ints, slices .map(lambda x: (x,) if not isinstance(x, tuple) else x) # skip empty tuple .filter(bool) ) (idxr,) = val zindexer.append(idxr) if isinstance(idxr, int): npindexer[axis] = np.array(idxr) continue if isinstance(idxr, slice): idxr = np.arange(*idxr.indices(size)) elif isinstance(idxr, tuple): idxr = np.array(idxr) kept.append((axis, idxr)) for pos, (axis, idxr) in enumerate(kept): newshape = [1] * len(kept) newshape[pos] = idxr.size npindexer[axis] = idxr.reshape(newshape) # casting the output of broadcast_arrays is needed for numpy < 2 return tuple(zindexer), tuple( np.broadcast_arrays(*(npindexer[axis] for axis in range(len(shape)))) ) @st.composite def block_indices( draw: st.DrawFn, *, chunk_sizes: tuple[tuple[int, ...], ...] ) -> tuple[tuple[int | slice, ...], tuple[slice, ...]]: """ Strategy for block-selection indexers over a chunk grid. Block indexing is basic indexing applied to the block grid (the grid of chunks), so each axis is drawn with ``basic_indices`` over that axis's chunk count, mirroring how ``orthogonal_indices`` reuses ``basic_indices`` per axis. ``chunk_sizes`` gives the per-chunk data sizes of the array's *outer* (block) grid for every axis — i.e. ``Array.write_chunk_sizes``, the grid that ``Array.blocks`` addresses (the shard grid when sharding is used). For example ``(3, 3, 3, 1)`` for a length-10 axis with a regular chunk size of 3, or the explicit edges of a rectilinear axis; ``nchunks`` for an axis is ``len(chunk_sizes[axis])``. The array-space translation uses the cumulative sum of those sizes, matching ``BlockIndexer``'s use of ``dim_grid.chunk_offset``. Because the sizes are clipped to the array extent, the final offset equals the extent and the translation is exact for regular (uniform), rectilinear, and sharded grids alike. Block indexing only supports integers and step-1 slices whose start references an existing chunk, so strided slices and slices starting at the grid edge are filtered out. Returns ------- block_indexer A per-axis tuple of ints / step-1 slices addressing whole chunks, suitable for ``Array.blocks`` / ``get_block_selection`` / ``set_block_selection``. array_indexer The equivalent array-space selection (a tuple of slices) for indexing the corresponding numpy array, used as the comparison oracle. """ def supported(nchunks: int) -> Callable[[tuple[Any, ...]], bool]: # Block indexing only accepts step-1 slices whose start references an # existing chunk (a slice starting at nchunks raises, unlike numpy). def predicate(value: tuple[Any, ...]) -> bool: dim_sel = value[0] if isinstance(dim_sel, slice): if dim_sel.step not in (None, 1): return False start = dim_sel.start or 0 return 0 <= (start + nchunks if start < 0 else start) < nchunks return True return predicate block_indexer: list[int | slice] = [] array_indexer: list[slice] = [] for sizes in chunk_sizes: nchunks = len(sizes) # offsets[i] is the array-space start of chunk i; length nchunks + 1. offsets = list(itertools.accumulate(sizes, initial=0)) dim_strategy = ( basic_indices(min_dims=1, shape=(nchunks,), allow_ellipsis=False) # normalize bare ints / slices to a 1-tuple, skip the empty tuple .map(lambda x: (x,) if not isinstance(x, tuple) else x) .filter(bool) .filter(supported(nchunks)) ) # basic_indices draws slices far more often than bare integers, so the # integer (single-block) branch below would only be hit on rare draws. # Union in an explicit integer so it is reliably exercised — keeping # coverage deterministic under the derandomized ``ci`` Hypothesis profile. (dim_sel,) = draw( dim_strategy | st.integers(min_value=0, max_value=nchunks - 1).map(lambda i: (i,)) ) block_indexer.append(dim_sel) if isinstance(dim_sel, slice): start, stop, _ = dim_sel.indices(nchunks) array_indexer.append(slice(offsets[start], offsets[stop])) else: block = dim_sel % nchunks array_indexer.append(slice(offsets[block], offsets[block + 1])) return tuple(block_indexer), tuple(array_indexer) @st.composite def block_test_arrays( draw: st.DrawFn, ) -> tuple[Array[Any], np.ndarray[Any, Any]]: """Draw an array for block-indexing property tests, with its source contents. Two arms, selected with equal probability: - **regular**: a regular chunk grid, optionally wrapped in sharding. - **rectilinear**: a variable (rectilinear) chunk grid, always unsharded. Returns ``(zarray, nparray)``. The per-axis block sizes the oracle needs are ``zarray.write_chunk_sizes`` — the array's *outer* (block / shard) grid, which is exactly the grid ``Array.blocks`` addresses; the caller reads it directly. """ chunks: tuple[int, ...] | list[int | list[int]] if draw(st.booleans()): # regular arm, optionally sharded nparray, chunks = draw( np_array_and_chunks( arrays=numpy_arrays(shapes=npst.array_shapes(max_dims=4, min_side=1)) ) ) # min_side=1 chunking guarantees shape // chunk >= 1 on every axis, which # shard_shapes requires. shards = draw(st.none() | shard_shapes(shape=nparray.shape, chunk_shape=chunks)) event("block regular sharded" if shards is not None else "block regular unsharded") rectilinear = False else: # rectilinear arm, always unsharded event("block rectilinear") shape = draw(_rectilinear_shapes) chunks = draw(_rectilinear_chunks(shape=shape)) nparray = draw(numpy_arrays(shapes=st.just(shape), dtype=draw(dtypes()))) shards, rectilinear = None, True store = draw(stores) with zarr.config.set({"array.rectilinear_chunks": rectilinear}): zarray = zarr.create_array( store=store, shape=nparray.shape, chunks=chunks, shards=shards, dtype=nparray.dtype, ) zarray[...] = nparray return zarray, nparray def key_ranges( keys: SearchStrategy[str] = node_names, max_size: int = sys.maxsize ) -> SearchStrategy[list[tuple[str, ByteRequest | None]]]: """ Function to generate key_ranges strategy for get_partial_values() returns list strategy w/ form:: [(key, byte_request), (key, byte_request),...] where ``byte_request`` is ``None`` or any of the concrete ``ByteRequest`` subtypes. The bounds are drawn independently of each value's length, so the offsets/suffixes routinely exceed the data and exercise the clamping logic in ``_normalize_byte_range_index``. """ def make_range(start: int, length: int) -> RangeByteRequest: return RangeByteRequest(start, end=min(start + length, max_size)) bound = st.integers(min_value=0, max_value=max_size) byte_ranges: SearchStrategy[ByteRequest | None] = st.one_of( st.none(), st.builds(make_range, start=bound, length=bound), st.builds(OffsetByteRequest, offset=bound), st.builds(SuffixByteRequest, suffix=bound), ) key_tuple = st.tuples(keys, byte_ranges) return st.lists(key_tuple, min_size=1, max_size=10) @st.composite def complex_rectilinear_arrays( draw: st.DrawFn, *, stores: st.SearchStrategy[StoreLike] = stores, paths: st.SearchStrategy[str] = paths(), # noqa: B008 array_names: st.SearchStrategy = array_names, attrs: st.SearchStrategy = attrs, ) -> tuple[npt.NDArray[Any], AnyArray]: """Generate a rectilinear array with many small chunks. The shape is derived from the chunk edges (5-10 chunks per dim, sizes 1-5), exercising higher chunk counts than ``rectilinear_arrays``. """ ndim = draw(st.integers(min_value=1, max_value=3)) nchunks = draw(st.integers(min_value=5, max_value=10)) dim_chunks = st.lists(st.integers(min_value=1, max_value=5), min_size=nchunks, max_size=nchunks) chunk_shapes = draw(st.lists(dim_chunks, min_size=ndim, max_size=ndim)) shape = tuple(sum(dim) for dim in chunk_shapes) nparray = draw(numpy_arrays(shapes=st.just(shape))) dim_names = draw(dimension_names(ndim=ndim)) fill_value = draw(st.one_of([st.none(), npst.from_dtype(nparray.dtype)])) attributes = draw(attrs) store = draw(stores, label="store") path = draw(paths, label="array parent") name = draw(array_names, label="array name") array_path = _join_paths([path, name]) root = zarr.open_group(store, mode="w", zarr_format=3) with zarr.config.set({"array.rectilinear_chunks": True}): a = root.create_array( array_path, shape=shape, chunks=chunk_shapes, dtype=nparray.dtype, fill_value=fill_value, dimension_names=dim_names, attributes=attributes, ) a[:] = nparray return nparray, a @st.composite def chunk_paths(draw: st.DrawFn, ndim: int, numblocks: tuple[int, ...], subset: bool = True) -> str: blockidx = draw( st.tuples(*tuple(st.integers(min_value=0, max_value=max(0, b - 1)) for b in numblocks)) ) subset_slicer = slice(draw(st.integers(min_value=0, max_value=ndim))) if subset else slice(None) return "/".join(map(str, blockidx[subset_slicer])) zarr-developers-zarr-python-a2f2415/src/zarr/testing/utils.py000066400000000000000000000021101527014137600244130ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING, cast import pytest from zarr.core.buffer import Buffer if TYPE_CHECKING: from zarr.core.common import BytesLike __all__ = ["assert_bytes_equal"] def assert_bytes_equal(b1: Buffer | BytesLike | None, b2: Buffer | BytesLike | None) -> None: """Help function to assert if two bytes-like or Buffers are equal Warnings -------- Always copies data, only use for testing and debugging """ if isinstance(b1, Buffer): b1 = b1.to_bytes() if isinstance(b2, Buffer): b2 = b2.to_bytes() assert b1 == b2 def has_cupy() -> bool: try: import cupy return cast("bool", cupy.cuda.runtime.getDeviceCount() > 0) except ImportError: return False except cupy.cuda.runtime.CUDARuntimeError: return False gpu_mark = pytest.mark.gpu skip_if_no_gpu = pytest.mark.skipif(not has_cupy(), reason="CuPy not installed or no GPU available") # Decorator for GPU tests def gpu_test[T](func: T) -> T: return cast(T, gpu_mark(skip_if_no_gpu(func))) zarr-developers-zarr-python-a2f2415/src/zarr/types.py000066400000000000000000000016231527014137600227520ustar00rootroot00000000000000from typing import Any from zarr.core.array import Array, AsyncArray from zarr.core.common import JSON, ZarrFormat from zarr.core.dtype.common import DTypeConfig_V2, DTypeJSON from zarr.core.metadata.v2 import ArrayV2Metadata from zarr.core.metadata.v3 import ArrayV3Metadata type AnyAsyncArray = AsyncArray[Any] """A Zarr format 2 or 3 `AsyncArray`""" type AsyncArrayV2 = AsyncArray[ArrayV2Metadata] """A Zarr format 2 `AsyncArray`""" type AsyncArrayV3 = AsyncArray[ArrayV3Metadata] """A Zarr format 3 `AsyncArray`""" type AnyArray = Array[Any] """A Zarr format 2 or 3 `Array`""" type ArrayV2 = Array[ArrayV2Metadata] """A Zarr format 2 `Array`""" type ArrayV3 = Array[ArrayV3Metadata] """A Zarr format 3 `Array`""" __all__ = ( "JSON", "AnyArray", "AnyAsyncArray", "ArrayV2", "ArrayV3", "AsyncArrayV2", "AsyncArrayV3", "DTypeConfig_V2", "DTypeJSON", "ZarrFormat", ) zarr-developers-zarr-python-a2f2415/tests/000077500000000000000000000000001527014137600206275ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/__init__.py000066400000000000000000000000001527014137600227260ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/benchmarks/000077500000000000000000000000001527014137600227445ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/benchmarks/__init__.py000066400000000000000000000000001527014137600250430ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/benchmarks/common.py000066400000000000000000000002621527014137600246060ustar00rootroot00000000000000from dataclasses import dataclass @dataclass(kw_only=True, frozen=True) class Layout: shape: tuple[int, ...] chunks: tuple[int, ...] shards: tuple[int, ...] | None zarr-developers-zarr-python-a2f2415/tests/benchmarks/conftest.py000066400000000000000000000007441527014137600251500ustar00rootroot00000000000000"""Pytest configuration for benchmark tests.""" import pytest # Filter CodSpeed instrumentation warnings that can occur intermittently # when registering benchmark results. This is a known issue with the # CodSpeed walltime instrumentation hooks. # See: https://github.com/CodSpeedHQ/pytest-codspeed def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line( "filterwarnings", "ignore:Failed to set executed benchmark:RuntimeWarning", ) zarr-developers-zarr-python-a2f2415/tests/benchmarks/test_e2e.py000066400000000000000000000215721527014137600250370ustar00rootroot00000000000000""" Benchmarks for end-to-end read/write performance of Zarr """ from __future__ import annotations import os import platform import subprocess import warnings from functools import lru_cache from operator import getitem, setitem from typing import TYPE_CHECKING, Any, Literal import numpy as np import pytest import zarr from tests.benchmarks.common import Layout from zarr import create_array from zarr.core.config import config as zarr_config from zarr.testing.store import LatencyStore def clear_cache() -> None: """Drop the OS page cache between benchmark rounds. Requires passwordless sudo, so it is opt-in: set `ZARR_BENCHMARK_CLEAR_CACHE=1` to enable it (as the benchmark CI jobs do). By default this is a no-op, so a plain `pytest` run never prompts for a sudo password (see issue #4199). `sudo -n` guarantees we fail instead of blocking on a password prompt even when the variable is set. """ if os.environ.get("ZARR_BENCHMARK_CLEAR_CACHE", "") not in ("1", "true"): return if platform.system() == "Darwin": subprocess.call(["sync"]) subprocess.call(["sudo", "-n", "purge"]) elif platform.system() == "Linux": subprocess.call(["sudo", "-n", "sh", "-c", "sync; echo 3 > /proc/sys/vm/drop_caches"]) else: warnings.warn( f"ZARR_BENCHMARK_CLEAR_CACHE is set but cache clearing is not supported on " f"{platform.system()}; skipping.", stacklevel=2, ) if TYPE_CHECKING: from collections.abc import Callable, Iterator from types import EllipsisType from pytest_benchmark.fixture import BenchmarkFixture from zarr.abc.store import Store from zarr.core.common import NamedConfig @lru_cache def _data(shape: tuple[int]) -> np.ndarray: n = shape[0] period = 256 noise_level = 1 pattern = (np.sin(np.linspace(0, 2 * np.pi, period)) * 50 + 128).round().astype(np.uint8) data = np.tile(pattern, int(np.ceil(n / period)))[:n].astype(np.int16) rng = np.random.default_rng(0) data += rng.integers(-noise_level, noise_level + 1, size=n, dtype=np.int16) return np.clip(data, 0, 255).astype(np.uint8) CompressorName = Literal["zstd"] | None compressors: dict[CompressorName, NamedConfig[Any, Any] | None] = { None: None, # Default v3 "zstd": {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, } layouts: tuple[Layout, ...] = ( # No shards, just 1000 chunks Layout(shape=(100_000_000,), chunks=(100_000,), shards=None), # 1:1 chunk:shard shape, should measure overhead of sharding Layout(shape=(100_000_000,), chunks=(100_000,), shards=(100_000,)), # One shard with all the chunks, should measure over/under-head of handling inner shard chunks Layout(shape=(100_000_000,), chunks=(100_000,), shards=(100_000 * 1_000,)), # Mixed layout balancing inner vs. outer concurrency (likely the most real-world case) Layout(shape=(1_000_000_000,), chunks=(100_000,), shards=(100_000 * 100,)), ) _PIPELINE_SETTINGS = { "batched": {"codec_pipeline.path": "zarr.core.codec_pipeline.BatchedCodecPipeline"}, "fused_full_threaded": { "codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline", "codec_pipeline.max_workers": None, }, "fused_single_threaded": { "codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline", "codec_pipeline.max_workers": 1, }, } _LATENCY_VALUES = (0, 0.03) @pytest.fixture(params=_LATENCY_VALUES, ids=lambda v: f"latency={v}") def latency(request: pytest.FixtureRequest) -> float: return request.param # type: ignore[no-any-return] @pytest.fixture def bench_store(store: Store, latency: float, request: pytest.FixtureRequest) -> Store: """Wraps the underlying store in LatencyStore when latency > 0. Local-store cases skip nonzero latency — synthetic latency on top of a real LocalStore is double-counting; latency simulation only applies to the in-process memory store. """ callspec = getattr(request.node, "callspec", None) store_kind = callspec.params.get("store", "memory") if callspec is not None else "memory" if latency > 0: if store_kind == "local": pytest.skip("latency injection only applies to in-memory store") return LatencyStore( store, get_latency=(latency, latency * 1.2), set_latency=(latency, latency * 1.2) ) if store_kind == "memory": pytest.skip("memory store doesn't offer much over local without latency") return store @pytest.fixture(params=["batched"]) # , "fused_full_threaded", "fused_single_threaded"]) def pipeline(request: pytest.FixtureRequest) -> Iterator[str]: """Set ``codec_pipeline.path`` for the duration of the benchmark. Yields the pipeline name so each parametrize cell has a distinct benchmark id. """ name = request.param with zarr_config.set(_PIPELINE_SETTINGS[name]): yield name @pytest.mark.parametrize( "get_data", [lambda shape: 1, lambda shape: _data(shape)], ids=["repeated", "semi_random"] ) @pytest.mark.parametrize("compression_name", ["zstd", None]) @pytest.mark.parametrize("layout", layouts, ids=str) @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) def test_write_array( bench_store: Store, layout: Layout, compression_name: CompressorName, pipeline: str, benchmark: BenchmarkFixture, get_data: Callable[[tuple[int]], np.ndarray | int], ) -> None: """ Test the time required to fill an array with a single value """ arr = create_array( bench_store, dtype="uint8", shape=layout.shape, chunks=layout.chunks, shards=layout.shards, compressors=compressors[compression_name], # type: ignore[arg-type] fill_value=0, ) def setup() -> tuple[tuple[zarr.Array, EllipsisType, int | np.ndarray], dict]: # type: ignore[type-arg] clear_cache() return (arr, Ellipsis, get_data(layout.shape)), {} # type: ignore[arg-type] benchmark.pedantic(setitem, setup=setup, rounds=3) # type: ignore[no-untyped-call] @pytest.mark.parametrize( "get_data", [lambda shape: 1, lambda shape: _data(shape)], ids=["repeated", "semi_random"] ) @pytest.mark.parametrize("compression_name", ["zstd", None]) @pytest.mark.parametrize("layout", layouts, ids=str) @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) def test_read_array( bench_store: Store, layout: Layout, compression_name: CompressorName, pipeline: str, benchmark: BenchmarkFixture, get_data: Callable[[tuple[int]], np.ndarray | int], ) -> None: """ Test the time required to read the entirety of an array """ arr = create_array( bench_store, dtype="uint8", shape=layout.shape, chunks=layout.chunks, shards=layout.shards, compressors=compressors[compression_name], # type: ignore[arg-type] fill_value=0, ) arr[:] = get_data(layout.shape) # type: ignore[arg-type] def setup() -> tuple[tuple[zarr.Array, EllipsisType], dict]: # type: ignore[type-arg] clear_cache() return (arr, Ellipsis), {} benchmark.pedantic(getitem, setup=setup, rounds=3) # type: ignore[no-untyped-call] _CONCURRENT_READ_THREADS = 8 _concurrent_layout = Layout(shape=(64_000_000,), chunks=(4_000_000,), shards=None) @pytest.mark.parametrize("pipeline", ["batched", "fused_full_threaded"], indirect=True) @pytest.mark.parametrize("compression_name", ["zstd", None]) @pytest.mark.parametrize("store", ["local"], indirect=["store"]) def test_read_array_concurrent( bench_store: Store, compression_name: CompressorName, pipeline: str, benchmark: BenchmarkFixture, ) -> None: """Dask-style access: several user threads each reading one chunk per call. All sync-API calls are serviced by the one global event loop, so this measures how much of each read's IO+compute the pipeline runs while holding the loop: anything inline serializes the readers. """ from concurrent.futures import ThreadPoolExecutor layout = _concurrent_layout arr = create_array( bench_store, dtype="uint8", shape=layout.shape, chunks=layout.chunks, shards=layout.shards, compressors=compressors[compression_name], # type: ignore[arg-type] fill_value=0, ) arr[:] = _data(layout.shape) selections = [ slice(start, start + layout.chunks[0]) for start in range(0, layout.shape[0], layout.chunks[0]) ] def read_all_chunks_concurrently() -> None: with ThreadPoolExecutor(max_workers=_CONCURRENT_READ_THREADS) as executor: list(executor.map(lambda sel: arr[sel], selections)) def setup() -> tuple[tuple[()], dict]: # type: ignore[type-arg] clear_cache() return (), {} benchmark.pedantic(read_all_chunks_concurrently, setup=setup, rounds=3) # type: ignore[no-untyped-call] zarr-developers-zarr-python-a2f2415/tests/benchmarks/test_indexing.py000066400000000000000000000255341527014137600261730ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING if TYPE_CHECKING: from pytest_benchmark.fixture import BenchmarkFixture from zarr.abc.store import Store from operator import getitem import pytest from zarr import create_array indexers = ( (0,) * 3, (slice(None),) * 3, (slice(0, None, 4),) * 3, (slice(10),) * 3, (slice(10, -10, 4),) * 3, (slice(None), slice(0, 3, 2), slice(0, 10)), ) shards = ( None, (50,) * 3, ) @pytest.mark.parametrize("store", ["memory", "memory_get_latency"], indirect=["store"]) @pytest.mark.parametrize("indexer", indexers, ids=str) @pytest.mark.parametrize("shards", shards, ids=str) def test_slice_indexing( store: Store, indexer: tuple[int | slice], shards: tuple[int, ...] | None, benchmark: BenchmarkFixture, ) -> None: data = create_array( store=store, shape=(105,) * 3, dtype="uint8", chunks=(10,) * 3, shards=shards, compressors=None, filters=None, fill_value=0, ) data[:] = 1 benchmark(getitem, data, indexer) # Benchmark for Morton order optimization with power-of-2 shards # Morton order is used internally by sharding codec for chunk iteration morton_shards = ( (16,) * 3, # With 2x2x2 chunks: 8x8x8 = 512 chunks per shard (32,) * 3, # With 2x2x2 chunks: 16x16x16 = 4096 chunks per shard ) @pytest.mark.parametrize("store", ["memory"], indirect=["store"]) @pytest.mark.parametrize("shards", morton_shards, ids=str) def test_sharded_morton_indexing( store: Store, shards: tuple[int, ...], benchmark: BenchmarkFixture, ) -> None: """Benchmark sharded array indexing with power-of-2 chunks per shard. This benchmark exercises the Morton order iteration path in the sharding codec, which benefits from the hypercube and vectorization optimizations. The Morton order cache is cleared before each iteration to measure the full computation cost. """ from zarr.core.indexing import _morton_order, morton_order_coords # Create array where each shard contains many small chunks # e.g., shards=(32,32,32) with chunks=(2,2,2) means 16x16x16 = 4096 chunks per shard shape = tuple(s * 2 for s in shards) # 2 shards per dimension chunks = (2,) * 3 # Small chunks to maximize chunks per shard data = create_array( store=store, shape=shape, dtype="uint8", chunks=chunks, shards=shards, compressors=None, filters=None, fill_value=0, ) data[:] = 1 # Read a sub-shard region to exercise Morton order iteration indexer = (slice(shards[0]),) * 3 def read_with_cache_clear() -> None: _morton_order.cache_clear() morton_order_coords.cache_clear() getitem(data, indexer) benchmark(read_with_cache_clear) # Benchmark with larger chunks_per_shard to make Morton order impact more visible large_morton_shards = ( (32,) * 3, # With 1x1x1 chunks: 32x32x32 = 32768 chunks per shard (power-of-2) (30,) * 3, # With 1x1x1 chunks: 30x30x30 = 27000 chunks per shard (non-power-of-2) (33,) * 3, # With 1x1x1 chunks: 33x33x33 = 35937 chunks per shard (near-miss: just above power-of-2) ) @pytest.mark.parametrize("store", ["memory"], indirect=["store"]) @pytest.mark.parametrize("shards", large_morton_shards, ids=str) def test_sharded_morton_indexing_large( store: Store, shards: tuple[int, ...], benchmark: BenchmarkFixture, ) -> None: """Benchmark sharded array indexing with large chunks_per_shard. Uses 1x1x1 chunks to maximize chunks_per_shard (32^3 = 32768), making the Morton order computation a more significant portion of total time. The Morton order cache is cleared before each iteration. """ from zarr.core.indexing import _morton_order, morton_order_coords # 1x1x1 chunks means chunks_per_shard equals shard shape shape = tuple(s * 2 for s in shards) # 2 shards per dimension chunks = (1,) * 3 # 1x1x1 chunks: chunks_per_shard = shards data = create_array( store=store, shape=shape, dtype="uint8", chunks=chunks, shards=shards, compressors=None, filters=None, fill_value=0, ) data[:] = 1 # Read one full shard indexer = (slice(shards[0]),) * 3 def read_with_cache_clear() -> None: _morton_order.cache_clear() morton_order_coords.cache_clear() getitem(data, indexer) benchmark(read_with_cache_clear) @pytest.mark.parametrize("store", ["memory"], indirect=["store"]) @pytest.mark.parametrize("shards", large_morton_shards, ids=str) def test_sharded_morton_single_chunk( store: Store, shards: tuple[int, ...], benchmark: BenchmarkFixture, ) -> None: """Benchmark reading a single chunk from a large shard. This isolates the Morton order computation overhead by minimizing I/O. Reading one chunk from a shard with 32^3 = 32768 chunks still requires computing the full Morton order, making the optimization impact clear. The Morton order cache is cleared before each iteration. """ from zarr.core.indexing import _morton_order, morton_order_coords # 1x1x1 chunks means chunks_per_shard equals shard shape shape = tuple(s * 2 for s in shards) # 2 shards per dimension chunks = (1,) * 3 # 1x1x1 chunks: chunks_per_shard = shards data = create_array( store=store, shape=shape, dtype="uint8", chunks=chunks, shards=shards, compressors=None, filters=None, fill_value=0, ) data[:] = 1 # Read only a single chunk (1x1x1) from the shard indexer = (slice(1),) * 3 def read_with_cache_clear() -> None: _morton_order.cache_clear() morton_order_coords.cache_clear() getitem(data, indexer) benchmark(read_with_cache_clear) # Benchmark for morton_order_coords directly (no I/O) morton_iter_shapes = ( (8, 8, 8), # 512 elements (power-of-2) (10, 10, 10), # 1000 elements (non-power-of-2) (16, 16, 16), # 4096 elements (power-of-2) (20, 20, 20), # 8000 elements (non-power-of-2) (32, 32, 32), # 32768 elements (power-of-2) (30, 30, 30), # 27000 elements (non-power-of-2) (33, 33, 33), # 35937 elements (near-miss: just above power-of-2, n_z=262144) ) @pytest.mark.parametrize("shape", morton_iter_shapes, ids=str) def test_morton_order( shape: tuple[int, ...], benchmark: BenchmarkFixture, ) -> None: """Benchmark morton_order_coords directly without I/O. This isolates the Morton order computation to measure the optimization impact without array read/write overhead. The cache is cleared before each iteration. """ from zarr.core.indexing import _morton_order, morton_order_coords def compute_morton_order() -> None: _morton_order.cache_clear() morton_order_coords.cache_clear() # Build the full sequence to force computation list(morton_order_coords(shape)) benchmark(compute_morton_order) @pytest.mark.parametrize("store", ["memory"], indirect=["store"]) @pytest.mark.parametrize("shards", large_morton_shards, ids=str) def test_sharded_morton_write_single_chunk( store: Store, shards: tuple[int, ...], benchmark: BenchmarkFixture, ) -> None: """Benchmark writing a single chunk to a large shard. This is the clearest end-to-end demonstration of Morton order optimization. Writing a single chunk to a shard with 32^3 = 32768 chunks requires computing the full Morton order, but minimizes I/O overhead. Expected improvement: ~160ms (matching Morton computation speedup of ~178ms). The Morton order cache is cleared before each iteration. """ import numpy as np from zarr.core.indexing import ( _lexicographic_order, _morton_order, lexicographic_order_coords, morton_order_coords, ) # 1x1x1 chunks means chunks_per_shard equals shard shape shape = tuple(s * 2 for s in shards) # 2 shards per dimension chunks = (1,) * 3 # 1x1x1 chunks: chunks_per_shard = shards data = create_array( store=store, shape=shape, dtype="uint8", chunks=chunks, shards=shards, compressors=None, filters=None, fill_value=0, ) # Write data for a single chunk write_data = np.ones((1, 1, 1), dtype="uint8") indexer = (slice(1), slice(1), slice(1)) def write_with_cache_clear() -> None: # Clear every coordinate cache the write path touches, not just morton: # the sharded write also builds the lexicographic grid (dict.fromkeys / # to_dict_vectorized), so a partial clear would leave that path warm and # under-report the cold build cost. _morton_order.cache_clear() morton_order_coords.cache_clear() _lexicographic_order.cache_clear() lexicographic_order_coords.cache_clear() data[indexer] = write_data benchmark(write_with_cache_clear) @pytest.mark.parametrize("store", ["memory"], indirect=["store"]) @pytest.mark.parametrize("shards", large_morton_shards, ids=str) def test_sharded_morton_write_single_chunk_warm_cache( store: Store, shards: tuple[int, ...], benchmark: BenchmarkFixture, ) -> None: """Benchmark a single-chunk shard write with the chunk-order cache warm. Unlike ``test_sharded_morton_write_single_chunk``, this does NOT clear the order cache between iterations: it warms the cache once, then repeatedly writes the same single chunk. This isolates the amortized per-write cost the cache exists to optimize — the regime where the coordinate grid was already built (by an earlier write to this shard, or to any same-shaped shard) and is reused rather than rebuilt. Repeated writes to one shard and writes spread across many same-shaped shards exercise that cache reuse identically. This is the regime the cold benchmark cannot see. A regression that rebuilds the per-shard coordinate tuples on every write (rather than reusing the cached sequence) is invisible to the cold benchmark but shows up here. """ import numpy as np from zarr.core.indexing import _morton_order, morton_order_coords shape = tuple(s * 2 for s in shards) # 2 shards per dimension chunks = (1,) * 3 # 1x1x1 chunks: chunks_per_shard = shards data = create_array( store=store, shape=shape, dtype="uint8", chunks=chunks, shards=shards, compressors=None, filters=None, fill_value=0, ) write_data = np.ones((1, 1, 1), dtype="uint8") indexer = (slice(1), slice(1), slice(1)) # Warm the cache once up front; the timed writes then hit the warm path. _morton_order.cache_clear() morton_order_coords.cache_clear() data[indexer] = write_data def write_warm() -> None: data[indexer] = write_data benchmark(write_warm) zarr-developers-zarr-python-a2f2415/tests/conftest.py000066400000000000000000000501131527014137600230260ustar00rootroot00000000000000from __future__ import annotations import itertools import math import os import pathlib import re import sys from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from typing import TYPE_CHECKING, cast import numpy as np import numpy.typing as npt import pytest from hypothesis import HealthCheck, Verbosity, settings from hypothesis import strategies as st import zarr.registry from zarr import AsyncGroup, config from zarr.abc.store import Store from zarr.codecs.sharding import IndexLocation, ShardingCodec from zarr.core.array import ( _parse_chunk_encoding_v2, _parse_chunk_encoding_v3, _parse_chunk_key_encoding, ) from zarr.core.chunk_grids import ( SHARDED_INNER_CHUNK_MAX_BYTES, guess_chunks, normalize_chunks_nd, resolve_outer_and_inner_chunks, ) from zarr.core.common import ( JSON, DimensionNamesLike, MemoryOrder, ShapeLike, ZarrFormat, parse_shapelike, ) from zarr.core.config import config as zarr_config from zarr.core.dtype import ( data_type_registry, get_data_type_from_native_dtype, ) from zarr.core.dtype.common import HasItemSize from zarr.core.metadata.v2 import ArrayV2Metadata from zarr.core.metadata.v3 import ArrayV3Metadata, create_chunk_grid_metadata from zarr.core.sync import sync from zarr.storage import FsspecStore, LocalStore, MemoryStore, StorePath, ZipStore from zarr.testing.store import LatencyStore if TYPE_CHECKING: from collections.abc import Generator from contextlib import AbstractContextManager from typing import Any, Literal from zarr.abc.codec import Codec from zarr.core.array import CompressorsLike, FiltersLike, SerializerLike, ShardsLike from zarr.core.chunk_key_encodings import ( ChunkKeyEncoding, ChunkKeyEncodingLike, V2ChunkKeyEncoding, ) from zarr.core.dtype.wrapper import ZDType @dataclass(frozen=True) class Expect[TIn, TOut]: """A test case with explicit input, expected output, and a human-readable id.""" input: TIn output: TOut id: str @dataclass(frozen=True) class ExpectFail[TIn]: """A test case that should raise an exception. `msg` is a regex matched against the exception text (pytest's native `match=` semantics). Leave it `None` to assert only the exception type. Set `escape=True` when `msg` is a literal that contains regex metacharacters such as `(`, `[`, or `.`; `escape` has no effect when `msg` is `None`. """ input: TIn exception: type[Exception] id: str msg: str | None = None escape: bool = False def raises(self) -> AbstractContextManager[pytest.ExceptionInfo[Exception]]: if self.msg is None: return pytest.raises(self.exception) pattern = re.escape(self.msg) if self.escape else self.msg return pytest.raises(self.exception, match=pattern) async def parse_store( store: Literal["local", "memory", "fsspec", "zip", "memory_get_latency"], path: str ) -> LocalStore | MemoryStore | FsspecStore | ZipStore | LatencyStore: if store == "local": return await LocalStore.open(path) if store == "memory": return await MemoryStore.open() if store == "fsspec": return await FsspecStore.open(url=path) if store == "zip": return await ZipStore.open(f"{path}/zarr.zip", mode="w") if store == "memory_get_latency": return LatencyStore(MemoryStore(), get_latency=0.0001, set_latency=0.0) raise AssertionError @pytest.fixture(params=[str, pathlib.Path]) def path_type(request: pytest.FixtureRequest) -> Any: return request.param # todo: harmonize this with local_store fixture @pytest.fixture async def store_path(tmp_path: pathlib.Path) -> StorePath: store = await LocalStore.open(str(tmp_path)) return StorePath(store) @pytest.fixture async def local_store(tmp_path: pathlib.Path) -> LocalStore: return await LocalStore.open(str(tmp_path)) @pytest.fixture async def remote_store(url: str) -> FsspecStore: return await FsspecStore.open(url) @pytest.fixture async def memory_store() -> MemoryStore: return await MemoryStore.open() @pytest.fixture async def zip_store(tmp_path: pathlib.Path) -> ZipStore: return await ZipStore.open(str(tmp_path / "zarr.zip"), mode="w") @pytest.fixture async def store(request: pytest.FixtureRequest, tmp_path: pathlib.Path) -> Store: param = request.param return await parse_store(param, str(tmp_path)) @pytest.fixture async def store2(request: pytest.FixtureRequest, tmp_path: pathlib.Path) -> Store: """Fixture to create a second store for testing copy operations between stores""" param = request.param store2_path = tmp_path / "store2" store2_path.mkdir() return await parse_store(param, str(store2_path)) @pytest.fixture(params=["local", "memory", "zip"]) def sync_store(request: pytest.FixtureRequest, tmp_path: pathlib.Path) -> Store: result = sync(parse_store(request.param, str(tmp_path))) if not isinstance(result, Store): raise TypeError(f"Wrong store class returned by test fixture! got {result} instead") return result @dataclass class AsyncGroupRequest: zarr_format: ZarrFormat store: Literal["local", "fsspec", "memory", "zip"] attributes: dict[str, Any] = field(default_factory=dict) @pytest.fixture async def async_group(request: pytest.FixtureRequest, tmp_path: pathlib.Path) -> AsyncGroup: param: AsyncGroupRequest = request.param store = await parse_store(param.store, str(tmp_path)) return await AsyncGroup.from_store( store, attributes=param.attributes, zarr_format=param.zarr_format, overwrite=False, ) @pytest.fixture(params=["numpy", "cupy"]) def xp(request: pytest.FixtureRequest) -> Any: """Fixture to parametrize over numpy-like libraries""" if request.param == "cupy": request.node.add_marker(pytest.mark.gpu) return pytest.importorskip(request.param) @pytest.fixture(autouse=True) def reset_config() -> Generator[None, None, None]: config.reset() yield config.reset() @dataclass class ArrayRequest: shape: tuple[int, ...] dtype: str order: MemoryOrder @pytest.fixture def array_fixture(request: pytest.FixtureRequest) -> npt.NDArray[Any]: array_request: ArrayRequest = request.param return ( np.arange(np.prod(array_request.shape)) .reshape(array_request.shape, order=array_request.order) .astype(array_request.dtype) ) @pytest.fixture(params=(2, 3), ids=["zarr2", "zarr3"]) def zarr_format(request: pytest.FixtureRequest) -> ZarrFormat: if request.param == 2: return 2 elif request.param == 3: return 3 msg = f"Invalid zarr format requested. Got {request.param}, expected on of (2,3)." raise ValueError(msg) def _clear_registries() -> None: registries = zarr.registry._collect_entrypoints() for registry in registries: registry.lazy_load_list.clear() data_type_registry._lazy_load_list.clear() @pytest.fixture def set_path() -> Generator[None, None, None]: tests_dir = str(pathlib.Path(__file__).parent.absolute()) sys.path.append(tests_dir) _clear_registries() zarr.registry._collect_entrypoints() yield sys.path.remove(tests_dir) _clear_registries() zarr.registry._collect_entrypoints() config.reset() def pytest_addoption(parser: Any) -> None: parser.addoption( "--run-slow-hypothesis", action="store_true", default=False, help="run slow hypothesis tests", ) def pytest_collection_modifyitems(config: Any, items: Any) -> None: if config.getoption("--run-slow-hypothesis"): return skip_slow_hyp = pytest.mark.skip(reason="need --run-slow-hypothesis option to run") for item in items: if "slow_hypothesis" in item.keywords: item.add_marker(skip_slow_hyp) settings.register_profile( "default", parent=settings.get_profile("default"), max_examples=300, suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], deadline=None, verbosity=Verbosity.verbose, ) settings.register_profile( "ci", parent=settings.get_profile("ci"), max_examples=300, derandomize=True, # more like regression testing deadline=None, suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], ) settings.register_profile( "nightly", max_examples=500, parent=settings.get_profile("ci"), derandomize=False, stateful_step_count=100, ) settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default")) @st.composite def json_attributes(draw: st.DrawFn, *, depth: int) -> dict[str, JSON]: """Generate JSON attribute dictionaries with additional nested object/array layers.""" keys = st.text(max_size=8) scalars = ( st.none() | st.booleans() | st.integers() | st.floats(allow_nan=False, allow_infinity=False) | st.text(max_size=8) ) values = st.recursive( scalars, lambda children: ( st.lists(children, max_size=3) | st.dictionaries(keys, children, max_size=3) ), max_leaves=8, ) attributes: dict[str, JSON] = draw(st.dictionaries(keys, values, max_size=3)) if depth: value: JSON = attributes for is_object in draw(st.lists(st.booleans(), min_size=depth, max_size=depth)): value = {draw(keys): value} if is_object else [value] attributes = {draw(keys): value} return attributes # TODO: uncomment these overrides when we can get mypy to accept them """ @overload def create_array_metadata( *, shape: ShapeLike, dtype: npt.DTypeLike, chunks: tuple[int, ...] | Literal["auto"], shards: None, filters: FiltersLike, compressors: CompressorsLike, serializer: SerializerLike, fill_value: Any | None, order: MemoryOrder | None, zarr_format: Literal[2], attributes: dict[str, JSON] | None, chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None, dimension_names: None, ) -> ArrayV2Metadata: ... @overload def create_array_metadata( *, shape: ShapeLike, dtype: npt.DTypeLike, chunks: tuple[int, ...] | Literal["auto"], shards: ShardsLike | None, filters: FiltersLike, compressors: CompressorsLike, serializer: SerializerLike, fill_value: Any | None, order: None, zarr_format: Literal[3], attributes: dict[str, JSON] | None, chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None, dimension_names: Iterable[str] | None, ) -> ArrayV3Metadata: ... """ def create_array_metadata( *, shape: ShapeLike, dtype: npt.DTypeLike, chunks: tuple[int, ...] | Literal["auto"] = "auto", shards: ShardsLike | None = None, filters: FiltersLike = "auto", compressors: CompressorsLike = "auto", serializer: SerializerLike = "auto", fill_value: Any = 0, order: MemoryOrder | None = None, zarr_format: ZarrFormat, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, ) -> ArrayV2Metadata | ArrayV3Metadata: """ Create array metadata """ dtype_parsed = get_data_type_from_native_dtype(dtype) shape_parsed = parse_shapelike(shape) chunk_key_encoding_parsed = _parse_chunk_key_encoding( chunk_key_encoding, zarr_format=zarr_format ) item_size = 1 if isinstance(dtype_parsed, HasItemSize): item_size = dtype_parsed.item_size if chunks == "auto": chunks_normalized = guess_chunks( shape_parsed, item_size, max_bytes=SHARDED_INNER_CHUNK_MAX_BYTES if shards is not None else None, ) else: chunks_normalized = normalize_chunks_nd(chunks, shape_parsed) outer_chunks, inner = resolve_outer_and_inner_chunks( array_shape=shape_parsed, chunks=chunks_normalized, shard_shape=shards, item_size=item_size, ) if order is None: order_parsed = zarr_config.get("array.order") else: order_parsed = order if zarr_format == 2: filters_parsed, compressor_parsed = _parse_chunk_encoding_v2( compressor=compressors, filters=filters, dtype=dtype_parsed ) chunk_key_encoding_parsed = cast("V2ChunkKeyEncoding", chunk_key_encoding_parsed) return ArrayV2Metadata( shape=shape_parsed, dtype=dtype_parsed, chunks=outer_chunks.chunk_shape, order=order_parsed, dimension_separator=chunk_key_encoding_parsed.separator, fill_value=fill_value, compressor=compressor_parsed, filters=filters_parsed, attributes=attributes, ) elif zarr_format == 3: array_array, array_bytes, bytes_bytes = _parse_chunk_encoding_v3( compressors=compressors, filters=filters, serializer=serializer, dtype=dtype_parsed, ) sub_codecs: tuple[Codec, ...] = (*array_array, array_bytes, *bytes_bytes) codecs_out: tuple[Codec, ...] if inner is not None: inner_chunks_flat = inner.outer_chunks.chunk_shape index_location: IndexLocation = "end" if isinstance(shards, dict): index_location = cast("IndexLocation", shards.get("index_location", "end")) sharding_codec = ShardingCodec( chunk_shape=inner_chunks_flat, codecs=sub_codecs, index_location=index_location, ) validation_grid = create_chunk_grid_metadata(outer_chunks) sharding_codec.validate( shape=inner_chunks_flat, dtype=dtype_parsed, chunk_grid=validation_grid, ) codecs_out = (sharding_codec,) else: codecs_out = sub_codecs return ArrayV3Metadata( shape=shape_parsed, data_type=dtype_parsed, chunk_grid=create_chunk_grid_metadata(outer_chunks), chunk_key_encoding=chunk_key_encoding_parsed, fill_value=fill_value, codecs=codecs_out, attributes=attributes, dimension_names=dimension_names, ) raise ValueError(f"Invalid Zarr format: {zarr_format}") # TODO: uncomment these overrides when we can get mypy to accept them """ @overload def meta_from_array( array: np.ndarray[Any, Any], chunks: tuple[int, ...] | Literal["auto"], shards: None, filters: FiltersLike, compressors: CompressorsLike, serializer: SerializerLike, fill_value: Any | None, order: MemoryOrder | None, zarr_format: Literal[2], attributes: dict[str, JSON] | None, chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None, dimension_names: Iterable[str] | None, ) -> ArrayV2Metadata: ... @overload def meta_from_array( array: np.ndarray[Any, Any], chunks: tuple[int, ...] | Literal["auto"], shards: ShardsLike | None, filters: FiltersLike, compressors: CompressorsLike, serializer: SerializerLike, fill_value: Any | None, order: None, zarr_format: Literal[3], attributes: dict[str, JSON] | None, chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None, dimension_names: Iterable[str] | None, ) -> ArrayV3Metadata: ... """ def meta_from_array( array: np.ndarray[Any, Any], *, chunks: tuple[int, ...] | Literal["auto"] = "auto", shards: ShardsLike | None = None, filters: FiltersLike = "auto", compressors: CompressorsLike = "auto", serializer: SerializerLike = "auto", fill_value: Any = 0, order: MemoryOrder | None = None, zarr_format: ZarrFormat = 3, attributes: dict[str, JSON] | None = None, chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, dimension_names: DimensionNamesLike = None, ) -> ArrayV3Metadata | ArrayV2Metadata: """ Create array metadata from an array """ return create_array_metadata( shape=array.shape, dtype=array.dtype, chunks=chunks, shards=shards, filters=filters, compressors=compressors, serializer=serializer, fill_value=fill_value, order=order, zarr_format=zarr_format, attributes=attributes, chunk_key_encoding=chunk_key_encoding, dimension_names=dimension_names, ) def skip_object_dtype(dtype: ZDType[Any, Any]) -> None: if dtype.dtype_cls is type(np.dtype("O")): msg = ( f"{dtype} uses the numpy object data type, which is not a valid target for data " "type resolution" ) pytest.skip(msg) def nan_equal(a: object, b: object) -> bool: """ Convenience function for equality comparison between two values ``a`` and ``b``, that might both be NaN. Returns True if both ``a`` and ``b`` are NaN, otherwise returns a == b """ if math.isnan(a) and math.isnan(b): # type: ignore[arg-type] return True return a == b def deep_nan_equal(a: object, b: object) -> bool: if isinstance(a, Mapping) and isinstance(b, Mapping): return all(deep_nan_equal(a[k], b[k]) for k in a) if isinstance(a, Sequence) and isinstance(b, Sequence): return all(deep_nan_equal(a[i], b[i]) for i in range(len(a))) return nan_equal(a, b) def declared_chunk_data_sizes(declared: int | Sequence[int], extent: int) -> tuple[int, ...]: """The data sizes of the chunks one declared chunk grid dimension places over `extent`, worked out from the declaration alone: a bare int repeats to cover the extent, explicit edges are clipped to it. An oracle that does not go through zarr's chunk grid code.""" sizes: list[int] = [] offset = 0 for edge in itertools.repeat(declared) if isinstance(declared, int) else declared: if offset >= extent: break sizes.append(min(edge, extent - offset)) offset += edge return tuple(sizes) def gzip_streams_equal_except_mtime(a: bytes, b: bytes) -> bool: """Compare two gzip streams, ignoring the MTIME field of the header. Per RFC 1952 the gzip header is [magic(2)][CM(1)][FLG(1)][MTIME(4)][XFL(1)][OS(1)], so bytes 4-8 are MTIME. The fixed offsets assume the standard 10-byte header with no FNAME/FEXTRA/FCOMMENT flags set, which holds here because numcodecs' ``GZip.encode`` wraps ``gzip.GzipFile`` without a filename. """ if len(a) != len(b): return False return a[:4] == b[:4] and a[8:] == b[8:] # Shared mock-S3 (moto) backend. A single server is reused across the whole test session by # every test that needs S3 -- both the fsspec store tests and the documentation examples -- # instead of each module standing up its own. Consumers create their own buckets and choose # how the endpoint reaches the client (explicit storage_options vs. the AWS_ENDPOINT_URL # env var) on top of this fixture. @pytest.fixture(scope="session") def moto_server() -> Generator[str, None, None]: """Start a session-scoped moto S3 server and yield its endpoint URL. The server binds an ephemeral port (port=0), so the endpoint is only known at runtime; consumers must take it from this fixture rather than a constant. A fixed port deadlocks under pytest-xdist: session-scoped fixtures run once per *worker*, so concurrent workers race to bind the same port, and the losers block forever inside ThreadedMotoServer.start(), whose server thread dies on "Address already in use" before ever setting the ready event that start() waits on. importorskip lives inside the fixture so moto is only required when a test actually requests an S3 backend, not for the whole test session.""" moto_server_mod = pytest.importorskip("moto.moto_server.threaded_moto_server") server = moto_server_mod.ThreadedMotoServer(ip_address="127.0.0.1", port=0) server.start() host, port = server.get_host_and_port() # moto needs *some* credentials present; use throwaway values if the environment has none. os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foo") os.environ.setdefault("AWS_ACCESS_KEY_ID", "foo") try: yield f"http://{host}:{port}/" finally: server.stop() zarr-developers-zarr-python-a2f2415/tests/package_with_entrypoint-0.1.dist-info/000077500000000000000000000000001527014137600277375ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt000066400000000000000000000012371527014137600332400ustar00rootroot00000000000000[zarr.codecs] test = package_with_entrypoint:TestEntrypointCodec [zarr.codecs.test] another_codec = package_with_entrypoint:TestEntrypointGroup.Codec [zarr] codec_pipeline = package_with_entrypoint:TestEntrypointCodecPipeline ndbuffer = package_with_entrypoint:TestEntrypointNDBuffer buffer = package_with_entrypoint:TestEntrypointBuffer [zarr.buffer] another_buffer = package_with_entrypoint:TestEntrypointGroup.Buffer [zarr.ndbuffer] another_ndbuffer = package_with_entrypoint:TestEntrypointGroup.NDBuffer [zarr.codec_pipeline] another_pipeline = package_with_entrypoint:TestEntrypointGroup.Pipeline [zarr.data_type] new_data_type = package_with_entrypoint:TestDataTypezarr-developers-zarr-python-a2f2415/tests/package_with_entrypoint/000077500000000000000000000000001527014137600255505ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/package_with_entrypoint/__init__.py000066400000000000000000000054671527014137600276750ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING import numpy as np import numpy.typing as npt import zarr.core.buffer from zarr.abc.codec import ArrayBytesCodec, CodecInput, CodecPipeline from zarr.codecs import BytesCodec from zarr.core.buffer import Buffer, NDBuffer from zarr.core.dtype.npy.bool import Bool from zarr.errors import DataTypeValidationError if TYPE_CHECKING: from collections.abc import Iterable from typing import Any, ClassVar, Literal, Self from zarr.core.array_spec import ArraySpec from zarr.core.common import ZarrFormat from zarr.core.dtype.common import DTypeJSON, DTypeSpec_V2 class TestEntrypointCodec(ArrayBytesCodec): is_fixed_size = True async def encode( self, chunks_and_specs: Iterable[tuple[CodecInput | None, ArraySpec]], ) -> Iterable[Buffer | None]: return [None] async def decode( self, chunks_and_specs: Iterable[tuple[CodecInput | None, ArraySpec]], ) -> npt.NDArray[Any]: return np.array(1) def compute_encoded_size(self, input_byte_length: int, chunk_spec: ArraySpec) -> int: return input_byte_length class TestEntrypointCodecPipeline(CodecPipeline): def __init__(self, batch_size: int = 1) -> None: pass async def encode( self, chunks_and_specs: Iterable[tuple[CodecInput | None, ArraySpec]] ) -> Iterable[Buffer | None]: return [None] async def decode( self, chunks_and_specs: Iterable[tuple[CodecInput | None, ArraySpec]] ) -> Iterable[NDBuffer | None]: return np.array(1) class TestEntrypointBuffer(Buffer): pass class TestEntrypointNDBuffer(NDBuffer): pass class TestEntrypointGroup: class Codec(BytesCodec): pass class Buffer(zarr.core.buffer.Buffer): pass class NDBuffer(zarr.core.buffer.NDBuffer): pass class Pipeline(CodecPipeline): pass class TestDataType(Bool): """ This is a "data type" that serializes to "test" """ _zarr_v3_name: ClassVar[Literal["test"]] = "test" # type: ignore[assignment] @classmethod def from_json(cls, data: DTypeJSON, *, zarr_format: ZarrFormat) -> Self: if zarr_format == 2 and data == {"name": cls._zarr_v3_name, "object_codec_id": None}: return cls() if zarr_format == 3 and data == cls._zarr_v3_name: return cls() raise DataTypeValidationError( f"Invalid JSON representation of {cls.__name__}. Got {data!r}" ) def to_json(self, zarr_format: ZarrFormat) -> str | DTypeSpec_V2: # type: ignore[override] if zarr_format == 2: return {"name": self._zarr_v3_name, "object_codec_id": None} if zarr_format == 3: return self._zarr_v3_name raise ValueError("zarr_format must be 2 or 3") zarr-developers-zarr-python-a2f2415/tests/test_abc/000077500000000000000000000000001527014137600224135ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/test_abc/__init__.py000066400000000000000000000000001527014137600245120ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/test_abc/test_codec.py000066400000000000000000000005221527014137600251000ustar00rootroot00000000000000from __future__ import annotations from zarr.abc.codec import _check_codecjson_v2 def test_check_codecjson_v2_valid() -> None: """ Test that the _check_codecjson_v2 function works """ assert _check_codecjson_v2({"id": "gzip"}) assert not _check_codecjson_v2({"id": 10}) assert not _check_codecjson_v2([10, 11]) zarr-developers-zarr-python-a2f2415/tests/test_api.py000066400000000000000000002236241527014137600230220ustar00rootroot00000000000000from __future__ import annotations import inspect import itertools import re from typing import TYPE_CHECKING, Any import zarr.codecs import zarr.storage from zarr.core.array import AsyncArray, init_array from zarr.storage import LocalStore, ZipStore from zarr.storage._common import StorePath if TYPE_CHECKING: from collections.abc import Callable from pathlib import Path import numpy.typing as npt from zarr.abc.store import Store from zarr.core.array_spec import ArrayConfigLike from zarr.core.common import JSON, AccessModeLiteral, MemoryOrder, ZarrFormat from zarr.types import AnyArray import contextlib from typing import Literal import numpy as np import pytest from numpy.testing import assert_array_equal from packaging.version import parse as parse_version import zarr import zarr.api.asynchronous import zarr.api.synchronous import zarr.core.group from zarr import Array, Group from zarr.api.synchronous import ( create, create_array, create_group, from_array, group, load, open_group, save, save_array, save_group, ) from zarr.core.array_spec import ArrayConfig from zarr.core.buffer import NDArrayLike, default_buffer_prototype from zarr.core.sync import sync from zarr.errors import ( ArrayNotFoundError, ContainsArrayError, MetadataValidationError, ZarrDeprecationWarning, ZarrUserWarning, ) from zarr.storage import MemoryStore from zarr.storage._utils import normalize_path from zarr.testing.utils import gpu_test def test_create(memory_store: Store) -> None: store = memory_store # create array z = create(shape=100, store=store) assert isinstance(z, Array) assert z.shape == (100,) # create array, overwrite, specify chunk shape z = create(shape=200, chunk_shape=20, store=store, overwrite=True) assert isinstance(z, Array) assert z.shape == (200,) assert z.chunks == (20,) # create array, overwrite, specify chunk shape via chunks param z = create(shape=400, chunks=40, store=store, overwrite=True) assert isinstance(z, Array) assert z.shape == (400,) assert z.chunks == (40,) # create array with float shape with pytest.raises(TypeError, match="Expected an iterable of integers"): z = create(shape=(400.5, 100), store=store, overwrite=True) # type: ignore[arg-type] # create array with float chunk shape with pytest.raises(TypeError, match="Chunk specification must be an integer or an iterable"): z = create(shape=(400, 100), chunks=(16, 16.5), store=store, overwrite=True) # type: ignore[arg-type] def _create_async(**kwargs: Any) -> AnyArray: return Array(sync(zarr.api.asynchronous.create(**kwargs))) CREATE_FUNCS = pytest.mark.parametrize( "create_func", [create, _create_async], ids=["sync", "async"] ) def _write_existing_array(store: MemoryStore) -> None: """ Store an array with written chunks, plus a key that belongs to no Zarr node. """ old = create_array(store, shape=(4,), chunks=(2,), dtype="int8") old[:] = [1, 2, 3, 4] sync(store.set("junk", default_buffer_prototype().buffer.from_bytes(b"junk"))) @CREATE_FUNCS @pytest.mark.parametrize( ("mode", "overwrite", "existing"), [ (None, False, False), ("a", False, False), ("r+", False, False), ("w", False, False), ("w-", False, False), ("a", True, True), ("r+", True, True), ("w", False, True), ("w", True, True), ], ids=lambda v: repr(v), ) @pytest.mark.parametrize("data", [None, np.arange(3, dtype="int8")], ids=["no-data", "int8-data"]) def test_create_mode_data( create_func: Callable[..., AnyArray], mode: AccessModeLiteral | None, overwrite: bool, existing: bool, data: npt.NDArray[Any] | None, ) -> None: """ `create` opens the store with `mode`, replaces what is stored under the path when `mode` or `overwrite` says so, and writes `data` without taking its data type. """ store = MemoryStore() if existing: _write_existing_array(store) reference = MemoryStore() create(shape=(3,), chunks=(2,), store=reference, data=data) z = create_func(shape=(3,), chunks=(2,), store=store, mode=mode, overwrite=overwrite, data=data) assert sorted(store._store_dict) == sorted(reference._store_dict) assert z.dtype == np.dtype("float64") assert_array_equal(z[:], np.zeros(3) if data is None else data) @pytest.mark.parametrize("mode", [None, "a", "r+"]) def test_create_existing_array_without_overwrite(mode: AccessModeLiteral | None) -> None: """ `create` refuses to replace an existing array unless `mode` or `overwrite` allows it. """ store = MemoryStore() _write_existing_array(store) with pytest.raises(ContainsArrayError): create(shape=(3,), store=store, mode=mode) @pytest.mark.parametrize("overwrite", [False, True]) def test_create_mode_w_minus_existing_data(overwrite: bool) -> None: """ `create` with mode `'w-'` refuses to write where data is stored, whatever `overwrite` says. """ store = MemoryStore() _write_existing_array(store) with pytest.raises(FileExistsError, match="mode 'w-'"): create(shape=(3,), store=store, mode="w-", overwrite=overwrite) def test_create_mode_r() -> None: """ `create` with mode `'r'` opens the store read-only, so creation fails. """ with pytest.raises(ValueError, match="read-only"): create(shape=(3,), store=MemoryStore(), mode="r") def test_create_invalid_mode() -> None: """ `create` rejects a mode that is not an access mode. """ with pytest.raises(ValueError, match="Invalid mode: x"): create(shape=(3,), store=MemoryStore(), mode="x") # type: ignore[arg-type] @CREATE_FUNCS def test_create_unknown_keyword(create_func: Callable[..., AnyArray]) -> None: """ `create` rejects keyword arguments it does not declare, naming itself in the error, before `mode="w"` deletes anything. """ store = MemoryStore() _write_existing_array(store) before = dict(store._store_dict) with pytest.raises(TypeError, match=r"^create\(\) got an unexpected keyword argument 'typo'"): create_func(shape=(3,), store=store, mode="w", typo=1) assert store._store_dict == before @pytest.mark.parametrize( "func", [ zarr.api.asynchronous.zeros_like, zarr.api.asynchronous.ones_like, zarr.api.asynchronous.empty_like, zarr.api.asynchronous.full_like, zarr.api.asynchronous.open_like, ], ) @pytest.mark.parametrize("out_shape", ["keep", (10, 10)]) @pytest.mark.parametrize("out_chunks", ["keep", (10, 10)]) @pytest.mark.parametrize("out_dtype", ["keep", "int8"]) @pytest.mark.parametrize("out_fill", ["keep", 4]) async def test_array_like_creation( zarr_format: ZarrFormat, func: Callable[[Any], Any], out_shape: Literal["keep"] | tuple[int, ...], out_chunks: Literal["keep"] | tuple[int, ...], out_dtype: str, out_fill: Literal["keep"] | int, ) -> None: """ Test zeros_like, ones_like, empty_like, full_like, ensuring that we can override the shape, chunks, dtype and fill_value of the array-like object provided to these functions with appropriate keyword arguments """ ref_fill = 100 ref_arr = zarr.create_array( store={}, shape=(11, 12), dtype="uint8", chunks=(11, 12), zarr_format=zarr_format, fill_value=ref_fill, ) kwargs: dict[str, object] = {} if func is zarr.api.asynchronous.full_like: if out_fill == "keep": expect_fill = ref_fill else: expect_fill = out_fill kwargs["fill_value"] = expect_fill elif func is zarr.api.asynchronous.zeros_like: expect_fill = 0 elif func is zarr.api.asynchronous.ones_like: expect_fill = 1 elif func is zarr.api.asynchronous.empty_like: if out_fill == "keep": expect_fill = ref_fill else: kwargs["fill_value"] = out_fill expect_fill = out_fill elif func is zarr.api.asynchronous.open_like: # type: ignore[comparison-overlap] if out_fill == "keep": # type: ignore[unreachable] expect_fill = ref_fill else: kwargs["fill_value"] = out_fill expect_fill = out_fill kwargs["mode"] = "w" else: raise AssertionError if out_shape != "keep": kwargs["shape"] = out_shape expect_shape = out_shape else: expect_shape = ref_arr.shape if out_chunks != "keep": kwargs["chunks"] = out_chunks expect_chunks = out_chunks else: expect_chunks = ref_arr.chunks if out_dtype != "keep": kwargs["dtype"] = out_dtype expect_dtype = out_dtype else: expect_dtype = ref_arr.dtype # type: ignore[assignment] new_arr = await func(ref_arr, path="foo", zarr_format=zarr_format, **kwargs) assert new_arr.shape == expect_shape assert new_arr.chunks == expect_chunks assert new_arr.dtype == expect_dtype assert np.all(Array(new_arr)[:] == expect_fill) @pytest.mark.parametrize("mode_kwargs", [{}, {"mode": None}]) async def test_open_like_creates_array_by_default( zarr_format: ZarrFormat, mode_kwargs: dict[str, None] ) -> None: ref_arr = zarr.create_array( store={}, shape=(11, 12), dtype="uint8", chunks=(11, 12), zarr_format=zarr_format, fill_value=100, ) new_arr = await zarr.api.asynchronous.open_like( ref_arr, path="foo", store={}, zarr_format=zarr_format, **mode_kwargs, ) assert new_arr.shape == ref_arr.shape assert new_arr.chunks == ref_arr.chunks assert new_arr.dtype == ref_arr.dtype assert np.all(Array(new_arr)[:] == ref_arr.fill_value) async def test_open_like_default_mode_rejects_read_only_store( zarr_format: ZarrFormat, ) -> None: ref_arr = zarr.create_array( store={}, shape=(11, 12), dtype="uint8", chunks=(11, 12), zarr_format=zarr_format, ) with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"): await zarr.api.asynchronous.open_like( ref_arr, path="foo", store=MemoryStore(read_only=True), zarr_format=zarr_format, ) # TODO: parametrize over everything this function takes @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_create_array(store: Store, zarr_format: ZarrFormat) -> None: attrs: dict[str, JSON] = {"foo": 100} # explicit type annotation to avoid mypy error shape = (10, 10) path = "foo" data_val = 1 array_w = create_array( store, name=path, shape=shape, attributes=attrs, chunks=shape, dtype="uint8", zarr_format=zarr_format, ) array_w[:] = data_val assert array_w.shape == shape assert array_w.attrs == attrs assert np.array_equal(array_w[:], np.zeros(shape, dtype=array_w.dtype) + data_val) # type: ignore[unreachable] @pytest.mark.parametrize("write_empty_chunks", [True, False]) def test_write_empty_chunks_warns(write_empty_chunks: bool, zarr_format: ZarrFormat) -> None: """ Test that using the `write_empty_chunks` kwarg on array access will raise a warning. """ match = "The `write_empty_chunks` keyword argument .*" with pytest.warns(RuntimeWarning, match=match): _ = zarr.array( data=np.arange(10), shape=(10,), dtype="uint8", write_empty_chunks=write_empty_chunks, zarr_format=zarr_format, ) with pytest.warns(RuntimeWarning, match=match): _ = zarr.create( shape=(10,), dtype="uint8", write_empty_chunks=write_empty_chunks, zarr_format=zarr_format, ) @pytest.mark.parametrize("zarr_format", [2, 3]) def test_open_array_respects_write_empty_chunks_config(zarr_format: ZarrFormat) -> None: """Test that zarr.open() respects write_empty_chunks config.""" store = MemoryStore() _ = zarr.create( store=store, path="test_array", shape=(10,), chunks=(5,), dtype="f8", fill_value=0.0, zarr_format=zarr_format, ) arr2 = zarr.open(store=store, path="test_array", config={"write_empty_chunks": True}) assert isinstance(arr2, zarr.Array) assert arr2.async_array.config.write_empty_chunks is True arr2[0:5] = np.zeros(5) assert arr2.nchunks_initialized == 1 @pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.parametrize( ("func", "exists"), [ *itertools.product(["open_array-sync", "open_array-async", "open-shape"], [True, False]), # without a shape, zarr.open can only open an existing array ("open", True), ], ) @pytest.mark.parametrize( ("config", "expected"), [ (None, ArrayConfig.from_dict({})), ( {"read_missing_chunks": False}, ArrayConfig.from_dict({"read_missing_chunks": False}), ), ( {"order": "F", "write_empty_chunks": True}, ArrayConfig.from_dict({"order": "F", "write_empty_chunks": True}), ), ( ArrayConfig.from_dict({"read_missing_chunks": False}), ArrayConfig.from_dict({"read_missing_chunks": False}), ), ], ) async def test_open_array_config( zarr_format: ZarrFormat, func: str, exists: bool, config: ArrayConfigLike | None, expected: ArrayConfig, ) -> None: """ open_array and zarr.open apply the config argument whether they open an existing array or create one, with keys missing from a partial config taken from the global defaults. """ store = MemoryStore() if exists: zarr.create_array(store, shape=(4,), dtype="uint8", zarr_format=zarr_format) kwargs: dict[str, Any] = {"shape": (4,), "dtype": "uint8", "zarr_format": zarr_format} if config is not None: kwargs["config"] = config if func == "open_array-sync": observed = zarr.api.synchronous.open_array(store, mode="a", **kwargs).config elif func == "open_array-async": observed = (await zarr.api.asynchronous.open_array(store=store, mode="a", **kwargs)).config else: if func == "open": del kwargs["shape"], kwargs["dtype"] node = zarr.open(store=store, mode="a", **kwargs) assert isinstance(node, Array) observed = node.config assert observed == expected @pytest.mark.parametrize("exists", [True, False]) def test_open_array_unknown_config_key(exists: bool) -> None: """ Opening or creating an array with a config containing an unknown key raises TypeError. """ store = MemoryStore() if exists: zarr.create_array(store, shape=(4,), dtype="uint8") with pytest.raises(TypeError, match=r"Unknown array config keys: \['nope'\]"): zarr.open_array( store, mode="a", shape=(4,), dtype="uint8", config={"nope": 1}, # type: ignore[arg-type] ) @pytest.mark.parametrize("exists", [True, False]) def test_open_array_invalid_config_value(exists: bool) -> None: """ Opening or creating an array with a config containing an invalid value raises ValueError. """ store = MemoryStore() if exists: zarr.create_array(store, shape=(4,), dtype="uint8") with pytest.raises(ValueError, match="Expected instance of bool"): zarr.open_array( store, mode="a", shape=(4,), dtype="uint8", config={"read_missing_chunks": "yes"}, # type: ignore[arg-type] ) @pytest.mark.parametrize("exists", [True, False]) def test_open_group_with_config(exists: bool) -> None: """ zarr.open with a config raises TypeError when it opens or creates a group. """ store = MemoryStore() if exists: zarr.create_group(store) with pytest.raises(TypeError, match="resolves to a group"): zarr.open(store=store, mode="a", config={"order": "F"}) @pytest.mark.parametrize("path", ["foo", "/", "/foo", "///foo/bar"]) @pytest.mark.parametrize("node_type", ["array", "group"]) def test_open_normalized_path( memory_store: MemoryStore, path: str, node_type: Literal["array", "group"] ) -> None: node: Group | AnyArray if node_type == "group": node = group(store=memory_store, path=path) elif node_type == "array": node = create(store=memory_store, path=path, shape=(2,)) assert node.path == normalize_path(path) async def test_open_array(memory_store: MemoryStore, zarr_format: ZarrFormat) -> None: store = memory_store # open array, create if doesn't exist z = zarr.api.synchronous.open(store=store, shape=100, zarr_format=zarr_format) assert isinstance(z, Array) assert z.shape == (100,) # open array, overwrite # store._store_dict = {} store = MemoryStore() z = zarr.api.synchronous.open(store=store, shape=200, zarr_format=zarr_format) assert isinstance(z, Array) assert z.shape == (200,) # open array, read-only store_cls = type(store) ro_store = await store_cls.open(store_dict=store._store_dict, read_only=True) z = zarr.api.synchronous.open(store=ro_store, mode="r") assert isinstance(z, Array) assert z.shape == (200,) assert z.read_only # path not found with pytest.raises(FileNotFoundError): zarr.api.synchronous.open(store="doesnotexist", mode="r", zarr_format=zarr_format) def test_open_array_rectilinear_chunks(tmp_path: Path) -> None: """zarr.open with rectilinear (dask-style) chunks preserves the chunk grid.""" from zarr.core.metadata.v3 import RectilinearChunkGridMetadata chunks = ((3, 3, 4), (5, 5)) with zarr.config.set({"array.rectilinear_chunks": True}): z = zarr.open(store=tmp_path, shape=(10, 10), dtype="float64", chunks=chunks, mode="w") assert isinstance(z, Array) assert z.shape == (10, 10) assert isinstance(z.metadata.chunk_grid, RectilinearChunkGridMetadata) assert z.read_chunk_sizes == ((3, 3, 4), (5, 5)) async def test_async_array_open_array_not_found() -> None: """Test that AsyncArray.open raises ArrayNotFoundError when array doesn't exist""" store = MemoryStore() # Try to open an array that does not exist with pytest.raises(ArrayNotFoundError): await AsyncArray.open(store, zarr_format=2) def test_array_open_array_not_found_sync() -> None: """Test that Array.open raises ArrayNotFoundError when array doesn't exist""" store = MemoryStore() # Try to open an array that does not exist with pytest.raises(ArrayNotFoundError): Array.open(store) @pytest.mark.parametrize("func", [zarr.empty_like, zarr.zeros_like, zarr.ones_like, zarr.full_like]) @pytest.mark.parametrize("source_format", [2, 3]) @pytest.mark.parametrize("zarr_format", [None, 2, 3]) def test_like_zarr_format( func: Callable[..., AnyArray], source_format: ZarrFormat, zarr_format: ZarrFormat | None ) -> None: """ An array created like a zarr array has the zarr format of the source unless another is requested, and keeps the codecs of the source only when the formats match. """ source = zarr.create_array( {}, shape=(4,), dtype="int32", compressors=None, zarr_format=source_format, fill_value=7, ) new = func(source, zarr_format=zarr_format) expected_format = source_format if zarr_format is None else zarr_format assert new.metadata.zarr_format == expected_format assert new.shape == source.shape assert new.dtype == source.dtype if expected_format == source_format: assert new.compressors == source.compressors else: assert ( new.compressors == zarr.create_array( {}, shape=(4,), dtype="int32", zarr_format=expected_format ).compressors ) @pytest.mark.parametrize("source_format", [2, 3]) @pytest.mark.parametrize("target_format", [None, 2, 3]) def test_open_like_zarr_format(source_format: ZarrFormat, target_format: ZarrFormat | None) -> None: """ open_like does not inherit the zarr format of the source: it opens an existing array of any format, and creates a missing one in the default zarr format. """ source = zarr.zeros(store={}, shape=(4,), dtype="int32", zarr_format=source_format) store = MemoryStore() if target_format is not None: zarr.create_array( store, name="existing", shape=(4,), dtype="int32", zarr_format=target_format ) opened = zarr.open_like(source, path="existing", store=store) assert opened.metadata.zarr_format == (target_format or zarr.config.get("default_zarr_format")) assert set(store._store_dict) & {"existing/zarr.json", "existing/.zarray"} == { "existing/zarr.json" if opened.metadata.zarr_format == 3 else "existing/.zarray" } async def _read_all(store: Store) -> dict[str, bytes]: contents = {} for key in sorted([key async for key in store.list()]): value = await store.get(key, prototype=default_buffer_prototype()) assert value is not None contents[key] = value.to_bytes() return contents def _store_contents(store: Store) -> dict[str, bytes]: """Every key in `store` with its bytes.""" return sync(_read_all(store)) def _seed_array(store: Store, zarr_format: ZarrFormat) -> None: """Write an int32 array of shape (4,), with its chunk, at the root of `store`.""" zarr.create_array(store, shape=(4,), dtype="int32", zarr_format=zarr_format)[:] = 1 def _seed_group(store: Store, zarr_format: ZarrFormat) -> None: """Write a group with attributes and a child array, with its chunk, at the root of `store`.""" root = zarr.create_group(store, zarr_format=zarr_format, attributes={"k": 1}) root.create_array("child", shape=(4,), dtype="int32")[:] = 1 _SEEDS: dict[str, Callable[[Store, ZarrFormat], None]] = { "array": _seed_array, "group": _seed_group, } # Each call replaces the node at the root of the store it is given. _OVERWRITE_CALLS: dict[str, Callable[..., object]] = { "open-array": lambda store, **kw: zarr.open(store=store, mode="w", shape=(2,), **kw), "open-group": lambda store, **kw: zarr.open(store=store, mode="w", **kw), "open_array": lambda store, **kw: zarr.open_array(store=store, mode="w", shape=(2,), **kw), "open_group": lambda store, **kw: zarr.open_group(store=store, mode="w", **kw), "create-mode": lambda store, **kw: zarr.create(store=store, mode="w", shape=(2,), **kw), "create-overwrite": lambda store, **kw: zarr.create( store=store, overwrite=True, shape=(2,), **kw ), "create_array": lambda store, **kw: zarr.create_array( store, overwrite=True, shape=(2,), dtype="int32", **kw ), "save_array": lambda store, **kw: zarr.save_array(store, np.arange(2), mode="w", **kw), "save_group": lambda store, **kw: zarr.save_group(store, np.arange(2), **kw), "group": lambda store, **kw: zarr.group(store=store, overwrite=True, **kw), } @pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.parametrize("existing", _SEEDS) @pytest.mark.parametrize("call", _OVERWRITE_CALLS) def test_overwrite_replaces_existing_node( zarr_format: ZarrFormat, existing: str, call: str ) -> None: """ Opening or creating a node in an overwriting mode replaces the node at the path, including its chunks or children: the store ends up exactly as the same call leaves an empty store. """ store = MemoryStore() _SEEDS[existing](store, zarr_format) _OVERWRITE_CALLS[call](store) reference = MemoryStore() _OVERWRITE_CALLS[call](reference) assert _store_contents(store) == _store_contents(reference) @pytest.mark.parametrize("call", _OVERWRITE_CALLS) def test_overwrite_without_deletes_creates_node(tmp_path: Path, call: str) -> None: """ On a store that cannot delete keys, an overwriting call still creates the node when nothing is stored under the path. """ with ZipStore(tmp_path / "store.zip", mode="w") as store: _OVERWRITE_CALLS[call](store) reference = MemoryStore() _OVERWRITE_CALLS[call](reference) assert _store_contents(store) == _store_contents(reference) @pytest.mark.parametrize("call", _OVERWRITE_CALLS) def test_overwrite_without_deletes_keeps_existing_node(tmp_path: Path, call: str) -> None: """ On a store that cannot delete keys, an overwriting call raises without modifying the store when a node already exists at the path. """ with ZipStore(tmp_path / "store.zip", mode="w") as store: _seed_array(store, 3) before = _store_contents(store) with pytest.raises(ContainsArrayError): _OVERWRITE_CALLS[call](store) assert _store_contents(store) == before @pytest.mark.parametrize( "call", [ "open-array", "open-group", "open_array", "open_group", "create-mode", "save_array", "save_group", ], ) def test_overwrite_read_only_store_names_mode(call: str) -> None: """ An overwriting mode on a read-only store is refused with an error that names the mode that was passed. """ with pytest.raises(ValueError, match="mode is 'w'"): _OVERWRITE_CALLS[call](MemoryStore(read_only=True)) @pytest.mark.parametrize( "call", ["open-array", "open-group", "open_array", "create-mode", "create-overwrite", "save_array"], ) def test_overwrite_invalid_keyword_keeps_existing_node(call: str) -> None: """ An overwriting call that is given a keyword argument its target cannot accept raises without modifying the existing node. """ store = MemoryStore() _seed_array(store, 3) before = _store_contents(store) if call == "open-group": # zarr.open refuses an array config for a group before open_group runs kwargs: dict[str, object] = {"config": {"order": "F"}} match = "resolves to a group" else: kwargs = {"bogus": 1} match = "unexpected keyword argument" with pytest.raises(TypeError, match=match): _OVERWRITE_CALLS[call](store, **kwargs) assert _store_contents(store) == before @pytest.mark.parametrize( "call", ["open-array", "open_array", "create-mode", "create-overwrite", "create_array", "save_array"], ) def test_overwrite_invalid_fill_value_keeps_existing_node(call: str) -> None: """ An overwriting call whose new array metadata cannot be built raises without modifying the existing node. """ store = MemoryStore() _seed_array(store, 3) before = _store_contents(store) kwargs: dict[str, Any] = {"fill_value": "not a number"} if call in {"open-array", "open_array", "create-mode", "create-overwrite"}: kwargs["dtype"] = "int32" with pytest.raises(ValueError, match="invalid literal"): _OVERWRITE_CALLS[call](store, **kwargs) assert _store_contents(store) == before @pytest.mark.parametrize( "call", ["open-array", "open-group", "open_array", "open_group", "create_array", "group"], ) def test_overwrite_unencodable_attributes_keeps_existing_node(call: str) -> None: """ An overwriting call whose new metadata cannot be encoded raises without modifying the existing node. """ store = MemoryStore() _seed_array(store, 3) before = _store_contents(store) with pytest.raises(TypeError, match="not JSON serializable"): _OVERWRITE_CALLS[call](store, attributes={"x": object()}) assert _store_contents(store) == before def test_save_group_non_array_argument_keeps_existing_node() -> None: """ save_group given an argument that is not an array raises without modifying the existing node. """ store = MemoryStore() _seed_group(store, 3) before = _store_contents(store) with pytest.raises(TypeError, match="must be a numpy or other NDArrayLike array"): zarr.save_group(store, a=np.arange(2), b="not an array") # type: ignore[arg-type] assert _store_contents(store) == before def test_save_group_unstorable_dtype_keeps_existing_node() -> None: """ save_group given an array whose data type Zarr cannot store raises without modifying the existing node. """ store = MemoryStore() _seed_group(store, 3) before = _store_contents(store) with pytest.raises(ValueError, match="data type resolution"): zarr.save_group(store, a=np.arange(2), b=np.array([object()], dtype=object)) assert _store_contents(store) == before @pytest.mark.parametrize("store", ["memory", "local", "zip"], indirect=True) def test_v2_and_v3_exist_at_same_path(store: Store) -> None: zarr.create_array(store, shape=(10,), dtype="uint8", zarr_format=3) zarr.create_array(store, shape=(10,), dtype="uint8", zarr_format=2) msg = f"Both zarr.json (Zarr format 3) and .zarray (Zarr format 2) metadata objects exist at {store}. Zarr v3 will be used." with pytest.warns(ZarrUserWarning, match=re.escape(msg)): zarr.open(store=store) @pytest.mark.parametrize("store", ["memory"], indirect=True) async def test_create_group(store: Store, zarr_format: ZarrFormat) -> None: attrs = {"foo": 100} path = "node" node = create_group(store, path=path, attributes=attrs, zarr_format=zarr_format) assert isinstance(node, Group) assert node.attrs == attrs assert node.metadata.zarr_format == zarr_format # type: ignore[unreachable] async def test_open_group(memory_store: MemoryStore) -> None: store = memory_store # open group, create if doesn't exist g = open_group(store=store) g.create_group("foo") assert isinstance(g, Group) assert "foo" in g # open group, overwrite g = open_group(store=store, mode="w") assert isinstance(g, Group) assert "foo" not in g # open group, read-only store_cls = type(store) ro_store = await store_cls.open(store_dict=store._store_dict, read_only=True) g = open_group(store=ro_store, mode="r") assert isinstance(g, Group) assert g.read_only @pytest.mark.parametrize("zarr_format", [None, 2, 3]) async def test_open_group_unspecified_version(tmp_path: Path, zarr_format: ZarrFormat) -> None: """Regression test for https://github.com/zarr-developers/zarr-python/issues/2175""" # create a group with specified zarr format (could be 2, 3, or None) _ = await zarr.api.asynchronous.open_group( store=str(tmp_path), mode="w", zarr_format=zarr_format, attributes={"foo": "bar"} ) # now open that group without specifying the format g2 = await zarr.api.asynchronous.open_group(store=str(tmp_path), mode="r") assert g2.attrs == {"foo": "bar"} if zarr_format is not None: assert g2.metadata.zarr_format == zarr_format @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) @pytest.mark.parametrize("n_args", [10, 1, 0]) @pytest.mark.parametrize("n_kwargs", [10, 1, 0]) @pytest.mark.parametrize("path", [None, "some_path"]) def test_save(store: Store, n_args: int, n_kwargs: int, path: str | None) -> None: data = np.arange(10) args = [np.arange(10) for _ in range(n_args)] kwargs = {f"arg_{i}": data for i in range(n_kwargs)} if n_kwargs == 0 and n_args == 0: with pytest.raises(ValueError, match="at least one array must be provided"): save(store, path=path) elif n_args == 1 and n_kwargs == 0: save(store, *args, path=path) array = zarr.api.synchronous.open(store, path=path) assert isinstance(array, Array) assert_array_equal(array[:], data) else: save(store, *args, path=path, **kwargs) # type: ignore[arg-type] group = zarr.api.synchronous.open(store, path=path) assert isinstance(group, Group) for array in group.array_values(): assert_array_equal(array[:], data) for k in kwargs: assert k in group assert group.nmembers() == n_args + n_kwargs @pytest.mark.parametrize( "data", [ np.array(42, dtype=np.int64), np.array("teststr", dtype=np.bytes_), ], ) @pytest.mark.filterwarnings("ignore::zarr.errors.UnstableSpecificationWarning") def test_group_setitem_loads_scalar_arrays(sync_store: Store, data: np.ndarray) -> None: root = zarr.open_group(store=sync_store) root["test"] = data assert_array_equal(root["test"][...], data) assert_array_equal(zarr.load(store=sync_store, path="test"), data) def test_save_errors() -> None: with pytest.raises(ValueError, match="at least one array must be provided"): # no arrays provided save_group("data/group.zarr") with pytest.raises(TypeError, match="missing 1 required positional argument: 'arr'"): # no array provided save_array("data/group.zarr") # type: ignore[call-arg] with pytest.raises(ValueError, match="at least one array must be provided"): # no arrays provided save("data/group.zarr") a = np.arange(10) with pytest.raises( TypeError, match="Keyword argument 'mode' must be a numpy or other NDArrayLike array" ): # mode is no valid argument and would get handled as an array zarr.save("data/example.zarr", a, mode="w") def test_save_storage_options_is_not_an_array(tmp_path: Path) -> None: # `storage_options` is an option of `save`, not one of the arrays to save, so a # single array still lands as an array and not as a group holding `arr_0`. data = np.arange(10) save(str(tmp_path / "a.zarr"), data, storage_options=None) node = zarr.api.synchronous.open(str(tmp_path / "a.zarr")) assert isinstance(node, Array) assert_array_equal(node[:], data) def test_save_group_storage_options_positional_args(tmp_path: Path) -> None: # `storage_options` has to reach the store for positional arrays too, exactly as it # does for keyword arrays. fsspec = pytest.importorskip("fsspec") # An fsspec URL for a sync filesystem needs AsyncFileSystemWrapper, which landed in # fsspec 2024.12.0. The min-deps CI job pins an older one. if parse_version(fsspec.__version__) < parse_version("2024.12.0"): pytest.skip("No AsyncFileSystemWrapper") data = np.arange(10) url = f"local://{tmp_path}/group.zarr" save_group(url, data, data, storage_options={"auto_mkdir": True}) group = zarr.api.synchronous.open(str(tmp_path / "group.zarr")) assert isinstance(group, Group) assert sorted(group) == ["arr_0", "arr_1"] def test_open_with_mode_r(tmp_path: Path) -> None: # 'r' means read only (must exist) with pytest.raises(FileNotFoundError): zarr.open(store=tmp_path, mode="r") z1 = zarr.ones(store=tmp_path, shape=(3, 3)) assert z1.fill_value == 1 z2 = zarr.open(store=tmp_path, mode="r") assert isinstance(z2, Array) assert z2.fill_value == 1 result = z2[:] assert isinstance(result, NDArrayLike) assert (result == 1).all() with pytest.raises(ValueError): z2[:] = 3 def test_open_with_mode_r_plus(tmp_path: Path) -> None: # 'r+' means read/write (must exist) new_store_path = tmp_path / "new_store.zarr" assert not new_store_path.exists(), "Test should operate on non-existent directory" with pytest.raises(FileNotFoundError): zarr.open(store=new_store_path, mode="r+") assert not new_store_path.exists(), "mode='r+' should not create directory" zarr.ones(store=tmp_path, shape=(3, 3)) z2 = zarr.open(store=tmp_path, mode="r+") assert isinstance(z2, Array) result = z2[:] assert isinstance(result, NDArrayLike) assert (result == 1).all() z2[:] = 3 async def test_open_with_mode_a(tmp_path: Path) -> None: # Open without shape argument should default to group g = zarr.open(store=tmp_path, mode="a") assert isinstance(g, Group) await g.store_path.delete() # 'a' means read/write (create if doesn't exist) arr = zarr.open(store=tmp_path, mode="a", shape=(3, 3)) assert isinstance(arr, Array) arr[...] = 1 z2 = zarr.open(store=tmp_path, mode="a") assert isinstance(z2, Array) result = z2[:] assert isinstance(result, NDArrayLike) assert (result == 1).all() z2[:] = 3 def test_open_with_mode_w(tmp_path: Path) -> None: # 'w' means create (overwrite if exists); arr = zarr.open(store=tmp_path, mode="w", shape=(3, 3)) assert isinstance(arr, Array) arr[...] = 3 z2 = zarr.open(store=tmp_path, mode="w", shape=(3, 3)) assert isinstance(z2, Array) result = z2[:] assert isinstance(result, NDArrayLike) assert not (result == 3).all() z2[:] = 3 def test_open_with_mode_w_minus(tmp_path: Path) -> None: # 'w-' means create (fail if exists) arr = zarr.open(store=tmp_path, mode="w-", shape=(3, 3)) assert isinstance(arr, Array) arr[...] = 1 with pytest.raises(FileExistsError): zarr.open(store=tmp_path, mode="w-") @pytest.mark.parametrize("order", ["C", "F", None]) @pytest.mark.parametrize("config", [{"order": "C"}, {"order": "F"}, {}], ids=["C", "F", "None"]) def test_array_order( order: MemoryOrder | None, config: dict[str, MemoryOrder | None], zarr_format: ZarrFormat ) -> None: """ Check that: - For v2, memory order is taken from the `order` keyword argument. - For v3, memory order is taken from `config`, and when order is passed a warning is raised - The numpy array returned has the expected order - For v2, the order metadata is set correctly """ default_order = zarr.config.get("array.order") ctx: contextlib.AbstractContextManager # type: ignore[type-arg] if zarr_format == 3: if order is None: ctx = contextlib.nullcontext() else: ctx = pytest.warns( RuntimeWarning, match="The `order` keyword argument has no effect for Zarr format 3 arrays", ) expected_order = config.get("order", default_order) if zarr_format == 2: ctx = contextlib.nullcontext() expected_order = order or config.get("order", default_order) with ctx: arr = zarr.ones(shape=(2, 2), order=order, zarr_format=zarr_format, config=config) assert arr.order == expected_order vals = np.asarray(arr) if expected_order == "C": assert vals.flags.c_contiguous elif expected_order == "F": assert vals.flags.f_contiguous else: raise AssertionError if zarr_format == 2: assert arr.metadata.zarr_format == 2 assert arr.metadata.order == expected_order async def test_init_order_warns() -> None: with pytest.warns( RuntimeWarning, match="The `order` keyword argument has no effect for Zarr format 3 arrays" ): await init_array( store_path=StorePath(store=MemoryStore()), shape=(1,), dtype="uint8", zarr_format=3, order="F", ) # def test_lazy_loader(): # foo = np.arange(100) # bar = np.arange(100, 0, -1) # store = "data/group.zarr" # save(store, foo=foo, bar=bar) # loader = load(store) # assert "foo" in loader # assert "bar" in loader # assert "baz" not in loader # assert len(loader) == 2 # assert sorted(loader) == ["bar", "foo"] # assert_array_equal(foo, loader["foo"]) # assert_array_equal(bar, loader["bar"]) # assert "LazyLoader: " in repr(loader) def test_load_array(sync_store: Store) -> None: store = sync_store foo = np.arange(100) bar = np.arange(100, 0, -1) save(store, foo=foo, bar=bar) # can also load arrays directly into a numpy array for array_name in ["foo", "bar"]: array = load(store, path=array_name) assert isinstance(array, np.ndarray) if array_name == "foo": assert_array_equal(foo, array) else: assert_array_equal(bar, array) @pytest.mark.parametrize("path", ["data", None]) @pytest.mark.parametrize("load_read_only", [True, False, None]) def test_load_zip(tmp_path: Path, path: str | None, load_read_only: bool | None) -> None: file = tmp_path / "test.zip" data = np.arange(100).reshape(10, 10) with ZipStore(file, mode="w", read_only=False) as zs: save(zs, data, path=path) with ZipStore(file, mode="r", read_only=load_read_only) as zs: result = zarr.load(store=zs, path=path) assert isinstance(result, np.ndarray) assert np.array_equal(result, data) with ZipStore(file, read_only=load_read_only) as zs: result = zarr.load(store=zs, path=path) assert isinstance(result, np.ndarray) assert np.array_equal(result, data) @pytest.mark.parametrize("path", ["data", None]) @pytest.mark.parametrize("load_read_only", [True, False]) def test_load_local(tmp_path: Path, path: str | None, load_read_only: bool) -> None: file = tmp_path / "test.zip" data = np.arange(100).reshape(10, 10) with LocalStore(file, read_only=False) as zs: save(zs, data, path=path) with LocalStore(file, read_only=load_read_only) as zs: result = zarr.load(store=zs, path=path) assert isinstance(result, np.ndarray) assert np.array_equal(result, data) def test_tree() -> None: g1 = zarr.group() g1.create_group("foo") g3 = g1.create_group("bar") g3.create_group("baz") g5 = g3.create_group("qux") g5.create_array("baz", shape=(100,), chunks=(10,), dtype="float64") with pytest.warns(ZarrDeprecationWarning, match=r"Group\.tree instead\."): # noqa: PT031 assert repr(zarr.tree(g1)) == repr(g1.tree()) assert str(zarr.tree(g1)) == str(g1.tree()) # @pytest.mark.parametrize("stores_from_path", [False, True]) # @pytest.mark.parametrize( # "with_chunk_store,listable", # [(False, True), (True, True), (False, False)], # ids=["default-listable", "with_chunk_store-listable", "default-unlistable"], # ) # def test_consolidate_metadata(with_chunk_store, listable, monkeypatch, stores_from_path): # # setup initial data # if stores_from_path: # store = tempfile.mkdtemp() # atexit.register(atexit_rmtree, store) # if with_chunk_store: # chunk_store = tempfile.mkdtemp() # atexit.register(atexit_rmtree, chunk_store) # else: # chunk_store = None # else: # store = MemoryStore() # chunk_store = MemoryStore() if with_chunk_store else None # path = None # z = group(store, chunk_store=chunk_store, path=path) # # Reload the actual store implementation in case str # store_to_copy = z.store # z.create_group("g1") # g2 = z.create_group("g2") # g2.attrs["hello"] = "world" # arr = g2.create_array("arr", shape=(20, 20), chunks=(5, 5), dtype="f8") # assert 16 == arr.nchunks # assert 0 == arr.nchunks_initialized # arr.attrs["data"] = 1 # arr[:] = 1.0 # assert 16 == arr.nchunks_initialized # if stores_from_path: # # get the actual store class for use with consolidate_metadata # store_class = z._store # else: # store_class = store # # perform consolidation # out = consolidate_metadata(store_class, path=path) # assert isinstance(out, Group) # assert ["g1", "g2"] == list(out) # if not stores_from_path: # assert isinstance(out._store, ConsolidatedMetadataStore) # assert ".zmetadata" in store # meta_keys = [ # ".zgroup", # "g1/.zgroup", # "g2/.zgroup", # "g2/.zattrs", # "g2/arr/.zarray", # "g2/arr/.zattrs", # ] # for key in meta_keys: # del store[key] # # https://github.com/zarr-developers/zarr-python/issues/993 # # Make sure we can still open consolidated on an unlistable store: # if not listable: # fs_memory = pytest.importorskip("fsspec.implementations.memory") # monkeypatch.setattr(fs_memory.MemoryFileSystem, "isdir", lambda x, y: False) # monkeypatch.delattr(fs_memory.MemoryFileSystem, "ls") # fs = fs_memory.MemoryFileSystem() # store_to_open = FSStore("", fs=fs) # # copy original store to new unlistable store # store_to_open.update(store_to_copy) # else: # store_to_open = store # # open consolidated # z2 = open_consolidated(store_to_open, chunk_store=chunk_store, path=path) # assert ["g1", "g2"] == list(z2) # assert "world" == z2.g2.attrs["hello"] # assert 1 == z2.g2.arr.attrs["data"] # assert (z2.g2.arr[:] == 1.0).all() # assert 16 == z2.g2.arr.nchunks # if listable: # assert 16 == z2.g2.arr.nchunks_initialized # else: # with pytest.raises(NotImplementedError): # _ = z2.g2.arr.nchunks_initialized # if stores_from_path: # # path string is note a BaseStore subclass so cannot be used to # # initialize a ConsolidatedMetadataStore. # with pytest.raises(ValueError): # cmd = ConsolidatedMetadataStore(store) # else: # # tests del/write on the store # cmd = ConsolidatedMetadataStore(store) # with pytest.raises(PermissionError): # del cmd[".zgroup"] # with pytest.raises(PermissionError): # cmd[".zgroup"] = None # # test getsize on the store # assert isinstance(getsize(cmd), Integral) # # test new metadata are not writeable # with pytest.raises(PermissionError): # z2.create_group("g3") # with pytest.raises(PermissionError): # z2.create_dataset("spam", shape=42, chunks=7, dtype="i4") # with pytest.raises(PermissionError): # del z2["g2"] # # test consolidated metadata are not writeable # with pytest.raises(PermissionError): # z2.g2.attrs["hello"] = "universe" # with pytest.raises(PermissionError): # z2.g2.arr.attrs["foo"] = "bar" # # test the data are writeable # z2.g2.arr[:] = 2 # assert (z2.g2.arr[:] == 2).all() # # test invalid modes # with pytest.raises(ValueError): # open_consolidated(store, chunk_store=chunk_store, mode="a", path=path) # with pytest.raises(ValueError): # open_consolidated(store, chunk_store=chunk_store, mode="w", path=path) # with pytest.raises(ValueError): # open_consolidated(store, chunk_store=chunk_store, mode="w-", path=path) # # make sure keyword arguments are passed through without error # open_consolidated( # store, # chunk_store=chunk_store, # path=path, # cache_attrs=True, # synchronizer=None, # ) # @pytest.mark.parametrize( # "options", # ( # {"dimension_separator": "/"}, # {"dimension_separator": "."}, # {"dimension_separator": None}, # ), # ) # def test_save_array_separator(tmpdir, options): # data = np.arange(6).reshape((3, 2)) # url = tmpdir.join("test.zarr") # save_array(url, data, **options) # class TestCopyStore(unittest.TestCase): # _version = 2 # def setUp(self): # source = dict() # source["foo"] = b"xxx" # source["bar/baz"] = b"yyy" # source["bar/qux"] = b"zzz" # self.source = source # def _get_dest_store(self): # return dict() # def test_no_paths(self): # source = self.source # dest = self._get_dest_store() # copy_store(source, dest) # assert len(source) == len(dest) # for key in source: # assert source[key] == dest[key] # def test_source_path(self): # source = self.source # # paths should be normalized # for source_path in "bar", "bar/", "/bar", "/bar/": # dest = self._get_dest_store() # copy_store(source, dest, source_path=source_path) # assert 2 == len(dest) # for key in source: # if key.startswith("bar/"): # dest_key = key.split("bar/")[1] # assert source[key] == dest[dest_key] # else: # assert key not in dest # def test_dest_path(self): # source = self.source # # paths should be normalized # for dest_path in "new", "new/", "/new", "/new/": # dest = self._get_dest_store() # copy_store(source, dest, dest_path=dest_path) # assert len(source) == len(dest) # for key in source: # if self._version == 3: # dest_key = f"{key[:10]}new/{key[10:]}" # else: # dest_key = f"new/{key}" # assert source[key] == dest[dest_key] # def test_source_dest_path(self): # source = self.source # # paths should be normalized # for source_path in "bar", "bar/", "/bar", "/bar/": # for dest_path in "new", "new/", "/new", "/new/": # dest = self._get_dest_store() # copy_store(source, dest, source_path=source_path, dest_path=dest_path) # assert 2 == len(dest) # for key in source: # if key.startswith("bar/"): # dest_key = "new/" + key.split("bar/")[1] # assert source[key] == dest[dest_key] # else: # assert key not in dest # assert (f"new/{key}") not in dest # def test_excludes_includes(self): # source = self.source # # single excludes # dest = self._get_dest_store() # excludes = "f.*" # copy_store(source, dest, excludes=excludes) # assert len(dest) == 2 # root = "" # assert "f{root}foo" not in dest # # multiple excludes # dest = self._get_dest_store() # excludes = "b.z", ".*x" # copy_store(source, dest, excludes=excludes) # assert len(dest) == 1 # assert f"{root}foo" in dest # assert f"{root}bar/baz" not in dest # assert f"{root}bar/qux" not in dest # # excludes and includes # dest = self._get_dest_store() # excludes = "b.*" # includes = ".*x" # copy_store(source, dest, excludes=excludes, includes=includes) # assert len(dest) == 2 # assert f"{root}foo" in dest # assert f"{root}bar/baz" not in dest # assert f"{root}bar/qux" in dest # def test_dry_run(self): # source = self.source # dest = self._get_dest_store() # copy_store(source, dest, dry_run=True) # assert 0 == len(dest) # def test_if_exists(self): # source = self.source # dest = self._get_dest_store() # root = "" # dest[f"{root}bar/baz"] = b"mmm" # # default ('raise') # with pytest.raises(CopyError): # copy_store(source, dest) # # explicit 'raise' # with pytest.raises(CopyError): # copy_store(source, dest, if_exists="raise") # # skip # copy_store(source, dest, if_exists="skip") # assert 3 == len(dest) # assert dest[f"{root}foo"] == b"xxx" # assert dest[f"{root}bar/baz"] == b"mmm" # assert dest[f"{root}bar/qux"] == b"zzz" # # replace # copy_store(source, dest, if_exists="replace") # assert 3 == len(dest) # assert dest[f"{root}foo"] == b"xxx" # assert dest[f"{root}bar/baz"] == b"yyy" # assert dest[f"{root}bar/qux"] == b"zzz" # # invalid option # with pytest.raises(ValueError): # copy_store(source, dest, if_exists="foobar") # def check_copied_array(original, copied, without_attrs=False, expect_props=None): # # setup # source_h5py = original.__module__.startswith("h5py.") # dest_h5py = copied.__module__.startswith("h5py.") # zarr_to_zarr = not (source_h5py or dest_h5py) # h5py_to_h5py = source_h5py and dest_h5py # zarr_to_h5py = not source_h5py and dest_h5py # h5py_to_zarr = source_h5py and not dest_h5py # if expect_props is None: # expect_props = dict() # else: # expect_props = expect_props.copy() # # common properties in zarr and h5py # for p in "dtype", "shape", "chunks": # expect_props.setdefault(p, getattr(original, p)) # # zarr-specific properties # if zarr_to_zarr: # for p in "compressor", "filters", "order", "fill_value": # expect_props.setdefault(p, getattr(original, p)) # # h5py-specific properties # if h5py_to_h5py: # for p in ( # "maxshape", # "compression", # "compression_opts", # "shuffle", # "scaleoffset", # "fletcher32", # "fillvalue", # ): # expect_props.setdefault(p, getattr(original, p)) # # common properties with some name differences # if h5py_to_zarr: # expect_props.setdefault("fill_value", original.fillvalue) # if zarr_to_h5py: # expect_props.setdefault("fillvalue", original.fill_value) # # compare properties # for k, v in expect_props.items(): # assert v == getattr(copied, k) # # compare data # assert_array_equal(original[:], copied[:]) # # compare attrs # if without_attrs: # for k in original.attrs.keys(): # assert k not in copied.attrs # else: # if dest_h5py and "filters" in original.attrs: # # special case in v3 (storing filters metadata under attributes) # # we explicitly do not copy this info over to HDF5 # original_attrs = original.attrs.asdict().copy() # original_attrs.pop("filters") # else: # original_attrs = original.attrs # assert sorted(original_attrs.items()) == sorted(copied.attrs.items()) # def check_copied_group(original, copied, without_attrs=False, expect_props=None, shallow=False): # # setup # if expect_props is None: # expect_props = dict() # else: # expect_props = expect_props.copy() # # compare children # for k, v in original.items(): # if hasattr(v, "shape"): # assert k in copied # check_copied_array(v, copied[k], without_attrs=without_attrs, expect_props=expect_props) # elif shallow: # assert k not in copied # else: # assert k in copied # check_copied_group( # v, # copied[k], # without_attrs=without_attrs, # shallow=shallow, # expect_props=expect_props, # ) # # compare attrs # if without_attrs: # for k in original.attrs.keys(): # assert k not in copied.attrs # else: # assert sorted(original.attrs.items()) == sorted(copied.attrs.items()) # def test_copy_all(): # """ # https://github.com/zarr-developers/zarr-python/issues/269 # copy_all used to not copy attributes as `.keys()` does not return hidden `.zattrs`. # """ # original_group = zarr.group(store=MemoryStore(), overwrite=True) # original_group.attrs["info"] = "group attrs" # original_subgroup = original_group.create_group("subgroup") # original_subgroup.attrs["info"] = "sub attrs" # destination_group = zarr.group(store=MemoryStore(), overwrite=True) # # copy from memory to directory store # copy_all( # original_group, # destination_group, # dry_run=False, # ) # assert "subgroup" in destination_group # assert destination_group.attrs["info"] == "group attrs" # assert destination_group.subgroup.attrs["info"] == "sub attrs" # class TestCopy: # @pytest.fixture(params=[False, True], ids=["zarr", "hdf5"]) # def source(self, request, tmpdir): # def prep_source(source): # foo = source.create_group("foo") # foo.attrs["experiment"] = "weird science" # baz = foo.create_dataset("bar/baz", data=np.arange(100), chunks=(50,)) # baz.attrs["units"] = "metres" # if request.param: # extra_kws = dict( # compression="gzip", # compression_opts=3, # fillvalue=84, # shuffle=True, # fletcher32=True, # ) # else: # extra_kws = dict(compressor=Zlib(3), order="F", fill_value=42, filters=[Adler32()]) # source.create_dataset( # "spam", # data=np.arange(100, 200).reshape(20, 5), # chunks=(10, 2), # dtype="i2", # **extra_kws, # ) # return source # if request.param: # h5py = pytest.importorskip("h5py") # fn = tmpdir.join("source.h5") # with h5py.File(str(fn), mode="w") as h5f: # yield prep_source(h5f) # else: # yield prep_source(group()) # @pytest.fixture(params=[False, True], ids=["zarr", "hdf5"]) # def dest(self, request, tmpdir): # if request.param: # h5py = pytest.importorskip("h5py") # fn = tmpdir.join("dest.h5") # with h5py.File(str(fn), mode="w") as h5f: # yield h5f # else: # yield group() # def test_copy_array(self, source, dest): # # copy array with default options # copy(source["foo/bar/baz"], dest) # check_copied_array(source["foo/bar/baz"], dest["baz"]) # copy(source["spam"], dest) # check_copied_array(source["spam"], dest["spam"]) # def test_copy_bad_dest(self, source, dest): # # try to copy to an array, dest must be a group # dest = dest.create_dataset("eggs", shape=(100,)) # with pytest.raises(ValueError): # copy(source["foo/bar/baz"], dest) # def test_copy_array_name(self, source, dest): # # copy array with name # copy(source["foo/bar/baz"], dest, name="qux") # assert "baz" not in dest # check_copied_array(source["foo/bar/baz"], dest["qux"]) # def test_copy_array_create_options(self, source, dest): # dest_h5py = dest.__module__.startswith("h5py.") # # copy array, provide creation options # compressor = Zlib(9) # create_kws = dict(chunks=(10,)) # if dest_h5py: # create_kws.update( # compression="gzip", compression_opts=9, shuffle=True, fletcher32=True, fillvalue=42 # ) # else: # create_kws.update(compressor=compressor, fill_value=42, order="F", filters=[Adler32()]) # copy(source["foo/bar/baz"], dest, without_attrs=True, **create_kws) # check_copied_array( # source["foo/bar/baz"], dest["baz"], without_attrs=True, expect_props=create_kws # ) # def test_copy_array_exists_array(self, source, dest): # # copy array, dest array in the way # dest.create_dataset("baz", shape=(10,)) # # raise # with pytest.raises(CopyError): # # should raise by default # copy(source["foo/bar/baz"], dest) # assert (10,) == dest["baz"].shape # with pytest.raises(CopyError): # copy(source["foo/bar/baz"], dest, if_exists="raise") # assert (10,) == dest["baz"].shape # # skip # copy(source["foo/bar/baz"], dest, if_exists="skip") # assert (10,) == dest["baz"].shape # # replace # copy(source["foo/bar/baz"], dest, if_exists="replace") # check_copied_array(source["foo/bar/baz"], dest["baz"]) # # invalid option # with pytest.raises(ValueError): # copy(source["foo/bar/baz"], dest, if_exists="foobar") # def test_copy_array_exists_group(self, source, dest): # # copy array, dest group in the way # dest.create_group("baz") # # raise # with pytest.raises(CopyError): # copy(source["foo/bar/baz"], dest) # assert not hasattr(dest["baz"], "shape") # with pytest.raises(CopyError): # copy(source["foo/bar/baz"], dest, if_exists="raise") # assert not hasattr(dest["baz"], "shape") # # skip # copy(source["foo/bar/baz"], dest, if_exists="skip") # assert not hasattr(dest["baz"], "shape") # # replace # copy(source["foo/bar/baz"], dest, if_exists="replace") # check_copied_array(source["foo/bar/baz"], dest["baz"]) # def test_copy_array_skip_initialized(self, source, dest): # dest_h5py = dest.__module__.startswith("h5py.") # dest.create_dataset("baz", shape=(100,), chunks=(10,), dtype="i8") # assert not np.all(source["foo/bar/baz"][:] == dest["baz"][:]) # if dest_h5py: # with pytest.raises(ValueError): # # not available with copy to h5py # copy(source["foo/bar/baz"], dest, if_exists="skip_initialized") # else: # # copy array, dest array exists but not yet initialized # copy(source["foo/bar/baz"], dest, if_exists="skip_initialized") # check_copied_array(source["foo/bar/baz"], dest["baz"]) # # copy array, dest array exists and initialized, will be skipped # dest["baz"][:] = np.arange(100, 200) # copy(source["foo/bar/baz"], dest, if_exists="skip_initialized") # assert_array_equal(np.arange(100, 200), dest["baz"][:]) # assert not np.all(source["foo/bar/baz"][:] == dest["baz"][:]) # def test_copy_group(self, source, dest): # # copy group, default options # copy(source["foo"], dest) # check_copied_group(source["foo"], dest["foo"]) # def test_copy_group_no_name(self, source, dest): # with pytest.raises(TypeError): # # need a name if copy root # copy(source, dest) # copy(source, dest, name="root") # check_copied_group(source, dest["root"]) # def test_copy_group_options(self, source, dest): # # copy group, non-default options # copy(source["foo"], dest, name="qux", without_attrs=True) # assert "foo" not in dest # check_copied_group(source["foo"], dest["qux"], without_attrs=True) # def test_copy_group_shallow(self, source, dest): # # copy group, shallow # copy(source, dest, name="eggs", shallow=True) # check_copied_group(source, dest["eggs"], shallow=True) # def test_copy_group_exists_group(self, source, dest): # # copy group, dest groups exist # dest.create_group("foo/bar") # copy(source["foo"], dest) # check_copied_group(source["foo"], dest["foo"]) # def test_copy_group_exists_array(self, source, dest): # # copy group, dest array in the way # dest.create_dataset("foo/bar", shape=(10,)) # # raise # with pytest.raises(CopyError): # copy(source["foo"], dest) # assert dest["foo/bar"].shape == (10,) # with pytest.raises(CopyError): # copy(source["foo"], dest, if_exists="raise") # assert dest["foo/bar"].shape == (10,) # # skip # copy(source["foo"], dest, if_exists="skip") # assert dest["foo/bar"].shape == (10,) # # replace # copy(source["foo"], dest, if_exists="replace") # check_copied_group(source["foo"], dest["foo"]) # def test_copy_group_dry_run(self, source, dest): # # dry run, empty destination # n_copied, n_skipped, n_bytes_copied = copy( # source["foo"], dest, dry_run=True, return_stats=True # ) # assert 0 == len(dest) # assert 3 == n_copied # assert 0 == n_skipped # assert 0 == n_bytes_copied # # dry run, array exists in destination # baz = np.arange(100, 200) # dest.create_dataset("foo/bar/baz", data=baz) # assert not np.all(source["foo/bar/baz"][:] == dest["foo/bar/baz"][:]) # assert 1 == len(dest) # # raise # with pytest.raises(CopyError): # copy(source["foo"], dest, dry_run=True) # assert 1 == len(dest) # # skip # n_copied, n_skipped, n_bytes_copied = copy( # source["foo"], dest, dry_run=True, if_exists="skip", return_stats=True # ) # assert 1 == len(dest) # assert 2 == n_copied # assert 1 == n_skipped # assert 0 == n_bytes_copied # assert_array_equal(baz, dest["foo/bar/baz"]) # # replace # n_copied, n_skipped, n_bytes_copied = copy( # source["foo"], dest, dry_run=True, if_exists="replace", return_stats=True # ) # assert 1 == len(dest) # assert 3 == n_copied # assert 0 == n_skipped # assert 0 == n_bytes_copied # assert_array_equal(baz, dest["foo/bar/baz"]) # def test_logging(self, source, dest, tmpdir): # # callable log # copy(source["foo"], dest, dry_run=True, log=print) # # file name # fn = str(tmpdir.join("log_name")) # copy(source["foo"], dest, dry_run=True, log=fn) # # file # with tmpdir.join("log_file").open(mode="w") as f: # copy(source["foo"], dest, dry_run=True, log=f) # # bad option # with pytest.raises(TypeError): # copy(source["foo"], dest, dry_run=True, log=True) def test_open_falls_back_to_open_group() -> None: # https://github.com/zarr-developers/zarr-python/issues/2309 store = MemoryStore() zarr.open_group(store, attributes={"key": "value"}) group = zarr.open(store) assert isinstance(group, Group) assert group.attrs == {"key": "value"} async def test_open_falls_back_to_open_group_async(zarr_format: ZarrFormat) -> None: # https://github.com/zarr-developers/zarr-python/issues/2309 store = MemoryStore() await zarr.api.asynchronous.open_group( store, attributes={"key": "value"}, zarr_format=zarr_format ) group = await zarr.api.asynchronous.open(store=store) assert isinstance(group, zarr.core.group.AsyncGroup) assert group.metadata.zarr_format == zarr_format assert group.attrs == {"key": "value"} @pytest.mark.parametrize("mode", ["r", "r+", "w", "a"]) def test_open_modes_creates_group(tmp_path: Path, mode: str) -> None: # https://github.com/zarr-developers/zarr-python/issues/2490 zarr_dir = tmp_path / f"mode-{mode}-test.zarr" if mode in ["r", "r+"]: # Expect FileNotFoundError to be raised if 'r' or 'r+' mode with pytest.raises(FileNotFoundError): zarr.open(store=zarr_dir, mode=mode) # type: ignore[arg-type] else: group = zarr.open(store=zarr_dir, mode=mode) # type: ignore[arg-type] assert isinstance(group, Group) async def test_metadata_validation_error() -> None: with pytest.raises( MetadataValidationError, match="Invalid value for 'zarr_format'. Expected 2, 3, or None. Got '3.0'.", ): await zarr.api.asynchronous.open_group(zarr_format="3.0") # type: ignore[arg-type] with pytest.raises( MetadataValidationError, match="Invalid value for 'zarr_format'. Expected 2, 3, or None. Got '3.0'.", ): await zarr.api.asynchronous.open_array(shape=(1,), zarr_format="3.0") # type: ignore[arg-type] @pytest.mark.parametrize( "store", ["local", "memory", "zip"], indirect=True, ) def test_open_array_with_mode_r_plus(store: Store, zarr_format: ZarrFormat) -> None: # 'r+' means read/write (must exist) with pytest.raises(ArrayNotFoundError): zarr.open_array(store=store, mode="r+", zarr_format=zarr_format) zarr.ones(store=store, shape=(3, 3), zarr_format=zarr_format) z2 = zarr.open_array(store=store, mode="r+") assert isinstance(z2, Array) assert z2.metadata.zarr_format == zarr_format result = z2[:] assert isinstance(result, NDArrayLike) assert (result == 1).all() z2[:] = 3 @pytest.mark.parametrize( ("a_func", "b_func"), [ (zarr.api.asynchronous.create_array, zarr.api.synchronous.create_array), (zarr.api.asynchronous.save, zarr.api.synchronous.save), (zarr.api.asynchronous.save_array, zarr.api.synchronous.save_array), (zarr.api.asynchronous.save_group, zarr.api.synchronous.save_group), (zarr.api.asynchronous.open_group, zarr.api.synchronous.open_group), (zarr.api.asynchronous.create, zarr.api.synchronous.create), ], ) def test_consistent_signatures( a_func: Callable[[object], object], b_func: Callable[[object], object] ) -> None: """ Ensure that pairs of functions have the same signature """ base_sig = inspect.signature(a_func) test_sig = inspect.signature(b_func) wrong: dict[str, list[object]] = { "missing_from_test": [], "missing_from_base": [], "wrong_type": [], } for key, value in base_sig.parameters.items(): if key not in test_sig.parameters: wrong["missing_from_test"].append((key, value)) for key, value in test_sig.parameters.items(): if key not in base_sig.parameters: wrong["missing_from_base"].append((key, value)) if base_sig.parameters[key] != value: wrong["wrong_type"].append({key: {"test": value, "base": base_sig.parameters[key]}}) assert wrong["missing_from_base"] == [] assert wrong["missing_from_test"] == [] assert wrong["wrong_type"] == [] def test_api_exports() -> None: """ Test that the sync API and the async API export the same objects """ assert zarr.api.asynchronous.__all__ == zarr.api.synchronous.__all__ @gpu_test @pytest.mark.parametrize( "store", ["local", "memory", "zip"], indirect=True, ) @pytest.mark.parametrize("zarr_format", [None, 2, 3]) def test_gpu_basic(store: Store, zarr_format: ZarrFormat | None) -> None: import cupy as cp if zarr_format == 2: # Without this, the zstd codec attempts to convert the cupy # array to bytes. compressors = None else: compressors = "auto" with zarr.config.enable_gpu(): src = cp.random.uniform(size=(100, 100)) # allocate on the device z = zarr.create_array( store, name="a", shape=src.shape, chunks=(10, 10), dtype=src.dtype, overwrite=True, zarr_format=zarr_format, compressors=compressors, # type: ignore[arg-type] ) z[:10, :10] = src[:10, :10] result = z[:10, :10] # assert_array_equal doesn't check the type assert isinstance(result, type(src)) cp.testing.assert_array_equal(result, src[:10, :10]) def test_v2_without_compressor() -> None: # Make sure it's possible to set no compressor for v2 arrays arr = zarr.create(store={}, shape=(1), dtype="uint8", zarr_format=2, compressor=None) assert arr.compressors == () def test_v2_with_v3_compressor() -> None: # Check trying to create a v2 array with a v3 compressor fails with pytest.raises( ValueError, match="Cannot use a BytesBytesCodec as a compressor for zarr v2 arrays. Use a numcodecs codec directly instead.", ): zarr.create( store={}, shape=(1), dtype="uint8", zarr_format=2, compressor=zarr.codecs.BloscCodec() ) def add_empty_file(path: Path) -> Path: fpath = path / "a.txt" fpath.touch() return fpath @pytest.mark.parametrize("create_function", [create_array, from_array]) @pytest.mark.parametrize("overwrite", [True, False]) def test_no_overwrite_array(tmp_path: Path, create_function: Callable, overwrite: bool) -> None: # type:ignore[type-arg] store = zarr.storage.LocalStore(tmp_path) existing_fpath = add_empty_file(tmp_path) assert existing_fpath.exists() create_function(store=store, data=np.ones(shape=(1,)), overwrite=overwrite) if overwrite: assert not existing_fpath.exists() else: assert existing_fpath.exists() @pytest.mark.parametrize("create_function", [create_group, group]) @pytest.mark.parametrize("overwrite", [True, False]) def test_no_overwrite_group(tmp_path: Path, create_function: Callable, overwrite: bool) -> None: # type:ignore[type-arg] store = zarr.storage.LocalStore(tmp_path) existing_fpath = add_empty_file(tmp_path) assert existing_fpath.exists() create_function(store=store, overwrite=overwrite) if overwrite: assert not existing_fpath.exists() else: assert existing_fpath.exists() @pytest.mark.parametrize("open_func", [zarr.open, open_group]) @pytest.mark.parametrize("mode", ["r", "r+", "a", "w", "w-"]) def test_no_overwrite_open(tmp_path: Path, open_func: Callable, mode: str) -> None: # type:ignore[type-arg] store = zarr.storage.LocalStore(tmp_path) existing_fpath = add_empty_file(tmp_path) assert existing_fpath.exists() with contextlib.suppress(FileExistsError, FileNotFoundError, ZarrUserWarning): open_func(store=store, mode=mode) if mode == "w": assert not existing_fpath.exists() else: assert existing_fpath.exists() def test_no_overwrite_load(tmp_path: Path) -> None: store = zarr.storage.LocalStore(tmp_path) existing_fpath = add_empty_file(tmp_path) assert existing_fpath.exists() with contextlib.suppress(NotImplementedError): zarr.load(store) assert existing_fpath.exists() @pytest.mark.parametrize( "f", [ zarr.array, zarr.create, zarr.create_array, zarr.ones, zarr.ones_like, zarr.empty, zarr.empty_like, zarr.full, zarr.full_like, zarr.zeros, zarr.zeros_like, ], ) def test_auto_chunks(f: Callable[..., AnyArray]) -> None: # Make sure chunks are set automatically across the public API # TODO: test shards with this test too shape = (1000, 1000) dtype = np.uint8 kwargs = {"shape": shape, "dtype": dtype} array = np.zeros(shape, dtype=dtype) store = zarr.storage.MemoryStore() # ruff: disable[FURB171] if f in [zarr.full, zarr.full_like]: kwargs["fill_value"] = 0 if f in [zarr.array]: kwargs["data"] = array if f in [zarr.empty_like, zarr.full_like, zarr.empty_like, zarr.ones_like, zarr.zeros_like]: kwargs["a"] = array if f in [zarr.create_array]: kwargs["store"] = store # ruff: enable[FURB171] a = f(**kwargs) assert a.chunks == (500, 500) @pytest.mark.parametrize("kwarg_name", ["synchronizer", "chunk_store", "cache_attrs", "meta_array"]) def test_unimplemented_kwarg_warnings(kwarg_name: str) -> None: kwargs = {kwarg_name: 1} with pytest.warns(RuntimeWarning, match=".* is not yet implemented"): zarr.create(shape=(1,), **kwargs) # type: ignore[arg-type] zarr-developers-zarr-python-a2f2415/tests/test_api/000077500000000000000000000000001527014137600224375ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/test_api/test_asynchronous.py000066400000000000000000000077361527014137600266200ustar00rootroot00000000000000from __future__ import annotations import json from dataclasses import dataclass from typing import TYPE_CHECKING import numpy as np import pytest from zarr import create_array from zarr.api.asynchronous import _get_shape_chunks, _like_args, group, open from zarr.core.buffer.core import default_buffer_prototype from zarr.core.group import AsyncGroup if TYPE_CHECKING: from pathlib import Path from typing import Any import numpy.typing as npt from zarr.core.array import AsyncArray from zarr.core.common import ZarrFormat from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.types import AnyArray @dataclass class WithShape: shape: tuple[int, ...] @dataclass class WithChunks(WithShape): chunks: tuple[int, ...] @dataclass class WithChunkLen(WithShape): chunklen: int @pytest.mark.parametrize( ("observed", "expected"), [ ({}, (None, None)), (WithShape(shape=(1, 2)), ((1, 2), None)), (WithChunks(shape=(1, 2), chunks=(1, 2)), ((1, 2), (1, 2))), (WithChunkLen(shape=(10, 10), chunklen=1), ((10, 10), (1, 10))), ], ) def test_get_shape_chunks( observed: object, expected: tuple[tuple[int, ...] | None, tuple[int, ...] | None] ) -> None: """ Test the _get_shape_chunks function """ assert _get_shape_chunks(observed) == expected _V2_ARRAY = create_array( {}, chunks=(10,), shape=(100,), dtype="f8", compressors=None, filters=None, zarr_format=2, )._async_array _V2_ARRAY_ARGS = { "chunks": (10,), "shape": (100,), "dtype": np.dtype("f8"), "fill_value": np.float64(0.0), } @pytest.mark.parametrize( ("observed", "zarr_format", "expected"), [ ( np.arange(10, dtype=np.dtype("int64")), None, {"shape": (10,), "dtype": np.dtype("int64")}, ), ( np.arange(10, dtype=np.dtype("int64")), 2, {"shape": (10,), "dtype": np.dtype("int64"), "zarr_format": 2}, ), (WithChunks(shape=(1, 2), chunks=(1, 2)), None, {"chunks": (1, 2), "shape": (1, 2)}), ( _V2_ARRAY, None, _V2_ARRAY_ARGS | {"zarr_format": 2, "compressor": None, "filters": None, "order": "C"}, ), ( _V2_ARRAY, 2, _V2_ARRAY_ARGS | {"zarr_format": 2, "compressor": None, "filters": None, "order": "C"}, ), (_V2_ARRAY, 3, _V2_ARRAY_ARGS | {"zarr_format": 3}), ], ) def test_like_args( observed: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AnyArray | npt.NDArray[Any], zarr_format: ZarrFormat | None, expected: object, ) -> None: """ Test the like_args function """ assert _like_args(observed, zarr_format) == expected async def test_open_no_array() -> None: """ Test that zarr.api.asynchronous.open attempts to open a group when no array is found, but shape was specified in kwargs. This behavior makes no sense but we should still test it. """ store = { "zarr.json": default_buffer_prototype().buffer.from_bytes( json.dumps({"zarr_format": 3, "node_type": "group"}).encode("utf-8") ) } with pytest.raises( TypeError, match=r"open_group\(\) got an unexpected keyword argument 'shape'" ): await open(store=store, shape=(1,)) async def test_open_group_new_path(tmp_path: Path) -> None: """ Test that zarr.api.asynchronous.group properly handles a string representation of a local file path that does not yet exist. See https://github.com/zarr-developers/zarr-python/issues/3406 """ # tmp_path exists, but tmp_path / "test.zarr" will not, which is important for this test path = tmp_path / "test.zarr" grp = await group(store=path, attributes={"a": 1}) assert isinstance(grp, AsyncGroup) # Calling group on an existing store should just open that store grp = await group(store=path) assert grp.attrs == {"a": 1} zarr-developers-zarr-python-a2f2415/tests/test_api/test_synchronous.py000066400000000000000000000124641527014137600264510ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING, Any, Final import pytest from numpydoc.docscrape import NumpyDocString import zarr from zarr.api import asynchronous, synchronous if TYPE_CHECKING: from collections.abc import Callable MATCHED_EXPORT_NAMES: Final[tuple[str, ...]] = tuple( sorted(set(synchronous.__all__) | set(asynchronous.__all__)) ) """A sorted tuple of names that are exported by both the sync and async APIs.""" MATCHED_CALLABLE_NAMES: Final[tuple[str, ...]] = tuple( x for x in MATCHED_EXPORT_NAMES if callable(getattr(synchronous, x)) ) """A sorted tuple of callable names that are exported by both the sync and async APIs.""" @pytest.mark.parametrize("callable_name", MATCHED_CALLABLE_NAMES) def test_docstrings_match(callable_name: str) -> None: """ Tests that the docstrings for the sync and async define identical parameters. """ callable_a = getattr(synchronous, callable_name) callable_b = getattr(asynchronous, callable_name) if callable_a.__doc__ is None: assert callable_b.__doc__ is None else: params_a = NumpyDocString(callable_a.__doc__)["Parameters"] params_b = NumpyDocString(callable_b.__doc__)["Parameters"] mismatch = [] for idx, (a, b) in enumerate(zip(params_a, params_b, strict=False)): if a != b: mismatch.append((idx, (a, b))) assert mismatch == [] CREATE_ARRAY_ROUTINES: Final[tuple[Callable[..., Any], ...]] = ( asynchronous.create_array, synchronous.create_array, zarr.AsyncGroup.create_array, zarr.Group.create_array, zarr.Group.create, ) """ Routines that share the `create_array` signature and must document it identically. The legacy `zarr.create` documents a different signature (e.g. `chunks : int or tuple of ints`) and is only checked for the parameters it genuinely shares, see the cases below. """ CREATE_ARRAY_EXEMPT_PARAMETERS: Final[frozenset[str]] = frozenset({"name"}) """ Parameters that the `create_array` routines document differently on purpose: `name` is relative to the store for the module-level functions and relative to the group for the group methods. """ def _documented_parameters(routines: tuple[Callable[..., Any], ...]) -> tuple[str, ...]: """The sorted union of the parameter names documented by `routines`.""" names: set[str] = set() for routine in routines: names.update(param.name for param in NumpyDocString(routine.__doc__)["Parameters"]) return tuple(sorted(names)) def _consistency_cases() -> list[Any]: """One test case per (parameter, routine set), so every mismatch is reported separately.""" groups: list[tuple[str, tuple[str, ...], tuple[Callable[..., Any], ...]]] = [ ( "store-path-create_array_group", ("store", "path"), ( asynchronous.create_array, synchronous.create_array, asynchronous.create_group, synchronous.create_group, zarr.AsyncGroup.create_array, zarr.Group.create_array, ), ), ( "store-path-create", ("store", "path"), (asynchronous.create, synchronous.create, zarr.Group.create), ), ( "create_array_variants", tuple( name for name in _documented_parameters(CREATE_ARRAY_ROUTINES) if name not in CREATE_ARRAY_EXEMPT_PARAMETERS ), CREATE_ARRAY_ROUTINES, ), ] return [ pytest.param(name, routines, id=f"{group_id}-{name}") for group_id, names, routines in groups for name in names ] @pytest.mark.parametrize(("parameter_name", "array_creation_routines"), _consistency_cases()) def test_docstring_consistent_parameters( parameter_name: str, array_creation_routines: tuple[Callable[..., Any], ...] ) -> None: """ Tests that array and group creation routines document the same parameter consistently. This test inspects the docstrings of a set of callables and generates two dicts: - a dict where the keys are parameter descriptions and the values are the names of the routines with those descriptions - a dict where the keys are parameter types and the values are the names of the routines with those types If each dict has at most 1 value, then the parameter description and type in the docstring must be identical across different routines. But if these dicts have multiple values, then there must be routines that use the same parameter but document it differently, which will trigger a test failure. """ descs: dict[tuple[str, ...], tuple[str, ...]] = {} types: dict[str, tuple[str, ...]] = {} for routine in array_creation_routines: key = f"{routine.__module__}.{routine.__qualname__}" param_dict = {d.name: d for d in NumpyDocString(routine.__doc__)["Parameters"]} if parameter_name in param_dict: val = param_dict[parameter_name] descs[tuple(val.desc)] = descs.get(tuple(val.desc), ()) + (key,) types[val.type] = types.get(val.type, ()) + (key,) assert len(descs) <= 1, f"parameter {parameter_name!r} has inconsistent descriptions: {descs}" assert len(types) <= 1, f"parameter {parameter_name!r} has inconsistent types: {types}" zarr-developers-zarr-python-a2f2415/tests/test_array.py000066400000000000000000002747101527014137600233710ustar00rootroot00000000000000import dataclasses import inspect import json import math import multiprocessing as mp import pickle import re import sys from itertools import accumulate, starmap from typing import TYPE_CHECKING, Any, Literal from unittest import mock import numcodecs import numpy as np import numpy.typing as npt import pytest from hypothesis import given, settings from hypothesis import strategies as st from packaging.version import Version import zarr.api.asynchronous import zarr.api.synchronous as sync_api from tests.conftest import json_attributes, skip_object_dtype from zarr import Array, Group from zarr.abc.store import Store from zarr.codecs import ( BytesCodec, GzipCodec, TransposeCodec, ZstdCodec, ) from zarr.core._info import ArrayInfo from zarr.core.array import ( AsyncArray, CompressorsLike, FiltersLike, ShardsLike, _iter_chunk_coords, _iter_chunk_regions, _iter_shard_coords, _iter_shard_keys, _iter_shard_regions, _parse_chunk_encoding_v2, _parse_chunk_encoding_v3, _shards_initialized, create_array, default_filters_v2, default_serializer_v3, ) from zarr.core.array_spec import ArrayConfig, ArrayConfigParams from zarr.core.buffer import NDArrayLike, NDArrayLikeOrScalar, cpu, default_buffer_prototype from zarr.core.chunk_grids import ( SHARDED_INNER_CHUNK_MAX_BYTES, guess_chunks, normalize_chunks_nd, resolve_outer_and_inner_chunks, ) from zarr.core.chunk_key_encodings import ChunkKeyEncodingParams from zarr.core.common import JSON, ChunksLike, ZarrFormat, ceildiv from zarr.core.dtype import ( DateTime64, Float32, Float64, Int16, Structured, TimeDelta64, UInt8, VariableLengthBytes, VariableLengthUTF8, ZDType, parse_dtype, ) from zarr.core.dtype.common import ENDIANNESS_STR, EndiannessStr from zarr.core.dtype.npy.common import NUMPY_ENDIANNESS_STR, endianness_from_numpy_str from zarr.core.group import AsyncGroup from zarr.core.indexing import BasicIndexer, _iter_grid, _iter_regions from zarr.core.metadata.v2 import ArrayV2Metadata from zarr.core.sync import sync from zarr.errors import ( ContainsArrayError, ContainsGroupError, ZarrUserWarning, ) from zarr.storage import LocalStore, MemoryStore, StorePath from zarr.storage._logging import LoggingStore from zarr.types import AnyArray, AnyAsyncArray from .test_dtype.conftest import zdtype_examples if TYPE_CHECKING: from zarr.abc.codec import CodecJSON_V3 @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) @pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.parametrize("overwrite", [True, False]) @pytest.mark.parametrize("extant_node", ["array", "group"]) def test_array_creation_existing_node( store: LocalStore | MemoryStore, zarr_format: ZarrFormat, overwrite: bool, extant_node: Literal["array", "group"], ) -> None: """ Check that an existing array or group is handled as expected during array creation. """ spath = StorePath(store) group = Group.from_store(spath, zarr_format=zarr_format) expected_exception: type[ContainsArrayError | ContainsGroupError] if extant_node == "array": expected_exception = ContainsArrayError _ = group.create_array("extant", shape=(10,), dtype="uint8") elif extant_node == "group": expected_exception = ContainsGroupError _ = group.create_group("extant") else: raise AssertionError new_shape = (2, 2) new_dtype = "float32" if overwrite: if not store.supports_deletes: pytest.skip("store does not support deletes") arr_new = zarr.create_array( spath / "extant", shape=new_shape, dtype=new_dtype, overwrite=overwrite, zarr_format=zarr_format, ) assert arr_new.shape == new_shape assert arr_new.dtype == new_dtype else: with pytest.raises(expected_exception): arr_new = zarr.create_array( spath / "extant", shape=new_shape, dtype=new_dtype, overwrite=overwrite, zarr_format=zarr_format, ) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) @pytest.mark.parametrize("zarr_format", [2, 3]) async def test_create_creates_parents( store: LocalStore | MemoryStore, zarr_format: ZarrFormat ) -> None: # prepare a root node, with some data set await zarr.api.asynchronous.open_group( store=store, path="a", zarr_format=zarr_format, attributes={"key": "value"} ) # create a child node with a couple intermediates await zarr.api.asynchronous.create( shape=(2, 2), store=store, path="a/b/c/d", zarr_format=zarr_format ) parts = ["a", "a/b", "a/b/c"] if zarr_format == 2: files = [".zattrs", ".zgroup"] else: files = ["zarr.json"] expected = [f"{part}/{file}" for file in files for part in parts] if zarr_format == 2: expected.extend([".zattrs", ".zgroup", "a/b/c/d/.zarray", "a/b/c/d/.zattrs"]) else: expected.extend(["zarr.json", "a/b/c/d/zarr.json"]) expected = sorted(expected) result = sorted([x async for x in store.list_prefix("")]) assert result == expected paths = ["a", "a/b", "a/b/c"] for path in paths: g = await zarr.api.asynchronous.open_group(store=store, path=path) assert isinstance(g, AsyncGroup) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) @pytest.mark.parametrize("zarr_format", [2, 3]) def test_array_name_properties_no_group( store: LocalStore | MemoryStore, zarr_format: ZarrFormat ) -> None: arr = zarr.create_array( store=store, shape=(100,), chunks=(10,), zarr_format=zarr_format, dtype=">i4" ) assert arr.path == "" assert arr.name == "/" assert arr.basename == "" @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) @pytest.mark.parametrize("zarr_format", [2, 3]) def test_array_name_properties_with_group( store: LocalStore | MemoryStore, zarr_format: ZarrFormat ) -> None: root = Group.from_store(store=store, zarr_format=zarr_format) foo = root.create_array("foo", shape=(100,), chunks=(10,), dtype="i4") assert foo.path == "foo" assert foo.name == "/foo" assert foo.basename == "foo" bar = root.create_group("bar") spam = bar.create_array("spam", shape=(100,), chunks=(10,), dtype="i4") assert spam.path == "bar/spam" assert spam.name == "/bar/spam" assert spam.basename == "spam" @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize("specify_fill_value", [True, False]) @pytest.mark.parametrize( "zdtype", zdtype_examples, ids=tuple(str(type(v)) for v in zdtype_examples) ) def test_array_fill_value_default( store: MemoryStore, specify_fill_value: bool, zdtype: ZDType[Any, Any] ) -> None: """ Test that creating an array with the fill_value parameter set to None, or unspecified, results in the expected fill_value attribute of the array, i.e. the default value of the dtype """ shape = (10,) if specify_fill_value: arr = zarr.create_array( store=store, shape=shape, dtype=zdtype, zarr_format=3, chunks=shape, fill_value=None, ) else: arr = zarr.create_array(store=store, shape=shape, dtype=zdtype, zarr_format=3, chunks=shape) expected_fill_value = zdtype.default_scalar() if isinstance(expected_fill_value, np.datetime64 | np.timedelta64): if np.isnat(expected_fill_value): assert np.isnat(arr.fill_value) elif isinstance(expected_fill_value, np.floating | np.complexfloating): if np.isnan(expected_fill_value): assert np.isnan(arr.fill_value) else: assert arr.fill_value == expected_fill_value # A simpler check would be to ensure that arr.fill_value.dtype == arr.dtype # But for some numpy data types (namely, U), scalars might not have length. An empty string # scalar from a `>U4` array would have dtype `>U`, and arr.fill_value.dtype == arr.dtype will fail. assert type(arr.fill_value) is type(np.array([arr.fill_value], dtype=arr.dtype)[0]) @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize( ("dtype_str", "fill_value"), [("bool", True), ("uint8", 99), ("float32", -99.9), ("complex64", 3 + 4j)], ) def test_array_v3_fill_value(store: MemoryStore, fill_value: int, dtype_str: str) -> None: shape = (10,) arr = zarr.create_array( store=store, shape=shape, dtype=dtype_str, zarr_format=3, chunks=shape, fill_value=fill_value, ) assert arr.fill_value == np.dtype(dtype_str).type(fill_value) assert arr.fill_value.dtype == arr.dtype @pytest.mark.parametrize("store", ["memory"], indirect=True) async def test_array_v3_nan_fill_value(store: MemoryStore) -> None: shape = (10,) arr = zarr.create_array( store=store, shape=shape, dtype=np.float64, zarr_format=3, chunks=shape, fill_value=np.nan, ) arr[:] = np.nan assert np.isnan(arr.fill_value) assert arr.fill_value.dtype == arr.dtype # all fill value chunk is an empty chunk, and should not be written assert len([a async for a in store.list_prefix("/")]) == 0 @pytest.mark.parametrize("store", ["local"], indirect=["store"]) @pytest.mark.parametrize("zarr_format", [2, 3]) async def test_serializable_async_array( store: LocalStore | MemoryStore, zarr_format: ZarrFormat ) -> None: expected = await zarr.api.asynchronous.create_array( store=store, shape=(100,), chunks=(10,), zarr_format=zarr_format, dtype="i4" ) # await expected.setitems(list(range(100))) p = pickle.dumps(expected) actual = pickle.loads(p) assert actual == expected # np.testing.assert_array_equal(await actual.getitem(slice(None)), await expected.getitem(slice(None))) # TODO: uncomment the parts of this test that will be impacted by the config/prototype changes in flight @pytest.mark.parametrize("store", ["local"], indirect=["store"]) @pytest.mark.parametrize("zarr_format", [2, 3]) def test_serializable_sync_array(store: LocalStore, zarr_format: ZarrFormat) -> None: expected = zarr.create_array( store=store, shape=(100,), chunks=(10,), zarr_format=zarr_format, dtype="i4" ) expected[:] = list(range(100)) p = pickle.dumps(expected) actual = pickle.loads(p) assert actual == expected np.testing.assert_array_equal(actual[:], expected[:]) @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize("zarr_format", [2, 3, "invalid"]) def test_storage_transformers(store: MemoryStore, zarr_format: ZarrFormat | str) -> None: """ Test that providing an actual storage transformer produces a warning and otherwise passes through """ metadata_dict: dict[str, JSON] if zarr_format == 3: metadata_dict = { "zarr_format": 3, "node_type": "array", "shape": (10,), "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, "data_type": "uint8", "chunk_key_encoding": {"name": "v2", "configuration": {"separator": "/"}}, "codecs": (BytesCodec().to_dict(),), "fill_value": 0, "storage_transformers": ({"test": "should_raise"}), } else: metadata_dict = { "zarr_format": zarr_format, "shape": (10,), "chunks": (1,), "dtype": "|u1", "dimension_separator": ".", "codecs": (BytesCodec().to_dict(),), "fill_value": 0, "order": "C", "storage_transformers": ({"test": "should_raise"}), } if zarr_format == 3: match = "Arrays with storage transformers are not supported in zarr-python at this time." with pytest.raises(ValueError, match=match): Array.from_dict(StorePath(store), data=metadata_dict) elif zarr_format == 2: # no warning Array.from_dict(StorePath(store), data=metadata_dict) else: match = f"Invalid zarr_format: {zarr_format}. Expected 2 or 3" with pytest.raises(ValueError, match=match): Array.from_dict(StorePath(store), data=metadata_dict) @pytest.mark.parametrize("test_cls", [AnyArray, AnyAsyncArray]) @pytest.mark.parametrize("nchunks", [2, 5, 10]) def test_nchunks(test_cls: type[AnyArray] | type[AnyAsyncArray], nchunks: int) -> None: """ Test that nchunks returns the number of chunks defined for the array. """ store = MemoryStore() shape = 100 arr = zarr.create_array(store, shape=(shape,), chunks=(ceildiv(shape, nchunks),), dtype="i4") expected = nchunks if test_cls == Array: observed = arr.nchunks else: observed = arr.async_array.nchunks assert observed == expected @pytest.mark.parametrize("test_cls", [Array, AsyncArray]) @pytest.mark.parametrize( ("shape", "shard_shape", "chunk_shape"), [((10,), None, (1,)), ((10,), (1,), (1,)), ((40,), (20,), (5,))], ) async def test_nchunks_initialized( test_cls: type[AnyArray] | type[AnyAsyncArray], shape: tuple[int, ...], shard_shape: tuple[int, ...] | None, chunk_shape: tuple[int, ...], ) -> None: """ Test that nchunks_initialized accurately returns the number of stored partitions. """ store = MemoryStore() if shard_shape is None: chunks_per_shard = 1 else: chunks_per_shard = np.prod(np.array(shard_shape) // np.array(chunk_shape)) arr = zarr.create_array(store, shape=shape, shards=shard_shape, chunks=chunk_shape, dtype="i1") # write chunks one at a time for idx, region in enumerate(arr._iter_shard_regions()): arr[region] = 1 expected = idx + 1 if test_cls == Array: observed = arr._nshards_initialized assert observed == arr.nchunks_initialized // chunks_per_shard else: observed = await arr.async_array._nshards_initialized() assert observed == await arr.async_array.nchunks_initialized() // chunks_per_shard assert observed == expected # delete chunks for idx, key in enumerate(arr._iter_shard_keys()): sync(arr.store_path.store.delete(key)) if test_cls == Array: observed = arr._nshards_initialized assert observed == arr.nchunks_initialized // chunks_per_shard else: observed = await arr.async_array._nshards_initialized() assert observed == await arr.async_array.nchunks_initialized() // chunks_per_shard expected = arr._nshards - idx - 1 assert observed == expected @pytest.mark.parametrize("path", ["", "foo"]) @pytest.mark.parametrize( ("shape", "shard_shape", "chunk_shape"), [((10,), None, (1,)), ((10,), (1,), (1,)), ((40,), (20,), (5,))], ) async def test_chunks_initialized( path: str, shape: tuple[int, ...], shard_shape: tuple[int, ...], chunk_shape: tuple[int, ...] ) -> None: """ Test that chunks_initialized accurately returns the keys of stored chunks. """ store = MemoryStore() arr = zarr.create_array( store, name=path, shape=shape, shards=shard_shape, chunks=chunk_shape, dtype="i1" ) if path: await store.set(path, cpu.Buffer.from_bytes(b"")) chunks_accumulated = tuple( accumulate(tuple(tuple(v.split(" ")) for v in arr._iter_shard_keys())) ) for keys, region in zip(chunks_accumulated, arr._iter_shard_regions(), strict=False): arr[region] = 1 observed = sorted(await _shards_initialized(arr.async_array)) expected = sorted(keys) assert observed == expected def test_nbytes_stored() -> None: arr = zarr.create(shape=(100,), chunks=(10,), dtype="i4", codecs=[BytesCodec(endian="little")]) result = arr.nbytes_stored() assert result == 502 # the size of the metadata document. This is a fragile test. arr[:50] = 1 result = arr.nbytes_stored() assert result == 702 # the size with 5 chunks filled. arr[50:] = 2 result = arr.nbytes_stored() assert result == 902 # the size with all chunks filled. async def test_nbytes_stored_async() -> None: arr = await zarr.api.asynchronous.create( shape=(100,), chunks=(10,), dtype="i4", codecs=[BytesCodec(endian="little")] ) result = await arr.nbytes_stored() assert result == 502 # the size of the metadata document. This is a fragile test. await arr.setitem(slice(50), 1) result = await arr.nbytes_stored() assert result == 702 # the size with 5 chunks filled. await arr.setitem(slice(50, 100), 2) result = await arr.nbytes_stored() assert result == 902 # the size with all chunks filled. @pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.parametrize("depth", [0, 1, 8, 32, 65, 100]) @settings(max_examples=20, deadline=None) @given(data=st.data()) def test_update_attrs(zarr_format: ZarrFormat, depth: int, data: st.DataObject) -> None: # regression test for https://github.com/zarr-developers/zarr-python/issues/2328 store = MemoryStore() arr = zarr.create_array( store=store, shape=(5,), chunks=(5,), dtype="f8", zarr_format=zarr_format ) attributes = data.draw(json_attributes(depth=depth)) for key, value in attributes.items(): arr.attrs[key] = value assert dict(arr.attrs) == attributes arr2 = zarr.open_array(store=store, zarr_format=zarr_format) assert dict(arr2.attrs) == attributes @pytest.mark.parametrize(("chunks", "shards"), [((2, 2), None), ((2, 2), (4, 4))]) class TestInfo: def test_info_v2(self, chunks: tuple[int, int], shards: tuple[int, int] | None) -> None: arr = zarr.create_array(store={}, shape=(8, 8), dtype="f8", chunks=chunks, zarr_format=2) result = arr.info expected = ArrayInfo( _zarr_format=2, _data_type=arr.async_array._zdtype, _fill_value=arr.fill_value, _shape=(8, 8), _chunk_shape=chunks, _shard_shape=None, _order="C", _read_only=False, _store_type="MemoryStore", _count_bytes=512, _compressors=(numcodecs.Zstd(),), ) assert result == expected def test_info_v3(self, chunks: tuple[int, int], shards: tuple[int, int] | None) -> None: arr = zarr.create_array(store={}, shape=(8, 8), dtype="f8", chunks=chunks, shards=shards) result = arr.info expected = ArrayInfo( _zarr_format=3, _data_type=arr.async_array._zdtype, _fill_value=arr.fill_value, _shape=(8, 8), _chunk_shape=chunks, _shard_shape=shards, _order="C", _read_only=False, _store_type="MemoryStore", _compressors=(ZstdCodec(),), _serializer=BytesCodec(endian="little"), _count_bytes=512, ) assert result == expected def test_info_complete(self, chunks: tuple[int, int], shards: tuple[int, int] | None) -> None: arr = zarr.create_array( store={}, shape=(8, 8), dtype="f8", chunks=chunks, shards=shards, compressors=(), ) result = arr.info_complete() expected = ArrayInfo( _zarr_format=3, _data_type=arr.async_array._zdtype, _fill_value=arr.fill_value, _shape=(8, 8), _chunk_shape=chunks, _shard_shape=shards, _order="C", _read_only=False, _store_type="MemoryStore", _serializer=BytesCodec(endian="little"), _count_bytes=512, _count_chunks_initialized=0, _count_bytes_stored=521 if shards is None else 982, # the metadata? ) assert result == expected arr[:4, :4] = 10 result = arr.info_complete() if shards is None: expected = dataclasses.replace( expected, _count_chunks_initialized=4, _count_bytes_stored=649 ) else: expected = dataclasses.replace( expected, _count_chunks_initialized=1, _count_bytes_stored=1178 ) assert result == expected async def test_info_v2_async( self, chunks: tuple[int, int], shards: tuple[int, int] | None ) -> None: arr = await zarr.api.asynchronous.create_array( store={}, shape=(8, 8), dtype="f8", chunks=chunks, zarr_format=2 ) result = arr.info expected = ArrayInfo( _zarr_format=2, _data_type=Float64(), _fill_value=arr.metadata.fill_value, _shape=(8, 8), _chunk_shape=(2, 2), _shard_shape=None, _order="C", _read_only=False, _store_type="MemoryStore", _count_bytes=512, _compressors=(numcodecs.Zstd(),), ) assert result == expected async def test_info_v3_async( self, chunks: tuple[int, int], shards: tuple[int, int] | None ) -> None: arr = await zarr.api.asynchronous.create_array( store={}, shape=(8, 8), dtype="f8", chunks=chunks, shards=shards, ) result = arr.info expected = ArrayInfo( _zarr_format=3, _data_type=arr._zdtype, _fill_value=arr.metadata.fill_value, _shape=(8, 8), _chunk_shape=chunks, _shard_shape=shards, _order="C", _read_only=False, _store_type="MemoryStore", _compressors=(ZstdCodec(),), _serializer=BytesCodec(endian="little"), _count_bytes=512, ) assert result == expected async def test_info_complete_async( self, chunks: tuple[int, int], shards: tuple[int, int] | None ) -> None: arr = await zarr.api.asynchronous.create_array( store={}, dtype="f8", shape=(8, 8), chunks=chunks, shards=shards, compressors=None, ) result = await arr.info_complete() expected = ArrayInfo( _zarr_format=3, _data_type=arr._zdtype, _fill_value=arr.metadata.fill_value, _shape=(8, 8), _chunk_shape=chunks, _shard_shape=shards, _order="C", _read_only=False, _store_type="MemoryStore", _serializer=BytesCodec(endian="little"), _count_bytes=512, _count_chunks_initialized=0, _count_bytes_stored=521 if shards is None else 982, # the metadata? ) assert result == expected await arr.setitem((slice(4), slice(4)), 10) result = await arr.info_complete() if shards is None: expected = dataclasses.replace( expected, _count_chunks_initialized=4, _count_bytes_stored=553 ) else: expected = dataclasses.replace( expected, _count_chunks_initialized=1, _count_bytes_stored=1178 ) @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_resize_wrong_ndim_raises(store: MemoryStore, zarr_format: ZarrFormat) -> None: """ Resizing to a shape with a different number of dimensions is a ValueError. """ z = zarr.create(shape=(10, 10), chunks=(5, 5), dtype="i4", store=store, zarr_format=zarr_format) with pytest.raises(ValueError, match="same number of dimensions"): z.resize((20,)) @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_resize_1d(store: MemoryStore, zarr_format: ZarrFormat) -> None: z = zarr.create( shape=105, chunks=10, dtype="i4", fill_value=0, store=store, zarr_format=zarr_format ) a = np.arange(105, dtype="i4") z[:] = a result = z[:] assert isinstance(result, NDArrayLike) assert (105,) == z.shape assert (105,) == result.shape assert np.dtype("i4") == z.dtype assert np.dtype("i4") == result.dtype assert (10,) == z.chunks np.testing.assert_array_equal(a, result) z.resize(205) result = z[:] assert isinstance(result, NDArrayLike) assert (205,) == z.shape assert (205,) == result.shape assert np.dtype("i4") == z.dtype assert np.dtype("i4") == result.dtype assert (10,) == z.chunks np.testing.assert_array_equal(a, z[:105]) np.testing.assert_array_equal(np.zeros(100, dtype="i4"), z[105:]) z.resize(55) result = z[:] assert isinstance(result, NDArrayLike) assert (55,) == z.shape assert (55,) == result.shape assert np.dtype("i4") == z.dtype assert np.dtype("i4") == result.dtype assert (10,) == z.chunks np.testing.assert_array_equal(a[:55], result) # via shape setter new_shape = (105,) z.shape = new_shape result = z[:] assert isinstance(result, NDArrayLike) assert new_shape == z.shape assert new_shape == result.shape @pytest.mark.parametrize("chunks", [(1,), (2,), (4,)]) def test_resize_sharded_keeps_cells_beyond_shape(chunks: tuple[int, ...]) -> None: """A shard kept by a shrinking resize keeps its cells beyond the new shape, and a later write to the shard leaves them alone, so they come back when the array grows.""" arr = zarr.create_array({}, shape=(4,), chunks=chunks, shards=(4,), dtype="int16", fill_value=0) arr[:] = [1, 2, 3, 4] arr.resize((2,)) arr[:] = [9, 9] arr.resize((4,)) np.testing.assert_array_equal(arr[:], [9, 9, 3, 4]) @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_resize_2d(store: MemoryStore, zarr_format: ZarrFormat) -> None: z = zarr.create( shape=(105, 105), chunks=(10, 10), dtype="i4", fill_value=0, store=store, zarr_format=zarr_format, ) a = np.arange(105 * 105, dtype="i4").reshape((105, 105)) z[:] = a result = z[:] assert isinstance(result, NDArrayLike) assert (105, 105) == z.shape assert (105, 105) == result.shape assert np.dtype("i4") == z.dtype assert np.dtype("i4") == result.dtype assert (10, 10) == z.chunks np.testing.assert_array_equal(a, result) z.resize((205, 205)) result = z[:] assert isinstance(result, NDArrayLike) assert (205, 205) == z.shape assert (205, 205) == result.shape assert np.dtype("i4") == z.dtype assert np.dtype("i4") == result.dtype assert (10, 10) == z.chunks np.testing.assert_array_equal(a, z[:105, :105]) np.testing.assert_array_equal(np.zeros((100, 205), dtype="i4"), z[105:, :]) np.testing.assert_array_equal(np.zeros((205, 100), dtype="i4"), z[:, 105:]) z.resize((55, 55)) result = z[:] assert isinstance(result, NDArrayLike) assert (55, 55) == z.shape assert (55, 55) == result.shape assert np.dtype("i4") == z.dtype assert np.dtype("i4") == result.dtype assert (10, 10) == z.chunks np.testing.assert_array_equal(a[:55, :55], result) z.resize((55, 1)) result = z[:] assert isinstance(result, NDArrayLike) assert (55, 1) == z.shape assert (55, 1) == result.shape assert np.dtype("i4") == z.dtype assert np.dtype("i4") == result.dtype assert (10, 10) == z.chunks np.testing.assert_array_equal(a[:55, :1], result) z.resize((1, 55)) result = z[:] assert isinstance(result, NDArrayLike) assert (1, 55) == z.shape assert (1, 55) == result.shape assert np.dtype("i4") == z.dtype assert np.dtype("i4") == result.dtype assert (10, 10) == z.chunks np.testing.assert_array_equal(a[:1, :10], z[:, :10]) np.testing.assert_array_equal(np.zeros((1, 55 - 10), dtype="i4"), z[:, 10:55]) # via shape setter new_shape = (105, 105) z.shape = new_shape result = z[:] assert isinstance(result, NDArrayLike) assert new_shape == z.shape assert new_shape == result.shape @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_resize_growing_skips_chunk_enumeration( store: MemoryStore, zarr_format: ZarrFormat ) -> None: """Growing an array should not enumerate chunk coords for deletion (#3650 mitigation).""" z = zarr.create( shape=(10, 10), chunks=(5, 5), dtype="i4", fill_value=0, store=store, zarr_format=zarr_format, ) z[:] = np.ones((10, 10), dtype="i4") grid_cls = type(z._chunk_grid) # growth only - ensure no chunk coords are enumerated with mock.patch.object( grid_cls, "all_chunk_coords", wraps=z._chunk_grid.all_chunk_coords, ) as mock_coords: z.resize((20, 20)) mock_coords.assert_not_called() assert z.shape == (20, 20) np.testing.assert_array_equal(np.ones((10, 10), dtype="i4"), z[:10, :10]) np.testing.assert_array_equal(np.zeros((10, 10), dtype="i4"), z[10:, 10:]) # Shrinking also avoids full-grid enumeration. with mock.patch.object( grid_cls, "all_chunk_coords", wraps=z._chunk_grid.all_chunk_coords, ) as mock_coords: z.resize((5, 5)) mock_coords.assert_not_called() assert z.shape == (5, 5) np.testing.assert_array_equal(np.ones((5, 5), dtype="i4"), z[:]) # mixed: grow dim 0, shrink dim 1 - ensure deletion path runs z2 = zarr.create( shape=(10, 10), chunks=(5, 5), dtype="i4", fill_value=0, store=store, zarr_format=zarr_format, overwrite=True, ) z2[:] = np.ones((10, 10), dtype="i4") with mock.patch.object( grid_cls, "all_chunk_coords", wraps=z2._chunk_grid.all_chunk_coords, ) as mock_coords: z2.resize((20, 5)) mock_coords.assert_not_called() assert z2.shape == (20, 5) np.testing.assert_array_equal(np.ones((10, 5), dtype="i4"), z2[:10, :]) np.testing.assert_array_equal(np.zeros((10, 5), dtype="i4"), z2[10:, :]) @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_append_1d(store: MemoryStore, zarr_format: ZarrFormat) -> None: a = np.arange(105) z = zarr.create(shape=a.shape, chunks=10, dtype=a.dtype, store=store, zarr_format=zarr_format) z[:] = a assert a.shape == z.shape assert a.dtype == z.dtype assert (10,) == z.chunks np.testing.assert_array_equal(a, z[:]) b = np.arange(105, 205) e = np.append(a, b) assert z.shape == (105,) z.append(b) assert e.shape == z.shape assert e.dtype == z.dtype assert (10,) == z.chunks np.testing.assert_array_equal(e, z[:]) # check append handles array-like c = [1, 2, 3] f = np.append(e, c) z.append(c) assert f.shape == z.shape assert f.dtype == z.dtype assert (10,) == z.chunks np.testing.assert_array_equal(f, z[:]) @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_append_2d(store: MemoryStore, zarr_format: ZarrFormat) -> None: a = np.arange(105 * 105, dtype="i4").reshape((105, 105)) z = zarr.create( shape=a.shape, chunks=(10, 10), dtype=a.dtype, store=store, zarr_format=zarr_format ) z[:] = a assert a.shape == z.shape assert a.dtype == z.dtype assert (10, 10) == z.chunks actual = z[:] np.testing.assert_array_equal(a, actual) b = np.arange(105 * 105, 2 * 105 * 105, dtype="i4").reshape((105, 105)) e = np.append(a, b, axis=0) z.append(b) assert e.shape == z.shape assert e.dtype == z.dtype assert (10, 10) == z.chunks actual = z[:] np.testing.assert_array_equal(e, actual) @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_append_2d_axis(store: MemoryStore, zarr_format: ZarrFormat) -> None: a = np.arange(105 * 105, dtype="i4").reshape((105, 105)) z = zarr.create( shape=a.shape, chunks=(10, 10), dtype=a.dtype, store=store, zarr_format=zarr_format ) z[:] = a assert a.shape == z.shape assert a.dtype == z.dtype assert (10, 10) == z.chunks np.testing.assert_array_equal(a, z[:]) b = np.arange(105 * 105, 2 * 105 * 105, dtype="i4").reshape((105, 105)) e = np.append(a, b, axis=1) z.append(b, axis=1) assert e.shape == z.shape assert e.dtype == z.dtype assert (10, 10) == z.chunks np.testing.assert_array_equal(e, z[:]) @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_append_bad_shape(store: MemoryStore, zarr_format: ZarrFormat) -> None: a = np.arange(100) z = zarr.create(shape=a.shape, chunks=10, dtype=a.dtype, store=store, zarr_format=zarr_format) z[:] = a b = a.reshape(10, 10) with pytest.raises(ValueError): z.append(b) @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize("write_empty_chunks", [True, False]) @pytest.mark.parametrize("fill_value", [0, 5]) def test_write_empty_chunks_behavior( zarr_format: ZarrFormat, store: MemoryStore, write_empty_chunks: bool, fill_value: int ) -> None: """ Check that the write_empty_chunks value of the config is applied correctly. We expect that when write_empty_chunks is True, writing chunks equal to the fill value will result in those chunks appearing in the store. When write_empty_chunks is False, writing chunks that are equal to the fill value will result in those chunks not being present in the store. In particular, they should be deleted if they were already present. """ arr = zarr.create_array( store=store, shape=(2,), zarr_format=zarr_format, dtype="i4", fill_value=fill_value, chunks=(1,), config={"write_empty_chunks": write_empty_chunks}, ) assert arr.async_array.config.write_empty_chunks == write_empty_chunks # initialize the store with some non-fill value chunks arr[:] = fill_value + 1 assert arr._nshards_initialized == arr._nshards arr[:] = fill_value if not write_empty_chunks: assert arr._nshards_initialized == 0 else: assert arr._nshards_initialized == arr._nshards @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize("fill_value", [0.0, -0.0]) @pytest.mark.parametrize("dtype", ["f4", "f2"]) def test_write_empty_chunks_negative_zero( zarr_format: ZarrFormat, store: MemoryStore, fill_value: float, dtype: str ) -> None: # regression test for https://github.com/zarr-developers/zarr-python/issues/3144 arr = zarr.create_array( store=store, shape=(2,), zarr_format=zarr_format, dtype=dtype, fill_value=fill_value, chunks=(1,), config={"write_empty_chunks": False}, ) assert arr.nchunks_initialized == 0 # initialize the with the negated fill value (-0.0 for +0.0, +0.0 for -0.0) arr[:] = -fill_value assert arr.nchunks_initialized == arr.nchunks @pytest.mark.parametrize( ("fill_value", "expected"), [ (np.nan * 1j, ["NaN", "NaN"]), (np.nan, ["NaN", 0.0]), (np.inf, ["Infinity", 0.0]), (np.inf * 1j, ["NaN", "Infinity"]), (-np.inf, ["-Infinity", 0.0]), (math.inf, ["Infinity", 0.0]), ], ) async def test_special_complex_fill_values_roundtrip(fill_value: Any, expected: list[Any]) -> None: store = MemoryStore() zarr.create_array(store=store, shape=(1,), dtype=np.complex64, fill_value=fill_value) content = await store.get("zarr.json", prototype=default_buffer_prototype()) assert content is not None actual = json.loads(content.to_bytes()) assert actual["fill_value"] == expected @pytest.mark.parametrize("shape", [(1,), (2, 3), (4, 5, 6)]) @pytest.mark.parametrize("dtype", ["uint8", "float32"]) @pytest.mark.parametrize("array_type", ["async", "sync"]) async def test_nbytes( shape: tuple[int, ...], dtype: str, array_type: Literal["async", "sync"] ) -> None: """ Test that the ``nbytes`` attribute of an Array or AsyncArray correctly reports the capacity of the chunks of that array. """ store = MemoryStore() arr = zarr.create_array(store=store, shape=shape, dtype=dtype, fill_value=0) if array_type == "async": assert arr.async_array.nbytes == np.prod(arr.shape) * arr.dtype.itemsize else: assert arr.nbytes == np.prod(arr.shape) * arr.dtype.itemsize @pytest.mark.parametrize( ("array_shape", "chunk_shape", "target_shard_size_bytes", "expected_shards"), [ pytest.param( (256, 256), (32, 32), 129 * 129, (128, 128), id="2d_chunking_max_byes_does_not_evenly_divide", ), pytest.param( (256, 256), (32, 32), 64 * 64, (64, 64), id="2d_chunking_max_byes_evenly_divides" ), pytest.param( (256, 256), (64, 32), 128 * 128, (128, 64), id="2d_non_square_chunking_max_byes_evenly_divides", ), pytest.param((256,), (2,), 255, (254,), id="max_bytes_just_below_array_shape"), pytest.param((256,), (2,), 256, (256,), id="max_bytes_equal_to_array_shape"), pytest.param((256,), (2,), 16, (16,), id="max_bytes_normal_val"), pytest.param((256,), (2,), 2, (2,), id="max_bytes_same_as_chunk"), pytest.param((256,), (2,), 1, (2,), id="max_bytes_less_than_chunk"), pytest.param((256,), (2,), None, (4,), id="use_default_auto_setting"), pytest.param((4,), (2,), None, (2,), id="small_array_shape_does_not_shard"), ], ) def test_auto_partition_auto_shards( array_shape: tuple[int, ...], chunk_shape: tuple[int, ...], target_shard_size_bytes: int | None, expected_shards: tuple[int, ...], ) -> None: """ Test that automatically picking a shard size returns a tuple of 2 * the chunk shape for any axis where there are 8 or more chunks. """ dtype = np.dtype("uint8") chunks_normalized = normalize_chunks_nd(chunk_shape, array_shape) with pytest.warns( ZarrUserWarning, match="Automatic shard shape inference is experimental and may change without notice.", ): with zarr.config.set({"array.target_shard_size_bytes": target_shard_size_bytes}): outer_chunks, _ = resolve_outer_and_inner_chunks( array_shape=array_shape, chunks=chunks_normalized, shard_shape="auto", item_size=dtype.itemsize, ) auto_shards = outer_chunks.chunk_shape assert auto_shards == expected_shards def test_auto_partition_auto_shards_with_auto_chunks_should_be_close_to_1MiB() -> None: """ Test that automatically picking chunk and shard sizes together produces chunks close to 1 MiB and shards that are a multiple of the chunk size. """ array_shape = (10_000_000,) item_size = 1 # Auto-chunks with sharding use the default inner chunk size target chunks_normalized = guess_chunks( array_shape, item_size, max_bytes=SHARDED_INNER_CHUNK_MAX_BYTES ) chunk_shape = chunks_normalized.chunk_shape chunk_bytes = np.prod(chunk_shape) * item_size assert chunk_bytes <= SHARDED_INNER_CHUNK_MAX_BYTES assert chunk_bytes > SHARDED_INNER_CHUNK_MAX_BYTES // 4 # should be in the right ballpark with pytest.warns( ZarrUserWarning, match="Automatic shard shape inference is experimental and may change without notice.", ): with zarr.config.set({"array.target_shard_size_bytes": 10_000_000}): outer_chunks, inner = resolve_outer_and_inner_chunks( array_shape=array_shape, chunks=chunks_normalized, shard_shape="auto", item_size=item_size, ) assert inner is not None shard_shape = outer_chunks.chunk_shape # Shard dimensions must be multiples of chunk dimensions assert all(s % c == 0 for s, c in zip(shard_shape, chunk_shape, strict=True)) @pytest.mark.parametrize( "chunks", [(5, 5), [5, 5], np.array([5, 5]), (np.int64(5), np.int64(5))], ids=["tuple", "list", "array", "numpy-scalars"], ) @pytest.mark.parametrize( "shards", [(10, 10), [10, 10], np.array([10, 10]), (np.int64(10), np.int64(10))], ids=["tuple", "list", "array", "numpy-scalars"], ) def test_chunks_and_shards(chunks: ChunksLike, shards: ChunksLike) -> None: store = StorePath(MemoryStore()) shape = (100, 100) expected_chunks = normalize_chunks_nd(chunks, shape).chunk_shape expected_shards = normalize_chunks_nd(shards, shape).chunk_shape arr_v3 = zarr.create_array(store=store / "v3", shape=shape, chunks=chunks, dtype="i4") assert arr_v3.chunks == expected_chunks assert arr_v3.shards is None arr_v3_sharding = zarr.create_array( store=store / "v3_sharding", shape=shape, chunks=chunks, shards=shards, dtype="i4", ) assert arr_v3_sharding.chunks == expected_chunks assert arr_v3_sharding.shards == expected_shards arr_v2 = zarr.create_array( store=store / "v2", shape=shape, chunks=chunks, zarr_format=2, dtype="i4" ) assert arr_v2.chunks == expected_chunks assert arr_v2.shards is None @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @pytest.mark.parametrize( ("dtype", "fill_value_expected"), [(" None: a = zarr.create_array(store, shape=(5,), chunks=(5,), dtype=dtype) assert a.fill_value == fill_value_expected @pytest.mark.parametrize("store", ["memory"], indirect=True) class TestCreateArray: @staticmethod def test_chunks_and_shards(store: Store) -> None: spath = StorePath(store) shape = (100, 100) chunks = (5, 5) shards = (10, 10) arr_v3 = zarr.create_array(store=spath / "v3", shape=shape, chunks=chunks, dtype="i4") assert arr_v3.chunks == chunks assert arr_v3.shards is None arr_v3_sharding = zarr.create_array( store=spath / "v3_sharding", shape=shape, chunks=chunks, shards=shards, dtype="i4", ) assert arr_v3_sharding.chunks == chunks assert arr_v3_sharding.shards == shards arr_v2 = zarr.create_array( store=spath / "v2", shape=shape, chunks=chunks, zarr_format=2, dtype="i4" ) assert arr_v2.chunks == chunks assert arr_v2.shards is None @staticmethod @pytest.mark.parametrize("dtype", zdtype_examples) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_default_fill_value(dtype: ZDType[Any, Any], store: Store) -> None: """ Test that the fill value of an array is set to the default value for the dtype object """ a = zarr.create_array(store, shape=(5,), chunks=(5,), dtype=dtype) if isinstance(dtype, DateTime64 | TimeDelta64) and np.isnat(a.fill_value): assert np.isnat(dtype.default_scalar()) else: assert a.fill_value == dtype.default_scalar() @staticmethod # @pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.parametrize("dtype", zdtype_examples) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_default_fill_value_None( dtype: ZDType[Any, Any], store: Store, zarr_format: ZarrFormat ) -> None: """ Test that the fill value of an array is set to the default value for an explicit None argument for Zarr Format 3, and to null for Zarr Format 2 """ a = zarr.create_array( store, shape=(5,), chunks=(5,), dtype=dtype, fill_value=None, zarr_format=zarr_format ) if zarr_format == 3: if isinstance(dtype, DateTime64 | TimeDelta64) and np.isnat(a.fill_value): assert np.isnat(dtype.default_scalar()) else: assert a.fill_value == dtype.default_scalar() elif zarr_format == 2: assert a.fill_value is None @staticmethod @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @pytest.mark.parametrize("dtype", zdtype_examples) def test_dtype_forms(dtype: ZDType[Any, Any], store: Store, zarr_format: ZarrFormat) -> None: """ Test that the same array is produced from a ZDType instance, a numpy dtype, or a numpy string """ skip_object_dtype(dtype) a = zarr.create_array( store, name="a", shape=(5,), chunks=(5,), dtype=dtype, zarr_format=zarr_format ) b = zarr.create_array( store, name="b", shape=(5,), chunks=(5,), dtype=dtype.to_native_dtype(), zarr_format=zarr_format, ) assert a.dtype == b.dtype # Structured dtypes do not have a numpy string representation that uniquely identifies them if not isinstance(dtype, Structured): if isinstance(dtype, VariableLengthUTF8): # in numpy 2.3, StringDType().str becomes the string 'StringDType()' which numpy # does not accept as a string representation of the dtype. c = zarr.create_array( store, name="c", shape=(5,), chunks=(5,), dtype=dtype.to_native_dtype().char, zarr_format=zarr_format, ) else: c = zarr.create_array( store, name="c", shape=(5,), chunks=(5,), dtype=dtype.to_native_dtype().str, zarr_format=zarr_format, ) assert a.dtype == c.dtype @staticmethod @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @pytest.mark.parametrize("dtype", zdtype_examples) def test_dtype_roundtrip( dtype: ZDType[Any, Any], store: Store, zarr_format: ZarrFormat ) -> None: """ Test that creating an array, then opening it, gets the same array. """ skip_object_dtype(dtype) a = zarr.create_array(store, shape=(5,), chunks=(5,), dtype=dtype, zarr_format=zarr_format) b = zarr.open_array(store) assert a.dtype == b.dtype @staticmethod @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @pytest.mark.parametrize("dtype", ["uint8", "float32", "U3", "S4", "V1"]) @pytest.mark.parametrize( "compressors", [ "auto", None, (), (ZstdCodec(level=3),), (ZstdCodec(level=3), GzipCodec(level=0)), ZstdCodec(level=3), {"name": "zstd", "configuration": {"level": 3}}, ({"name": "zstd", "configuration": {"level": 3}},), ], ) @pytest.mark.parametrize( "filters", [ "auto", None, (), ( TransposeCodec( order=[ 0, ] ), ), ( TransposeCodec( order=[ 0, ] ), TransposeCodec( order=[ 0, ] ), ), TransposeCodec( order=[ 0, ] ), {"name": "transpose", "configuration": {"order": [0]}}, ({"name": "transpose", "configuration": {"order": [0]}},), ], ) @pytest.mark.parametrize(("chunks", "shards"), [((6,), None), ((3,), (6,))]) async def test_v3_chunk_encoding( store: MemoryStore, compressors: CompressorsLike, filters: FiltersLike, dtype: str, chunks: tuple[int, ...], shards: tuple[int, ...] | None, ) -> None: """ Test various possibilities for the compressors and filters parameter to create_array """ arr = await create_array( store=store, dtype=dtype, shape=(12,), chunks=chunks, shards=shards, zarr_format=3, filters=filters, compressors=compressors, ) filters_expected, _, compressors_expected = _parse_chunk_encoding_v3( filters=filters, compressors=compressors, serializer="auto", dtype=arr._zdtype, ) assert arr.filters == filters_expected assert arr.compressors == compressors_expected @staticmethod @pytest.mark.parametrize("name", ["v2", "default", "invalid"]) @pytest.mark.parametrize("separator", [".", "/"]) async def test_chunk_key_encoding( name: str, separator: Literal[".", "/"], zarr_format: ZarrFormat, store: MemoryStore ) -> None: chunk_key_encoding = ChunkKeyEncodingParams(name=name, separator=separator) # type: ignore[typeddict-item] error_msg = "" if name == "invalid": error_msg = r'Unknown chunk key encoding: "Chunk key encoding \'invalid\' not found in registered chunk key encodings: \[.*\]."' if zarr_format == 2 and name == "default": error_msg = "Invalid chunk key encoding. For Zarr format 2 arrays, the `name` field of the chunk key encoding must be 'v2'." if error_msg: with pytest.raises(ValueError, match=error_msg): arr = await create_array( store=store, dtype="uint8", shape=(10,), chunks=(1,), zarr_format=zarr_format, chunk_key_encoding=chunk_key_encoding, ) else: arr = await create_array( store=store, dtype="uint8", shape=(10,), chunks=(1,), zarr_format=zarr_format, chunk_key_encoding=chunk_key_encoding, ) if isinstance(arr.metadata, ArrayV2Metadata): assert arr.metadata.dimension_separator == separator @staticmethod @pytest.mark.parametrize( ("kwargs", "error_msg"), [ ({"serializer": "bytes"}, "Zarr format 2 arrays do not support `serializer`."), ({"dimension_names": ["test"]}, "Zarr format 2 arrays do not support dimension names."), ], ) async def test_create_array_invalid_v2_arguments( kwargs: dict[str, Any], error_msg: str, store: MemoryStore ) -> None: with pytest.raises(ValueError, match=re.escape(error_msg)): await zarr.api.asynchronous.create_array( store=store, dtype="uint8", shape=(10,), chunks=(1,), zarr_format=2, **kwargs ) @staticmethod @pytest.mark.parametrize( ("kwargs", "error_msg"), [ ( {"dimension_names": ["test"]}, "dimension_names cannot be used for arrays with zarr_format 2.", ), ( {"chunk_key_encoding": {"name": "default", "separator": "/"}}, "chunk_key_encoding cannot be used for arrays with zarr_format 2. Use dimension_separator instead.", ), ( {"codecs": "bytes"}, "codecs cannot be used for arrays with zarr_format 2. Use filters and compressor instead.", ), ], ) async def test_create_invalid_v2_arguments( kwargs: dict[str, Any], error_msg: str, store: MemoryStore ) -> None: with pytest.raises(ValueError, match=re.escape(error_msg)): await zarr.api.asynchronous.create( store=store, dtype="uint8", shape=(10,), chunks=(1,), zarr_format=2, **kwargs ) @staticmethod @pytest.mark.parametrize( ("kwargs", "error_msg"), [ ( {"chunk_shape": (1,), "chunks": (2,)}, "Only one of chunk_shape or chunks can be provided.", ), ( {"dimension_separator": "/"}, "dimension_separator cannot be used for arrays with zarr_format 3. Use chunk_key_encoding instead.", ), ( {"filters": []}, "filters cannot be used for arrays with zarr_format 3. Use array-to-array codecs instead", ), ( {"compressor": "blosc"}, "compressor cannot be used for arrays with zarr_format 3. Use bytes-to-bytes codecs instead", ), ], ) async def test_invalid_v3_arguments( kwargs: dict[str, Any], error_msg: str, store: MemoryStore ) -> None: kwargs.setdefault("chunks", (1,)) with pytest.raises(ValueError, match=re.escape(error_msg)): zarr.create(store=store, dtype="uint8", shape=(10,), zarr_format=3, **kwargs) @staticmethod @pytest.mark.parametrize("dtype", ["uint8", "float32", "str", "U10", "S10", ">M8[10s]"]) @pytest.mark.parametrize( "compressors", [ "auto", None, numcodecs.Zstd(level=3), (), (numcodecs.Zstd(level=3),), ], ) @pytest.mark.parametrize( "filters", ["auto", None, numcodecs.GZip(level=1), (numcodecs.GZip(level=1),)] ) async def test_v2_chunk_encoding( store: MemoryStore, compressors: CompressorsLike, filters: FiltersLike, dtype: str ) -> None: if dtype == "str" and filters != "auto": pytest.skip("Only the auto filters are compatible with str dtype in this test.") arr: AsyncArray[ArrayV2Metadata] = await create_array( store=store, dtype=dtype, shape=(10,), zarr_format=2, compressors=compressors, filters=filters, ) filters_expected, compressor_expected = _parse_chunk_encoding_v2( filters=filters, compressor=compressors, dtype=parse_dtype(dtype, zarr_format=2) ) assert arr.metadata.zarr_format == 2 # guard for mypy assert arr.metadata.compressor == compressor_expected assert arr.metadata.filters == filters_expected # Normalize for property getters arr_compressors_expected = () if compressor_expected is None else (compressor_expected,) arr_filters_expected = () if filters_expected is None else filters_expected assert arr.compressors == arr_compressors_expected assert arr.filters == arr_filters_expected @staticmethod @pytest.mark.parametrize("dtype", [UInt8(), Float32(), VariableLengthUTF8()]) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") async def test_default_filters_compressors( store: MemoryStore, dtype: UInt8 | Float32 | VariableLengthUTF8, zarr_format: ZarrFormat ) -> None: """ Test that the default ``filters`` and ``compressors`` are used when ``create_array`` is invoked with ``filters`` and ``compressors`` unspecified. """ arr = await create_array( store=store, dtype=dtype, # type: ignore[arg-type] shape=(10,), zarr_format=zarr_format, ) sig = inspect.signature(create_array) if zarr_format == 3: expected_filters, expected_serializer, expected_compressors = _parse_chunk_encoding_v3( compressors=sig.parameters["compressors"].default, filters=sig.parameters["filters"].default, serializer=sig.parameters["serializer"].default, dtype=dtype, # type: ignore[arg-type] ) elif zarr_format == 2: default_filters, default_compressors = _parse_chunk_encoding_v2( compressor=sig.parameters["compressors"].default, filters=sig.parameters["filters"].default, dtype=dtype, # type: ignore[arg-type] ) if default_filters is None: expected_filters = () else: expected_filters = default_filters # type: ignore[assignment] if default_compressors is None: expected_compressors = () else: expected_compressors = (default_compressors,) # type: ignore[assignment] expected_serializer = None else: raise ValueError(f"Invalid zarr_format: {zarr_format}") assert arr.filters == expected_filters assert arr.serializer == expected_serializer assert arr.compressors == expected_compressors @staticmethod async def test_v2_no_shards(store: Store) -> None: """ Test that creating a Zarr v2 array with ``shard_shape`` set to a non-None value raises an error. """ msg = re.escape( "Zarr format 2 arrays can only be created with `shard_shape` set to `None`. Got `shard_shape=(5,)` instead." ) with pytest.raises(ValueError, match=msg): _ = await create_array( store=store, dtype="uint8", shape=(10,), shards=(5,), zarr_format=2, ) @staticmethod @pytest.mark.parametrize("impl", ["sync", "async"]) async def test_with_data(impl: Literal["sync", "async"], store: Store) -> None: """ Test that we can invoke ``create_array`` with a ``data`` parameter. """ data = np.arange(10) name = "foo" arr: AnyAsyncArray | AnyArray if impl == "sync": arr = sync_api.create_array(store, name=name, data=data) stored = arr[:] elif impl == "async": arr = await create_array(store, name=name, data=data, zarr_format=3) stored = await arr._get_selection( BasicIndexer(..., shape=arr.shape, chunk_grid=arr._chunk_grid), prototype=default_buffer_prototype(), ) else: raise ValueError(f"Invalid impl: {impl}") assert np.array_equal(stored, data) @staticmethod async def test_with_data_invalid_params(store: Store) -> None: """ Test that failing to specify data AND shape / dtype results in a ValueError """ with pytest.raises(ValueError, match="shape was not specified"): await create_array(store, data=None, shape=None, dtype=None) # we catch shape=None first, so specifying a dtype should raise the same exception as before with pytest.raises(ValueError, match="shape was not specified"): await create_array(store, data=None, shape=None, dtype="uint8") with pytest.raises(ValueError, match="dtype was not specified"): await create_array(store, data=None, shape=(10, 10)) @staticmethod async def test_data_ignored_params(store: Store) -> None: """ Test that specifying data AND shape AND dtype results in a ValueError """ data = np.arange(10) with pytest.raises( ValueError, match="The data parameter was used, but the shape parameter was also used." ): await create_array(store, data=data, shape=data.shape, dtype=None, overwrite=True) # we catch shape first, so specifying a dtype should raise the same warning as before with pytest.raises( ValueError, match="The data parameter was used, but the shape parameter was also used." ): await create_array(store, data=data, shape=data.shape, dtype=data.dtype, overwrite=True) with pytest.raises( ValueError, match="The data parameter was used, but the dtype parameter was also used." ): await create_array(store, data=data, shape=None, dtype=data.dtype, overwrite=True) @staticmethod @pytest.mark.parametrize("write_empty_chunks", [True, False]) async def test_write_empty_chunks_config(write_empty_chunks: bool, store: Store) -> None: """ Test that the value of write_empty_chunks is sensitive to the global config when not set explicitly """ with zarr.config.set({"array.write_empty_chunks": write_empty_chunks}): arr = await create_array(store, shape=(2, 2), dtype="i4") assert arr.config.write_empty_chunks == write_empty_chunks @staticmethod @pytest.mark.parametrize("path", [None, "", "/", "/foo", "foo", "foo/bar"]) async def test_name(store: Store, zarr_format: ZarrFormat, path: str | None) -> None: arr = await create_array( store, shape=(2, 2), dtype="i4", name=path, zarr_format=zarr_format ) if path is None: expected_path = "" elif path.startswith("/"): expected_path = path.lstrip("/") else: expected_path = path assert arr.path == expected_path assert arr.name == f"/{expected_path}" # test that implicit groups were created path_parts = expected_path.split("/") if len(path_parts) > 1: *parents, _ = ["", *accumulate(path_parts, lambda x, y: "/".join([x, y]))] # noqa: FLY002 for parent_path in parents: # this will raise if these groups were not created _ = await zarr.api.asynchronous.open_group( store=store, path=parent_path, zarr_format=zarr_format ) @staticmethod @pytest.mark.parametrize("endianness", ENDIANNESS_STR) def test_default_endianness( store: Store, zarr_format: ZarrFormat, endianness: EndiannessStr ) -> None: """ Test that endianness is correctly set when creating an array when not specifying a serializer. """ dtype = Int16(endianness=endianness) arr = zarr.create_array(store=store, shape=(1,), dtype=dtype, zarr_format=zarr_format) byte_order: str = arr[:].dtype.byteorder # type: ignore[union-attr] assert byte_order in NUMPY_ENDIANNESS_STR assert endianness_from_numpy_str(byte_order) == endianness # type: ignore[arg-type] # The explicit id for b"a" avoids colliding with the auto-generated id for "a". @pytest.mark.parametrize("value", [1, 1.4, "a", pytest.param(b"a", id="a-bytes"), np.array(1)]) @pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_scalar_array(value: Any, zarr_format: ZarrFormat) -> None: arr = zarr.array(value, zarr_format=zarr_format) assert arr[...] == value assert arr.shape == () assert arr.ndim == 0 assert isinstance(arr[()], NDArrayLikeOrScalar) @pytest.mark.parametrize("store", ["local"], indirect=True) @pytest.mark.parametrize("store2", ["local"], indirect=["store2"]) @pytest.mark.parametrize("src_format", [2, 3]) @pytest.mark.parametrize("new_format", [2, 3, None]) async def test_creation_from_other_zarr_format( store: Store, store2: Store, src_format: ZarrFormat, new_format: ZarrFormat | None, ) -> None: if src_format == 2: src = zarr.create( (50, 50), chunks=(10, 10), store=store, zarr_format=src_format, dimension_separator="/" ) else: src = zarr.create( (50, 50), chunks=(10, 10), store=store, zarr_format=src_format, chunk_key_encoding=("default", "."), ) src[:] = np.arange(50 * 50).reshape((50, 50)) result = zarr.from_array( store=store2, data=src, zarr_format=new_format, ) np.testing.assert_array_equal(result[:], src[:]) assert result.fill_value == src.fill_value assert result.dtype == src.dtype assert result.chunks == src.chunks expected_format = src_format if new_format is None else new_format assert result.metadata.zarr_format == expected_format if src_format == new_format: assert result.metadata == src.metadata result2 = zarr.array( data=src, store=store2, overwrite=True, zarr_format=new_format, ) np.testing.assert_array_equal(result2[:], src[:]) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=True) @pytest.mark.parametrize("store2", ["local", "memory", "zip"], indirect=["store2"]) @pytest.mark.parametrize("src_chunks", [(40, 10), (11, 50)]) @pytest.mark.parametrize( "new_chunks", [(40, 10), (11, 50), [40, 10], np.array([11, 50]), (np.int64(40), np.int64(10))] ) @pytest.mark.parametrize( "new_shards", [None, (440, 100), [440, 100], np.array([440, 100]), (np.int64(440), np.int64(100))], ids=["none", "tuple", "list", "array", "numpy-scalars"], ) @pytest.mark.parametrize("source_as_numpy", [False, True], ids=["zarr", "numpy"]) async def test_from_array( store: Store, store2: Store, src_chunks: tuple[int, int], new_chunks: ChunksLike, new_shards: ShardsLike | None, source_as_numpy: bool, zarr_format: ZarrFormat, ) -> None: if zarr_format == 2 and new_shards is not None: pytest.skip("Zarr format 2 does not support sharding") src_fill_value = 2 src_dtype = np.dtype("uint8") src_attributes = None src = zarr.create( (100, 10), chunks=src_chunks, dtype=src_dtype, store=store, fill_value=src_fill_value, attributes=src_attributes, zarr_format=zarr_format, ) src[:] = np.arange(1000).reshape((100, 10)) new_fill_value = 3 new_attributes: dict[str, JSON] = {"foo": "bar"} result = zarr.from_array( data=np.asarray(src) if source_as_numpy else src, store=store2, chunks=new_chunks, shards=new_shards, fill_value=new_fill_value, attributes=new_attributes, zarr_format=zarr_format, ) np.testing.assert_array_equal(result[:], src[:]) assert result.fill_value == new_fill_value assert result.dtype == src_dtype assert result.attrs == new_attributes np.testing.assert_array_equal(result.chunks, new_chunks) np.testing.assert_array_equal(result.shards, new_shards) @pytest.mark.parametrize("zdtype", zdtype_examples, ids=str) @pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_from_array_preserves_dtype(zdtype: ZDType[Any, Any], zarr_format: ZarrFormat) -> None: source = zarr.create_array({}, shape=(4,), chunks=(2,), dtype=zdtype, zarr_format=zarr_format) expected = source[:] source[:] = expected result = zarr.from_array({}, data=source) assert result.dtype == source.dtype assert result._async_array._zdtype == source._async_array._zdtype np.testing.assert_array_equal(result[:], expected) reopened = zarr.open_array(result.store, mode="r") assert reopened._async_array._zdtype == source._async_array._zdtype np.testing.assert_array_equal(reopened[:], expected) @pytest.mark.parametrize("store", ["local"], indirect=True) @pytest.mark.parametrize("chunks", ["keep", "auto"]) @pytest.mark.parametrize("write_data", [True, False]) @pytest.mark.parametrize( "src", [ np.arange(1000).reshape(10, 10, 10), zarr.ones((10, 10, 10)), 5, [1, 2, 3], [[1, 2, 3], [4, 5, 6]], ], ) # add other npt.ArrayLike? async def test_from_array_arraylike( store: Store, chunks: Literal["auto", "keep"] | tuple[int, int], write_data: bool, src: AnyArray | npt.ArrayLike, ) -> None: fill_value = 42 result = zarr.from_array( store, data=src, chunks=chunks, write_data=write_data, fill_value=fill_value ) if write_data: np.testing.assert_array_equal(result[...], np.array(src)) else: np.testing.assert_array_equal(result[...], np.full_like(src, fill_value)) @pytest.mark.parametrize("store", ["local", "memory"], indirect=True) def test_from_array_keeps_fill_value_and_attributes(store: Store, zarr_format: ZarrFormat) -> None: """`from_array` defaults to the fill value and attributes of the source array.""" attributes: dict[str, JSON] = {"units": "K", "nested": {"x": [1]}, "tags": ["a"]} src = zarr.create_array( store, name="src", shape=(4,), dtype="int32", fill_value=42, attributes=attributes, zarr_format=zarr_format, ) src[:] = np.arange(4, dtype="int32") result = zarr.from_array({}, data=src) assert result.fill_value == 42 assert dict(result.attrs) == attributes # The copied attributes must not alias the source's nested containers. nested = result.attrs["nested"] assert isinstance(nested, dict) nested_x = nested["x"] assert isinstance(nested_x, list) nested_x.append(99) tags = result.attrs["tags"] assert isinstance(tags, list) tags.append("b") assert src.attrs["nested"] == {"x": [1]} assert src.attrs["tags"] == ["a"] # A metadata-only copy must read back the source's fill value, not the dtype default. meta_only = zarr.from_array({}, data=src, write_data=False) np.testing.assert_array_equal(meta_only[:], np.full((4,), 42, dtype="int32")) @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_from_array_explicit_fill_value_and_attributes_override( store: Store, zarr_format: ZarrFormat ) -> None: """Explicit `fill_value` / `attributes` arguments take precedence over the source. An explicit ``fill_value=None`` selects the dtype's default scalar for Zarr format 3 and a null fill value for Zarr format 2, matching `create_array`, rather than being treated as "keep the source's fill value". An empty ``attributes`` dict is likewise honoured, so it is possible to drop the source's attributes. """ src = zarr.create_array( store, name="src", shape=(4,), dtype="int32", fill_value=42, attributes={"units": "K"}, zarr_format=zarr_format, ) assert zarr.from_array({}, data=src, fill_value=7).fill_value == 7 assert dict(zarr.from_array({}, data=src, attributes={}).attrs) == {} assert dict(zarr.from_array({}, data=src, attributes={"a": 1}).attrs) == {"a": 1} explicit_none = zarr.from_array({}, data=src, fill_value=None) if zarr_format == 2: assert explicit_none.fill_value is None else: assert explicit_none.fill_value == 0 # int32 default scalar def test_from_array_arraylike_gains_no_attributes() -> None: """A non-Array source has no attributes or fill value to keep.""" result = zarr.from_array({}, data=np.arange(4, dtype="int32")) assert dict(result.attrs) == {} assert result.fill_value == 0 @pytest.mark.parametrize("store", ["local", "memory"], indirect=True) @pytest.mark.parametrize( ("src_name", "dest_name"), [("a", "a"), ("g/a", "g"), ("g/a", ""), ("a", "a/c")], ) def test_from_array_overwrite_overlapping_source_raises( store: Store, src_name: str, dest_name: str ) -> None: """Overwriting a destination that overlaps the source array raises instead of deleting the source before its data is copied.""" src = zarr.create_array(store, name=src_name, data=np.arange(4.0), fill_value=-1.0) with pytest.raises(ValueError, match="paths overlap"): zarr.from_array(store, name=dest_name, data=src, overwrite=True) np.testing.assert_array_equal(src[...], np.arange(4.0)) @pytest.mark.parametrize("store", ["local", "memory"], indirect=True) def test_from_array_overwrite_read_only_source_raises(store: Store) -> None: """A read-only view of the destination store still counts as overlapping.""" zarr.create_array(store, name="a", data=np.arange(4.0), fill_value=-1.0) src = zarr.open_array(store.with_read_only(True), path="a") with pytest.raises(ValueError, match="paths overlap"): zarr.from_array(store, name="a", data=src, overwrite=True) np.testing.assert_array_equal(src[...], np.arange(4.0)) def test_from_array_overwrite_equal_memory_stores() -> None: """Distinct MemoryStores with equal contents don't overlap.""" src_store, dest_store = MemoryStore(), MemoryStore() src = zarr.create_array(src_store, name="a", data=np.arange(4.0), fill_value=-1.0) zarr.create_array(dest_store, name="a", data=np.arange(4.0), fill_value=-1.0) result = zarr.from_array(dest_store, name="a", data=src, overwrite=True) np.testing.assert_array_equal(result[...], np.arange(4.0)) @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_from_array_overwrite_non_overlapping_source(store: Store) -> None: """Overlap checks respect path boundaries, and metadata-only copies are allowed.""" src = zarr.create_array( store, name="ab", data=np.arange(4.0), chunks=(2,), fill_value=-1.0, attributes={"units": "K"}, ) zarr.create_array(store, name="a", shape=(2,), dtype="int8") result = zarr.from_array(store, name="a", data=src, overwrite=True) np.testing.assert_array_equal(result[...], np.arange(4.0)) meta_only = zarr.from_array(store, name="ab", data=src, overwrite=True, write_data=False) np.testing.assert_array_equal(meta_only[...], np.full(4, -1.0)) assert zarr.open_array(store, path="ab").metadata == src.metadata def test_from_array_F_order() -> None: arr = zarr.create_array(store={}, data=np.array([1]), order="F", zarr_format=2) with pytest.warns( ZarrUserWarning, match="The existing order='F' of the source Zarr format 2 array will be ignored.", ): zarr.from_array(store={}, data=arr, zarr_format=3) async def test_orthogonal_set_total_slice() -> None: """Ensure that a whole chunk overwrite does not read chunks""" store = MemoryStore() array = zarr.create_array(store, shape=(20, 20), chunks=(1, 2), dtype=int, fill_value=-1) with mock.patch("zarr.storage.MemoryStore.get", side_effect=RuntimeError): array[0, slice(4, 10)] = np.arange(6) array = zarr.create_array( store, shape=(20, 21), chunks=(1, 2), dtype=int, fill_value=-1, overwrite=True ) with mock.patch("zarr.storage.MemoryStore.get", side_effect=RuntimeError): array[0, :] = np.arange(21) with mock.patch("zarr.storage.MemoryStore.get", side_effect=RuntimeError): array[:] = 1 @pytest.mark.skipif( Version(numcodecs.__version__) < Version("0.15.1"), reason="codec configuration is overwritten on older versions. GH2800", ) def test_roundtrip_numcodecs() -> None: store = MemoryStore() compressors = [ {"name": "numcodecs.shuffle", "configuration": {"elementsize": 2}}, {"name": "numcodecs.zlib", "configuration": {"level": 4}}, ] filters: list[CodecJSON_V3] = [ { "name": "numcodecs.fixedscaleoffset", "configuration": { "scale": 100.0, "offset": 0.0, "dtype": " Any: return arr[index] @pytest.mark.parametrize( "method", [ pytest.param( "fork", marks=[ pytest.mark.skipif( sys.platform in ("win32", "darwin"), reason="fork not supported on Windows or OSX", ), # Python 3.15 deprecates fork() in multi-threaded processes, and zarr's # sync event-loop thread is always running here. Fork-safety despite # those threads is exactly what this test pins down, so keep running it. pytest.mark.filterwarnings( r"ignore:This process \(pid=\d+\) is multi-threaded, use of fork\(\):DeprecationWarning" ), ], ), "spawn", pytest.param( "forkserver", marks=pytest.mark.skipif( sys.platform == "win32", reason="forkserver not supported on Windows" ), ), ], ) @pytest.mark.parametrize("store", ["local"], indirect=True) @pytest.mark.parametrize("shards", [None, (20,)]) def test_multiprocessing( store: Store, method: Literal["fork", "spawn", "forkserver"], shards: tuple[int, ...] | None ) -> None: """ Test that arrays can be pickled and indexed in child processes """ data = np.arange(100) chunks: Literal["auto"] | tuple[int, ...] if shards is None: chunks = "auto" else: chunks = (1,) arr = zarr.create_array(store=store, data=data, shards=shards, chunks=chunks) ctx = mp.get_context(method) with ctx.Pool() as pool: results = pool.starmap(_index_array, [(arr, slice(len(data)))]) assert all(np.array_equal(r, data) for r in results) @pytest.mark.skipif( sys.platform in ("win32", "darwin"), reason="fork not supported on Windows or OSX", ) @pytest.mark.filterwarnings( r"ignore:This process \(pid=\d+\) is multi-threaded, use of fork\(\):DeprecationWarning" ) @pytest.mark.parametrize("store", ["local"], indirect=True) def test_multiprocessing_fork_codec_pipeline_pool(store: Store) -> None: """A forked child must not reuse the parent's codec-pipeline thread pool. Regression test for https://github.com/zarr-developers/zarr-python/issues/4478: submitting to the inherited pool hung forever because its threads did not survive the fork. The child must read FEWER chunks than the pool's inherited idle permits: once submissions exceed the stale permits, a real thread is spawned and the queued work drains normally. Single-chunk batches bypass the pool entirely. """ data = np.arange(100).reshape(10, 10) with zarr.config.set({"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"}): arr = zarr.create_array(store=store, shape=data.shape, chunks=(5, 5), dtype=data.dtype) arr[:] = data # Two multi-chunk reads: spawn 4 pool threads and leave all of their # idle-semaphore permits released before the fork. arr[:] arr[:] ctx = mp.get_context("fork") with ctx.Pool() as pool: # [:5] spans 2 chunks — below the ~4 inherited idle permits, so on # unfixed code both submissions are swallowed by the dead pool. result = pool.starmap_async(_index_array, [(arr, np.s_[:5])]) (read_back,) = result.get(timeout=30) assert np.array_equal(read_back, data[:5]) def test_create_array_method_signature() -> None: """ Test that the signature of the ``AsyncGroup.create_array`` function has nearly the same signature as the ``create_array`` function. ``AsyncGroup.create_array`` should take all of the same keyword arguments as ``create_array`` except ``store``. """ base_sig = inspect.signature(create_array) meth_sig = inspect.signature(AsyncGroup.create_array) # ignore keyword arguments that are either missing or have different semantics when # create_array is invoked as a group method ignore_kwargs = {"zarr_format", "store", "name"} # TODO: make this test stronger. right now, it only checks that all the parameters in the # function signature are used in the method signature. we can be more strict and check that # the method signature uses no extra parameters. base_params = dict(filter(lambda kv: kv[0] not in ignore_kwargs, base_sig.parameters.items())) assert (set(base_params.items()) - set(meth_sig.parameters.items())) == set() async def test_sharding_coordinate_selection() -> None: store = MemoryStore() g = zarr.open_group(store, mode="w") arr = g.create_array( name="a", shape=(2, 3, 4), chunks=(1, 2, 2), overwrite=True, dtype=np.float32, shards=(2, 4, 4), ) arr[:] = np.arange(2 * 3 * 4).reshape((2, 3, 4)) result = arr[1, [0, 1]] # type: ignore[index] assert isinstance(result, NDArrayLike) assert (result == np.array([[12, 13, 14, 15], [16, 17, 18, 19]])).all() @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) def test_array_repr(store: Store) -> None: shape = (2, 3, 4) dtype = "uint8" arr = zarr.create_array(store, shape=shape, dtype=dtype) assert str(arr) == f"" class UnknownObjectCodecDtype(VariableLengthUTF8): """A data type that requires an object codec with an unknown id, used for error-path tests.""" object_codec_id = "unknown" # type: ignore[assignment] @pytest.mark.parametrize( "dtype", [VariableLengthUTF8(), VariableLengthBytes(), UnknownObjectCodecDtype()] ) def test_chunk_encoding_no_object_codec_errors(dtype: ZDType[Any, Any]) -> None: """ Test that a valuerror is raised when checking the chunk encoding for a v2 array with a data type that requires an object codec, but where no object codec is specified """ if isinstance(dtype, VariableLengthUTF8): codec_name = "the numcodecs.VLenUTF8 codec" elif isinstance(dtype, VariableLengthBytes): codec_name = "the numcodecs.VLenBytes codec" else: codec_name = f"an unknown object codec with id {dtype.object_codec_id!r}" # type: ignore[attr-defined] msg = ( f"Data type {dtype} requires {codec_name}, " "but no such codec was specified in the filters or compressor parameters for " "this array. " ) with pytest.raises(ValueError, match=re.escape(msg)): _parse_chunk_encoding_v2(filters=None, compressor=None, dtype=dtype) def test_unknown_object_codec_default_serializer_v3() -> None: """ Test that we get a valueerrror when trying to create the default serializer for a data type that requires an unknown object codec """ dtype = UnknownObjectCodecDtype() msg = f"Data type {dtype} requires an unknown object codec: {dtype.object_codec_id!r}." with pytest.raises(ValueError, match=re.escape(msg)): default_serializer_v3(dtype) def test_unknown_object_codec_default_filters_v2() -> None: """ Test that we get a valueerrror when trying to create the default serializer for a data type that requires an unknown object codec """ dtype = UnknownObjectCodecDtype() msg = f"Data type {dtype} requires an unknown object codec: {dtype.object_codec_id!r}." with pytest.raises(ValueError, match=re.escape(msg)): default_filters_v2(dtype) @pytest.mark.parametrize( ("array_shape", "shard_shape", "chunk_shape"), [ ((10,), None, (1,)), ((10,), (1,), (1,)), ((30, 10), None, (2, 5)), ((30, 10), (4, 10), (2, 5)), ], ) def test_chunk_grid_shape( array_shape: tuple[int, ...], shard_shape: tuple[int, ...] | None, chunk_shape: tuple[int, ...], zarr_format: ZarrFormat, ) -> None: """ Test that the shape of the chunk grid and the shard grid are correctly indicated """ if zarr_format == 2 and shard_shape is not None: with pytest.raises( ValueError, match="Zarr format 2 arrays can only be created with `shard_shape` set to `None`.", ): arr = zarr.create_array( {}, dtype="uint8", shape=array_shape, chunks=chunk_shape, shards=shard_shape, zarr_format=zarr_format, ) pytest.skip("Zarr format 2 arrays can only be created with `shard_shape` set to `None`.") else: arr = zarr.create_array( {}, dtype="uint8", shape=array_shape, chunks=chunk_shape, shards=shard_shape, zarr_format=zarr_format, ) chunk_grid_shape = tuple(starmap(ceildiv, zip(array_shape, chunk_shape, strict=True))) if shard_shape is None: _shard_shape = chunk_shape else: _shard_shape = shard_shape shard_grid_shape = tuple(starmap(ceildiv, zip(array_shape, _shard_shape, strict=True))) assert arr._chunk_grid_shape == chunk_grid_shape assert arr.cdata_shape == chunk_grid_shape assert arr.async_array.cdata_shape == chunk_grid_shape assert arr._shard_grid_shape == shard_grid_shape assert arr._nshards == np.prod(shard_grid_shape) @pytest.mark.parametrize( ("array_shape", "shard_shape", "chunk_shape"), [((10,), None, (1,)), ((30, 10), None, (2, 5))] ) def test_iter_chunk_coords( array_shape: tuple[int, ...], shard_shape: tuple[int, ...] | None, chunk_shape: tuple[int, ...], zarr_format: ZarrFormat, ) -> None: """ Test that we can use the various invocations of iter_chunk_coords to iterate over the coordinates of the origin of each chunk. """ arr = zarr.create_array( {}, dtype="uint8", shape=array_shape, chunks=chunk_shape, shards=shard_shape, zarr_format=zarr_format, ) expected = tuple(_iter_grid(arr._shard_grid_shape)) observed = tuple(_iter_chunk_coords(arr)) assert observed == expected assert observed == tuple(arr._iter_chunk_coords()) assert observed == tuple(arr.async_array._iter_chunk_coords()) @pytest.mark.parametrize( ("array_shape", "shard_shape", "chunk_shape"), [((10,), (1,), (1,)), ((10,), None, (1,)), ((30, 10), (10, 5), (2, 5))], ) def test_iter_shard_coords( array_shape: tuple[int, ...], shard_shape: tuple[int, ...] | None, chunk_shape: tuple[int, ...], zarr_format: ZarrFormat, ) -> None: """ Test that we can use the various invocations of iter_shard_coords to iterate over the coordinates of the origin of each shard. """ if zarr_format == 2 and shard_shape is not None: pytest.skip("Zarr format 2 does not support shard shape.") arr = zarr.create_array( {}, dtype="uint8", shape=array_shape, chunks=chunk_shape, shards=shard_shape, zarr_format=zarr_format, ) expected = tuple(_iter_grid(arr._shard_grid_shape)) observed = tuple(_iter_shard_coords(arr)) assert observed == expected assert observed == tuple(arr._iter_shard_coords()) assert observed == tuple(arr.async_array._iter_shard_coords()) @pytest.mark.parametrize( ("array_shape", "shard_shape", "chunk_shape"), [((10,), (1,), (1,)), ((10,), None, (1,)), ((30, 10), (10, 5), (2, 5))], ) def test_iter_shard_keys( array_shape: tuple[int, ...], shard_shape: tuple[int, ...] | None, chunk_shape: tuple[int, ...], zarr_format: ZarrFormat, ) -> None: """ Test that we can use the various invocations of iter_shard_keys to iterate over the stored keys of the shards of an array. """ if zarr_format == 2 and shard_shape is not None: pytest.skip("Zarr format 2 does not support shard shape.") arr = zarr.create_array( {}, dtype="uint8", shape=array_shape, chunks=chunk_shape, shards=shard_shape, zarr_format=zarr_format, ) expected = tuple( arr.metadata.encode_chunk_key(key) for key in _iter_grid(arr._shard_grid_shape) ) observed = tuple(_iter_shard_keys(arr)) assert observed == expected assert observed == tuple(arr._iter_shard_keys()) assert observed == tuple(arr.async_array._iter_shard_keys()) @pytest.mark.parametrize( ("array_shape", "shard_shape", "chunk_shape"), [((10,), None, (1,)), ((10,), (1,), (1,)), ((30, 10), (10, 5), (2, 5))], ) def test_iter_shard_regions( array_shape: tuple[int, ...], shard_shape: tuple[int, ...] | None, chunk_shape: tuple[int, ...], zarr_format: ZarrFormat, ) -> None: """ Test that we can use the various invocations of iter_shard_regions to iterate over the regions spanned by the shards of an array. """ if zarr_format == 2 and shard_shape is not None: pytest.skip("Zarr format 2 does not support shard shape.") arr = zarr.create_array( {}, dtype="uint8", shape=array_shape, chunks=chunk_shape, shards=shard_shape, zarr_format=zarr_format, ) if shard_shape is None: _shard_shape = chunk_shape else: _shard_shape = shard_shape expected = tuple(_iter_regions(arr.shape, _shard_shape)) observed = tuple(_iter_shard_regions(arr)) assert observed == expected assert observed == tuple(arr._iter_shard_regions()) assert observed == tuple(arr.async_array._iter_shard_regions()) @pytest.mark.parametrize( ("array_shape", "shard_shape", "chunk_shape"), [((10,), None, (1,)), ((30, 10), None, (2, 5))] ) def test_iter_chunk_regions( array_shape: tuple[int, ...], shard_shape: tuple[int, ...] | None, chunk_shape: tuple[int, ...], zarr_format: ZarrFormat, ) -> None: """ Test that we can use the various invocations of iter_chunk_regions to iterate over the regions spanned by the chunks of an array. """ arr = zarr.create_array( {}, dtype="uint8", shape=array_shape, chunks=chunk_shape, shards=shard_shape, zarr_format=zarr_format, ) expected = tuple(_iter_regions(arr.shape, chunk_shape)) observed = tuple(_iter_chunk_regions(arr)) assert observed == expected assert observed == tuple(arr._iter_chunk_regions()) assert observed == tuple(arr.async_array._iter_chunk_regions()) @pytest.mark.parametrize("num_shards", [1, 3]) @pytest.mark.parametrize("array_type", ["numpy", "zarr"]) def test_create_array_with_data_num_gets( num_shards: int, array_type: Literal["numpy", "zarr"] ) -> None: """ Test that creating an array with data only invokes a single get request per stored object """ store = LoggingStore(store=MemoryStore()) chunk_shape = (1,) shard_shape = (100,) shape = (shard_shape[0] * num_shards,) data: AnyArray | npt.NDArray[np.int64] if array_type == "numpy": data = np.zeros(shape[0], dtype="int64") else: data = zarr.zeros(shape, dtype="int64") zarr.create_array(store, data=data, chunks=chunk_shape, shards=shard_shape, fill_value=-1) # type: ignore[arg-type] # One get for the metadata; full-shard writes should not read shard payloads. assert store.counter["get"] == 1 @pytest.mark.parametrize( ("selection", "expected_gets"), [(slice(None), 0), (slice(1, 9), 1)], ) def test_shard_write_num_gets(selection: slice, expected_gets: int) -> None: """ Test that partial-shard writes read the existing data and full-shard writes don't. """ store = LoggingStore(store=MemoryStore()) arr = zarr.create_array( store, shape=(10,), chunks=(1,), shards=(10,), dtype="int64", fill_value=-1, ) arr[:] = 0 store.counter.clear() arr[selection] = 1 assert store.counter["get"] == expected_gets @pytest.mark.parametrize("config", [{}, {"write_empty_chunks": True}, {"order": "C"}]) def test_with_config(config: ArrayConfigParams) -> None: """ Test that `AsyncArray.with_config` and `Array.with_config` create a copy of the source array with a new runtime configuration. """ # the config we start with source_config: ArrayConfigParams = {"write_empty_chunks": False, "order": "F"} source_array = zarr.create_array({}, shape=(1,), dtype="uint8", config=source_config) new_async_array_config_dict = source_array._async_array.with_config(config).config.to_dict() new_array_config_dict = source_array.with_config(config).config.to_dict() for key in source_config: if key in config: assert new_async_array_config_dict[key] == config[key] # type: ignore[literal-required] assert new_array_config_dict[key] == config[key] # type: ignore[literal-required] else: assert new_async_array_config_dict[key] == source_config[key] # type: ignore[literal-required] assert new_array_config_dict[key] == source_config[key] # type: ignore[literal-required] def test_with_config_polymorphism() -> None: """ Test that `AsyncArray.with_config` and `Array.with_config` accept dicts and full array config objects. """ source_config: ArrayConfig = ArrayConfig.from_dict({"write_empty_chunks": False, "order": "F"}) source_config_dict = source_config.to_dict() arr = zarr.create_array({}, shape=(1,), dtype="uint8") arr_source_config = arr.with_config(source_config) arr_source_config_dict = arr.with_config(source_config_dict) assert arr_source_config.config == arr_source_config_dict.config @pytest.mark.parametrize( ("chunk_input", "expected"), [ (-1, ((10,),)), ((-1,), ((10,),)), ((10,), ((10,),)), ((5,), ((5, 5),)), ((3,), ((3, 3, 3, 1),)), ], ids=["scalar-neg1", "tuple-neg1", "exact", "half", "remainder"], ) async def test_create_array_chunks_1d( chunk_input: int | tuple[int, ...], expected: tuple[tuple[int, ...], ...], ) -> None: """Test that chunk normalization produces the expected chunk sizes for 1D arrays.""" arr = await create_array(store={}, shape=(10,), chunks=chunk_input, dtype="uint8") assert arr.write_chunk_sizes == expected @pytest.mark.parametrize( ("chunk_input", "expected"), [ (-1, ((10,), (12,), (15,))), ((3, 4, 5), ((3, 3, 3, 1), (4, 4, 4), (5, 5, 5))), ((-1, 4, -1), ((10,), (4, 4, 4), (15,))), ((10, 12, 15), ((10,), (12,), (15,))), ((7, 3, 2), ((7, 3), (3, 3, 3, 3), (2, 2, 2, 2, 2, 2, 2, 1))), ], ids=["all-neg1", "mixed", "neg1-middle", "exact", "remainder"], ) async def test_create_array_chunks_3d( chunk_input: int | tuple[int, ...], expected: tuple[tuple[int, ...], ...], ) -> None: """Test that chunk normalization produces the expected chunk sizes for 3D arrays.""" shape = (10, 12, 15) arr = await create_array(store={}, shape=shape, chunks=chunk_input, dtype="float64") assert arr.write_chunk_sizes == expected async def test_create_array_huge_chunk_count() -> None: """Array creation must be O(1) in the number of chunks per dimension. With `shape=(2**62,)` and `chunks=(1,)` this dimension has 2**62 chunks; materializing one entry per chunk would raise ("array is too big"). Companion to the indexing-time fix from gh-4174. """ arr = await create_array(store={}, shape=(2**62,), chunks=(1,), dtype="int32") assert arr.shape == (2**62,) assert arr.chunks == (1,) zarr-developers-zarr-python-a2f2415/tests/test_array_spec.py000066400000000000000000000171541527014137600244000ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING, Any import numpy as np import pytest import zarr from zarr.core.array_spec import ArrayConfig, ArrayConfigParams, ArraySpec from zarr.core.buffer import BufferPrototype, default_buffer_prototype from zarr.core.buffer.cpu import NDBuffer from zarr.core.dtype import get_data_type_from_native_dtype if TYPE_CHECKING: from collections.abc import Callable from zarr.core.common import MemoryOrder def _make_spec( *, shape: tuple[int, ...] = (4, 4), native_dtype: Any = "int16", fill_value: Any = 0, order: MemoryOrder = "C", write_empty_chunks: bool = False, prototype: BufferPrototype | None = None, ) -> ArraySpec: """Creates an ArraySpec with common defaults""" zdtype = get_data_type_from_native_dtype(np.dtype(native_dtype)) fill_value = zdtype.cast_scalar(fill_value) # mirrors ArrayV3Metadata's fill_value return ArraySpec( shape=shape, dtype=zdtype, fill_value=fill_value, config=ArrayConfig(order=order, write_empty_chunks=write_empty_chunks), prototype=prototype if prototype is not None else default_buffer_prototype(), ) class _AltNDBuffer(NDBuffer): """A distinct NDBuffer subclass""" _ALT_PROTOTYPE = BufferPrototype( buffer=default_buffer_prototype().buffer, nd_buffer=_AltNDBuffer, ) # a distinct BufferPrototype with a different nd_buffer subclass # Difficult / important cases: # issue #3054: np.void is unhashable when writeable # nan/NaT aren't self-equal yet must compare equal for a ArraySpec SPECS = [ pytest.param({"native_dtype": "int16", "fill_value": 7}, id="int16"), pytest.param({"native_dtype": "float64", "fill_value": 1.5}, id="float64"), pytest.param({"native_dtype": "float64", "fill_value": float("nan")}, id="float64-nan"), pytest.param({"native_dtype": "float64", "fill_value": -0.0}, id="float64-negzero"), pytest.param({"native_dtype": "complex128", "fill_value": 1 + 2j}, id="complex128"), pytest.param( {"native_dtype": "complex128", "fill_value": complex(-0.0, -0.0)}, id="complex128-negzero", ), pytest.param({"native_dtype": "bool", "fill_value": True}, id="bool"), pytest.param( {"native_dtype": "datetime64[s]", "fill_value": np.datetime64("2020-01-01")}, id="datetime64", ), pytest.param( {"native_dtype": "datetime64[s]", "fill_value": np.datetime64("NaT", "s")}, id="datetime64-NaT", ), pytest.param( {"native_dtype": [("a", "f8"), ("b", "i8")], "fill_value": (1.0, 2)}, id="structured-void", ), pytest.param({"native_dtype": "U5", "fill_value": "hello"}, id="fixed-string"), pytest.param({"shape": ()}, id="scalar-shape"), pytest.param({"shape": (0,)}, id="zero-size"), pytest.param({"order": "F"}, id="order-F"), ] # Mutations: each mutate kwargs to an unequal version def _grow_shape(kw: dict[str, Any]) -> dict[str, Any]: return {"shape": (*kw.get("shape", (4, 4)), 1)} def _flip_order(kw: dict[str, Any]) -> dict[str, Any]: return {"order": "F" if kw.get("order", "C") == "C" else "C"} def _swap_prototype(_kw: dict[str, Any]) -> dict[str, Any]: return {"prototype": _ALT_PROTOTYPE} MUTATIONS = [ pytest.param(_grow_shape, id="shape"), pytest.param(_flip_order, id="order"), pytest.param(_swap_prototype, id="prototype"), ] @pytest.mark.parametrize("kwargs", SPECS) def test_hashable(kwargs: dict[str, Any]) -> None: """Every ArraySpec is hashable, including structured (np.void) fill values.""" assert isinstance(hash(_make_spec(**kwargs)), int) @pytest.mark.parametrize("kwargs", SPECS) def test_equal_specs_hash_equal(kwargs: dict[str, Any]) -> None: """Independently built specs with identical fields are equal and hash equal.""" a = _make_spec(**kwargs) b = _make_spec(**kwargs) assert a == b assert hash(a) == hash(b) @pytest.mark.parametrize("kwargs", SPECS) @pytest.mark.parametrize("mutate", MUTATIONS) def test_distinct_specs_unequal( mutate: Callable[[dict[str, Any]], dict[str, Any]], kwargs: dict[str, Any], ) -> None: """Changing one dtype-independent field makes a spec unequal to its base.""" base = _make_spec(**kwargs) variant = _make_spec(**{**kwargs, **mutate(kwargs)}) assert base != variant @pytest.mark.parametrize( ("base", "variant"), [ pytest.param({"fill_value": 0}, {"fill_value": 1}, id="fill_value"), pytest.param({"native_dtype": "int16"}, {"native_dtype": "int32"}, id="dtype"), pytest.param( {"native_dtype": "float32", "fill_value": 1.0}, {"native_dtype": "float64", "fill_value": 1.0}, id="dtype-float-promote", ), ], ) def test_dtype_and_fill_value_matter(base: dict[str, Any], variant: dict[str, Any]) -> None: """dtype and fill_value participate in equality; they can't join the cross product because fill_value is coupled to dtype.""" assert _make_spec(**base) != _make_spec(**variant) @pytest.mark.parametrize( ("native_dtype", "neg_fill", "pos_fill"), [ pytest.param("float16", -0.0, 0.0, id="float16"), pytest.param("float32", -0.0, 0.0, id="float32"), pytest.param("float64", -0.0, 0.0, id="float64"), pytest.param("complex128", complex(-0.0, -0.0), 0j, id="complex128-both"), pytest.param("complex128", complex(0.0, -0.0), 0j, id="complex128-imag"), pytest.param("complex128", complex(-0.0, 0.0), 0j, id="complex128-real"), pytest.param([("a", "f8")], (-0.0,), (0.0,), id="structured"), ], ) def test_signed_zero_fills_are_distinct(native_dtype: Any, neg_fill: Any, pos_fill: Any) -> None: """A -0.0 fill writes different bytes than +0.0, so the specs are not equal.""" neg = _make_spec(native_dtype=native_dtype, fill_value=neg_fill) pos = _make_spec(native_dtype=native_dtype, fill_value=pos_fill) assert neg != pos @pytest.mark.parametrize( ("obj"), [ pytest.param(None, id="None"), pytest.param(42, id="int"), pytest.param("hello", id="str"), pytest.param([1, 2, 3], id="list"), pytest.param({"a": 1}, id="dict"), ], ) def test_unequal_with_invalid_type(obj: Any) -> None: assert (_make_spec() == obj) is False assert _make_spec() != obj @pytest.mark.parametrize( "data", [ {}, {"order": "F"}, {"write_empty_chunks": True, "read_missing_chunks": False}, { "order": "F", "write_empty_chunks": True, "read_missing_chunks": False, "sharding_coalesce_max_gap_bytes": 1, "sharding_coalesce_max_bytes": 2, }, ], ) def test_array_config_from_dict(data: ArrayConfigParams) -> None: """ ArrayConfig.from_dict takes the given keys from `data` and the rest from the global configuration. """ observed = ArrayConfig.from_dict(data).to_dict() for key, value in observed.items(): assert value == data.get(key, zarr.config.get(f"array.{key}")) @pytest.mark.parametrize( "build", [ lambda config: ArrayConfig.from_dict(config), lambda config: zarr.create_array({}, shape=(4,), dtype="uint8", config=config), lambda config: zarr.create(shape=(4,), dtype="uint8", config=config), ], ids=["from_dict", "create_array", "create"], ) def test_array_config_unknown_key(build: Callable[[Any], object]) -> None: """ An array config with a key that is not an ArrayConfig attribute raises TypeError instead of being ignored. """ with pytest.raises(TypeError, match=r"Unknown array config keys: \['nope'\]"): build({"order": "C", "nope": 1}) zarr-developers-zarr-python-a2f2415/tests/test_array_stateful.py000066400000000000000000000331111527014137600252640ustar00rootroot00000000000000"""A stateful test of one array's life: create, append, resize, write, reopen. The model is a NumPy array of what the store holds, not a second zarr array, so a bug in zarr's chunk grid logic cannot hide by being made on both sides. Zero-length axes are drawn on purpose, both at creation and by resizing and appending, and so are stored chunk sizes of 0. The model tracks cells beyond the array's shape too, because chunks do: a shrinking `resize` deletes exactly the chunks outside the new grid, a chunk it keeps keeps its cells beyond the new shape (and they come back if the array grows again), and a write that covers every in-bounds cell of an unsharded chunk rewrites the whole chunk, resetting its cells beyond the shape to the fill value. A sharded array rewrites a shard through its inner chunks, each judged against the shard rather than the array shape, so a write never resets cells beyond the shape. """ from __future__ import annotations import itertools import json import warnings from typing import Any, Literal import hypothesis.extra.numpy as npst import hypothesis.strategies as st import numpy as np import pytest from hypothesis import event, note from hypothesis.stateful import ( RuleBasedStateMachine, initialize, invariant, precondition, rule, ) import zarr from zarr.core.buffer import cpu, default_buffer_prototype from zarr.core.chunk_grids import ChunkGrid from zarr.core.sync import sync from zarr.errors import ZarrUserWarning from zarr.storage import MemoryStore from zarr.testing.strategies import _rectilinear_chunks pytestmark = [ pytest.mark.slow_hypothesis, pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning"), ] DTYPE = np.dtype("int16") METADATA_KEYS = (".zarray", ".zattrs", "zarr.json") MAX_SIDE = 6 # `append` may grow an axis past `MAX_SIDE`, but not without bound: with chunks of 1, # the cells an extent of 100 steps could reach make each full-array read take seconds, # and the nightly run of every example together exceeded pytest's faulthandler_timeout. MAX_APPEND_EXTENT = 2 * MAX_SIDE async def _list(store: MemoryStore, prefix: str) -> list[str]: return [key async for key in store.list_prefix(prefix)] class ArrayLifecycle(RuleBasedStateMachine): def __init__(self) -> None: super().__init__() self._rectilinear = zarr.config.set({"array.rectilinear_chunks": True}) self._rectilinear.__enter__() self.store = MemoryStore() self.path = "a" self.shape: tuple[int, ...] = (0,) self.fill = 0 # What the store holds, indexed like the array and extending past its shape. self.stored: np.ndarray[Any, np.dtype[np.int16]] = np.zeros((0,), dtype=DTYPE) # Axes stored with chunk size 0; the store warns until its metadata is re-saved. self.legacy_axes: list[int] = [] # -------------------------------------------------------------- creation @initialize(data=st.data()) def create(self, data: st.DataObject) -> None: zarr_format: Literal[2, 3] = data.draw(st.sampled_from([3, 2]), label="zarr_format") shape = data.draw( npst.array_shapes(min_dims=1, max_dims=3, min_side=0, max_side=MAX_SIDE), label="shape", ) self.fill = data.draw(st.integers(-3, 3), label="fill_value") # sampled_from favours early entries; the less common spellings go first. spellings = ["ints", "-1", "False", "auto"] if zarr_format == 3: spellings[:0] = ["sharded", "rectilinear"] spelling = data.draw(st.sampled_from(spellings), label="chunk spelling") event(f"chunks: {spelling}") chunks: Any shards: Any = None if spelling in ("-1", "False"): chunks = {"-1": -1, "False": False}[spelling] elif spelling == "auto": chunks = "auto" elif spelling == "rectilinear": chunks = data.draw(_rectilinear_chunks(shape=shape), label="rectilinear chunks") else: chunks = tuple(data.draw(st.integers(1, 3)) for _ in shape) if spelling == "sharded": shards = tuple(c * data.draw(st.integers(1, 3)) for c in chunks) note(f"create {shape=} {chunks=} {shards=} {zarr_format=} fill={self.fill}") zarr.create_array( self.store, name=self.path, shape=shape, chunks=chunks, shards=shards, dtype=DTYPE, fill_value=self.fill, zarr_format=zarr_format, ) self.shape = shape self.stored = np.full(shape, self.fill, dtype=DTYPE) if spelling != "rectilinear" and data.draw(st.booleans(), label="legacy zero"): # A stored chunk size of 0, as zarr-python wrote for arrays created with a # zero-length axis; older releases could then grow the axis without storing # a chunk, so any extent is possible, but zero-length axes come first. # Sharded arrays store it in the outer grid. axes = st.lists(st.integers(0, len(shape) - 1), min_size=1, unique=True) if zero_axes := [axis for axis, extent in enumerate(shape) if extent == 0]: axes = st.lists(st.sampled_from(zero_axes), min_size=1, unique=True) | axes self.legacy_axes = data.draw(axes, label="axes stored with chunk size 0") stored_zero = data.draw(st.sampled_from([0, False]), label="stored zero") self._rewrite_stored_chunks(zarr_format, self.legacy_axes, stored_zero) event("legacy zero chunk size") else: # Re-saving valid metadata, as the warning tells users to, changes nothing. arr = self._open() arr.update_attributes({}) assert self._open().metadata == arr.metadata def _rewrite_stored_chunks( self, zarr_format: Literal[2, 3], axes: list[int], value: Any ) -> None: key = f"{self.path}/{'.zarray' if zarr_format == 2 else 'zarr.json'}" buf = sync(self.store.get(key, prototype=default_buffer_prototype())) assert buf is not None doc = json.loads(buf.to_bytes()) sizes = ( doc["chunks"] if zarr_format == 2 else doc["chunk_grid"]["configuration"]["chunk_shape"] ) for axis in axes: sizes[axis] = value sync(self.store.set(key, cpu.Buffer.from_bytes(json.dumps(doc).encode()))) def _open(self) -> zarr.Array[Any]: with warnings.catch_warnings(record=True) as record: warnings.simplefilter("always", ZarrUserWarning) arr = zarr.open_array(self.store, path=self.path, mode="r+") # A stored chunk size of 0 is read silently on an empty axis; on a non-empty one # it warns that the axis holds only the fill value. Either way it is repaired. warned = any(issubclass(w.category, ZarrUserWarning) for w in record) must_warn = any(self.shape[axis] > 0 for axis in self.legacy_axes) assert warned is must_warn, [str(w.message) for w in record] assert (arr.metadata._stored_document is not None) is bool(self.legacy_axes) return arr # ----------------------------------------------------------------- model def _cover(self, shape: tuple[int, ...]) -> None: """Grow `stored` with the fill value so that it covers `shape`.""" pad = [(0, max(0, n - s)) for n, s in zip(shape, self.stored.shape, strict=True)] if any(after for _, after in pad): self.stored = np.pad(self.stored, pad, constant_values=self.fill) def _model_resize(self, grid: ChunkGrid, new_shape: tuple[int, ...]) -> None: """Delete exactly the chunks of `grid` outside the grid for `new_shape`.""" kept = [] for dim, old, new in zip(grid.dimensions, self.shape, new_shape, strict=True): if new >= old: kept.append(slice(None)) elif new == 0: kept.append(slice(0, 0)) else: last = dim.index_to_chunk(new - 1) kept.append(slice(0, dim.chunk_offset(last) + dim.chunk_size(last))) stored = np.full_like(self.stored, self.fill) stored[tuple(kept)] = self.stored[tuple(kept)] self.stored = stored self.shape = new_shape self._cover(new_shape) def _model_write(self, arr: zarr.Array[Any], region: tuple[slice, ...], values: Any) -> None: """Write `values`; an unsharded chunk whose in-bounds cells are all written is rewritten whole.""" grid = ChunkGrid.from_metadata(arr.metadata) if arr.shards is None and all(r.stop > r.start for r in region): # Per axis, each chunk the region touches: (start, stop, all in-bounds cells written). per_axis: list[list[tuple[int, int, bool]]] = [] for dim, r, extent in zip(grid.dimensions, region, self.shape, strict=True): spans = [] for c in range(dim.index_to_chunk(r.start), dim.index_to_chunk(r.stop - 1) + 1): lo = dim.chunk_offset(c) hi = lo + dim.chunk_size(c) spans.append((lo, hi, r.start <= lo and r.stop >= min(hi, extent))) per_axis.append(spans) self._cover(tuple(max(hi for _, hi, _ in axis) for axis in per_axis)) for combo in itertools.product(*per_axis): if all(complete for *_, complete in combo): self.stored[tuple(slice(lo, hi) for lo, hi, _ in combo)] = self.fill self.stored[region] = values # ----------------------------------------------------------------- rules @rule(data=st.data()) def append(self, data: st.DataObject) -> None: arr = self._open() axes = st.integers(0, len(self.shape) - 1) if self.legacy_axes: # What a user of an older release did next: grow an axis stored with chunk size 0. axes = st.sampled_from(self.legacy_axes) | axes axis = data.draw(axes, label="axis") block_shape = list(self.shape) max_rows = min(4, max(0, MAX_APPEND_EXTENT - self.shape[axis])) block_shape[axis] = data.draw(st.integers(0, max_rows), label="rows") block = data.draw(npst.arrays(DTYPE, tuple(block_shape)), label="block") note(f"append {block.shape} along {axis} to {self.shape}") if self.shape[axis] == 0 and block.shape[axis]: event("append to a zero-length axis") if axis in self.legacy_axes and block.shape[axis]: event("grow an axis stored with chunk size 0") old_extent = self.shape[axis] arr.append(block, axis=axis) self._model_resize(ChunkGrid.from_metadata(arr.metadata), arr.shape) region = tuple( slice(old_extent, s) if i == axis else slice(0, s) for i, s in enumerate(arr.shape) ) self._model_write(arr, region, block) # Growing the array rewrites its metadata, which stores any correction. self.legacy_axes = [] @rule(data=st.data()) def resize(self, data: st.DataObject) -> None: arr = self._open() extents = [st.integers(0, MAX_SIDE) for _ in self.shape] for axis in self.legacy_axes: # What a user of an older release did next: grow an axis stored with chunk size 0. extents[axis] = st.integers(self.shape[axis] + 1, MAX_SIDE + 1) | extents[axis] new_shape = data.draw(st.tuples(*extents), label="new shape") note(f"resize {self.shape} -> {new_shape}") if any(new_shape[axis] > self.shape[axis] for axis in self.legacy_axes): event("grow an axis stored with chunk size 0") grid = ChunkGrid.from_metadata(arr.metadata) arr.resize(new_shape) self._model_resize(grid, new_shape) self.legacy_axes = [] @rule(data=st.data()) def write(self, data: st.DataObject) -> None: arr = self._open() region = tuple( slice(*sorted(data.draw(st.tuples(st.integers(0, s), st.integers(0, s))))) for s in self.shape ) shape = tuple(r.stop - r.start for r in region) values = data.draw(npst.arrays(DTYPE, shape), label="values") note(f"write {region}") arr[region] = values self._model_write(arr, region, values) if all(shape): # Writing chunks first stores the metadata they are written under; a write # of nothing stores nothing. self.legacy_axes = [] @precondition(lambda self: self.legacy_axes) @rule() def resave_metadata(self) -> None: """What the warning for an invalid stored chunk size tells users to do: store the metadata as read.""" arr = self._open() read = arr.metadata arr.update_attributes({}) self.legacy_axes = [] assert self._open().metadata == read def teardown(self) -> None: self._rectilinear.__exit__(None, None, None) # ------------------------------------------------------------ invariants @invariant() def no_chunk_under_an_invalid_document(self) -> None: """While the stored document is still one that is repaired on read, which other readers may reject or read differently, no chunk is stored under it.""" if self.legacy_axes: keys = sync(_list(self.store, f"{self.path}/")) assert set(keys) <= {f"{self.path}/{name}" for name in METADATA_KEYS}, keys @invariant() def matches_model(self) -> None: arr = self._open() assert arr.shape == self.shape expected = self.stored[tuple(slice(0, s) for s in self.shape)] np.testing.assert_array_equal(np.asarray(arr[...]), expected) TestArrayLifecycle = ArrayLifecycle.TestCase zarr-developers-zarr-python-a2f2415/tests/test_attributes.py000066400000000000000000000053331527014137600244320ustar00rootroot00000000000000import json from typing import TYPE_CHECKING, Any import numpy as np import pytest import zarr.core import zarr.core.attributes import zarr.storage from tests.conftest import deep_nan_equal from zarr.core.common import ZarrFormat if TYPE_CHECKING: from zarr.types import AnyArray @pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.parametrize( "data", [{"inf": np.inf, "-inf": -np.inf, "nan": np.nan}, {"a": 3, "c": 4}] ) def test_put(data: dict[str, Any], zarr_format: ZarrFormat) -> None: store = zarr.storage.MemoryStore() attrs = zarr.core.attributes.Attributes(zarr.Group.from_store(store, zarr_format=zarr_format)) attrs.put(data) expected = json.loads(json.dumps(data, allow_nan=True)) assert deep_nan_equal(dict(attrs), expected) def test_asdict() -> None: store = zarr.storage.MemoryStore() attrs = zarr.core.attributes.Attributes( zarr.Group.from_store(store, attributes={"a": 1, "b": 2}) ) result = attrs.asdict() assert result == {"a": 1, "b": 2} def test_update_attributes_preserves_existing() -> None: """ Test that `update_attributes` only updates the specified attributes and preserves existing ones. """ store = zarr.storage.MemoryStore() z = zarr.create(10, store=store, overwrite=True) z.attrs["a"] = [] z.attrs["b"] = 3 assert dict(z.attrs) == {"a": [], "b": 3} z.update_attributes({"a": [3, 4], "c": 4}) assert dict(z.attrs) == {"a": [3, 4], "b": 3, "c": 4} def test_update_empty_attributes() -> None: """ Ensure updating when initial attributes are empty works. """ store = zarr.storage.MemoryStore() z = zarr.create(10, store=store, overwrite=True) assert dict(z.attrs) == {} z.update_attributes({"a": [3, 4], "c": 4}) assert dict(z.attrs) == {"a": [3, 4], "c": 4} def test_update_no_changes() -> None: """ Ensure updating when no new or modified attributes does not alter existing ones. """ store = zarr.storage.MemoryStore() z = zarr.create(10, store=store, overwrite=True) z.attrs["a"] = [] z.attrs["b"] = 3 z.update_attributes({}) assert dict(z.attrs) == {"a": [], "b": 3} @pytest.mark.parametrize("group", [True, False]) def test_del_works(group: bool) -> None: store = zarr.storage.MemoryStore() z: zarr.Group | AnyArray if group: z = zarr.create_group(store) else: z = zarr.create_array(store=store, shape=10, dtype=int) assert dict(z.attrs) == {} z.update_attributes({"a": [3, 4], "c": 4}) del z.attrs["a"] assert dict(z.attrs) == {"c": 4} z2: zarr.Group | AnyArray if group: z2 = zarr.open_group(store) else: z2 = zarr.open_array(store) assert dict(z2.attrs) == {"c": 4} zarr-developers-zarr-python-a2f2415/tests/test_buffer.py000066400000000000000000000305701527014137600235160ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING, Literal import numpy as np import pytest import zarr from zarr.abc.buffer import ArrayLike, BufferPrototype, NDArrayLike from zarr.buffer import cpu, gpu from zarr.codecs.blosc import BloscCodec from zarr.codecs.crc32c_ import Crc32cCodec from zarr.codecs.gzip import GzipCodec from zarr.codecs.transpose import TransposeCodec from zarr.codecs.zstd import ZstdCodec from zarr.errors import ZarrUserWarning from zarr.storage import MemoryStore, StorePath from zarr.testing.buffer import ( NDBufferUsingTestNDArrayLike, StoreExpectingTestBuffer, TestBuffer, TestNDArrayLike, ) from zarr.testing.utils import gpu_mark, gpu_test, skip_if_no_gpu if TYPE_CHECKING: import types try: import cupy as cp except ImportError: cp = None import zarr.api.asynchronous if TYPE_CHECKING: import types def test_nd_array_like(xp: types.ModuleType) -> None: ary = xp.arange(10) assert isinstance(ary, ArrayLike) assert isinstance(ary, NDArrayLike) async def test_async_array_prototype() -> None: """Test the use of a custom buffer prototype""" expect = np.zeros((9, 9), dtype="uint16", order="F") a = await zarr.api.asynchronous.create_array( StorePath(StoreExpectingTestBuffer()) / "test_async_array_prototype", shape=expect.shape, chunks=(5, 5), dtype=expect.dtype, fill_value=0, ) expect[1:4, 3:6] = np.ones((3, 3)) my_prototype = BufferPrototype(buffer=TestBuffer, nd_buffer=NDBufferUsingTestNDArrayLike) await a.setitem( selection=(slice(1, 4), slice(3, 6)), value=np.ones((3, 3)), prototype=my_prototype, ) got = await a.getitem(selection=(slice(0, 9), slice(0, 9)), prototype=my_prototype) # ignoring a mypy error here that TestNDArrayLike doesn't meet the NDArrayLike protocol # The test passes, so it clearly does. assert isinstance(got, TestNDArrayLike) assert np.array_equal(expect, got) # type: ignore[unreachable] @gpu_test async def test_async_array_gpu_prototype() -> None: """Test the use of the GPU buffer prototype""" expect = cp.zeros((9, 9), dtype="uint16", order="F") a = await zarr.api.asynchronous.create_array( StorePath(MemoryStore()) / "test_async_array_gpu_prototype", shape=expect.shape, chunks=(5, 5), dtype=expect.dtype, fill_value=0, ) expect[1:4, 3:6] = cp.ones((3, 3)) await a.setitem( selection=(slice(1, 4), slice(3, 6)), value=cp.ones((3, 3)), prototype=gpu.buffer_prototype, ) got = await a.getitem(selection=(slice(0, 9), slice(0, 9)), prototype=gpu.buffer_prototype) assert isinstance(got, cp.ndarray) assert cp.array_equal(expect, got) async def test_codecs_use_of_prototype() -> None: expect = np.zeros((10, 10), dtype="uint16", order="F") a = await zarr.api.asynchronous.create_array( StorePath(StoreExpectingTestBuffer()) / "test_codecs_use_of_prototype", shape=expect.shape, chunks=(5, 5), dtype=expect.dtype, fill_value=0, compressors=[BloscCodec(), Crc32cCodec(), GzipCodec(), ZstdCodec()], filters=[TransposeCodec(order=(1, 0))], ) expect[:] = np.arange(100).reshape(10, 10) my_prototype = BufferPrototype(buffer=TestBuffer, nd_buffer=NDBufferUsingTestNDArrayLike) await a.setitem( selection=(slice(0, 10), slice(0, 10)), value=expect[:], prototype=my_prototype, ) got = await a.getitem(selection=(slice(0, 10), slice(0, 10)), prototype=my_prototype) # ignoring a mypy error here that TestNDArrayLike doesn't meet the NDArrayLike protocol # The test passes, so it clearly does. assert isinstance(got, TestNDArrayLike) assert np.array_equal(expect, got) # type: ignore[unreachable] @gpu_test async def test_codecs_use_of_gpu_prototype() -> None: expect = cp.zeros((10, 10), dtype="uint16", order="F") a = await zarr.api.asynchronous.create_array( StorePath(MemoryStore()) / "test_codecs_use_of_gpu_prototype", shape=expect.shape, chunks=(5, 5), dtype=expect.dtype, fill_value=0, compressors=[BloscCodec(), Crc32cCodec(), GzipCodec(), ZstdCodec()], filters=[TransposeCodec(order=(1, 0))], ) expect[:] = cp.arange(100).reshape(10, 10) msg = "Creating a zarr.buffer.gpu.Buffer with an array that does not support the __cuda_array_interface__ for zero-copy transfers, falling back to slow copy based path" with pytest.warns(ZarrUserWarning, match=msg): await a.setitem( selection=(slice(0, 10), slice(0, 10)), value=expect[:], prototype=gpu.buffer_prototype, ) with pytest.warns(ZarrUserWarning, match=msg): got = await a.getitem( selection=(slice(0, 10), slice(0, 10)), prototype=gpu.buffer_prototype ) assert isinstance(got, cp.ndarray) assert cp.array_equal(expect, got) @gpu_test async def test_sharding_use_of_gpu_prototype() -> None: with zarr.config.enable_gpu(): expect = cp.zeros((10, 10), dtype="uint16", order="F") a = await zarr.api.asynchronous.create_array( StorePath(MemoryStore()) / "test_codecs_use_of_gpu_prototype", shape=expect.shape, chunks=(5, 5), shards=(10, 10), dtype=expect.dtype, fill_value=0, ) expect[:] = cp.arange(100).reshape(10, 10) msg = "Creating a zarr.buffer.gpu.Buffer with an array that does not support the __cuda_array_interface__ for zero-copy transfers, falling back to slow copy based path" with pytest.warns(ZarrUserWarning, match=msg): await a.setitem( selection=(slice(0, 10), slice(0, 10)), value=expect[:], prototype=gpu.buffer_prototype, ) with pytest.warns(ZarrUserWarning, match=msg): got = await a.getitem( selection=(slice(0, 10), slice(0, 10)), prototype=gpu.buffer_prototype ) assert isinstance(got, cp.ndarray) assert cp.array_equal(expect, got) def test_numpy_buffer_prototype() -> None: buffer = cpu.buffer_prototype.buffer.create_zero_length() ndbuffer = cpu.buffer_prototype.nd_buffer.create(shape=(1, 2), dtype=np.dtype("int64")) assert isinstance(buffer.as_array_like(), np.ndarray) assert isinstance(ndbuffer.as_ndarray_like(), np.ndarray) with pytest.raises(ValueError, match="Buffer does not contain a single scalar value"): ndbuffer.as_scalar() @gpu_test def test_gpu_buffer_prototype() -> None: buffer = gpu.buffer_prototype.buffer.create_zero_length() ndbuffer = gpu.buffer_prototype.nd_buffer.create(shape=(1, 2), dtype=cp.dtype("int64")) assert isinstance(buffer.as_array_like(), cp.ndarray) assert isinstance(ndbuffer.as_ndarray_like(), cp.ndarray) with pytest.raises(ValueError, match="Buffer does not contain a single scalar value"): ndbuffer.as_scalar() @pytest.mark.parametrize(("shape", "key"), [((), ()), ((3, 4), (1, 2))]) def test_cpu_ndbuffer_getitem_returns_view(shape: tuple[int, ...], key: tuple[int, ...]) -> None: # https://github.com/zarr-developers/zarr-python/issues/4464 data = np.full(shape, b"x" * 10) item = cpu.NDBuffer.from_numpy_array(data)[key].as_numpy_array() assert item.shape == () assert np.shares_memory(item, data) # TODO: the same test for other buffer classes def test_cpu_buffer_as_scalar() -> None: buf = cpu.buffer_prototype.nd_buffer.create(shape=(), dtype="int64") assert buf.as_scalar() == buf.as_ndarray_like()[()] # type: ignore[index] @pytest.mark.parametrize( "prototype", [ cpu.buffer_prototype, pytest.param( gpu.buffer_prototype, marks=[gpu_mark, skip_if_no_gpu], ), BufferPrototype( buffer=cpu.Buffer, nd_buffer=NDBufferUsingTestNDArrayLike, ), ], ) @pytest.mark.parametrize( "shape", [ (1, 2), (1, 2, 3), ], ) @pytest.mark.parametrize("dtype", ["int32", "float64"]) @pytest.mark.parametrize("order", ["C", "F"]) def test_empty( prototype: BufferPrototype, shape: tuple[int, ...], dtype: str, order: Literal["C", "F"] ) -> None: buf = prototype.nd_buffer.empty(shape=shape, dtype=dtype, order=order) result = buf.as_ndarray_like() assert result.shape == shape assert result.dtype == dtype if order == "C": assert result.flags.c_contiguous # type: ignore[attr-defined] else: assert result.flags.f_contiguous # type: ignore[attr-defined] def _all_equal_reference(data: np.ndarray, fill: float) -> bool: """`NDBuffer.all_equal` as it was before the fast paths: one full scan, comparing a zero `fill` bitwise through a void view.""" if np.asarray(fill).dtype.kind == "f" and fill == 0.0: void = f"V{data.dtype.itemsize}" expected = np.broadcast_to(np.asarray(fill, data.dtype), data.shape) return bool(np.array_equal(data.view(void), expected.view(void))) return bool(np.array_equal(data, np.broadcast_to(fill, data.shape), equal_nan=True)) _BLOCK = 1 << 14 def _layouts(dtype: str) -> dict[str, np.ndarray]: """Buffers larger than one scan block and not a whole number of blocks.""" side = 211 return { "1d": np.zeros(_BLOCK * 3 + 7, dtype=dtype), "C": np.zeros((side, side), dtype=dtype), "F": np.zeros((side, side), dtype=dtype, order="F"), # a chunk carved out of a larger array, which is what the write path passes "strided": np.zeros((2 * side, 2 * side), dtype=dtype)[5 : side + 5, 7 : side + 7], "leading-1": np.zeros((1, _BLOCK * 2), dtype=dtype), } @pytest.mark.parametrize( "dtype", ["float16", "float32", "float64", "complex64", "longdouble", "int32"] ) @pytest.mark.parametrize("layout", ["1d", "C", "F", "strided", "leading-1"]) @pytest.mark.parametrize("contents", [0.0, -0.0, np.nan, 1.0]) @pytest.mark.parametrize("mismatch", [None, "first", "last"]) @pytest.mark.parametrize("fill", [0.0, -0.0, np.nan, 1.0, 0]) def test_all_equal( dtype: str, layout: str, contents: float, mismatch: str | None, fill: float ) -> None: """`all_equal` agrees with a full scan, whatever the layout or where a mismatch sits.""" data = _layouts(dtype)[layout] if np.dtype(dtype).kind == "i" and contents != 1.0: contents = 0 data[...] = contents if mismatch is not None: index = tuple(0 if mismatch == "first" else n - 1 for n in data.shape) data[index] = 7 assert cpu.NDBuffer.from_numpy_array(data).all_equal(fill) == _all_equal_reference(data, fill) @pytest.mark.parametrize("dtype", ["float16", "float32", "float64"]) def test_all_equal_distinguishes_negative_zero(dtype: str) -> None: """Regression test for #3144: -0.0 is not the same chunk as 0.0.""" positive = np.zeros(_BLOCK * 2 + 3, dtype=dtype) negative = np.full_like(positive, -0.0) assert cpu.NDBuffer.from_numpy_array(positive).all_equal(0.0) assert not cpu.NDBuffer.from_numpy_array(negative).all_equal(0.0) assert cpu.NDBuffer.from_numpy_array(negative).all_equal(-0.0) assert not cpu.NDBuffer.from_numpy_array(positive).all_equal(-0.0) @pytest.mark.parametrize("kind", ["M8", "m8"]) @pytest.mark.parametrize("byteorder", ["<", ">"]) @pytest.mark.parametrize("scale_factor", [2, 2**31 - 1]) @pytest.mark.parametrize("fill_value", [None, 0, "NaT"]) @pytest.mark.parametrize("order", ["C", "F"]) @pytest.mark.filterwarnings( "ignore:The 'generic' unit for NumPy timedelta is deprecated:DeprecationWarning" ) def test_cpu_generic_time_allocation( kind: str, byteorder: str, scale_factor: int, fill_value: int | str | None, order: Literal["C", "F"], ) -> None: """Both allocation paths retain generic scale even when NumPy drops it.""" dtype = np.dtype(f"{byteorder}{kind}[{scale_factor}generic]") empty = cpu.NDBuffer.empty((2, 3), dtype=dtype, order=order) filled = cpu.NDBuffer.create(shape=(2, 3), dtype=dtype, order=order, fill_value=fill_value) for buffer in (empty, filled): assert buffer.dtype == dtype array = buffer.as_numpy_array() assert array.flags.c_contiguous if order == "C" else array.flags.f_contiguous expected = -(2**63) if fill_value == "NaT" else 0 np.testing.assert_array_equal( filled.as_numpy_array().view(dtype.byteorder + "i8"), np.full((2, 3), expected) ) zarr-developers-zarr-python-a2f2415/tests/test_chunk_grids.py000066400000000000000000000517001527014137600245430ustar00rootroot00000000000000import contextlib from typing import Any, Literal, cast import numpy as np import pytest import zarr from tests.conftest import Expect, ExpectFail from zarr.core.chunk_grids import ( ChunkGrid, ChunkLayout, FixedDimension, VaryingDimension, _guess_num_chunks_per_axis_shard, _guess_regular_chunks, full_span_chunk_size, normalize_chunks_1d, normalize_chunks_nd, resolve_outer_and_inner_chunks, ) from zarr.errors import ZarrUserWarning def _assert_chunks_equal( actual: ChunkGrid, expected: tuple[int | tuple[int, ...], ...], shape: tuple[int, ...], ) -> None: """Compare a normalized ChunkGrid against per-dimension expectations. An expected bare `int` requires a `FixedDimension` with that uniform size; an expected tuple requires a `VaryingDimension` with those exact edges. Every dimension must carry the corresponding extent from `shape`. """ dims = actual.dimensions assert len(dims) == len(expected) == len(shape), ( f"axis count mismatch: {len(dims)} vs {len(expected)} vs {len(shape)}" ) for axis, (a, e, span) in enumerate(zip(dims, expected, shape, strict=True)): if isinstance(e, int): assert isinstance(a, FixedDimension), f"axis {axis}: expected FixedDimension, got {a!r}" assert a.size == e, f"axis {axis}: size {a.size} != {e}" else: assert isinstance(a, VaryingDimension), ( f"axis {axis}: expected VaryingDimension, got {a!r}" ) assert a.edges == tuple(e), f"axis {axis}: edges {a.edges} != {tuple(e)}" assert a.extent == span, f"axis {axis}: extent {a.extent} != {span}" @pytest.mark.parametrize( "shape", [(0,), (0,) * 2, (1, 2, 0, 4, 5), (10, 0), (10,), (100,) * 3, (1000000,), (10000,) * 2] ) @pytest.mark.parametrize("itemsize", [1, 2, 4]) def test_guess_chunks(shape: tuple[int, ...], itemsize: int) -> None: chunks = _guess_regular_chunks(shape, itemsize) chunk_size = np.prod(chunks) * itemsize assert isinstance(chunks, tuple) assert len(chunks) == len(shape) assert chunk_size < (64 * 1024 * 1024) # doesn't make any sense to allow chunks to have zero length dimension assert all(0 < c <= max(s, 1) for c, s in zip(chunks, shape, strict=False)) class _IndexedSequence: """A sequence with only `__getitem__` and `__len__`: `list` takes it, but `collections.abc.Iterable` does not recognize it.""" def __init__(self, *items: int) -> None: self._items = items def __getitem__(self, index: int) -> int: return self._items[index] def __len__(self) -> int: return len(self._items) @pytest.mark.parametrize( ("chunks", "shape", "expected"), [ # 1D cases (uniform sizes stay bare ints) ((10,), (100,), (10,)), ([10], (100,), (10,)), (10, (100,), (10,)), # 2D cases ((10, 10), (100, 10), (10, 10)), (10, (100, 10), (10, 10)), ((10, -1), (100, 10), (10, 10)), # 3D cases (30, (100, 20, 10), (30, 30, 30)), ((30, -1, -1), (100, 20, 10), (30, 20, 10)), ((30, 20, -1), (100, 20, 10), (30, 20, 10)), ((30, 20, 10), (100, 20, 10), (30, 20, 10)), # dask-style explicit lists always keep the per-chunk rectilinear form, # even when the sizes describe a regular grid (gh-4272) (((100, 100, 100), (50, 50)), (300, 100), ((100, 100, 100), (50, 50))), (((100, 100, 50),), (250,), ((100, 100, 50),)), (((100,),), (100,), ((100,),)), (((10, 20, 70), (50, 50)), (100, 100), ((10, 20, 70), (50, 50))), # no chunking (False means each dimension is one chunk spanning the full extent) (False, (100,), (100,)), (False, (100, 50), (100, 50)), # sentinel values (-1, (100,), (100,)), # False and -1 on a zero-length axis clamp to chunk size 1 (chunk sizes must be positive) (False, (0,), (1,)), (False, (0, 4), (1, 4)), (-1, (4, 0), (4, 1)), # zero-length dimensions preserve the declared chunk size (10, (0,), (10,)), ((5, 10), (0, 100), (5, 10)), ((5, 10), (20, 0), (5, 10)), # numpy integers are accepted anywhere a python int is, whether as the scalar # convenience form, as per-dimension entries, or as the `-1` sentinel. (np.int64(10), (100,), (10,)), ((np.int64(2), np.int64(2)), (4, 4), (2, 2)), ((1, 3, np.int64(16), np.int64(16)), (1, 3, 32, 32), (1, 3, 16, 16)), ((np.int32(30), np.int64(-1)), (100, 20), (30, 20)), (np.array([10, 10]), (100, 100), (10, 10)), # 0-d integer arrays are integers, as the scalar form and per dimension (np.array(10), (100, 100), (10, 10)), ((np.array(5), np.int64(-1)), (10, 6), (5, 6)), # rectilinear chunks given as numpy arrays ((np.array([60, 40]), np.array([50, 50])), (100, 100), ((60, 40), (50, 50))), # a Python `bool` inside a specification is an `int`: `True` is a chunk size of 1 ((True, 5), (10, 10), (1, 5)), # a sequence with only `__getitem__` and `__len__` is a list of chunk sizes (_IndexedSequence(5, 5), (10, 10), (5, 5)), ((_IndexedSequence(6, 4), 5), (10, 10), ((6, 4), 5)), ([[True, 9], 10], (10, 10), ((1, 9), 10)), ], ) def test_normalize_chunks( chunks: Any, shape: tuple[int, ...], expected: tuple[int | tuple[int, ...], ...] ) -> None: _assert_chunks_equal(normalize_chunks_nd(chunks, shape), expected, shape) @pytest.mark.parametrize( ("array_shape", "chunks_input", "shard_shape", "expected_outer", "expected_inner_outer"), [ # no sharding: outer = chunks, inner = None ((100,), (10,), None, (10,), None), # explicit regular shards ((100,), (10,), (50,), (50,), (10,)), # rectilinear shards keep the per-chunk form even when the sizes # describe a regular-with-boundary grid (gh-4272) ((100,), (10,), ((60, 40),), ((60, 40),), (10,)), ((100,), (10,), ((30, 60, 10),), ((30, 60, 10),), (10,)), # dict-style shards ((100, 100), (10, 10), {"shape": (50, 50)}, (50, 50), (10, 10)), ], ) def test_resolve_outer_and_inner_chunks( array_shape: tuple[int, ...], chunks_input: tuple[int, ...], shard_shape: Any, expected_outer: tuple[int | tuple[int, ...], ...], expected_inner_outer: tuple[int | tuple[int, ...], ...] | None, ) -> None: chunks = normalize_chunks_nd(chunks_input, array_shape) outer_chunks, inner = resolve_outer_and_inner_chunks( array_shape=array_shape, chunks=chunks, shard_shape=shard_shape, item_size=1 ) _assert_chunks_equal(outer_chunks, expected_outer, array_shape) if expected_inner_outer is None: assert inner is None else: assert inner is not None _assert_chunks_equal(inner.outer_chunks, expected_inner_outer, array_shape) assert inner.inner is None def test_chunk_layout_nested() -> None: """Test that ChunkLayout supports recursive nesting for nested sharding.""" leaf = normalize_chunks_nd((5, 5), (100, 100)) mid = ChunkLayout( outer_chunks=normalize_chunks_nd((25, 25), (100, 100)), inner=ChunkLayout(outer_chunks=leaf), ) top = ChunkLayout(outer_chunks=normalize_chunks_nd((50, 50), (100, 100)), inner=mid) # Three levels: top -> mid -> leaf _assert_chunks_equal(top.outer_chunks, (50, 50), (100, 100)) assert top.inner is not None _assert_chunks_equal(top.inner.outer_chunks, (25, 25), (100, 100)) assert top.inner.inner is not None _assert_chunks_equal(top.inner.inner.outer_chunks, (5, 5), (100, 100)) assert top.inner.inner.inner is None @pytest.mark.parametrize( "case", [ ExpectFail( input=(0, 100), exception=ValueError, id="zero-uniform", msg="Chunk size must be positive", ), ExpectFail( input=(-2, 100), exception=ValueError, id="negative-uniform", msg="Chunk size must be positive", ), ExpectFail( input=(np.int64(0), 100), exception=ValueError, id="zero-uniform-numpy", msg="Chunk size must be positive", ), ExpectFail(input=([], 100), exception=ValueError, id="empty-list", msg="must not be empty"), # Scalars that are neither integers nor iterable name themselves in the error, # rather than surfacing an opaque "object is not iterable" from `list(chunks)`. ExpectFail( input=(2.5, 100), exception=TypeError, id="non-iterable-scalar", msg="must be an integer or an iterable of integers; got 2.5 of type float", escape=True, ), # an integral float is still not an integer ExpectFail( input=(10.0, 100), exception=TypeError, id="integral-float-scalar", msg="got 10.0 of type float", ), ExpectFail( input=([10, -1, 10], 100), exception=ValueError, id="negative-element", msg="must be positive", ), ExpectFail( input=([10, 0, 10], 20), exception=ValueError, id="zero-element", msg="must be positive" ), ExpectFail( input=([10, 20], 100), exception=ValueError, id="wrong-sum", msg="do not sum to span" ), # Only a zero-length span keeps edges that do not sum to it. ExpectFail( input=([3, 5], 1), exception=ValueError, id="over-sum", msg="do not sum to span 1" ), # Nested/RLE form for a single dim is rejected with offending indices. ExpectFail( input=([[3, 3], 1], 7), exception=TypeError, id="rle-single-dim", msg="non-integer element(s) ([3, 3],) at indices (0,)", escape=True, ), # Multiple non-int elements: all offending indices reported. ExpectFail( input=([1, [2, 2], 1, [3]], 9), exception=TypeError, id="multiple-non-ints", msg="non-integer element(s) ([2, 2], [3]) at indices (1, 3)", escape=True, ), # Strings are non-integers and should be reported the same way. ExpectFail( input=([2, "3", 5], 10), exception=TypeError, id="string-element", msg="non-integer element(s) ('3',) at indices (1,)", escape=True, ), ], ids=lambda c: c.id, ) def test_normalize_chunks_1d_errors(case: ExpectFail[tuple[Any, int]]) -> None: """Invalid 1D chunk specifications are rejected with informative error messages.""" chunks, span = case.input with case.raises(): normalize_chunks_1d(chunks, span=span) @pytest.mark.parametrize( "case", [ # `None` and `True` mean auto-chunking only to the top-level API. ExpectFail( input=(None, (100,)), exception=ValueError, id="none", msg="None is not a valid chunk input", ), ExpectFail( input=(True, (100,)), exception=ValueError, id="true", msg="True is not a valid chunk input", ), # `False` means one chunk spanning the array only as the whole specification. ExpectFail( input=((False,), (100,)), exception=ValueError, id="false-size", msg="Chunk size must be positive or -1, got 0", ), ExpectFail(input=("foo", (100,)), exception=ValueError, id="string", msg="dimensions"), # A 0-d array is an integer only if its dtype is. ExpectFail( input=(np.array(2.0), (100,)), exception=TypeError, id="0-d-float-array", msg="must be an integer or an iterable of integers", ), ExpectFail( input=((100, 10), (100,)), exception=ValueError, id="too-many-dims", msg="dimensions" ), ExpectFail( input=((10,), (100, 100)), exception=ValueError, id="too-few-dims", msg="dimensions" ), # End-to-end: per-dim RLE surfaces through normalize_chunks_nd. ExpectFail( input=([[6, 4], [[3, 3], 1]], (10, 10)), exception=TypeError, id="rle-inner-dim", msg="non-integer element(s) ([3, 3],) at indices (0,)", escape=True, ), ], ids=lambda c: c.id, ) def test_normalize_chunks_nd_errors(case: ExpectFail[tuple[Any, tuple[int, ...]]]) -> None: """Invalid N-D chunk specifications are rejected with informative error messages.""" chunks, shape = case.input with case.raises(): normalize_chunks_nd(chunks, shape) @pytest.mark.parametrize( "chunks", [ np.True_, np.array(True), np.array([True, True]), (np.True_, 5), (np.array(True), 5), [[np.True_, 9], 10], ], ids=repr, ) def test_normalize_chunks_nd_rejects_numpy_bool(chunks: Any) -> None: """A numpy boolean is not an integer, whether it is the whole specification, a dimension's size, or an edge in a dimension's list.""" with pytest.raises(TypeError, match="integer"): normalize_chunks_nd(chunks, (10, 10)) @pytest.mark.parametrize( "case", [ # uniform-chunks branch: O(1) size+extent record, never one entry per chunk. Expect( input=(1000, 100_000), output=FixedDimension(size=1000, extent=100_000), id="uniform" ), # uniform chunks on a span too large to expand per-chunk (creation-time # counterpart of the gh-4174 indexing fix). Expect(input=(1, 2**62), output=FixedDimension(size=1, extent=2**62), id="uniform-huge"), # -1 sentinel branch: one chunk covering the full span. Expect(input=(-1, 100), output=FixedDimension(size=100, extent=100), id="full-span"), # -1 on a zero-length span clamps to chunk size 1 (chunk sizes must be positive). Expect(input=(-1, 0), output=FixedDimension(size=1, extent=0), id="full-span-empty"), # zero-length span preserves the declared chunk size. Expect(input=(10, 0), output=FixedDimension(size=10, extent=0), id="uniform-zero-span"), # explicit lists always keep the per-chunk form, even when the sizes # describe a regular grid (gh-4272). Expect( input=([10, 10, 10], 30), output=VaryingDimension([10, 10, 10], extent=30), id="explicit-regular", ), Expect( input=([10, 10, 5], 25), output=VaryingDimension([10, 10, 5], extent=25), id="explicit-boundary", ), Expect( input=([10, 20, 70], 100), output=VaryingDimension([10, 20, 70], extent=100), id="explicit-irregular", ), # on a zero-length span any non-empty list of positive edges is kept: the # chunks the axis grows into. Expect( input=([3, 5], 0), output=VaryingDimension([3, 5], extent=0), id="explicit-zero-span" ), ], ids=lambda c: c.id, ) def test_normalize_chunks_1d( case: Expect[tuple[Any, int], FixedDimension | VaryingDimension], ) -> None: """Both output variants bind chunk sizes to the span: scalar specs become `FixedDimension` (O(1) regardless of chunk count), explicit per-chunk lists become `VaryingDimension`.""" chunks, span = case.input assert normalize_chunks_1d(chunks, span) == case.output @pytest.mark.parametrize( ("chunk_shape", "array_shape"), [((), ()), ((0,), (0,)), ((0, 0), (0, 0))], ids=["0d", "zero-1d", "zero-2d"], ) def test_guess_num_chunks_per_axis_shard_degenerate( chunk_shape: tuple[int, ...], array_shape: tuple[int, ...] ) -> None: """Degenerate chunk shapes must return 1 instead of hanging the search loop. Regression test for https://github.com/zarr-developers/zarr-python/issues/4304. """ assert ( _guess_num_chunks_per_axis_shard( chunk_shape=chunk_shape, item_size=8, max_bytes=128 * 1024 * 1024, array_shape=array_shape, ) == 1 ) def test_create_0d_array_auto_shards_with_target_shard_size() -> None: """A 0-dimensional array with shards="auto" and a shard size budget must not hang. Regression test for https://github.com/zarr-developers/zarr-python/issues/4304. """ with ( zarr.config.set({"array.target_shard_size_bytes": 128 * 1024 * 1024}), pytest.warns(ZarrUserWarning, match="Automatic shard shape inference is experimental"), ): arr = zarr.create_array(store={}, shape=(), dtype="int64", shards="auto") assert arr.shards == () # -- Zero-length dimensions -- @pytest.mark.parametrize( ("span", "unit", "expected"), [(0, 1, 1), (5, 1, 5), (0, 4, 4), (8, 4, 8), (10, 4, 12)], ) def test_full_span_chunk_size(span: int, unit: int, expected: int) -> None: """One chunk spanning an axis is the smallest positive multiple of `unit` covering it.""" assert full_span_chunk_size(span, unit) == expected @pytest.mark.parametrize("chunks", [-1, False, "auto"]) @pytest.mark.parametrize( "shape", [(0,), (0, 4), (4, 0), (0, 0), ()], ids=["1d", "2d-lead", "2d-trail", "2d-both", "0d"], ) @pytest.mark.parametrize( ("zarr_format", "shards", "target_shard_size_bytes"), [(2, None, None), (3, None, None), (3, "auto", None), (3, "auto", 128 * 1024 * 1024)], ids=["v2", "v3", "v3-auto-shards", "v3-auto-shards-budget"], ) def test_create_zero_length_array( chunks: Any, shape: tuple[int, ...], zarr_format: Literal[2, 3], shards: Literal["auto"] | None, target_shard_size_bytes: int | None, ) -> None: """Every spelling of one chunk spanning the axis gives chunk size 1 on a zero-length axis, and the array can grow along that axis and shrink back.""" expected = tuple(max(s, 1) for s in shape) warns = ( pytest.warns(ZarrUserWarning, match="Automatic shard shape inference is experimental") if shards == "auto" else contextlib.nullcontext() ) with zarr.config.set({"array.target_shard_size_bytes": target_shard_size_bytes}), warns: arr = zarr.create_array( store={}, shape=shape, dtype="int64", chunks=chunks, shards=shards, zarr_format=zarr_format, ) assert arr.chunks == expected assert arr.shards == (None if shards is None else expected) meta = cast(dict[str, Any], arr.metadata.to_dict()) stored = ( meta["chunks"] if zarr_format == 2 else meta["chunk_grid"]["configuration"]["chunk_shape"] ) assert tuple(stored) == expected if not shape: arr[...] = 7 assert arr[...] == 7 return axis = shape.index(0) grown = tuple(2 if i == axis else s for i, s in enumerate(shape)) data = np.full(grown, 7, dtype="int64") arr.append(data, axis=axis) np.testing.assert_array_equal(arr[...], data) arr.resize(shape) assert np.asarray(arr[...]).shape == shape @pytest.mark.parametrize( ("shape", "chunks", "shards", "expected"), [ ((0, 20), (5, 5), -1, (5, 20)), ((0,), (4,), False, (4,)), ((10,), (4,), -1, (12,)), ((8, 0), (4, 3), (-1, 6), (8, 6)), ], ) def test_create_full_span_shards( shape: tuple[int, ...], chunks: tuple[int, ...], shards: Any, expected: tuple[int, ...] ) -> None: """A shard spanning the axis is a multiple of the inner chunk, even on a zero-length axis.""" arr = zarr.create_array(store={}, shape=shape, chunks=chunks, shards=shards, dtype="int8") assert arr.shards == expected assert arr.chunks == chunks @pytest.mark.parametrize("zarr_format", [2, 3]) def test_create_zero_chunk_rejected(zarr_format: Literal[2, 3]) -> None: """An explicit chunk size of 0 is rejected up front, even for a zero-length axis.""" with pytest.raises(ValueError, match="Chunk size must be positive or -1, got 0"): zarr.create_array(store={}, shape=(0,), chunks=(0,), dtype="int64", zarr_format=zarr_format) def test_rectilinear_zero_extent_matches_resize() -> None: """Creating a rectilinear axis at length 0 equals resizing one down to 0.""" with zarr.config.set({"array.rectilinear_chunks": True}): created = zarr.create_array(store={}, shape=(0,), chunks=[[2, 2]], dtype="int64") resized = zarr.create_array(store={}, shape=(4,), chunks=[[2, 2]], dtype="int64") resized.resize((0,)) created_meta = cast(dict[str, Any], created.metadata.to_dict()) resized_meta = cast(dict[str, Any], resized.metadata.to_dict()) assert created_meta["chunk_grid"] == resized_meta["chunk_grid"] assert created_meta["shape"] == resized_meta["shape"] == (0,) created.append(np.arange(3, dtype="int64")) resized.append(np.arange(3, dtype="int64")) np.testing.assert_array_equal(created[...], np.arange(3)) np.testing.assert_array_equal(resized[...], np.arange(3)) assert created.write_chunk_sizes == resized.write_chunk_sizes == ((2, 1),) zarr-developers-zarr-python-a2f2415/tests/test_chunk_key_encodings.py000066400000000000000000000047131527014137600262560ustar00rootroot00000000000000from __future__ import annotations import pytest from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding, V2ChunkKeyEncoding @pytest.mark.parametrize("separator", ["/", "."]) @pytest.mark.parametrize( "coords", [(), (0,), (1, 2), (10, 0, 3)], ) def test_default_encoding_round_trips(separator: str, coords: tuple[int, ...]) -> None: """Encoding coordinates and decoding the result returns the coordinates.""" encoding = DefaultChunkKeyEncoding(separator=separator) # type: ignore[arg-type] key = encoding.encode_chunk_key(coords) assert encoding.decode_chunk_key(key) == coords @pytest.mark.parametrize("separator", ["/", "."]) @pytest.mark.parametrize("coords", [(0,), (1, 2), (10, 0, 3)]) def test_v2_encoding_round_trips(separator: str, coords: tuple[int, ...]) -> None: """The v2 encoding round-trips coordinates for either separator.""" encoding = V2ChunkKeyEncoding(separator=separator) # type: ignore[arg-type] key = encoding.encode_chunk_key(coords) assert encoding.decode_chunk_key(key) == coords @pytest.mark.parametrize("separator", ["/", "."]) def test_v2_zero_dimensional_key_is_ambiguous(separator: str) -> None: """A 0-d v2 array stores its sole chunk under `"0"`, the same key a 1-d array uses for chunk 0, so decoding cannot recover the empty tuple on its own -- the array's dimensionality is what disambiguates it.""" encoding = V2ChunkKeyEncoding(separator=separator) # type: ignore[arg-type] assert encoding.encode_chunk_key(()) == "0" assert encoding.decode_chunk_key("0") == (0,) @pytest.mark.parametrize( "chunk_key", [ "0/1", # no "c" prefix at all "c0/1", # "c" not followed by the separator "x/0/1", # wrong prefix character "", ], ) def test_default_encoding_rejects_key_without_prefix(chunk_key: str) -> None: """A key that does not carry the `c` prefix is not a chunk key for this encoding, and must be rejected rather than silently decoded.""" encoding = DefaultChunkKeyEncoding(separator="/") with pytest.raises(ValueError, match="Invalid chunk key"): encoding.decode_chunk_key(chunk_key) def test_default_encoding_rejects_key_using_the_other_separator() -> None: """A key encoded with `.` is not valid for a `/`-separated encoding.""" encoding = DefaultChunkKeyEncoding(separator="/") with pytest.raises(ValueError, match="Invalid chunk key"): encoding.decode_chunk_key("c.0.1") zarr-developers-zarr-python-a2f2415/tests/test_chunk_transform.py000066400000000000000000000132231527014137600254440ustar00rootroot00000000000000"""Unit tests for ChunkTransform -- the per-chunk synchronous codec chain. ChunkTransform is the data structure FusedCodecPipeline uses to encode/decode a single chunk through a sequence of codecs synchronously. These tests exercise it directly (no pipeline, no store): construction and its rejection of codecs that lack a synchronous implementation, encode/decode roundtrips across codec chains, compute_encoded_size, and None short-circuiting when an array->array codec returns None. End-to-end pipeline behavior lives in the pipeline test modules. """ from __future__ import annotations from typing import Any import numpy as np import pytest from zarr.abc.codec import ArrayBytesCodec, Codec from zarr.codecs.bytes import BytesCodec from zarr.codecs.crc32c_ import Crc32cCodec from zarr.codecs.gzip import GzipCodec from zarr.codecs.transpose import TransposeCodec from zarr.codecs.zstd import ZstdCodec from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer import Buffer, NDBuffer, default_buffer_prototype from zarr.core.chunk_utils import ChunkTransform from zarr.core.dtype import get_data_type_from_native_dtype class AsyncOnlyCodec(ArrayBytesCodec): """A codec that only supports async, for testing rejection of non-sync codecs.""" is_fixed_size = True async def _decode_single(self, chunk_data: Buffer, chunk_spec: ArraySpec) -> NDBuffer: raise NotImplementedError # pragma: no cover async def _encode_single(self, chunk_data: NDBuffer, chunk_spec: ArraySpec) -> Buffer | None: raise NotImplementedError # pragma: no cover def compute_encoded_size(self, input_byte_length: int, chunk_spec: ArraySpec) -> int: return input_byte_length # pragma: no cover def _make_array_spec(shape: tuple[int, ...], dtype: np.dtype[np.generic]) -> ArraySpec: zdtype = get_data_type_from_native_dtype(dtype) return ArraySpec( shape=shape, dtype=zdtype, fill_value=zdtype.cast_scalar(0), config=ArrayConfig(order="C", write_empty_chunks=True), prototype=default_buffer_prototype(), ) def _make_nd_buffer(arr: np.ndarray[Any, np.dtype[Any]]) -> NDBuffer: return default_buffer_prototype().nd_buffer.from_numpy_array(arr) @pytest.mark.parametrize( ("shape", "codecs"), [ ((100,), (BytesCodec(),)), ((100,), (BytesCodec(), GzipCodec())), ((3, 4), (TransposeCodec(order=(1, 0)), BytesCodec(), ZstdCodec())), ], ids=["bytes-only", "with-compression", "full-chain"], ) def test_construction(shape: tuple[int, ...], codecs: tuple[Codec, ...]) -> None: """Construction succeeds when all codecs implement SupportsSyncCodec.""" _ = _make_array_spec(shape, np.dtype("float64")) ChunkTransform(codecs=codecs) @pytest.mark.parametrize( ("shape", "codecs"), [ ((100,), (AsyncOnlyCodec(),)), ((3, 4), (TransposeCodec(order=(1, 0)), AsyncOnlyCodec())), ], ids=["async-only", "mixed-sync-and-async"], ) def test_construction_rejects_non_sync(shape: tuple[int, ...], codecs: tuple[Codec, ...]) -> None: """Construction raises TypeError when any codec lacks SupportsSyncCodec.""" _ = _make_array_spec(shape, np.dtype("float64")) with pytest.raises(TypeError, match="AsyncOnlyCodec"): ChunkTransform(codecs=codecs) @pytest.mark.parametrize( ("arr", "codecs"), [ (np.arange(100, dtype="float64"), (BytesCodec(),)), (np.arange(100, dtype="float64"), (BytesCodec(), GzipCodec(level=1))), ( np.arange(12, dtype="float64").reshape(3, 4), (TransposeCodec(order=(1, 0)), BytesCodec(), ZstdCodec(level=1)), ), (np.arange(100, dtype="float64"), (BytesCodec(), Crc32cCodec())), (np.arange(50, dtype="int32"), (BytesCodec(), ZstdCodec(level=1))), ], ids=["bytes-only", "gzip", "transpose+zstd", "crc32c", "int32"], ) def test_encode_decode_roundtrip( arr: np.ndarray[Any, np.dtype[Any]], codecs: tuple[Codec, ...] ) -> None: """Data survives a full encode/decode cycle.""" spec = _make_array_spec(arr.shape, arr.dtype) chain = ChunkTransform(codecs=codecs) nd_buf = _make_nd_buffer(arr) encoded = chain.encode_chunk(nd_buf, spec) assert encoded is not None decoded = chain.decode_chunk(encoded, spec) np.testing.assert_array_equal(arr, decoded.as_numpy_array()) @pytest.mark.parametrize( ("shape", "codecs", "input_size", "expected_size"), [ ((100,), (BytesCodec(),), 800, 800), ((100,), (BytesCodec(), Crc32cCodec()), 800, 804), ((3, 4), (TransposeCodec(order=(1, 0)), BytesCodec()), 96, 96), ], ids=["bytes-only", "crc32c", "transpose"], ) def test_compute_encoded_size( shape: tuple[int, ...], codecs: tuple[Codec, ...], input_size: int, expected_size: int, ) -> None: """compute_encoded_size returns the correct byte length.""" spec = _make_array_spec(shape, np.dtype("float64")) chain = ChunkTransform(codecs=codecs) assert chain.compute_encoded_size(input_size, spec) == expected_size def test_encode_returns_none_propagation() -> None: """When an AA codec returns None, encode short-circuits and returns None.""" class NoneReturningAACodec(TransposeCodec): """An ArrayArrayCodec that always returns None from encode.""" def _encode_sync(self, chunk_array: NDBuffer, chunk_spec: ArraySpec) -> NDBuffer | None: return None spec = _make_array_spec((3, 4), np.dtype("float64")) chain = ChunkTransform( codecs=(NoneReturningAACodec(order=(1, 0)), BytesCodec()), ) arr = np.arange(12, dtype="float64").reshape(3, 4) nd_buf = _make_nd_buffer(arr) assert chain.encode_chunk(nd_buf, spec) is None zarr-developers-zarr-python-a2f2415/tests/test_cli/000077500000000000000000000000001527014137600224355ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/test_cli/conftest.py000066400000000000000000000104231527014137600246340ustar00rootroot00000000000000from pathlib import Path from typing import Any, Literal import pytest import zarr from zarr.abc.store import Store from zarr.core.common import ZarrFormat def create_nested_zarr( store: Store, attributes: dict[str, Any] | None = None, separator: Literal[".", "/"] = ".", zarr_format: ZarrFormat = 2, ) -> list[str]: """Create a zarr with nested groups / arrays for testing, returning the paths to all.""" if attributes is None: attributes = {"baz": 42, "qux": [1, 4, 7, 12]} # 3 levels of nested groups group_0 = zarr.create_group(store=store, zarr_format=zarr_format, attributes=attributes) group_1 = group_0.create_group(name="group_1", attributes=attributes) group_2 = group_1.create_group(name="group_2", attributes=attributes) paths = [group_0.path, group_1.path, group_2.path] # 1 array per group for i, group in enumerate([group_0, group_1, group_2]): array = group.create_array( name=f"array_{i}", shape=(10, 10), chunks=(5, 5), dtype="uint16", attributes=attributes, chunk_key_encoding={"name": "v2", "separator": separator}, ) array[:] = 1 paths.append(array.path) return paths @pytest.fixture def expected_paths() -> list[Path]: """Expected paths for create_nested_zarr, with no metadata files or chunks""" return [ Path("array_0"), Path("group_1"), Path("group_1/array_1"), Path("group_1/group_2"), Path("group_1/group_2/array_2"), ] @pytest.fixture def expected_chunks() -> list[Path]: """Expected chunks for create_nested_zarr""" return [ Path("array_0/0.0"), Path("array_0/0.1"), Path("array_0/1.0"), Path("array_0/1.1"), Path("group_1/array_1/0.0"), Path("group_1/array_1/0.1"), Path("group_1/array_1/1.0"), Path("group_1/array_1/1.1"), Path("group_1/group_2/array_2/0.0"), Path("group_1/group_2/array_2/0.1"), Path("group_1/group_2/array_2/1.0"), Path("group_1/group_2/array_2/1.1"), ] @pytest.fixture def expected_v3_metadata() -> list[Path]: """Expected v3 metadata for create_nested_zarr""" return sorted( [ Path("zarr.json"), Path("array_0/zarr.json"), Path("group_1/zarr.json"), Path("group_1/array_1/zarr.json"), Path("group_1/group_2/zarr.json"), Path("group_1/group_2/array_2/zarr.json"), ] ) @pytest.fixture def expected_v2_metadata() -> list[Path]: """Expected v2 metadata for create_nested_zarr""" return sorted( [ Path(".zgroup"), Path(".zattrs"), Path("array_0/.zarray"), Path("array_0/.zattrs"), Path("group_1/.zgroup"), Path("group_1/.zattrs"), Path("group_1/array_1/.zarray"), Path("group_1/array_1/.zattrs"), Path("group_1/group_2/.zgroup"), Path("group_1/group_2/.zattrs"), Path("group_1/group_2/array_2/.zarray"), Path("group_1/group_2/array_2/.zattrs"), ] ) @pytest.fixture def expected_paths_no_metadata( expected_paths: list[Path], expected_chunks: list[Path] ) -> list[Path]: return sorted(expected_paths + expected_chunks) @pytest.fixture def expected_paths_v3_metadata( expected_paths: list[Path], expected_chunks: list[Path], expected_v3_metadata: list[Path] ) -> list[Path]: return sorted(expected_paths + expected_chunks + expected_v3_metadata) @pytest.fixture def expected_paths_v3_metadata_no_chunks( expected_paths: list[Path], expected_v3_metadata: list[Path] ) -> list[Path]: return sorted(expected_paths + expected_v3_metadata) @pytest.fixture def expected_paths_v2_metadata( expected_paths: list[Path], expected_chunks: list[Path], expected_v2_metadata: list[Path] ) -> list[Path]: return sorted(expected_paths + expected_chunks + expected_v2_metadata) @pytest.fixture def expected_paths_v2_v3_metadata( expected_paths: list[Path], expected_chunks: list[Path], expected_v2_metadata: list[Path], expected_v3_metadata: list[Path], ) -> list[Path]: return sorted(expected_paths + expected_chunks + expected_v2_metadata + expected_v3_metadata) zarr-developers-zarr-python-a2f2415/tests/test_cli/test_migrate_v3.py000066400000000000000000000543511527014137600261160ustar00rootroot00000000000000import lzma from pathlib import Path from typing import Literal, cast import numcodecs import numcodecs.abc import numpy as np import pytest import zarr from tests.test_cli.conftest import create_nested_zarr from zarr.abc.codec import Codec from zarr.codecs.blosc import BloscCodec from zarr.codecs.bytes import BytesCodec from zarr.codecs.gzip import GzipCodec from zarr.codecs.numcodecs import LZMA, Delta from zarr.codecs.transpose import TransposeCodec from zarr.codecs.zstd import ZstdCodec from zarr.core.chunk_key_encodings import V2ChunkKeyEncoding from zarr.core.common import JSON, ZarrFormat from zarr.core.dtype.npy.int import UInt8, UInt16 from zarr.core.group import Group, GroupMetadata from zarr.core.metadata.v3 import ArrayV3Metadata from zarr.storage._local import LocalStore from zarr.types import AnyArray typer_testing = pytest.importorskip( "typer.testing", reason="optional cli dependencies aren't installed" ) cli = pytest.importorskip("zarr._cli.cli", reason="optional cli dependencies aren't installed") runner = typer_testing.CliRunner() def test_migrate_array(local_store: LocalStore) -> None: shape = (10, 10) chunks = (10, 10) dtype = " None: attributes = {"baz": 42, "qux": [1, 4, 7, 12]} zarr.create_group(store=local_store, zarr_format=2, attributes=attributes) result = runner.invoke(cli.app, ["migrate", "v3", str(local_store.root)]) assert result.exit_code == 0 assert (local_store.root / "zarr.json").exists() zarr_array = zarr.open(local_store.root, zarr_format=3) expected_metadata = GroupMetadata( attributes=attributes, zarr_format=3, consolidated_metadata=None ) assert zarr_array.metadata == expected_metadata @pytest.mark.parametrize("separator", [".", "/"]) def test_migrate_nested_groups_and_arrays_in_place( local_store: LocalStore, separator: str, expected_v3_metadata: list[Path] ) -> None: """Test that zarr.json are made at the correct points in a hierarchy of groups and arrays (including when there are additional dirs due to using a / separator)""" attributes = {"baz": 42, "qux": [1, 4, 7, 12]} paths = create_nested_zarr(local_store, attributes=attributes, separator=separator) result = runner.invoke(cli.app, ["migrate", "v3", str(local_store.root)]) assert result.exit_code == 0 zarr_json_paths = sorted(local_store.root.rglob("zarr.json")) expected_zarr_json_paths = [local_store.root / p for p in expected_v3_metadata] assert zarr_json_paths == expected_zarr_json_paths # Check converted zarr can be opened + metadata accessed at all levels zarr_array = zarr.open(local_store.root, zarr_format=3) for path in paths: zarr_v3 = cast(AnyArray | Group, zarr_array[path]) metadata = zarr_v3.metadata assert metadata.zarr_format == 3 assert metadata.attributes == attributes @pytest.mark.parametrize("separator", [".", "/"]) async def test_migrate_nested_groups_and_arrays_separate_location( tmp_path: Path, separator: str, expected_v2_metadata: list[Path], expected_v3_metadata: list[Path], ) -> None: """Test that zarr.json are made at the correct paths, when saving to a separate output location.""" input_zarr_path = tmp_path / "input.zarr" output_zarr_path = tmp_path / "output.zarr" local_store = await LocalStore.open(str(input_zarr_path)) create_nested_zarr(local_store, separator=separator) result = runner.invoke(cli.app, ["migrate", "v3", str(input_zarr_path), str(output_zarr_path)]) assert result.exit_code == 0 # Files in input zarr should be unchanged i.e. still v2 only zarr_json_paths = sorted(input_zarr_path.rglob("zarr.json")) assert len(zarr_json_paths) == 0 paths = [ path for path in input_zarr_path.rglob("*") if path.stem in [".zarray", ".zgroup", ".zattrs"] ] expected_paths = [input_zarr_path / p for p in expected_v2_metadata] assert sorted(paths) == expected_paths # Files in output zarr should only contain v3 metadata zarr_json_paths = sorted(output_zarr_path.rglob("zarr.json")) expected_zarr_json_paths = [output_zarr_path / p for p in expected_v3_metadata] assert zarr_json_paths == expected_zarr_json_paths def test_remove_v2_metadata_option_in_place( local_store: LocalStore, expected_paths_v3_metadata: list[Path] ) -> None: create_nested_zarr(local_store) # convert v2 metadata to v3, then remove v2 metadata result = runner.invoke( cli.app, ["migrate", "v3", str(local_store.root), "--remove-v2-metadata"] ) assert result.exit_code == 0 paths = sorted(local_store.root.rglob("*")) expected_paths = [local_store.root / p for p in expected_paths_v3_metadata] assert paths == expected_paths async def test_remove_v2_metadata_option_separate_location( tmp_path: Path, expected_paths_v2_metadata: list[Path], expected_paths_v3_metadata_no_chunks: list[Path], ) -> None: """Check that when using --remove-v2-metadata with a separate output location, no v2 metadata is removed from the input location.""" input_zarr_path = tmp_path / "input.zarr" output_zarr_path = tmp_path / "output.zarr" local_store = await LocalStore.open(str(input_zarr_path)) create_nested_zarr(local_store) result = runner.invoke( cli.app, ["migrate", "v3", str(input_zarr_path), str(output_zarr_path), "--remove-v2-metadata"], ) assert result.exit_code == 0 # input image should be unchanged paths = sorted(input_zarr_path.rglob("*")) expected_paths = [input_zarr_path / p for p in expected_paths_v2_metadata] assert paths == expected_paths # output image should be only v3 metadata paths = sorted(output_zarr_path.rglob("*")) expected_paths = [output_zarr_path / p for p in expected_paths_v3_metadata_no_chunks] assert paths == expected_paths def test_overwrite_option_in_place( local_store: LocalStore, expected_paths_v2_v3_metadata: list[Path] ) -> None: create_nested_zarr(local_store) # add v3 metadata in place result = runner.invoke(cli.app, ["migrate", "v3", str(local_store.root)]) assert result.exit_code == 0 # check that v3 metadata can be overwritten with --overwrite result = runner.invoke(cli.app, ["migrate", "v3", str(local_store.root), "--overwrite"]) assert result.exit_code == 0 paths = sorted(local_store.root.rglob("*")) expected_paths = [local_store.root / p for p in expected_paths_v2_v3_metadata] assert paths == expected_paths async def test_overwrite_option_separate_location( tmp_path: Path, expected_paths_v2_metadata: list[Path], expected_paths_v3_metadata_no_chunks: list[Path], ) -> None: input_zarr_path = tmp_path / "input.zarr" output_zarr_path = tmp_path / "output.zarr" local_store = await LocalStore.open(str(input_zarr_path)) create_nested_zarr(local_store) # create v3 metadata at output_zarr_path result = runner.invoke( cli.app, ["migrate", "v3", str(input_zarr_path), str(output_zarr_path)], ) assert result.exit_code == 0 # re-run with --overwrite option result = runner.invoke( cli.app, ["migrate", "v3", str(input_zarr_path), str(output_zarr_path), "--overwrite", "--force"], ) assert result.exit_code == 0 # original image should be un-changed paths = sorted(input_zarr_path.rglob("*")) expected_paths = [input_zarr_path / p for p in expected_paths_v2_metadata] assert paths == expected_paths # output image is only v3 metadata paths = sorted(output_zarr_path.rglob("*")) expected_paths = [output_zarr_path / p for p in expected_paths_v3_metadata_no_chunks] assert paths == expected_paths @pytest.mark.parametrize("separator", [".", "/"]) def test_migrate_sub_group( local_store: LocalStore, separator: str, expected_v3_metadata: list[Path] ) -> None: """Test that only arrays/groups within group_1 are converted (+ no other files in store)""" create_nested_zarr(local_store, separator=separator) group_path = local_store.root / "group_1" result = runner.invoke(cli.app, ["migrate", "v3", str(group_path)]) assert result.exit_code == 0 zarr_json_paths = sorted(local_store.root.rglob("zarr.json")) expected_zarr_json_paths = [ local_store.root / p for p in expected_v3_metadata if group_path in (local_store.root / p).parents ] assert zarr_json_paths == expected_zarr_json_paths @pytest.mark.parametrize( ("compressor_v2", "compressor_v3"), [ ( numcodecs.Blosc(cname="zstd", clevel=3, shuffle=1), BloscCodec(typesize=2, cname="zstd", clevel=3, shuffle="shuffle", blocksize=0), ), (numcodecs.Zstd(level=3), ZstdCodec(level=3)), (numcodecs.GZip(level=3), GzipCodec(level=3)), ], ids=["blosc", "zstd", "gzip"], ) def test_migrate_compressor( local_store: LocalStore, compressor_v2: numcodecs.abc.Codec, compressor_v3: Codec ) -> None: zarr_array = zarr.create_array( store=local_store, shape=(10, 10), chunks=(10, 10), dtype=" None: """Test migration of a numcodecs compressor without a zarr.codecs equivalent.""" lzma_settings = { "format": lzma.FORMAT_RAW, "check": -1, "preset": None, "filters": [ {"id": lzma.FILTER_DELTA, "dist": 4}, {"id": lzma.FILTER_LZMA2, "preset": 1}, ], } zarr_array = zarr.create_array( store=local_store, shape=(10, 10), chunks=(10, 10), dtype=" None: filter_v2 = numcodecs.Delta(dtype=" None: zarr_array = zarr.create_array( store=local_store, shape=(10, 10), chunks=(10, 10), dtype="u2", UInt16(), (BytesCodec(endian="big"),)), ], ids=["single_byte", "little_endian", "big_endian"], ) def test_migrate_endian( local_store: LocalStore, dtype: str, expected_data_type: UInt8 | UInt16, expected_codecs: tuple[Codec], ) -> None: zarr_array = zarr.create_array( store=local_store, shape=(10, 10), chunks=(10, 10), dtype=dtype, compressors=None, zarr_format=2, fill_value=0, ) zarr_array[:] = 1 result = runner.invoke(cli.app, ["migrate", "v3", str(local_store.root)]) assert result.exit_code == 0 assert (local_store.root / "zarr.json").exists() zarr_array = zarr.open_array(local_store.root, zarr_format=3) metadata = zarr_array.metadata assert metadata.zarr_format == 3 assert metadata.data_type == expected_data_type assert metadata.codecs == expected_codecs assert np.all(zarr_array[:] == 1) @pytest.mark.parametrize("node_type", ["array", "group"]) def test_migrate_v3(local_store: LocalStore, node_type: str) -> None: """Attempting to convert a v3 array/group should always fail""" if node_type == "array": zarr.create_array( store=local_store, shape=(10, 10), chunks=(10, 10), zarr_format=3, dtype="uint16" ) else: zarr.create_group(store=local_store, zarr_format=3) result = runner.invoke(cli.app, ["migrate", "v3", str(local_store.root)]) assert result.exit_code == 1 assert isinstance(result.exception, TypeError) assert str(result.exception) == "Only arrays / groups with zarr v2 metadata can be converted" def test_migrate_consolidated_metadata(local_store: LocalStore) -> None: """Attempting to convert a group with consolidated metadata should always fail""" group = zarr.create_group(store=local_store, zarr_format=2) group.create_array(shape=(1,), name="a", dtype="uint8") zarr.consolidate_metadata(local_store) result = runner.invoke(cli.app, ["migrate", "v3", str(local_store.root)]) assert result.exit_code == 1 assert isinstance(result.exception, NotImplementedError) assert str(result.exception) == "Migration of consolidated metadata isn't supported." def test_migrate_unknown_codec(local_store: LocalStore) -> None: """Attempting to convert a codec without a v3 equivalent should always fail""" zarr.create_array( store=local_store, shape=(10, 10), chunks=(10, 10), dtype="uint16", filters=[numcodecs.Categorize(labels=["a", "b"], dtype=object)], zarr_format=2, fill_value=0, ) result = runner.invoke(cli.app, ["migrate", "v3", str(local_store.root)]) assert result.exit_code == 1 assert isinstance(result.exception, ValueError) assert ( str(result.exception) == "Couldn't find corresponding zarr.codecs.numcodecs codec for categorize" ) def test_migrate_incorrect_filter(local_store: LocalStore) -> None: """Attempting to convert a filter (which is the wrong type of codec) should always fail""" zarr.create_array( store=local_store, shape=(10, 10), chunks=(10, 10), dtype="uint16", filters=[numcodecs.Zstd(level=3)], zarr_format=2, fill_value=0, ) result = runner.invoke(cli.app, ["migrate", "v3", str(local_store.root)]) assert result.exit_code == 1 assert isinstance(result.exception, TypeError) assert ( str(result.exception) == "Filter is not an ArrayArrayCodec" ) def test_migrate_incorrect_compressor(local_store: LocalStore) -> None: """Attempting to convert a compressor (which is the wrong type of codec) should always fail""" zarr.create_array( store=local_store, shape=(10, 10), chunks=(10, 10), dtype="uint16", compressors=numcodecs.Delta(dtype=" is not a BytesBytesCodec" ) @pytest.mark.parametrize("zarr_format", [2, 3]) def test_remove_metadata_fails_without_force( local_store: LocalStore, zarr_format: ZarrFormat ) -> None: """Test removing metadata (when no alternate metadata is present) fails without --force.""" create_nested_zarr(local_store, zarr_format=zarr_format) result = runner.invoke(cli.app, ["remove-metadata", f"v{zarr_format}", str(local_store.root)]) assert result.exit_code == 1 assert isinstance(result.exception, ValueError) assert str(result.exception).startswith(f"Cannot remove v{zarr_format} metadata at file") @pytest.mark.parametrize("zarr_format", [2, 3]) def test_remove_metadata_succeeds_with_force( local_store: LocalStore, zarr_format: ZarrFormat, expected_paths_no_metadata: list[Path] ) -> None: """Test removing metadata (when no alternate metadata is present) succeeds with --force.""" create_nested_zarr(local_store, zarr_format=zarr_format) result = runner.invoke( cli.app, ["remove-metadata", f"v{zarr_format}", str(local_store.root), "--force"] ) assert result.exit_code == 0 paths = sorted(local_store.root.rglob("*")) expected_paths = [local_store.root / p for p in expected_paths_no_metadata] assert paths == expected_paths def test_remove_metadata_sub_group( local_store: LocalStore, expected_paths_no_metadata: list[Path] ) -> None: """Test only v2 metadata within group_1 is removed and rest remains un-changed.""" create_nested_zarr(local_store) result = runner.invoke( cli.app, ["remove-metadata", "v2", str(local_store.root / "group_1"), "--force"] ) assert result.exit_code == 0 # check all metadata files inside group_1 are removed (.zattrs / .zgroup / .zarray should remain only inside the top # group) paths = sorted(local_store.root.rglob("*")) expected_paths = [local_store.root / p for p in expected_paths_no_metadata] expected_paths.append(local_store.root / ".zattrs") expected_paths.append(local_store.root / ".zgroup") expected_paths.append(local_store.root / "array_0" / ".zarray") expected_paths.append(local_store.root / "array_0" / ".zattrs") assert paths == sorted(expected_paths) @pytest.mark.parametrize( ("zarr_format", "expected_output_paths"), [("v2", "expected_paths_v3_metadata"), ("v3", "expected_paths_v2_metadata")], ) def test_remove_metadata_after_conversion( local_store: LocalStore, request: pytest.FixtureRequest, zarr_format: str, expected_output_paths: str, ) -> None: """Test all v2/v3 metadata can be removed after metadata conversion (all groups / arrays / metadata of other versions should remain as-is)""" create_nested_zarr(local_store) # convert v2 metadata to v3 (so now both v2 and v3 metadata present!), then remove either the v2 or v3 metadata result = runner.invoke(cli.app, ["migrate", "v3", str(local_store.root)]) assert result.exit_code == 0 result = runner.invoke(cli.app, ["remove-metadata", zarr_format, str(local_store.root)]) assert result.exit_code == 0 paths = sorted(local_store.root.rglob("*")) expected_paths = request.getfixturevalue(expected_output_paths) expected_paths = [local_store.root / p for p in expected_paths] assert paths == expected_paths @pytest.mark.parametrize("cli_command", ["migrate", "remove-metadata"]) def test_dry_run( local_store: LocalStore, cli_command: str, expected_paths_v2_metadata: list[Path] ) -> None: """Test that all files are un-changed after a dry run""" create_nested_zarr(local_store) if cli_command == "migrate": result = runner.invoke( cli.app, ["migrate", "v3", str(local_store.root), "--overwrite", "--force", "--dry-run"] ) else: result = runner.invoke( cli.app, ["remove-metadata", "v2", str(local_store.root), "--force", "--dry-run"] ) assert result.exit_code == 0 paths = sorted(local_store.root.rglob("*")) expected_paths = [local_store.root / p for p in expected_paths_v2_metadata] assert paths == expected_paths zarr-developers-zarr-python-a2f2415/tests/test_coalesce.py000066400000000000000000000577731527014137600240410ustar00rootroot00000000000000# tests/test_coalesce.py from __future__ import annotations import asyncio from dataclasses import dataclass, field from typing import TYPE_CHECKING import pytest from zarr.abc.store import ( ByteRequest, OffsetByteRequest, RangeByteRequest, SuffixByteRequest, ) from zarr.core._coalesce import ( coalesce_ranges, coalesced_get, ) from zarr.core.buffer import Buffer, default_buffer_prototype if TYPE_CHECKING: from collections.abc import AsyncIterator, Callable, Mapping, Sequence def _buf(data: bytes) -> Buffer: return default_buffer_prototype().buffer.from_bytes(data) @dataclass class FakeFetch: """Records every call and serves canned bytes from an in-memory blob.""" blob: bytes key_exists: bool = True raise_on: Callable[[ByteRequest | None], bool] | None = None calls: list[ByteRequest | None] = field(default_factory=list) async def __call__(self, byte_range: ByteRequest | None) -> Buffer | None: self.calls.append(byte_range) if not self.key_exists: return None if self.raise_on is not None and self.raise_on(byte_range): raise OSError("injected") if byte_range is None: return _buf(self.blob) if isinstance(byte_range, RangeByteRequest): return _buf(self.blob[byte_range.start : byte_range.end]) if isinstance(byte_range, OffsetByteRequest): return _buf(self.blob[byte_range.offset :]) if isinstance(byte_range, SuffixByteRequest): return _buf(self.blob[-byte_range.suffix :]) raise AssertionError(f"unknown byte_range {byte_range!r}") async def _collect( agen: AsyncIterator[Sequence[tuple[int, Buffer | None]]], ) -> list[list[tuple[int, Buffer | None]]]: """Drain an async generator of groups into a list of lists of tuples.""" return [list(group) async for group in agen] def _contents(groups: list[list[tuple[int, Buffer | None]]]) -> dict[int, bytes]: """Flatten to {index: bytes}.""" result: dict[int, bytes] = {} for group in groups: for idx, buf in group: assert buf is not None result[idx] = buf.to_bytes() return result # --------------------------------------------------------------------------- # Shared coalescing-knob bundles. Each is a complete mapping of all three # kwargs to splat into `coalesced_get`; `coalesce_ranges` ignores # `max_concurrency`. The leaf functions in `_coalesce.py` require all knobs # explicitly — `Store.get_ranges` is the public entry point and owns the # canonical defaults. Tests pick their own values appropriate to the scenario. # --------------------------------------------------------------------------- # Permissive default for tests that don't care about specific thresholds. Mirrors # `Store.get_ranges`'s public defaults but the test file owns this independently # of any production constants. DEFAULT: Mapping[str, int] = { "max_concurrency": 10, "max_gap_bytes": 1 << 20, "max_coalesced_bytes": 16 << 20, } """Permissive defaults; mirrors `Store.get_ranges`'s baseline.""" MERGE_GAP_50: Mapping[str, int] = { "max_concurrency": 10, "max_gap_bytes": 50, "max_coalesced_bytes": 1 << 20, } """Merge ranges within 50 bytes of each other.""" NO_MERGE: Mapping[str, int] = { "max_concurrency": 10, "max_gap_bytes": -1, "max_coalesced_bytes": 1 << 20, } """No merging: any positive gap is > -1, so no pair ever coalesces.""" CAP_50: Mapping[str, int] = { "max_concurrency": 10, "max_gap_bytes": 1000, "max_coalesced_bytes": 50, } """Gap permissive but merged size capped at 50 bytes.""" def _grouping(opts: Mapping[str, int]) -> dict[str, int]: """Return only the grouping knobs from a full options bundle. `coalesce_ranges` rejects `max_concurrency`; this lets test bundles be full kwargs maps (for `coalesced_get`) and still be passed to the pure planner via splat. """ return {k: v for k, v in opts.items() if k != "max_concurrency"} # A deterministic blob used for content-sensitive cases: byte i == (i % 256). _INDEXED_BLOB = bytes(i % 256 for i in range(10_000)) # --------------------------------------------------------------------------- # Parametrized structural/content tests (cases without async timing or errors). # --------------------------------------------------------------------------- @dataclass(frozen=True) class StructuralCase: """One row of the parametrized structure-and-contents table.""" id: str """pytest id for the case.""" ranges: list[ByteRequest | None] """Input to coalesced_get.""" options: Mapping[str, int] """Coalescing knobs to splat into `coalesced_get`.""" expected_group_sizes: list[int] """Sorted list of group tuple-counts (order-independent).""" expected_contents: dict[int, bytes] | None = None """{input_index: bytes} to verify bytes, or None to skip the content check.""" expected_n_fetches: int | None = None """Exact number of calls to the fetch callable, or None to skip the check.""" _STRUCTURAL_CASES: list[StructuralCase] = [ StructuralCase( id="empty-input", ranges=[], options=DEFAULT, expected_group_sizes=[], expected_n_fetches=0, ), StructuralCase( id="single-range", ranges=[RangeByteRequest(2, 5)], options=DEFAULT, expected_group_sizes=[1], expected_contents={0: _INDEXED_BLOB[2:5]}, expected_n_fetches=1, ), StructuralCase( id="disjoint-3-no-merge", ranges=[ RangeByteRequest(0, 10), RangeByteRequest(200, 210), RangeByteRequest(500, 510), ], options=MERGE_GAP_50, expected_group_sizes=[1, 1, 1], expected_contents={ 0: _INDEXED_BLOB[0:10], 1: _INDEXED_BLOB[200:210], 2: _INDEXED_BLOB[500:510], }, expected_n_fetches=3, ), StructuralCase( id="adjacent-3-one-merged-group", ranges=[ RangeByteRequest(0, 5), RangeByteRequest(10, 15), RangeByteRequest(20, 25), ], options=MERGE_GAP_50, expected_group_sizes=[3], expected_contents={ 0: _INDEXED_BLOB[0:5], 1: _INDEXED_BLOB[10:15], 2: _INDEXED_BLOB[20:25], }, expected_n_fetches=1, ), StructuralCase( id="two-clusters-one-singleton", ranges=[ RangeByteRequest(0, 10), RangeByteRequest(20, 30), RangeByteRequest(500, 510), ], options=MERGE_GAP_50, expected_group_sizes=[1, 2], expected_contents={ 0: _INDEXED_BLOB[0:10], 1: _INDEXED_BLOB[20:30], 2: _INDEXED_BLOB[500:510], }, expected_n_fetches=2, ), StructuralCase( id="uncoalescable-mixed-with-range", ranges=[ RangeByteRequest(0, 3), OffsetByteRequest(5), SuffixByteRequest(2), None, ], options=DEFAULT, expected_group_sizes=[1, 1, 1, 1], expected_contents={ 0: _INDEXED_BLOB[0:3], 1: _INDEXED_BLOB[5:], 2: _INDEXED_BLOB[-2:], 3: _INDEXED_BLOB, }, expected_n_fetches=4, ), StructuralCase( id="shuffled-input-indices-preserved", ranges=[ RangeByteRequest(500, 510), RangeByteRequest(0, 10), RangeByteRequest(200, 210), RangeByteRequest(300, 310), ], options=MERGE_GAP_50, expected_group_sizes=[1, 1, 1, 1], expected_contents={ 0: _INDEXED_BLOB[500:510], 1: _INDEXED_BLOB[0:10], 2: _INDEXED_BLOB[200:210], 3: _INDEXED_BLOB[300:310], }, expected_n_fetches=4, ), StructuralCase( id="cap-prevents-merge-of-close-ranges", # 20 + 20 gap + 20 = 60-byte merged span > cap of 50. ranges=[RangeByteRequest(0, 20), RangeByteRequest(40, 60)], options=CAP_50, expected_group_sizes=[1, 1], expected_n_fetches=2, ), StructuralCase( id="single-range-larger-than-cap-passes-through", # Cap only applies to MERGE decisions; a lone oversized range still fetches. ranges=[RangeByteRequest(0, 200)], options=CAP_50, expected_group_sizes=[1], expected_contents={0: _INDEXED_BLOB[0:200]}, expected_n_fetches=1, ), ] @pytest.mark.parametrize("case", _STRUCTURAL_CASES, ids=lambda c: c.id) async def test_coalescing_structure_and_contents(case: StructuralCase) -> None: """Group structure, byte contents, and fetch-call count for the deterministic cases.""" fetch = FakeFetch(_INDEXED_BLOB) groups = await _collect(coalesced_get(fetch, case.ranges, **case.options)) assert sorted(len(g) for g in groups) == sorted(case.expected_group_sizes) if case.expected_contents is not None: assert _contents(groups) == case.expected_contents if case.expected_n_fetches is not None: assert len(fetch.calls) == case.expected_n_fetches # --------------------------------------------------------------------------- # Focused non-parametrized tests for cases with distinctive assertion shapes. # --------------------------------------------------------------------------- async def test_within_group_ordering_is_start_offset() -> None: """Within a merged group, tuples are ordered by start offset, not input order.""" fetch = FakeFetch(_INDEXED_BLOB) # Two ranges that merge; one has a later start but is listed first in input. ranges: list[ByteRequest | None] = [RangeByteRequest(20, 25), RangeByteRequest(0, 5)] groups = await _collect(coalesced_get(fetch, ranges, **MERGE_GAP_50)) assert len(groups) == 1 # Input index 1 (start=0) comes first, then 0 (start=20). assert [idx for idx, _ in groups[0]] == [1, 0] async def test_adjacent_ranges_fire_single_fetch_spanning_merged_region() -> None: """Verify the merged fetch covers exactly the span from min-start to max-end.""" fetch = FakeFetch(_INDEXED_BLOB) ranges: list[ByteRequest | None] = [ RangeByteRequest(0, 5), RangeByteRequest(10, 15), RangeByteRequest(20, 25), ] await _collect(coalesced_get(fetch, ranges, **MERGE_GAP_50)) assert len(fetch.calls) == 1 call = fetch.calls[0] assert isinstance(call, RangeByteRequest) assert call.start == 0 assert call.end == 25 # --------------------------------------------------------------------------- # Concurrency and cancellation. # --------------------------------------------------------------------------- async def test_max_concurrency_is_honored() -> None: """With 10 non-mergeable ranges and max_concurrency=3, peak in-flight must not exceed 3.""" in_flight = 0 peak = 0 lock = asyncio.Lock() async def fetch(byte_range: ByteRequest | None) -> Buffer | None: nonlocal in_flight, peak async with lock: in_flight += 1 peak = max(peak, in_flight) # give the scheduler a chance to run other tasks await asyncio.sleep(0.01) async with lock: in_flight -= 1 return _buf(b"x") ranges: list[ByteRequest | None] = [RangeByteRequest(i * 1000, i * 1000 + 1) for i in range(10)] opts: Mapping[str, int] = { "max_gap_bytes": 0, # force no merging "max_coalesced_bytes": 1 << 20, "max_concurrency": 3, } async for _group in coalesced_get(fetch, ranges, **opts): pass assert peak <= 3 assert peak >= 2 # must have been some real concurrency async def test_consumer_break_cancels_pending_fetches() -> None: """Breaking out of the async for should cancel pending fetches rather than let them complete.""" completed_calls = 0 cancelled_calls = 0 async def fetch(byte_range: ByteRequest | None) -> Buffer | None: nonlocal completed_calls, cancelled_calls assert isinstance(byte_range, RangeByteRequest) start = byte_range.start try: # First fetch returns fast so the async-for body runs and can break. # Later fetches sleep long enough that cancellation has room to land. await asyncio.sleep(0.001 if start == 0 else 2.0) except asyncio.CancelledError: cancelled_calls += 1 raise completed_calls += 1 return _buf(b"x") opts: Mapping[str, int] = { "max_gap_bytes": -1, # no merging "max_coalesced_bytes": 1 << 20, "max_concurrency": 3, } ranges: list[ByteRequest | None] = [RangeByteRequest(i * 1000, i * 1000 + 1) for i in range(6)] agen = coalesced_get(fetch, ranges, **opts) async for _group in agen: break # Explicitly close the generator so its finally block runs (cancelling # in-flight tasks) before we make assertions. await agen.aclose() # The fast task completes; the remaining tasks are either cancelled while # sleeping (raising CancelledError into the user try block) or cancelled # while still waiting on the semaphore (which doesn't enter the try at all). # Either way, none of them should be allowed to complete. assert completed_calls == 1 assert cancelled_calls >= 1 assert completed_calls + cancelled_calls <= len(ranges) # --------------------------------------------------------------------------- # Key-missing semantics. # --------------------------------------------------------------------------- async def test_key_missing_from_first_call_raises() -> None: """If the very first fetch returns None, the iterator raises an ExceptionGroup containing FileNotFoundError.""" fetch = FakeFetch(b"x" * 100, key_exists=False) ranges: list[ByteRequest | None] = [RangeByteRequest(0, 10), RangeByteRequest(20, 30)] with pytest.RaisesGroup(pytest.RaisesExc(FileNotFoundError)): await _collect(coalesced_get(fetch, ranges, **DEFAULT)) @pytest.mark.parametrize( "byte_range", [OffsetByteRequest(5), SuffixByteRequest(5), None], ids=["offset", "suffix", "none"], ) async def test_key_missing_on_uncoalescable_input_raises( byte_range: ByteRequest | None, ) -> None: """Uncoalescable inputs take a distinct path; key-missing must still raise (wrapped in a group).""" fetch = FakeFetch(b"x" * 100, key_exists=False) with pytest.RaisesGroup(pytest.RaisesExc(FileNotFoundError)): await _collect(coalesced_get(fetch, [byte_range], **DEFAULT)) async def test_key_missing_mid_stream_raises_after_earlier_groups() -> None: """If a later fetch returns None, earlier-completed groups yield before the raise.""" call_count = 0 async def fetch(byte_range: ByteRequest | None) -> Buffer | None: nonlocal call_count call_count += 1 # Deterministic: first call serves, second returns None. await asyncio.sleep(0.01 if call_count == 1 else 0.02) if call_count >= 2: return None return _buf(b"ok") opts: Mapping[str, int] = { "max_gap_bytes": -1, "max_coalesced_bytes": 1 << 20, "max_concurrency": 1, # serialize for determinism } ranges: list[ByteRequest | None] = [RangeByteRequest(0, 2), RangeByteRequest(100, 102)] agen = coalesced_get(fetch, ranges, **opts) first = await anext(agen) assert len(first) == 1 with pytest.RaisesGroup(pytest.RaisesExc(FileNotFoundError)): await anext(agen) async def test_key_missing_mid_stream_with_concurrency_cancels_late_arrivals() -> None: """ Under max_concurrency > 1, a mid-stream miss should raise FileNotFoundError and cancel still-in-flight unrelated tasks rather than wait for them. """ late_gate = asyncio.Event() miss_fired = asyncio.Event() # Driven by the test body after the first successful yield, so the miss # task can't race past the start=0 result. fire_miss = asyncio.Event() async def fetch(byte_range: ByteRequest | None) -> Buffer | None: assert isinstance(byte_range, RangeByteRequest) start = byte_range.start if start == 0: return _buf(b"ok") if start == 1000: # Wait for the test to give the green light before returning None. # This makes ordering deterministic regardless of scheduling. await asyncio.wait_for(fire_miss.wait(), timeout=5.0) miss_fired.set() return None # Late arrivals would block on this gate; they should be cancelled # before they ever return. await asyncio.wait_for(late_gate.wait(), timeout=5.0) return _buf(b"ok") opts: Mapping[str, int] = { "max_gap_bytes": -1, "max_coalesced_bytes": 1 << 20, "max_concurrency": 3, } ranges: list[ByteRequest | None] = [RangeByteRequest(i * 1000, i * 1000 + 1) for i in range(7)] agen = coalesced_get(fetch, ranges, **opts) first = await anext(agen) assert len(first) == 1 idx, buf = first[0] assert idx == 0 assert buf is not None # Now that #0 has yielded, signal the miss task to return None. fire_miss.set() with pytest.RaisesGroup(pytest.RaisesExc(FileNotFoundError)): await anext(agen) assert miss_fired.is_set() # Sanity: late_gate was never set, so the cancellation path is what completed the test. assert not late_gate.is_set() # --------------------------------------------------------------------------- # Error propagation. # --------------------------------------------------------------------------- async def test_fetch_raises_propagates() -> None: """An exception raised by fetch propagates on the yield that produced the failing group.""" fetch = FakeFetch( _INDEXED_BLOB, raise_on=lambda r: isinstance(r, RangeByteRequest) and r.start >= 100, ) opts: Mapping[str, int] = { "max_gap_bytes": -1, "max_coalesced_bytes": 1 << 20, "max_concurrency": 1, } ranges: list[ByteRequest | None] = [RangeByteRequest(0, 10), RangeByteRequest(200, 210)] with pytest.RaisesGroup(pytest.RaisesExc(OSError, match="injected")): await _collect(coalesced_get(fetch, ranges, **opts)) # --------------------------------------------------------------------------- # Property-style coverage invariant. # --------------------------------------------------------------------------- async def test_coverage_invariant_random_inputs() -> None: """For any random RangeByteRequest input, every input index appears exactly once.""" import random rng = random.Random(42) fetch = FakeFetch(_INDEXED_BLOB) ranges: list[ByteRequest | None] = [] for _ in range(50): start = rng.randint(0, 9000) length = rng.randint(1, 500) ranges.append(RangeByteRequest(start, start + length)) groups = await _collect(coalesced_get(fetch, ranges, **DEFAULT)) seen: list[int] = [idx for group in groups for idx, _buf in group] assert sorted(seen) == list(range(len(ranges))) flat = _contents(groups) for i, r in enumerate(ranges): assert isinstance(r, RangeByteRequest) assert flat[i] == _INDEXED_BLOB[r.start : r.end] # --------------------------------------------------------------------------- # Pure-function tests for coalesce_ranges (no async, no fetch). # --------------------------------------------------------------------------- def test_coalesce_ranges_empty_input() -> None: groups, uncoalescable = coalesce_ranges([], max_gap_bytes=1 << 20, max_coalesced_bytes=16 << 20) assert groups == [] assert uncoalescable == [] def test_coalesce_ranges_separates_coalescable_from_uncoalescable() -> None: ranges: list[ByteRequest | None] = [ RangeByteRequest(0, 10), OffsetByteRequest(100), SuffixByteRequest(5), None, RangeByteRequest(20, 30), ] groups, uncoalescable = coalesce_ranges(ranges, **_grouping(MERGE_GAP_50)) # Both range requests fall within MERGE_GAP_50's gap budget. assert len(groups) == 1 assert [idx for idx, _ in groups[0]] == [0, 4] # Non-RangeByteRequest entries preserve their original input indices. assert [(idx, type(req).__name__ if req else None) for idx, req in uncoalescable] == [ (1, "OffsetByteRequest"), (2, "SuffixByteRequest"), (3, None), ] def test_coalesce_ranges_no_merge_when_gap_exceeds_budget() -> None: ranges: list[ByteRequest | None] = [ RangeByteRequest(0, 10), RangeByteRequest(200, 210), RangeByteRequest(500, 510), ] groups, uncoalescable = coalesce_ranges(ranges, **_grouping(MERGE_GAP_50)) assert uncoalescable == [] assert [len(g) for g in groups] == [1, 1, 1] assert [idx for g in groups for idx, _ in g] == [0, 1, 2] def test_coalesce_ranges_merges_within_gap_budget() -> None: ranges: list[ByteRequest | None] = [ RangeByteRequest(0, 5), RangeByteRequest(10, 15), RangeByteRequest(20, 25), ] groups, _ = coalesce_ranges(ranges, **_grouping(MERGE_GAP_50)) assert len(groups) == 1 assert [idx for idx, _ in groups[0]] == [0, 1, 2] def test_coalesce_ranges_respects_max_coalesced_bytes() -> None: # Gap budget is permissive (1000), but the merged span would exceed CAP_50's # 50-byte cap, so the second range starts a new group. ranges: list[ByteRequest | None] = [ RangeByteRequest(0, 30), RangeByteRequest(40, 80), ] groups, _ = coalesce_ranges(ranges, **_grouping(CAP_50)) assert [len(g) for g in groups] == [1, 1] def test_coalesce_ranges_groups_are_sorted_by_start() -> None: """Input order is irrelevant; groups always emerge in start-offset order.""" ranges: list[ByteRequest | None] = [ RangeByteRequest(500, 510), RangeByteRequest(0, 10), RangeByteRequest(20, 30), RangeByteRequest(200, 210), ] groups, _ = coalesce_ranges(ranges, **_grouping(MERGE_GAP_50)) # First group is the {0-10, 20-30} cluster (from input indices 1, 2). # Then the {200-210} singleton, then {500-510}. flat = [idx for g in groups for idx, _ in g] assert flat == [1, 2, 3, 0] # Within each group, members are sorted by start. for g in groups: starts = [r.start for _, r in g] assert starts == sorted(starts) def test_coalesce_ranges_overlapping_ranges_merge() -> None: """Nested/overlapping ranges have a non-positive 'gap' and always merge.""" ranges: list[ByteRequest | None] = [ RangeByteRequest(0, 100), RangeByteRequest(50, 60), # nested RangeByteRequest(80, 120), # overlaps ] groups, _ = coalesce_ranges(ranges, **_grouping(MERGE_GAP_50)) assert len(groups) == 1 assert [idx for idx, _ in groups[0]] == [0, 1, 2] def test_coalesce_ranges_running_end_handles_nesting() -> None: """A subsequent range fully inside the running span must not extend group_end backwards.""" ranges: list[ByteRequest | None] = [ RangeByteRequest(0, 1000), # group_end=1000 RangeByteRequest(100, 200), # nested; group_end stays at 1000 RangeByteRequest(990, 1010), # gap = -10 from running end, still merges ] groups, _ = coalesce_ranges(ranges, **_grouping(MERGE_GAP_50)) assert len(groups) == 1 assert {idx for idx, _ in groups[0]} == {0, 1, 2} def test_coalesce_ranges_only_uncoalescable_inputs() -> None: ranges: list[ByteRequest | None] = [None, OffsetByteRequest(10), SuffixByteRequest(5)] groups, uncoalescable = coalesce_ranges( ranges, max_gap_bytes=1 << 20, max_coalesced_bytes=16 << 20 ) assert groups == [] assert [idx for idx, _ in uncoalescable] == [0, 1, 2] def test_coalesce_ranges_total_index_coverage() -> None: """Every input index appears exactly once across groups + uncoalescable.""" ranges: list[ByteRequest | None] = [ RangeByteRequest(0, 10), None, RangeByteRequest(15, 25), OffsetByteRequest(100), RangeByteRequest(30, 40), ] groups, uncoalescable = coalesce_ranges(ranges, **_grouping(MERGE_GAP_50)) seen = sorted([idx for g in groups for idx, _ in g] + [idx for idx, _ in uncoalescable]) assert seen == list(range(len(ranges))) zarr-developers-zarr-python-a2f2415/tests/test_codec_entrypoints.py000066400000000000000000000023011527014137600257670ustar00rootroot00000000000000import pytest import zarr.registry from zarr import config @pytest.mark.usefixtures("set_path") @pytest.mark.parametrize("codec_name", ["TestEntrypointCodec", "TestEntrypointGroup.Codec"]) def test_entrypoint_codec(codec_name: str) -> None: config.set({"codecs.test": f"package_with_entrypoint.{codec_name}"}) cls_test = zarr.registry.get_codec_class("test") assert cls_test.__qualname__ == codec_name @pytest.mark.usefixtures("set_path") def test_entrypoint_pipeline() -> None: config.set({"codec_pipeline.path": "package_with_entrypoint.TestEntrypointCodecPipeline"}) cls = zarr.registry.get_pipeline_class() assert cls.__name__ == "TestEntrypointCodecPipeline" @pytest.mark.usefixtures("set_path") @pytest.mark.parametrize("buffer_name", ["TestEntrypointBuffer", "TestEntrypointGroup.Buffer"]) def test_entrypoint_buffer(buffer_name: str) -> None: config.set( { "buffer": f"package_with_entrypoint.{buffer_name}", "ndbuffer": "package_with_entrypoint.TestEntrypointNDBuffer", } ) assert zarr.registry.get_buffer_class().__qualname__ == buffer_name assert zarr.registry.get_ndbuffer_class().__name__ == "TestEntrypointNDBuffer" zarr-developers-zarr-python-a2f2415/tests/test_codec_pipeline.py000066400000000000000000000270241527014137600252070ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING, Any import numpy as np import pytest pytest.importorskip("hypothesis") import hypothesis.strategies as st from hypothesis import given import zarr from zarr.codecs import BytesCodec, CastValue, GzipCodec, TransposeCodec from zarr.core.array import _get_chunk_spec from zarr.core.buffer.core import default_buffer_prototype from zarr.core.codec_pipeline import codecs_from_list from zarr.core.config import config as zarr_config from zarr.core.indexing import BasicIndexer from zarr.storage import MemoryStore if TYPE_CHECKING: from collections.abc import Callable, Generator from zarr.abc.codec import Codec @pytest.fixture(autouse=True) def _enable_rectilinear_chunks() -> Generator[None]: """Enable rectilinear chunks for all tests in this module.""" with zarr_config.set({"array.rectilinear_chunks": True}): yield pipeline_paths = [ "zarr.core.codec_pipeline.BatchedCodecPipeline", "zarr.core.codec_pipeline.FusedCodecPipeline", ] @pytest.fixture(params=pipeline_paths, ids=["batched", "sync"]) def pipeline_class(request: pytest.FixtureRequest) -> Generator[str]: """Temporarily set the codec pipeline class for the test.""" path = request.param with zarr_config.set({"codec_pipeline.path": path}): yield path # --------------------------------------------------------------------------- # GetResult status tests (low-level pipeline API) # --------------------------------------------------------------------------- @pytest.mark.parametrize( ("write_slice", "read_slice", "expected_statuses"), [ (slice(None), slice(None), ("present", "present", "present")), (slice(0, 2), slice(None), ("present", "missing", "missing")), (None, slice(None), ("missing", "missing", "missing")), ], ) async def test_read_returns_get_results( pipeline_class: str, write_slice: slice | None, read_slice: slice, expected_statuses: tuple[str, ...], ) -> None: """CodecPipeline.read returns GetResult with correct statuses.""" store = MemoryStore() arr = zarr.open_array(store, mode="w", shape=(6,), chunks=(2,), dtype="int64", fill_value=-1) if write_slice is not None: arr[write_slice] = 0 async_arr = arr._async_array pipeline = async_arr.codec_pipeline metadata = async_arr.metadata prototype = default_buffer_prototype() config = async_arr.config indexer = BasicIndexer( read_slice, shape=metadata.shape, chunk_grid=async_arr._chunk_grid, ) out_buffer = prototype.nd_buffer.empty( shape=indexer.shape, dtype=metadata.dtype.to_native_dtype(), order=config.order, ) results = await pipeline.read( [ ( async_arr.store_path / metadata.encode_chunk_key(chunk_coords), _get_chunk_spec(metadata, async_arr._chunk_grid, chunk_coords, config, prototype), chunk_selection, out_selection, is_complete_chunk, ) for chunk_coords, chunk_selection, out_selection, is_complete_chunk in indexer ], out_buffer, drop_axes=indexer.drop_axes, ) assert len(results) == len(expected_statuses) for result, expected_status in zip(results, expected_statuses, strict=True): assert result["status"] == expected_status # --------------------------------------------------------------------------- # write_empty_chunks / read_missing_chunks config tests # --------------------------------------------------------------------------- async def test_write_empty_chunks_false_no_store(pipeline_class: str) -> None: """With write_empty_chunks=False, fill_value-only chunks should not be stored.""" store: dict[str, Any] = {} arr = zarr.create_array( store=store, shape=(20,), dtype="float64", chunks=(10,), shards=None, compressors=None, fill_value=0.0, config={"write_empty_chunks": False}, ) arr[:] = 0.0 # all fill_value # Chunks should NOT be persisted assert "c/0" not in store assert "c/1" not in store # But reading should still return fill values np.testing.assert_array_equal(arr[:], np.zeros(20, dtype="float64")) try: import cast_value_rs # noqa: F401 _HAS_CAST_VALUE_RS = True except ModuleNotFoundError: _HAS_CAST_VALUE_RS = False requires_cast_value_rs = pytest.mark.skipif( not _HAS_CAST_VALUE_RS, reason="cast-value-rs not installed" ) @requires_cast_value_rs @pytest.mark.parametrize( ("source_dtype", "target_dtype"), [ # Source is single-byte (no endianness); target is multi-byte (has endianness). # Without the fix, BytesCodec.evolve_from_array_spec sees the source dtype, # strips its `endian` to None, and then chokes when the chunk_spec dtype # gets transformed to the multi-byte target before bytes-decoding. ("int8", "int16"), ("uint8", "int32"), ("int8", "float32"), # Source is multi-byte; target is single-byte (the reverse direction also # exercises the spec-threading logic). ("int16", "int8"), ], ) def test_codec_pipeline_threads_dtype_through_evolve(source_dtype: str, target_dtype: str) -> None: """Regression for #3937: each codec must be evolved against the spec it will see at runtime, not the original array spec. cast_value transforms the dtype between AA codecs and the array->bytes serializer.""" arr = zarr.create_array( store={}, shape=(4,), chunks=(4,), dtype=source_dtype, fill_value=0, filters=[CastValue(data_type=target_dtype)], serializer=BytesCodec(endian="little"), compressors=[], zarr_format=3, overwrite=True, ) arr[:] = np.asarray([0, 1, 2, 3], dtype=source_dtype) np.testing.assert_array_equal(arr[:], np.asarray([0, 1, 2, 3], dtype=source_dtype)) def test_evolve_threads_spec_preserving_serializer_endian(pipeline_class: str) -> None: """Regression for #3937, dependency-free variant. `evolve_from_array_spec` must thread the spec FORWARD through the codec chain: each codec is evolved against the spec produced by the previous one, not the original array spec. An array->array codec that widens the dtype from a single-byte type (no endianness) to a multi-byte type means the BytesCodec serializer must be evolved against the *widened* dtype — otherwise it sees the single-byte source, strips its `endian` to None, and later fails to decode the multi-byte data. The original regression test for this needs `cast_value_rs` (so it only runs in the optional-deps CI job). This variant uses a minimal dtype-widening AA codec stub, so it runs everywhere and on both pipelines via `pipeline_class`. """ from dataclasses import dataclass from zarr.abc.codec import ArrayArrayCodec from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.dtype import get_data_type_from_native_dtype from zarr.registry import get_pipeline_class @dataclass(frozen=True) class _WidenToInt16(ArrayArrayCodec): """Test-only AA codec: reports the encoded dtype as int16 (no real encode).""" is_fixed_size = True def to_dict(self) -> dict[str, Any]: return {"name": "_widen_to_int16"} @classmethod def from_dict(cls, data: dict[str, Any]) -> _WidenToInt16: return cls() def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: from dataclasses import replace return replace(chunk_spec, dtype=get_data_type_from_native_dtype(np.dtype("int16"))) def compute_encoded_size(self, input_byte_length: int, _spec: ArraySpec) -> int: return input_byte_length async def _decode_single(self, chunk_array: Any, chunk_spec: ArraySpec) -> Any: return chunk_array # pragma: no cover async def _encode_single(self, chunk_array: Any, chunk_spec: ArraySpec) -> Any: return chunk_array # pragma: no cover zdtype = get_data_type_from_native_dtype(np.dtype("int8")) # single-byte source spec = ArraySpec( shape=(4,), dtype=zdtype, fill_value=zdtype.cast_scalar(0), config=ArrayConfig(order="C", write_empty_chunks=False), prototype=default_buffer_prototype(), ) from zarr.core.codec_pipeline import BatchedCodecPipeline, FusedCodecPipeline pipeline = get_pipeline_class().from_codecs((_WidenToInt16(), BytesCodec(endian="little"))) evolved = pipeline.evolve_from_array_spec(spec) # Both concrete pipelines expose `array_bytes_codec`; narrow off the ABC. assert isinstance(evolved, (BatchedCodecPipeline, FusedCodecPipeline)) serializer = evolved.array_bytes_codec # The serializer must keep its little-endian setting: it is evolved against # the widened (int16) dtype, not the single-byte source. assert isinstance(serializer, BytesCodec) assert serializer.endian is not None, ( "BytesCodec serializer lost its `endian` — evolve_from_array_spec did not " "thread the dtype-widening AA codec's spec into the serializer" ) # Property-based check of codecs_from_list ordering validation. # # Valid codec orderings are exactly: (ArrayArrayCodec)* (ArrayBytesCodec) # (BytesBytesCodec)*. codecs_from_list walks adjacent pairs and must raise # TypeError the moment a codec appears in a structurally invalid position -- # notably, a BytesBytesCodec immediately following an ArrayArrayCodec with no # ArrayBytesCodec in between (which previously built an error message but never # raised it, falling through to an unrelated ValueError instead). _AA = "AA" # ArrayArrayCodec -> TransposeCodec _AB = "AB" # ArrayBytesCodec -> BytesCodec _BB = "BB" # BytesBytesCodec -> GzipCodec _CODEC_FACTORY: dict[str, Callable[[], Codec]] = { _AA: lambda: TransposeCodec(order=(0, 1)), _AB: BytesCodec, _BB: GzipCodec, } def _expected_codec_order_outcome(labels: list[str]) -> str: """Independently predict codecs_from_list's outcome: 'TypeError', 'ValueError' or 'ok', mirroring its left-to-right scan and the order in which it checks ordering violations (TypeError) vs. the ArrayBytes-count constraints (ValueError).""" prev = None seen_array_bytes = False for cur in labels: if cur == _AA: if prev in (_AB, _BB): return "TypeError" elif cur == _AB: if prev == _BB: return "TypeError" if seen_array_bytes: return "ValueError" # two ArrayBytesCodecs seen_array_bytes = True else: # _BB if prev == _AA: return "TypeError" prev = cur if not seen_array_bytes: return "ValueError" # Required ArrayBytesCodec was not found return "ok" @given(labels=st.lists(st.sampled_from([_AA, _AB, _BB]), min_size=1, max_size=5)) def test_codecs_from_list_outcome_matches_order_rules(labels: list[str]) -> None: codecs = [_CODEC_FACTORY[label]() for label in labels] expected = _expected_codec_order_outcome(labels) if expected == "TypeError": with pytest.raises(TypeError): codecs_from_list(codecs) elif expected == "ValueError": with pytest.raises(ValueError): codecs_from_list(codecs) else: # Valid ordering: must classify without raising. aa, _ab, bb = codecs_from_list(codecs) assert labels.count(_AA) == len(aa) assert labels.count(_BB) == len(bb) zarr-developers-zarr-python-a2f2415/tests/test_codec_pipeline_suite.py000066400000000000000000000536201527014137600264210ustar00rootroot00000000000000"""Shared codec-pipeline behavior suite, run against EVERY codec pipeline. The defining property of a codec pipeline is that the array semantics it produces must be identical no matter which pipeline is configured. To make "one pipeline diverges from the others" structurally hard to ship, every pipeline-agnostic behavior test lives as a method on ``CodecPipelineTests`` and is instantiated once per pipeline (``TestBatchedPipeline`` / ``TestFusedPipeline``). Each test also runs over a *store axis* that exercises both code paths the synchronous pipelines branch on: * ``sync`` -> ``MemoryStore`` (full sync surface: fast path) * ``async`` -> ``_NoSyncIOStore(MemoryStore())`` (NOT sync-capable: async fallback) The async axis is deliberate: a regression that only affects the async fallback of the default pipeline (e.g. a codec-spec-evolution bug that surfaces only on remote stores) is invisible if every test runs on MemoryStore. Running the same battery over a non-sync store closes that gap. Pipeline-specific tests (construction, ``from_codecs``, the byte-range write fast path, etc.) stay in their own modules; only behavior that ALL pipelines must share belongs here. """ from __future__ import annotations from dataclasses import dataclass from typing import TYPE_CHECKING, Any import numcodecs import numpy as np import pytest import zarr from zarr.codecs import BytesCodec, GzipCodec, ShardingCodec, TransposeCodec from zarr.core.config import config as zarr_config from zarr.errors import ChunkNotFoundError from zarr.storage import MemoryStore from zarr.testing.store import LatencyStore if TYPE_CHECKING: from collections.abc import Iterator from zarr.abc.store import Store from zarr.codecs.sharding import SubchunkWriteOrder # --- store axis: a sync store and a non-sync (async-fallback) store ---------- STORE_KINDS = ["sync", "async"] class _NoSyncIOStore(LatencyStore): """An in-memory store that advertises no sync IO capability, so a synchronous pipeline must fall back to its async path. (A plain wrapper won't do: `WrapperStore` forwards the wrapped store's sync capability.)""" @property def _supports_sync_io(self) -> bool: return False def _make_store(kind: str) -> Store: if kind == "sync": # MemoryStore supports get_sync/set_sync -> synchronous fast path. return MemoryStore() if kind == "async": return _NoSyncIOStore(MemoryStore(), get_latency=0.0, set_latency=0.0) raise AssertionError(kind) # --- scenario model ---------------------------------------------------------- # # Most pipeline behavior tests have one shape: # create an array, apply some writes, (optionally) assert which chunk keys # exist, then assert reads come back correct. A Scenario captures exactly those # variables so one parametrized test covers them all. Correctness is checked # against a numpy reference array that the scenario mutates in lock-step with # the zarr array, so cases don't hand-maintain expected values. @dataclass(frozen=True) class Scenario: id: str array_kwargs: dict[str, Any] # (selection, value) writes applied in order. value may be a scalar or array. writes: tuple[tuple[Any, Any], ...] = () # selections to read back and check against the reference. () means "read all". reads: tuple[Any, ...] = (slice(None),) # substrings of chunk keys that must be present / absent after the writes. # Only checked on the sync store (key layout is identical across stores, but # we keep it to one axis to avoid asserting store internals twice). keys_present: tuple[str, ...] = () keys_absent: tuple[str, ...] = () def reference(self) -> np.ndarray: """The numpy array the scenario's writes should produce, starting from the array's fill value.""" kw = self.array_kwargs shape = kw["shape"] dtype = np.dtype(kw["dtype"]) fill = kw.get("fill_value", 0) ref = np.full(shape, fill, dtype=dtype) for sel, value in self.writes: ref[sel] = value return ref def _val(n: int, dtype: str, offset: int = 1) -> np.ndarray: return np.arange(offset, offset + n, dtype=dtype) # Common dtype/chunk presets reused below. _F64 = {"dtype": "float64", "fill_value": 0.0} _I32 = {"dtype": "int32", "fill_value": -1} SCENARIOS: tuple[Scenario, ...] = ( # --- full-array roundtrips across layouts/codecs ------------------------ Scenario( "1d-unsharded-roundtrip", {"shape": (100,), "chunks": (10,), "shards": None, "compressors": None, **_F64}, writes=((slice(None), _val(100, "float64")),), ), Scenario( "1d-sharded-roundtrip", {"shape": (100,), "chunks": (10,), "shards": (100,), "compressors": None, **_F64}, writes=((slice(None), _val(100, "float64")),), ), Scenario( "1d-multi-chunk-shard-roundtrip", {"shape": (100,), "chunks": (10,), "shards": (50,), "compressors": None, **_F64}, writes=((slice(None), _val(100, "float64")),), ), Scenario( "2d-unsharded-roundtrip", {"shape": (10, 20), "chunks": (5, 10), "shards": None, "compressors": None, **_I32}, writes=((slice(None), np.arange(200, dtype="int32").reshape(10, 20)),), ), Scenario( "2d-sharded-roundtrip", {"shape": (20, 20), "chunks": (5, 5), "shards": (10, 10), "compressors": None, **_I32}, writes=((slice(None), np.arange(400, dtype="int32").reshape(20, 20)),), ), Scenario( "1d-gzip-roundtrip", { "shape": (100,), "chunks": (10,), "shards": None, "compressors": {"name": "gzip", "configuration": {"level": 1}}, **_F64, }, writes=((slice(None), _val(100, "float64")),), ), Scenario( "1d-zstd-roundtrip", { "shape": (100,), "chunks": (10,), "shards": None, "compressors": {"name": "zstd", "configuration": {"level": 1}}, **_F64, }, writes=((slice(None), _val(100, "float64")),), ), Scenario( "1d-float32-roundtrip", { "shape": (50,), "chunks": (10,), "shards": None, "compressors": None, "dtype": "float32", "fill_value": 0.0, }, writes=((slice(None), _val(50, "float32")),), ), # zarr v2 goes through the V2Codec wrapper (filters + compressor), a # different codec path than the v3 AA/AB/BB chain — and a different sync # implementation under FusedCodecPipeline. Without these scenarios, v2 was # only exercised implicitly via whichever pipeline is the global default. Scenario( "v2-roundtrip", { "shape": (100,), "chunks": (10,), "shards": None, "compressors": None, "zarr_format": 2, **_F64, }, writes=((slice(None), _val(100, "float64")),), ), Scenario( "v2-gzip-roundtrip", { "shape": (100,), "chunks": (10,), "shards": None, "compressors": numcodecs.GZip(level=1), "zarr_format": 2, **_F64, }, writes=((slice(None), _val(100, "float64")),), ), # v2 filters are the other half of the V2Codec wrapper (numcodecs # array->array filters, a distinct branch from the compressor in # _encode_sync/_decode_sync). Scenario( "v2-filter-gzip-roundtrip", { "shape": (100,), "chunks": (10,), "shards": None, "filters": numcodecs.Delta(dtype="float64"), "compressors": numcodecs.GZip(level=1), "zarr_format": 2, **_F64, }, writes=((slice(None), _val(100, "float64")),), ), # --- read unwritten chunks -> fill value -------------------------------- Scenario( "missing-chunks-fill", { "shape": (100,), "chunks": (10,), "shards": None, "compressors": None, "dtype": "float64", "fill_value": -7.0, }, writes=(), ), Scenario( "missing-chunks-fill-sharded", { "shape": (100,), "chunks": (10,), "shards": (100,), "compressors": None, "dtype": "float64", "fill_value": -7.0, }, writes=(), ), # --- partial write, varied read selections ------------------------------ Scenario( "partial-write-full-read", {"shape": (100,), "chunks": (10,), "shards": None, "compressors": None, **_F64}, writes=((slice(5, 15), _val(10, "float64")),), reads=(slice(None),), ), Scenario( "full-write-strided-read", {"shape": (100,), "chunks": (10,), "shards": None, "compressors": None, **_F64}, writes=((slice(None), _val(100, "float64")),), reads=(np.s_[::3], np.s_[10:20]), ), Scenario( "partial-write-partial-read-sharded", {"shape": (100,), "chunks": (10,), "shards": (100,), "compressors": None, **_F64}, writes=((slice(20, 70), _val(50, "float64")),), reads=(np.s_[30:60], slice(None)), ), # scalar single-element reads from a sharded array hit the sharding codec's # partial-decode path (_decode_partial_single), distinct from slice reads. Scenario( "sharded-scalar-reads-1d", {"shape": (100,), "chunks": (10,), "shards": (50,), "compressors": None, **_F64}, writes=((slice(None), _val(100, "float64")),), reads=(np.s_[0], np.s_[50], np.s_[99], np.s_[::3]), ), Scenario( "sharded-scalar-reads-2d", {"shape": (20, 20), "chunks": (5, 5), "shards": (10, 10), "compressors": None, **_I32}, writes=((slice(None), np.arange(400, dtype="int32").reshape(20, 20)),), reads=(np.s_[0, 0], np.s_[10, 10], np.s_[19, 19]), ), # --- spec-changing codec (transpose): the async-spec-evolution guard ---- Scenario( "transpose", { "shape": (8, 12), "chunks": (2, 4), "shards": None, "filters": [TransposeCodec(order=(1, 0))], "serializer": BytesCodec(), **_I32, }, writes=((slice(None), np.arange(96, dtype="int32").reshape(8, 12)),), reads=(slice(None), np.s_[1:7, 2:10]), ), Scenario( "transpose-gzip", { "shape": (8, 12), "chunks": (2, 4), "shards": None, "filters": [TransposeCodec(order=(1, 0))], "serializer": BytesCodec(), "compressors": GzipCodec(level=1), **_I32, }, writes=((slice(None), np.arange(96, dtype="int32").reshape(8, 12)),), reads=(slice(None), np.s_[1:7, 2:10]), ), # --- nested sharding ---------------------------------------------------- Scenario( "nested-sharding", { "shape": (20, 20), "chunks": (10, 10), "shards": None, "compressors": None, **_I32, "fill_value": 0, "serializer": ShardingCodec( chunk_shape=(10, 10), codecs=[ShardingCodec(chunk_shape=(5, 5))] ), }, writes=((slice(None), np.arange(400, dtype="int32").reshape(20, 20)),), ), # --- partial overwrite of an existing shard (merge) --------------------- Scenario( "partial-shard-overwrite", { "shape": (40,), "chunks": (4,), "shards": (40,), "compressors": None, **_I32, "config": {"write_empty_chunks": True}, }, writes=( (slice(None), np.arange(40, dtype="int32")), (slice(7, 18), _val(11, "int32", 700)), ), ), # --- write_empty_chunks: storage-key presence/absence ------------------- Scenario( "write-empty-false-omits-fill-chunk", { "shape": (20,), "chunks": (10,), "shards": None, "compressors": None, **_F64, "config": {"write_empty_chunks": False}, }, writes=((slice(0, 10), _val(10, "float64")), (slice(10, 20), np.zeros(10, "float64"))), keys_present=("c/0",), keys_absent=("c/1",), ), Scenario( "write-empty-true-persists-fill-chunk", { "shape": (20,), "chunks": (10,), "shards": None, "compressors": None, **_F64, "config": {"write_empty_chunks": True}, }, writes=((slice(None), np.zeros(20, "float64")),), keys_present=("c/0", "c/1"), ), # default config (no explicit write_empty_chunks) must still skip fill chunks Scenario( "default-config-omits-fill-chunk", {"shape": (20,), "chunks": (10,), "shards": None, "compressors": None, **_F64}, writes=((slice(10, 20), np.zeros(10, "float64")),), keys_absent=("c/1",), ), ) class CodecPipelineTests: """Behavior every codec pipeline must satisfy, on sync and async stores. Subclasses set ``pipeline_path`` to the fully-qualified pipeline class. """ pipeline_path: str @pytest.fixture(autouse=True) def _use_pipeline(self) -> Iterator[None]: with zarr_config.set({"codec_pipeline.path": self.pipeline_path}): yield @pytest.fixture(params=STORE_KINDS) def store(self, request: pytest.FixtureRequest) -> Store: return _make_store(request.param) @staticmethod def _chunk_keys(store: Store) -> set[str]: """All non-metadata keys currently in the store (v3 and v2 metadata).""" import asyncio def _is_metadata(key: str) -> bool: tail = key.rsplit("/", 1)[-1] return tail in ("zarr.json", ".zarray", ".zattrs", ".zgroup", ".zmetadata") async def _list() -> set[str]: return {k async for k in store.list() if not _is_metadata(k)} return asyncio.run(_list()) # -- the common shape: create -> write -> [assert keys] -> assert reads ---- @pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) def test_scenario(self, store: Store, scenario: Scenario) -> None: """Create an array, apply the scenario's writes, optionally assert which chunk keys exist, then assert each read selection matches a numpy reference. Run against every pipeline (subclass) and store kind (fixture). """ arr = zarr.create_array(store=store, **scenario.array_kwargs) for sel, value in scenario.writes: arr[sel] = value ref = scenario.reference() for sel in scenario.reads: np.testing.assert_array_equal( arr[sel], ref[sel], err_msg=f"{scenario.id}: read {sel!r} mismatch" ) if scenario.keys_present or scenario.keys_absent: keys = self._chunk_keys(store) for present in scenario.keys_present: assert any(present in k for k in keys), (present, keys) for absent in scenario.keys_absent: assert not any(absent in k for k in keys), (absent, keys) # -- outliers that don't fit the create/write/read scenario shape ---------- def test_read_missing_chunks_false_raises(self, store: Store) -> None: """read_missing_chunks=False makes reading an unwritten chunk an error, not a fill — a different assertion (raises) than the scenario shape.""" arr = zarr.create_array( store=store, shape=(20,), dtype="float64", chunks=(10,), shards=None, compressors=None, fill_value=0.0, config={"read_missing_chunks": False}, ) with pytest.raises(ChunkNotFoundError): arr[:] def test_read_missing_chunks_false_sharded_semantics(self, store: Store) -> None: """read_missing_chunks=False is a STORE-KEY-level promise on sharded arrays. The config exists to help consumers distinguish a transport error from a truly missing chunk. That distinction applies to store keys: a missing SHARD key raises ChunkNotFoundError. It does not cleanly apply to inner subchunks of a shard that was fetched successfully — there is no transport ambiguity there, the shard index simply records the subchunk as absent — so missing inner subchunks fill with the fill value rather than raising. This pins that asymmetry as intentional. """ arr = zarr.create_array( store=store, shape=(100,), dtype="float64", chunks=(10,), shards=(50,), compressors=None, fill_value=-1.0, config={"read_missing_chunks": False}, ) # No shard key exists yet: reading is a missing-store-key error. with pytest.raises(ChunkNotFoundError): arr[:] # Write one inner chunk of the first shard. The shard key now exists, # but most inner subchunks are absent from its index. arr[20:30] = np.arange(10, dtype="float64") # Reading across written + absent inner subchunks of the EXISTING shard # fills rather than raises. out = arr[15:35] expected = np.full(20, -1.0) expected[5:15] = np.arange(10, dtype="float64") np.testing.assert_array_equal(out, expected) # Both halves of the asymmetry in ONE read against the SAME partially # written array: shard 0 exists (absent subchunks fill), shard 1 has no # store key (raises) — pins that the raise still fires once some shard # exists, and not only on a fully-empty array. with pytest.raises(ChunkNotFoundError): arr[:] with pytest.raises(ChunkNotFoundError): arr[45:55] # spans the existing and the missing shard @pytest.mark.parametrize("subchunk_write_order", ["morton", "lexicographic", "colexicographic"]) def test_partial_write_after_reopen_is_correct( self, store: Store, subchunk_write_order: SubchunkWriteOrder ) -> None: """Has an extra step the scenario shape lacks — a REOPEN between writes. Reopening a sharded array and partially overwriting it must read back correctly regardless of the original subchunk_write_order. subchunk_write_ order is intentionally NOT recoverable on reopen, so chunk locations on a write to an existing shard must come from the STORED shard index, not the (now-default) live order. A non-square inner grid makes the orders physically distinct, so a wrong offset would corrupt data and fail here. """ shape, shard, inner = (6, 4), (6, 4), (2, 2) arr = zarr.create_array( store=store, shape=shape, dtype="int32", chunks=shard, fill_value=-1, compressors=None, config={"write_empty_chunks": True}, serializer=ShardingCodec( chunk_shape=inner, codecs=[BytesCodec()], subchunk_write_order=subchunk_write_order ), ) ref = np.arange(24, dtype="int32").reshape(shape) arr[:] = ref reopened = zarr.open_array(store=store, mode="r+") reopened[1:5, 0:3] = 777 # partial overwrite into the existing shard ref[1:5, 0:3] = 777 np.testing.assert_array_equal(reopened[:], ref) def test_empty_shard_deleted_after_overwrite_to_fill(self, store: Store) -> None: """A shard written with real data and then fully overwritten back to the fill value must have its store key deleted, not left as a stale blob. This has a mid-sequence key assertion (present after write 1, absent after write 2) that the create/write/read scenario shape can't express. """ arr = zarr.create_array( store=store, shape=(16,), chunks=(4,), shards=(8,), dtype="float64", compressors=None, fill_value=0.0, ) arr[0:8] = np.arange(8, dtype="float64") + 1 assert any("c/0" in k for k in self._chunk_keys(store)) arr[0:8] = 0.0 assert not any("c/0" in k for k in self._chunk_keys(store)), ( "shard should be deleted when fully overwritten to fill value" ) def test_read_write_methods_do_not_branch_on_sharding_codec_type(self) -> None: """Pipeline read/write must dispatch on supports_partial_encode/decode, not isinstance(ShardingCodec) — a static guard against type-branching. Scoped to this pipeline's own read/write methods (other helpers, e.g. metadata validation, may legitimately isinstance-check ShardingCodec). """ import inspect import re from zarr.registry import get_pipeline_class # The autouse _use_pipeline fixture has set codec_pipeline.path to this # subclass's pipeline; resolve the class it points at and guard that. # reload_config=False so the fixture's config override is honored # (reload_config=True re-reads the base config, ignoring the override). cls = get_pipeline_class(reload_config=False) pattern = re.compile(r"isinstance\s*\([^)]*ShardingCodec[^)]*\)") for method_name in ("read", "write", "read_sync", "write_sync"): method = getattr(cls, method_name, None) if method is None: continue matches = pattern.findall(inspect.getsource(method)) assert not matches, ( f"{cls.__name__}.{method_name} contains an isinstance check on " f"ShardingCodec; use supports_partial_encode/decode instead. " f"Matches: {matches}" ) class TestBatchedPipeline(CodecPipelineTests): pipeline_path = "zarr.core.codec_pipeline.BatchedCodecPipeline" class TestFusedPipeline(CodecPipelineTests): pipeline_path = "zarr.core.codec_pipeline.FusedCodecPipeline" zarr-developers-zarr-python-a2f2415/tests/test_codecs/000077500000000000000000000000001527014137600231265ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/test_codecs/__init__.py000066400000000000000000000000001527014137600252250ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/test_codecs/test_blosc.py000066400000000000000000000234001527014137600256400ustar00rootroot00000000000000import enum import json import warnings from typing import Any, cast import numcodecs import numpy as np import pytest from packaging.version import Version import zarr from zarr.abc.codec import SupportsSyncCodec from zarr.codecs import BloscCodec from zarr.codecs.blosc import ( BLOSC_CNAME, BLOSC_SHUFFLE, BloscCname, BloscCnameLiteral, BloscShuffle, BloscShuffleLiteral, ) from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer import default_buffer_prototype from zarr.core.dtype import UInt16, get_data_type_from_native_dtype from zarr.storage import MemoryStore, StorePath @pytest.mark.parametrize("dtype", ["uint8", "uint16"]) async def test_blosc_evolve(dtype: str) -> None: typesize = np.dtype(dtype).itemsize path = "blosc_evolve" store = MemoryStore() spath = StorePath(store, path) zarr.create_array( spath, shape=(16, 16), chunks=(16, 16), dtype=dtype, fill_value=0, compressors=BloscCodec(), ) buf = await store.get(f"{path}/zarr.json", prototype=default_buffer_prototype()) assert buf is not None zarr_json = json.loads(buf.to_bytes()) blosc_configuration_json = zarr_json["codecs"][1]["configuration"] assert blosc_configuration_json["typesize"] == typesize if typesize == 1: assert blosc_configuration_json["shuffle"] == "bitshuffle" else: assert blosc_configuration_json["shuffle"] == "shuffle" path2 = "blosc_evolve_sharding" spath2 = StorePath(store, path2) zarr.create_array( spath2, shape=(16, 16), chunks=(16, 16), shards=(16, 16), dtype=dtype, fill_value=0, compressors=BloscCodec(), ) buf = await store.get(f"{path2}/zarr.json", prototype=default_buffer_prototype()) assert buf is not None zarr_json = json.loads(buf.to_bytes()) blosc_configuration_json = zarr_json["codecs"][0]["configuration"]["codecs"][1]["configuration"] assert blosc_configuration_json["typesize"] == typesize if typesize == 1: assert blosc_configuration_json["shuffle"] == "bitshuffle" else: assert blosc_configuration_json["shuffle"] == "shuffle" @pytest.mark.parametrize("shuffle", [None, "bitshuffle", "legacy-enum"]) @pytest.mark.parametrize("typesize", [None, 1, 2]) def test_tunable_attrs_param( shuffle: BloscShuffleLiteral | str | None, typesize: int | None ) -> None: """ Test that the tunable_attrs parameter is set as expected when creating a BloscCodec. """ # Materialize BloscShuffle.shuffle via the deprecation shim without # contaminating the BloscCodec construction below with that warning. if shuffle == "legacy-enum": with pytest.warns(DeprecationWarning, match="BloscShuffle.shuffle"): shuffle_arg: BloscShuffleLiteral | str | None = BloscShuffle.shuffle else: shuffle_arg = shuffle codec = BloscCodec(typesize=typesize, shuffle=cast(BloscShuffleLiteral | None, shuffle_arg)) if shuffle_arg is None: assert codec.shuffle == "bitshuffle" # default shuffle assert "shuffle" in codec._tunable_attrs if typesize is None: assert codec.typesize == 1 # default typesize assert "typesize" in codec._tunable_attrs new_dtype = UInt16() array_spec = ArraySpec( shape=(1,), dtype=new_dtype, fill_value=1, prototype=default_buffer_prototype(), config=cast(ArrayConfig, {}), ) evolved_codec = codec.evolve_from_array_spec(array_spec=array_spec) if typesize is None: assert evolved_codec.typesize == new_dtype.item_size else: assert evolved_codec.typesize == codec.typesize if shuffle_arg is None: assert evolved_codec.shuffle == "shuffle" else: assert evolved_codec.shuffle == codec.shuffle async def test_typesize() -> None: a = np.arange(1000000, dtype=np.uint64) codecs = [zarr.codecs.BytesCodec(endian="little"), zarr.codecs.BloscCodec()] z = zarr.array(a, chunks=(10000), codecs=codecs) data = await z.store.get("c/0", prototype=default_buffer_prototype()) assert data is not None bytes = data.to_bytes() size = len(bytes) msg = f"Blosc size mismatch. First 10 bytes: {bytes[:20]!r} and last 10 bytes: {bytes[-20:]!r}" if Version(numcodecs.__version__) >= Version("0.16.0"): expected_size = 402 assert size == expected_size, msg else: expected_size = 10216 assert size == expected_size, msg def test_blosc_codec_supports_sync() -> None: assert isinstance(BloscCodec(), SupportsSyncCodec) def test_blosc_codec_sync_roundtrip() -> None: codec = BloscCodec(typesize=8) arr = np.arange(100, dtype="float64") zdtype = get_data_type_from_native_dtype(arr.dtype) spec = ArraySpec( shape=arr.shape, dtype=zdtype, fill_value=zdtype.cast_scalar(0), config=ArrayConfig(order="C", write_empty_chunks=True), prototype=default_buffer_prototype(), ) buf = default_buffer_prototype().buffer.from_array_like(arr.view("B")) encoded = codec._encode_sync(buf, spec) assert encoded is not None decoded = codec._decode_sync(encoded, spec) result = np.frombuffer(decoded.as_numpy_array(), dtype="float64") np.testing.assert_array_equal(arr, result) @pytest.mark.parametrize("cname", BLOSC_CNAME) def test_blosc_codec_accepts_all_cnames(cname: BloscCnameLiteral) -> None: """ Every compressor name in BLOSC_CNAME is accepted by BloscCodec and round-trips to the same value on the stored attribute. Adding a new value to the BloscCnameLiteral type alias without also adding it to BLOSC_CNAME (or vice versa) is caught here. """ codec = BloscCodec(cname=cname) assert codec.cname == cname @pytest.mark.parametrize("shuffle", BLOSC_SHUFFLE) def test_blosc_codec_accepts_all_shuffles(shuffle: BloscShuffleLiteral) -> None: """ Every shuffle mode in BLOSC_SHUFFLE is accepted by BloscCodec and round-trips to the same value on the stored attribute. Adding a new value to the BloscShuffleLiteral type alias without also adding it to BLOSC_SHUFFLE (or vice versa) is caught here. """ codec = BloscCodec(shuffle=shuffle) assert codec.shuffle == shuffle @pytest.mark.parametrize("shuffle", BLOSC_SHUFFLE) @pytest.mark.parametrize("cname", BLOSC_CNAME) def test_blosc_codec_json_roundtrip(cname: BloscCnameLiteral, shuffle: BloscShuffleLiteral) -> None: """ JSON serialization (to_dict / from_dict) preserves every (cname, shuffle) pair drawn from BLOSC_CNAME x BLOSC_SHUFFLE. Guards against drift in the codec's V3 JSON form for any combination of compressor and shuffle option. The non-varied fields are fully specified so the codec has no tunable attributes; tunability is not part of the JSON form and would otherwise cause spurious round-trip mismatches. """ codec = BloscCodec(typesize=1, cname=cname, clevel=5, shuffle=shuffle, blocksize=0) restored = BloscCodec.from_dict(codec.to_dict()) assert restored == codec @pytest.mark.parametrize( ("enum_cls", "member", "expected"), [ (BloscShuffle, "shuffle", "shuffle"), (BloscCname, "zstd", "zstd"), ], ) def test_blosc_enum_member_access_warns(enum_cls: type, member: str, expected: str) -> None: """ Accessing a member on the deprecated BloscShuffle / BloscCname classes emits a DeprecationWarning and resolves to the equivalent literal string. """ match = f"{enum_cls.__name__}.{member}" with pytest.warns(DeprecationWarning, match=match): value = getattr(enum_cls, member) assert value == expected def test_blosc_enum_classes_import_silently() -> None: """ Importing the deprecated enum classes by name must not emit a warning; only member access does. This guards against the blosc module accidentally triggering its own deprecation warnings when it (or zarr) is imported. """ with warnings.catch_warnings(): warnings.simplefilter("error") from zarr.codecs.blosc import BloscCname as _BloscCname # noqa: F401 from zarr.codecs.blosc import BloscShuffle as _BloscShuffle # noqa: F401 def test_blosc_codec_init_with_enum_instance_warns() -> None: """ Passing a real `enum.Enum` instance to BloscCodec.__init__ (e.g. an instance materialized before the deprecation shim was introduced) must trigger the init-level deprecation warning and still normalize the value to the corresponding literal string. """ class LegacyShuffle(enum.Enum): bitshuffle = "bitshuffle" class LegacyCname(enum.Enum): zstd = "zstd" with pytest.warns(DeprecationWarning, match="enum"): codec = BloscCodec( cname=cast(BloscCname, LegacyCname.zstd), shuffle=cast(BloscShuffle, LegacyShuffle.bitshuffle), ) assert codec.cname == "zstd" assert codec.shuffle == "bitshuffle" @pytest.mark.parametrize("param", ["cname", "shuffle"]) def test_blosc_codec_rejects_unknown(param: str) -> None: """ BloscCodec.__init__ raises ValueError when given a string outside the allowed set for `cname` or `shuffle`, and the error message names the offending parameter. """ kwargs: dict[str, Any] = {param: f"not-a-{param}"} with pytest.raises(ValueError, match=f"{param} must be one of"): BloscCodec(**kwargs) @pytest.mark.parametrize("enum_cls", [BloscShuffle, BloscCname]) def test_blosc_enum_attribute_error_for_unknown_member(enum_cls: type) -> None: """ Attribute access for a name that is not a known member of the deprecated enum classes falls through to AttributeError, matching the behavior of a regular class. """ unknown_name = "not_a_member" with pytest.raises(AttributeError): getattr(enum_cls, unknown_name) zarr-developers-zarr-python-a2f2415/tests/test_codecs/test_bytes.py000066400000000000000000000306121527014137600256670ustar00rootroot00000000000000"""Tests for `BytesCodec` and the deprecation of the `Endian` enum.""" from __future__ import annotations import enum import sys import warnings from typing import TYPE_CHECKING, Any, Literal, cast import numpy as np import pytest import zarr from tests.conftest import Expect, ExpectFail from zarr.abc.codec import SupportsSyncCodec from zarr.codecs.bytes import ( ENDIAN, BytesCodec, Endian, EndianLiteral, ) from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer import NDBuffer, default_buffer_prototype from zarr.core.dtype import get_data_type_from_native_dtype from zarr.core.dtype.npy.int import Int8, Int32 from zarr.core.dtype.npy.structured import Struct from zarr.storage import StorePath from .test_codecs import _AsyncArrayProxy if TYPE_CHECKING: from zarr.abc.store import Store @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) @pytest.mark.parametrize( "input_dtype", [ ">u2", "f4"), ("mask", ">i4")], [("flux", " None: """ The `bytes` codec stores multi-byte data in the byte order configured on the codec, regardless of the input array's byte order, and reads it back to the original values. For structured dtypes this applies to every multi-byte field, per the `struct` data type spec; the struct cases guard against the endianness bugs from https://github.com/zarr-developers/zarr-python/issues/4141, where the encode path never byte-swapped struct fields (numpy reports byteorder '|' for void dtypes) and the decode path ignored the codec's endian entirely. The input-dtype/store-endian cross-product exercises the encode-side byteswap (input byte order != store byte order) and the no-op case alike. Compression is disabled so the stored chunk is the codec's raw output and its byte layout can be asserted directly. """ dtype = np.dtype(input_dtype) if dtype.fields is None: data = np.arange(0, 256, dtype=dtype).reshape((16, 16)) else: data = np.zeros((16, 16), dtype=dtype) data["flux"] = np.arange(0, 256).reshape((16, 16)) data["mask"] = np.arange(256, 512).reshape((16, 16)) path = "endian" spath = StorePath(store, path) a = await zarr.api.asynchronous.create_array( spath, shape=data.shape, chunks=(16, 16), dtype=dtype, fill_value=0, compressors=None, serializer=BytesCodec(endian=store_endian), ) await _AsyncArrayProxy(a)[:, :].set(data) # The stored chunk is laid out in the byte order configured on the codec. stored = await store.get(f"{path}/c/0/0", prototype=default_buffer_prototype()) assert stored is not None assert stored.to_bytes() == data.astype(dtype.newbyteorder(store_endian)).tobytes() # ... and the data reads back to the original values. readback_data = await _AsyncArrayProxy(a)[:, :].get() assert np.array_equal(data, readback_data) def test_bytes_codec_supports_sync() -> None: assert isinstance(BytesCodec(), SupportsSyncCodec) @pytest.mark.parametrize("endian", ENDIAN) @pytest.mark.parametrize( "native_dtype", [np.dtype("float64"), np.dtype(">u2"), np.dtype([("a", ">f4"), ("b", " None: """ The synchronous encode/decode path round-trips data, and the two byte orders involved are independent: the codec's `endian` configuration governs only the stored byte layout (every multi-byte value, including struct fields, is laid out in the codec's byte order regardless of the input array's byte order), while the decoded buffer's byte order is governed by the array's data type regardless of the codec's. The mixed-endian struct case pins that per-field byte order of the in-memory dtype survives a roundtrip through a single stored byte order. """ if native_dtype.fields is None: arr = np.arange(100, dtype=native_dtype) else: arr = np.array([(1.5, 2), (3.5, 4), (5.5, 6), (7.5, 8)], dtype=native_dtype) zdtype = get_data_type_from_native_dtype(arr.dtype) spec = ArraySpec( shape=arr.shape, dtype=zdtype, fill_value=zdtype.cast_scalar(0), config=ArrayConfig(order="C", write_empty_chunks=True), prototype=default_buffer_prototype(), ) nd_buf: NDBuffer = default_buffer_prototype().nd_buffer.from_numpy_array(arr) codec = BytesCodec(endian=endian).evolve_from_array_spec(spec) encoded = codec._encode_sync(nd_buf, spec) assert encoded is not None assert encoded.to_bytes() == arr.astype(native_dtype.newbyteorder(endian)).tobytes() decoded = codec._decode_sync(encoded, spec) assert decoded.dtype == zdtype.to_native_dtype() np.testing.assert_array_equal(arr, decoded.as_numpy_array()) @pytest.mark.parametrize("endian", ENDIAN) def test_bytes_codec_accepts_all_endians(endian: EndianLiteral) -> None: """ Every endian value in ENDIAN is accepted by BytesCodec and round-trips to the same value on the stored attribute. Catches drift between the EndianLiteral type alias and the runtime ENDIAN tuple. """ codec = BytesCodec(endian=endian) assert codec.endian == endian @pytest.mark.parametrize("endian", ENDIAN) def test_bytes_codec_json_roundtrip(endian: EndianLiteral) -> None: """ BytesCodec.to_dict produces the spec-defined wire shape and the round-trip through from_dict preserves equality. Asserting the literal JSON shape catches drift between BytesCodec's runtime representation and the codec's V3 on-disk form. """ codec = BytesCodec(endian=endian) assert codec.to_dict() == {"name": "bytes", "configuration": {"endian": endian}} restored = BytesCodec.from_dict(codec.to_dict()) assert restored == codec # to_dict and from_dict are inverses over this (endian setting, wire dict) mapping: # to_dict turns the endian setting into the dict; from_dict recovers it. _ENDIAN_DICT_CASES: list[Expect[EndianLiteral | None, dict[str, Any]]] = [ Expect( input="little", output={"name": "bytes", "configuration": {"endian": "little"}}, id="little", ), Expect( input="big", output={"name": "bytes", "configuration": {"endian": "big"}}, id="big", ), Expect(input=None, output={"name": "bytes"}, id="missing"), ] @pytest.mark.parametrize("case", _ENDIAN_DICT_CASES, ids=lambda c: c.id) def test_to_dict(case: Expect[EndianLiteral | None, dict[str, Any]]) -> None: assert BytesCodec(endian=case.input).to_dict() == case.output @pytest.mark.parametrize("case", _ENDIAN_DICT_CASES, ids=lambda c: c.id) def test_from_dict(case: Expect[EndianLiteral | None, dict[str, Any]]) -> None: assert BytesCodec.from_dict(case.output).endian == case.input @pytest.mark.parametrize("endian", ["little", "big", pytest.param(None, id="missing")]) def test_roundtrip(endian: EndianLiteral | None) -> None: codec = BytesCodec(endian=endian) encoded = codec.to_dict() roundtripped = BytesCodec.from_dict(encoded) assert codec == roundtripped @pytest.mark.parametrize( ("member", "expected"), [("little", "little"), ("big", "big")], ) def test_endian_member_access_warns(member: str, expected: str) -> None: """ Accessing a member on the deprecated `Endian` class emits a `DeprecationWarning` and resolves to the equivalent literal string. """ with pytest.warns(DeprecationWarning, match=rf"Endian\.{member}"): value = getattr(Endian, member) assert value == expected def test_endian_class_imports_silently() -> None: """ Importing the deprecated `Endian` class by name must not emit a warning; only member access does. Guards against `bytes.py` accidentally triggering its own deprecation warnings at import time. """ with warnings.catch_warnings(): warnings.simplefilter("error") from zarr.codecs.bytes import Endian as _Endian # noqa: F401 def test_bytes_codec_init_with_enum_instance_warns() -> None: """ Passing a foreign `enum.Enum` instance to `BytesCodec.__init__` triggers the init-level deprecation warning (from `_coerce_enum_input`) and normalizes the value to the corresponding literal string. Covers the case where a downstream package defined its own enum-shaped class to bridge between zarr's old API and its own. """ class LegacyEndian(enum.Enum): little = "little" with pytest.warns(DeprecationWarning, match=r"Passing an enum to BytesCodec"): codec = BytesCodec(endian=cast(Endian, LegacyEndian.little)) assert codec.endian == "little" def test_bytes_codec_init_with_deprecated_class_member() -> None: """ The realistic legacy-upgrade idiom: `BytesCodec(endian=Endian.little)`. Member access on `Endian` emits one `DeprecationWarning` (from the metaclass) and resolves to the bare string, which `BytesCodec` then accepts without further warning. No second warning from `_coerce_enum_input` because the metaclass already produced a string. The `cast` is necessary because the metaclass `__getattr__` is typed as returning `str`, which does not statically match the codec's `EndianLiteral` parameter even though the runtime value does. """ with pytest.warns(DeprecationWarning, match=r"Endian\.little"): codec = BytesCodec(endian=cast(EndianLiteral, Endian.little)) assert codec.endian == "little" @pytest.mark.parametrize( "case", [ ExpectFail( input="north", exception=ValueError, id="unknown-string", msg="endian must be one of", ), ], ids=lambda c: c.id, ) def test_bytes_codec_rejects_unknown_endian(case: ExpectFail[Any]) -> None: """ `BytesCodec.__init__` raises `ValueError` when given a value outside `ENDIAN`, and the error message names the offending parameter. """ with case.raises(): BytesCodec(endian=case.input) def test_endian_attribute_error_for_unknown_member() -> None: """ Attribute access for a name that is not a known member of the deprecated `Endian` class falls through to `AttributeError`, matching the behavior of a regular class. """ with pytest.raises(AttributeError): getattr(Endian, "not_a_member") # noqa: B009 def test_bytes_codec_default_endian_matches_system() -> None: """ Constructing `BytesCodec()` with no arguments yields a codec whose `endian` matches `sys.byteorder`. This replaces the previous `default_system_endian = Endian(sys.byteorder)` module-level binding. """ codec = BytesCodec() assert codec.endian == sys.byteorder def _make_array_spec(dtype: Any) -> ArraySpec: """Build a minimal ArraySpec around the given dtype for codec.evolve testing.""" return ArraySpec( shape=(1,), dtype=dtype, fill_value=0, config=cast(ArrayConfig, {}), prototype=default_buffer_prototype(), ) def test_bytes_codec_evolve_structured_multi_byte_fields_warns_and_defaults() -> None: """ BytesCodec(endian=None).evolve_from_array_spec(spec) with a structured dtype whose fields contain multi-byte members emits a UserWarning about the missing endian and returns a codec with endian set to "little" for legacy compatibility. """ codec = BytesCodec(endian=None) dtype = Struct(fields=(("a", Int32()), ("b", Int32()))) spec = _make_array_spec(dtype) with pytest.warns(UserWarning, match=r"Missing 'endian' for structured dtype"): evolved = codec.evolve_from_array_spec(spec) assert evolved.endian == "little" def test_bytes_codec_evolve_structured_single_byte_fields_clears_endian() -> None: """ For a structured dtype whose fields are all single-byte, BytesCodec drops its endian on evolve (endian is meaningless for single-byte content). """ codec = BytesCodec(endian="little") dtype = Struct(fields=(("a", Int8()), ("b", Int8()))) spec = _make_array_spec(dtype) evolved = codec.evolve_from_array_spec(spec) assert evolved.endian is None zarr-developers-zarr-python-a2f2415/tests/test_codecs/test_cast_value.py000066400000000000000000000460241527014137600266730ustar00rootroot00000000000000from __future__ import annotations from typing import Any import numpy as np import pytest from numpy.testing import assert_array_equal import zarr from tests.conftest import Expect, ExpectFail from zarr.codecs import BytesCodec, TransposeCodec from zarr.codecs.cast_value import CastValue from zarr.storage import MemoryStore try: import cast_value_rs # noqa: F401 _HAS_CAST_VALUE_RS = True except ModuleNotFoundError: _HAS_CAST_VALUE_RS = False requires_cast_value_rs = pytest.mark.skipif( not _HAS_CAST_VALUE_RS, reason="cast-value-rs not installed" ) # --------------------------------------------------------------------------- # Serialization # --------------------------------------------------------------------------- @pytest.mark.parametrize( "case", [ Expect( input=CastValue(data_type="uint8"), output={"name": "cast_value", "configuration": {"data_type": "uint8"}}, id="minimal", ), Expect( input=CastValue( data_type="uint8", rounding="towards-zero", out_of_range="clamp", scalar_map={"encode": [("NaN", 0)]}, ), output={ "name": "cast_value", "configuration": { "data_type": "uint8", "rounding": "towards-zero", "out_of_range": "clamp", "scalar_map": {"encode": [("NaN", 0)]}, }, }, id="full", ), ], ids=lambda c: c.id, ) def test_to_dict(case: Expect[CastValue, dict[str, Any]]) -> None: """to_dict produces the expected JSON structure.""" assert case.input.to_dict() == case.output @pytest.mark.parametrize( "case", [ Expect( input={"name": "cast_value", "configuration": {"data_type": "float32"}}, output=("float32", "nearest-even", None), id="defaults", ), Expect( input={ "name": "cast_value", "configuration": { "data_type": "int16", "rounding": "towards-zero", "out_of_range": "clamp", }, }, output=("int16", "towards-zero", "clamp"), id="explicit", ), ], ids=lambda c: c.id, ) def test_from_dict(case: Expect[dict[str, Any], tuple[str, str, str | None]]) -> None: """from_dict deserializes configuration with correct values and defaults.""" codec = CastValue.from_dict(case.input) dtype_name, rounding, out_of_range = case.output assert codec.dtype.to_native_dtype() == np.dtype(dtype_name) assert codec.rounding == rounding assert codec.out_of_range == out_of_range @pytest.mark.parametrize( "codec", [ CastValue(data_type="int16", rounding="towards-zero", out_of_range="clamp"), CastValue( data_type="uint8", out_of_range="clamp", scalar_map={"encode": [("NaN", 0)], "decode": [(0, "NaN")]}, ), ], ids=["no-scalar-map", "with-scalar-map"], ) def test_serialization_roundtrip(codec: CastValue) -> None: """to_dict followed by from_dict produces an equal codec.""" restored = CastValue.from_dict(codec.to_dict()) assert codec == restored # --------------------------------------------------------------------------- # Construction # --------------------------------------------------------------------------- def test_construction_accepts_zdtype_object() -> None: """data_type can be a ZDType instance, not just a JSON string.""" from zarr.core.dtype import UInt8 codec = CastValue(data_type=UInt8()) assert codec.dtype.to_native_dtype() == np.dtype("uint8") def test_construction_rejects_invalid_target_dtype() -> None: """Construction rejects target dtypes not in PERMITTED_DATA_TYPE_NAMES.""" with pytest.raises(ValueError, match="Invalid target data type"): CastValue(data_type="complex64") # --------------------------------------------------------------------------- # Validation # --------------------------------------------------------------------------- @pytest.mark.parametrize( "case", [ ExpectFail( input={"dtype": "complex128", "target": "float64"}, msg="only supports integer and floating-point", exception=ValueError, id="complex-source", ), ExpectFail( input={"dtype": "int32", "target": "float64", "out_of_range": "wrap"}, msg="only valid for integer", exception=ValueError, id="wrap-float-target", ), ], ids=lambda c: c.id, ) def test_validation_rejects_invalid(case: ExpectFail[dict[str, Any]]) -> None: """Invalid dtype or out_of_range combinations are rejected at array creation.""" with case.raises(): zarr.create_array( store={}, shape=(10,), dtype=case.input["dtype"], chunks=(10,), filters=[ CastValue( data_type=case.input["target"], out_of_range=case.input.get("out_of_range"), ) ], compressors=None, fill_value=0, ) @requires_cast_value_rs @pytest.mark.parametrize( ("source_dtype", "target_dtype"), [ ("float16", "int8"), ("float32", "int32"), ("float64", "int64"), ("int32", "uint8"), ], ) def test_validation_accepts_wrap_with_integer_target(source_dtype: str, target_dtype: str) -> None: """Regression for #3936: `out_of_range="wrap"` is permitted when the cast TARGET (not the source array dtype) is an integer type.""" zarr.create_array( store={}, shape=(1,), dtype=source_dtype, chunks=(1,), filters=[CastValue(data_type=target_dtype, out_of_range="wrap")], compressors=None, fill_value=0, ) def test_zero_itemsize_raises() -> None: """Variable-length dtypes (itemsize=0) are rejected by compute_encoded_size.""" from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer import default_buffer_prototype from zarr.core.dtype.npy.string import VariableLengthUTF8 codec = CastValue(data_type="uint8") spec = ArraySpec( shape=(10,), dtype=VariableLengthUTF8(), # type: ignore[arg-type] fill_value="", config=ArrayConfig(order="C", write_empty_chunks=True), prototype=default_buffer_prototype(), ) with pytest.raises(ValueError, match="fixed-size integer and floating-point data types"): codec.compute_encoded_size(100, spec) # --------------------------------------------------------------------------- # Encode / decode # --------------------------------------------------------------------------- @requires_cast_value_rs @pytest.mark.parametrize( "case", [ Expect(input=("float64", "float32"), output=np.arange(50, dtype="float64"), id="f64→f32"), Expect(input=("float32", "float64"), output=np.arange(50, dtype="float32"), id="f32→f64"), Expect(input=("int32", "int64"), output=np.arange(50, dtype="int32"), id="i32→i64"), Expect(input=("int64", "int16"), output=np.arange(50, dtype="int64"), id="i64→i16"), Expect(input=("float64", "int32"), output=np.arange(50, dtype="float64"), id="f64→i32"), Expect(input=("int32", "float64"), output=np.arange(50, dtype="int32"), id="i32→f64"), ], ids=lambda c: c.id, ) def test_encode_decode_roundtrip( case: Expect[tuple[str, str], np.ndarray[Any, np.dtype[Any]]], ) -> None: """Small integer data survives encode → decode for each dtype pair.""" import zarr source_dtype, target_dtype = case.input arr = zarr.create_array( store={}, shape=(50,), dtype=source_dtype, chunks=(50,), filters=[CastValue(data_type=target_dtype)], compressors=None, fill_value=0, ) arr[:] = case.output np.testing.assert_array_equal(arr[:], case.output) @requires_cast_value_rs @pytest.mark.parametrize( "case", [ Expect( input=np.array([1.7, -1.7, 2.5, -2.5], dtype="float64"), output=np.array([1, -1, 2, -2], dtype="float64"), id="towards-zero", ), ], ids=lambda c: c.id, ) def test_float_to_int_rounding( case: Expect[np.ndarray[Any, np.dtype[Any]], np.ndarray[Any, np.dtype[Any]]], ) -> None: """Fractional float values are truncated towards zero when cast to int32.""" import zarr arr = zarr.create_array( store={}, shape=case.input.shape, dtype=case.input.dtype, chunks=case.input.shape, filters=[CastValue(data_type="int32", rounding="towards-zero", out_of_range="clamp")], compressors=None, fill_value=0, ) arr[:] = case.input np.testing.assert_array_equal(arr[:], case.output) @requires_cast_value_rs @pytest.mark.parametrize( "case", [ Expect( input=np.array([0, 200, -200], dtype="int32"), output=np.array([0, 127, -128], dtype="int32"), id="int32→int8", ), ], ids=lambda c: c.id, ) def test_out_of_range_clamp( case: Expect[np.ndarray[Any, np.dtype[Any]], np.ndarray[Any, np.dtype[Any]]], ) -> None: """Values outside the int8 range are clamped to [-128, 127].""" import zarr arr = zarr.create_array( store={}, shape=case.input.shape, dtype=case.input.dtype, chunks=case.input.shape, filters=[CastValue(data_type="int8", out_of_range="clamp")], compressors=None, fill_value=0, ) arr[:] = case.input np.testing.assert_array_equal(arr[:], case.output) def test_compute_encoded_size() -> None: """compute_encoded_size correctly scales byte length by itemsize ratio.""" from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer import default_buffer_prototype from zarr.core.dtype import get_data_type_from_json codec = CastValue(data_type="int16") spec = ArraySpec( shape=(10,), dtype=get_data_type_from_json("float64", zarr_format=3), fill_value=0, config=ArrayConfig(order="C", write_empty_chunks=True), prototype=default_buffer_prototype(), ) # 10 float64 elements = 80 bytes -> 10 int16 elements = 20 bytes assert codec.compute_encoded_size(80, spec) == 20 @requires_cast_value_rs def test_scalar_map_encode_decode_roundtrip() -> None: """Scalar map entries are applied during encode and decode.""" import zarr data = np.array([1.0, float("nan"), 3.0], dtype="float64") arr = zarr.create_array( store={}, shape=data.shape, dtype="float64", chunks=data.shape, filters=[ CastValue( data_type="int32", rounding="nearest-even", out_of_range="clamp", scalar_map={"encode": [("NaN", -999)], "decode": [(-999, "NaN")]}, ), ], compressors=None, fill_value=1, ) arr[:] = data result = np.asarray(arr[:]) np.testing.assert_equal(result[0], 1.0) np.testing.assert_equal(result[2], 3.0) assert np.isnan(result[1]) @pytest.mark.parametrize( "case", [ ExpectFail( input={ "dtype": "int32", "target": "int8", "scalar_map": {"encode": [("NaN", 0)]}, }, msg="not representable in dtype int32", exception=ValueError, id="nan-key-for-int-source", ), ExpectFail( input={ "dtype": "int32", "target": "float64", "scalar_map": {"decode": [(0, "NaN")]}, }, msg="not representable in dtype int32", exception=ValueError, id="nan-value-for-int-decode-target", ), ExpectFail( input={ "dtype": "float64", "target": "int8", "scalar_map": {"encode": [("NaN", 999)]}, }, msg="not representable in dtype int8", exception=ValueError, id="encode-value-out-of-range", ), ExpectFail( input={ "dtype": "float64", "target": "int8", "scalar_map": {"encode": [("NaN", 1.5)]}, }, msg="not representable in dtype int8", exception=ValueError, id="encode-value-not-integer", ), ], ids=lambda c: c.id, ) def test_scalar_map_validation_rejects_invalid(case: ExpectFail[dict[str, Any]]) -> None: """Invalid scalar_map entries are rejected at array creation.""" import zarr with case.raises(): zarr.create_array( store={}, shape=(10,), dtype=case.input["dtype"], chunks=(10,), filters=[ CastValue( data_type=case.input["target"], out_of_range="clamp", scalar_map=case.input["scalar_map"], ) ], compressors=None, fill_value=0, ) @requires_cast_value_rs def test_combined_with_scale_offset() -> None: """scale_offset followed by cast_value compresses float64 into int16 and round-trips.""" import zarr from zarr.codecs.scale_offset import ScaleOffset arr = zarr.create_array( store={}, shape=(100,), dtype="float64", chunks=(100,), filters=[ ScaleOffset(offset=0, scale=10), CastValue(data_type="int16", rounding="nearest-even", out_of_range="clamp"), ], compressors=None, fill_value=0, ) data = np.arange(100, dtype="float64") * 0.1 arr[:] = data result = arr[:] np.testing.assert_array_almost_equal(result, data, decimal=1) # type: ignore[arg-type] @pytest.mark.parametrize( "case", [ Expect( input={"encode": [("NaN", 0)]}, output={"encode": {"NaN": 0}}, id="encode-only", ), Expect( input={"encode": [("NaN", 0)], "decode": [(0, "NaN")]}, output={"encode": {"NaN": 0}, "decode": {0: "NaN"}}, id="both-directions", ), Expect( input={"encode": {"NaN": 0}}, output={"encode": {"NaN": 0}}, id="already-normalized", ), ], ids=lambda c: c.id, ) def test_parse_scalar_map(case: Expect[Any, Any]) -> None: from zarr.codecs.cast_value import parse_scalar_map assert parse_scalar_map(case.input) == case.output # --------------------------------------------------------------------------- # Backend version guard # --------------------------------------------------------------------------- @pytest.mark.parametrize( "case", [ Expect(input="0.4.2", output=None, id="exactly-minimum"), Expect(input="0.4.3", output=None, id="newer-patch"), Expect(input="0.5.0", output=None, id="newer-minor"), Expect(input="1.0.0", output=None, id="newer-major"), Expect(input="0.4.2.post1", output=None, id="post-release"), Expect(input="0.4.0", output="0.4.0", id="known-corrupting"), Expect(input="0.4.1", output="0.4.1", id="one-patch-below"), Expect(input="0.3.0", output="0.3.0", id="older-minor"), Expect(input="0.4.2.dev1", output="0.4.2.dev1", id="pre-release-of-minimum"), ], ids=lambda c: c.id, ) def test_check_backend_version( case: Expect[str, str | None], monkeypatch: pytest.MonkeyPatch ) -> None: """Versions at or above the minimum pass; older ones report the installed version.""" from zarr.codecs import cast_value as mod monkeypatch.setattr(mod, "version", lambda _: case.input) result = mod._check_backend_version() if case.output is None: assert result is None else: assert result is not None assert case.output in result assert mod.CAST_VALUE_RS_MIN_VERSION in result def test_check_backend_version_allows_missing_metadata(monkeypatch: pytest.MonkeyPatch) -> None: """A backend without distribution metadata is allowed: there is no version to compare.""" from importlib.metadata import PackageNotFoundError from zarr.codecs import cast_value as mod def raise_not_found(_: str) -> str: raise PackageNotFoundError monkeypatch.setattr(mod, "version", raise_not_found) assert mod._check_backend_version() is None def test_encode_rejects_outdated_backend(monkeypatch: pytest.MonkeyPatch) -> None: """Using the codec with an outdated backend raises instead of corrupting data.""" from zarr.codecs import cast_value as mod monkeypatch.setattr(mod, "_BACKEND_ERROR", "outdated backend") codec = CastValue(data_type="uint16") with pytest.raises(ImportError, match="outdated backend"): codec._do_cast( np.arange(4, dtype=np.float32), target_dtype=np.dtype("uint16"), scalar_map=None ) def test_min_version_matches_pyproject() -> None: """The runtime floor and the packaging floor must not drift apart.""" import re import tomllib from pathlib import Path from zarr.codecs.cast_value import CAST_VALUE_RS_MIN_VERSION pyproject = Path(__file__).parents[2] / "pyproject.toml" if not pyproject.is_file(): pytest.skip("pyproject.toml is not available in an installed checkout") with pyproject.open("rb") as f: extras = tomllib.load(f)["project"]["optional-dependencies"] (requirement,) = extras["cast-value-rs"] match = re.fullmatch(r"cast-value-rs>=(?P[\w.]+)", requirement) assert match is not None, f"unexpected requirement form: {requirement!r}" assert match.group("version") == CAST_VALUE_RS_MIN_VERSION # --------------------------------------------------------------------------- # Non-contiguous input (regression for #4237) # --------------------------------------------------------------------------- @requires_cast_value_rs def test_enforce_contiguous_arrays() -> None: """ Transpose codec produces non-contiguous arrays. Ensure cast_value makes them contiguous before processing. """ data = np.arange(20, dtype=np.float32).reshape(5, 2, 2) def make_array(filters: list[Any]) -> Any: return zarr.create_array( store=MemoryStore(), shape=data.shape, dtype=data.dtype, chunks=data.shape, filters=filters, serializer=BytesCodec(endian="little"), compressors=None, zarr_format=3, ) # Cast before transpose array = make_array( [ CastValue(data_type="uint16"), TransposeCodec(order=(1, 2, 0)), ] ) array[:] = data assert_array_equal(array[:], data) # Cast after transpose array = make_array( [ TransposeCodec(order=(1, 2, 0)), CastValue(data_type="uint16"), ] ) array[:] = data assert_array_equal(array[:], data) zarr-developers-zarr-python-a2f2415/tests/test_codecs/test_codecs.py000066400000000000000000000372001527014137600260010ustar00rootroot00000000000000from __future__ import annotations import json import warnings from dataclasses import dataclass from typing import TYPE_CHECKING, Any import numpy as np import pytest import zarr import zarr.api import zarr.api.asynchronous from zarr import Array, AsyncArray, config from zarr.codecs import ( BytesCodec, GzipCodec, ShardingCodec, TransposeCodec, ) from zarr.core.buffer import default_buffer_prototype from zarr.core.indexing import BasicSelection, decode_morton, morton_order_coords from zarr.core.metadata.v3 import ArrayV3Metadata from zarr.dtype import UInt8 from zarr.errors import ZarrUserWarning from zarr.storage import MemoryStore, StorePath if TYPE_CHECKING: from zarr.abc.codec import Codec from zarr.abc.store import Store from zarr.core.buffer.core import NDArrayLikeOrScalar from zarr.core.common import MemoryOrder from zarr.types import AnyAsyncArray @dataclass(frozen=True) class _AsyncArrayProxy: array: AnyAsyncArray def __getitem__(self, selection: BasicSelection) -> _AsyncArraySelectionProxy: return _AsyncArraySelectionProxy(self.array, selection) @dataclass(frozen=True) class _AsyncArraySelectionProxy: array: AnyAsyncArray selection: BasicSelection async def get(self) -> NDArrayLikeOrScalar: return await self.array.getitem(self.selection) async def set(self, value: np.ndarray[Any, Any]) -> None: return await self.array.setitem(self.selection, value) def order_from_dim(order: MemoryOrder, ndim: int) -> tuple[int, ...]: if order == "F": return tuple(ndim - x - 1 for x in range(ndim)) else: return tuple(range(ndim)) def test_sharding_pickle() -> None: """ Test that sharding codecs can be pickled """ @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) @pytest.mark.parametrize("input_order", ["F", "C"]) @pytest.mark.parametrize("store_order", ["F", "C"]) @pytest.mark.parametrize("runtime_write_order", ["F", "C"]) @pytest.mark.parametrize("runtime_read_order", ["F", "C"]) @pytest.mark.parametrize("with_sharding", [True, False]) async def test_order( store: Store, input_order: MemoryOrder, store_order: MemoryOrder, runtime_write_order: MemoryOrder, runtime_read_order: MemoryOrder, with_sharding: bool, ) -> None: data = np.arange(0, 256, dtype="uint16").reshape((32, 8), order=input_order) path = "order" spath = StorePath(store, path=path) a = await zarr.api.asynchronous.create_array( spath, shape=data.shape, chunks=(16, 8) if with_sharding else (32, 8), shards=(32, 8) if with_sharding else None, dtype=data.dtype, fill_value=0, chunk_key_encoding={"name": "v2", "separator": "."}, filters=[TransposeCodec(order=order_from_dim(store_order, data.ndim))], config={"order": runtime_write_order}, ) await _AsyncArrayProxy(a)[:, :].set(data) read_data = await _AsyncArrayProxy(a)[:, :].get() assert np.array_equal(data, read_data) with config.set({"array.order": runtime_read_order}): a = await AsyncArray.open( spath, ) read_data = await _AsyncArrayProxy(a)[:, :].get() assert np.array_equal(data, read_data) assert isinstance(read_data, np.ndarray) if runtime_read_order == "F": assert read_data.flags["F_CONTIGUOUS"] assert not read_data.flags["C_CONTIGUOUS"] else: assert not read_data.flags["F_CONTIGUOUS"] assert read_data.flags["C_CONTIGUOUS"] @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) @pytest.mark.parametrize("input_order", ["F", "C"]) @pytest.mark.parametrize("runtime_write_order", ["F", "C"]) @pytest.mark.parametrize("runtime_read_order", ["F", "C"]) @pytest.mark.parametrize("with_sharding", [True, False]) def test_order_implicit( store: Store, input_order: MemoryOrder, runtime_write_order: MemoryOrder, runtime_read_order: MemoryOrder, with_sharding: bool, ) -> None: data = np.arange(0, 256, dtype="uint16").reshape((16, 16), order=input_order) path = "order_implicit" spath = StorePath(store, path) with config.set({"array.order": runtime_write_order}): a = zarr.create_array( spath, shape=data.shape, chunks=(8, 8) if with_sharding else (16, 16), shards=(16, 16) if with_sharding else None, dtype=data.dtype, fill_value=0, ) a[:, :] = data with config.set({"array.order": runtime_read_order}): a = Array.open(spath) read_data = a[:, :] assert np.array_equal(data, read_data) assert isinstance(read_data, np.ndarray) if runtime_read_order == "F": assert read_data.flags["F_CONTIGUOUS"] assert not read_data.flags["C_CONTIGUOUS"] else: assert not read_data.flags["F_CONTIGUOUS"] assert read_data.flags["C_CONTIGUOUS"] @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) def test_open(store: Store) -> None: spath = StorePath(store) a = zarr.create_array( spath, shape=(16, 16), chunks=(16, 16), dtype="int32", fill_value=0, ) b = Array.open(spath) assert a.metadata == b.metadata def test_morton_exact_order() -> None: """Test exact morton ordering for power-of-2 shapes.""" assert list(morton_order_coords((2, 2))) == [(0, 0), (1, 0), (0, 1), (1, 1)] assert list(morton_order_coords((2, 2, 2))) == [ (0, 0, 0), (1, 0, 0), (0, 1, 0), (1, 1, 0), (0, 0, 1), (1, 0, 1), (0, 1, 1), (1, 1, 1), ] assert list(morton_order_coords((2, 2, 2, 2))) == [ (0, 0, 0, 0), (1, 0, 0, 0), (0, 1, 0, 0), (1, 1, 0, 0), (0, 0, 1, 0), (1, 0, 1, 0), (0, 1, 1, 0), (1, 1, 1, 0), (0, 0, 0, 1), (1, 0, 0, 1), (0, 1, 0, 1), (1, 1, 0, 1), (0, 0, 1, 1), (1, 0, 1, 1), (0, 1, 1, 1), (1, 1, 1, 1), ] @pytest.mark.parametrize( "shape", [ (2, 2, 2), (5, 2), (2, 5), (2, 9, 2), (3, 2, 12), (2, 5, 1), (4, 3, 6, 2, 7), (3, 2, 1, 6, 4, 5, 2), (1,), (1, 1), (5, 1, 3), (1, 4, 1, 2), (5, 5, 5), # triggers argsort strategy (n_z/n_total > 4) ], ) def test_morton_is_permutation(shape: tuple[int, ...]) -> None: """Test that morton_order_coords produces every valid coordinate exactly once.""" import itertools from zarr.core.common import product order = list(morton_order_coords(shape)) expected_len = product(shape) # completeness: every valid coordinate is present assert len(order) == expected_len # no duplicates assert len(set(order)) == expected_len # all coordinates are within bounds assert all(all(c < s for c, s in zip(coord, shape, strict=True)) for coord in order) # the set of coordinates equals the full cartesian product assert set(order) == set(itertools.product(*(range(s) for s in shape))) @pytest.mark.parametrize( "shape", [ (2, 2), (4, 4), (2, 2, 2), (4, 4, 4), (2, 2, 2, 2), ], ) def test_morton_ordering(shape: tuple[int, ...]) -> None: """Test that the iteration order matches consecutive decode_morton outputs. For power-of-2 shapes, every decode_morton output is in-bounds, so the ordering should be exactly decode_morton(0), decode_morton(1), ... """ order = list(morton_order_coords(shape)) for i, coord in enumerate(order): assert coord == decode_morton(i, shape) @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) def test_write_partial_chunks(store: Store) -> None: data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) spath = StorePath(store) a = zarr.create_array( spath, shape=data.shape, chunks=(20, 20), dtype=data.dtype, fill_value=1, ) a[0:16, 0:16] = data assert np.array_equal(a[0:16, 0:16], data) @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) async def test_delete_empty_chunks(store: Store) -> None: data = np.ones((16, 16)) path = "delete_empty_chunks" spath = StorePath(store, path) a = await zarr.api.asynchronous.create_array( spath, shape=data.shape, chunks=(32, 32), dtype=data.dtype, fill_value=1, ) await _AsyncArrayProxy(a)[:16, :16].set(np.zeros((16, 16))) await _AsyncArrayProxy(a)[:16, :16].set(data) assert np.array_equal(await _AsyncArrayProxy(a)[:16, :16].get(), data) assert await store.get(f"{path}/c0/0", prototype=default_buffer_prototype()) is None @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) async def test_dimension_names(store: Store) -> None: data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) path = "dimension_names" spath = StorePath(store, path) await zarr.api.asynchronous.create_array( spath, shape=data.shape, chunks=(16, 16), dtype=data.dtype, fill_value=0, dimension_names=("x", "y"), ) assert isinstance( meta := (await zarr.api.asynchronous.open_array(store=spath)).metadata, ArrayV3Metadata ) assert meta.dimension_names == ( "x", "y", ) path2 = "dimension_names2" spath2 = StorePath(store, path2) await zarr.api.asynchronous.create_array( spath2, shape=data.shape, chunks=(16, 16), dtype=data.dtype, fill_value=0, ) assert isinstance(meta := (await AsyncArray.open(spath2)).metadata, ArrayV3Metadata) assert meta.dimension_names is None zarr_json_buffer = await store.get(f"{path2}/zarr.json", prototype=default_buffer_prototype()) assert zarr_json_buffer is not None assert "dimension_names" not in json.loads(zarr_json_buffer.to_bytes()) @pytest.mark.parametrize( "codecs", [ (BytesCodec(), TransposeCodec(order=order_from_dim("F", 2))), (TransposeCodec(order=order_from_dim("F", 2)),), ], ) def test_invalid_metadata(codecs: tuple[Codec, ...]) -> None: shape = (16,) chunks = (16,) data_type = UInt8() with pytest.raises(ValueError, match="The `order` tuple must have as many entries"): ArrayV3Metadata( shape=shape, chunk_grid={"name": "regular", "configuration": {"chunk_shape": chunks}}, chunk_key_encoding={"name": "default", "configuration": {"separator": "/"}}, fill_value=0, data_type=data_type, codecs=codecs, attributes={}, dimension_names=None, ) def test_invalid_metadata_create_array() -> None: with pytest.warns( ZarrUserWarning, match="codec disables partial reads and writes, which may lead to inefficient performance", ): zarr.create_array( {}, shape=(16, 16), chunks=(16, 16), dtype=np.dtype("uint8"), fill_value=0, serializer=ShardingCodec(chunk_shape=(8, 8)), compressors=[ GzipCodec(), ], ) @pytest.mark.parametrize( "pipeline_path", [ "zarr.core.codec_pipeline.BatchedCodecPipeline", "zarr.core.codec_pipeline.FusedCodecPipeline", ], ) def test_sharding_warning_fires_once_per_open(pipeline_path: str) -> None: """Construction-time codec warnings (e.g. sharding's partial-reads warning) must fire exactly once per array open, not once per internal codec-chain reconstruction. `create_codec_pipeline` builds a throwaway pipeline via `from_codecs` (which warns) and then calls `evolve_from_array_spec` on it, which re-splits the (already-warned-about) codec chain against the evolved spec. That re-split goes through `codecs_from_list_unchecked` rather than `codecs_from_list`, so it does not re-emit the warning. `FusedCodecPipeline` additionally builds a `ChunkTransform` (and, on the async fallback path, an `AsyncChunkTransform` per call) from the same evolved codec chain, which must use the same quiet variant. """ with config.set({"codec_pipeline.path": pipeline_path}): store = MemoryStore() with warnings.catch_warnings(): warnings.simplefilter("ignore") zarr.create_array( store, shape=(16, 16), chunks=(16, 16), dtype=np.dtype("uint8"), fill_value=0, serializer=ShardingCodec(chunk_shape=(8, 8)), compressors=[GzipCodec()], ) with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") zarr.open_array(store, mode="r") matches = [w for w in caught if "disables partial reads" in str(w.message)] assert len(matches) == 1 @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) async def test_resize(store: Store) -> None: data = np.zeros((16, 18), dtype="uint16") path = "resize" spath = StorePath(store, path) a = await zarr.api.asynchronous.create_array( spath, shape=data.shape, chunks=(10, 10), dtype=data.dtype, chunk_key_encoding={"name": "v2", "separator": "."}, fill_value=1, ) await _AsyncArrayProxy(a)[:16, :18].set(data) assert await store.get(f"{path}/1.1", prototype=default_buffer_prototype()) is not None assert await store.get(f"{path}/0.0", prototype=default_buffer_prototype()) is not None assert await store.get(f"{path}/0.1", prototype=default_buffer_prototype()) is not None assert await store.get(f"{path}/1.0", prototype=default_buffer_prototype()) is not None await a.resize((10, 12)) assert a.metadata.shape == (10, 12) assert a.shape == (10, 12) assert await store.get(f"{path}/0.0", prototype=default_buffer_prototype()) is not None assert await store.get(f"{path}/0.1", prototype=default_buffer_prototype()) is not None assert await store.get(f"{path}/1.0", prototype=default_buffer_prototype()) is None assert await store.get(f"{path}/1.1", prototype=default_buffer_prototype()) is None def _resolve_metadata_codecs() -> list[Codec]: from zarr.codecs.crc32c_ import Crc32cCodec from zarr.codecs.zstd import ZstdCodec return [ BytesCodec(), GzipCodec(level=1), TransposeCodec(order=(0,)), Crc32cCodec(), ZstdCodec(level=1), ] @pytest.mark.parametrize("codec", _resolve_metadata_codecs(), ids=lambda c: type(c).__name__) def test_resolve_metadata_only_mutates_shape(codec: Codec) -> None: """A codec's resolve_metadata may change a chunk's `shape` but must leave the prototype, dtype, fill_value, and config untouched -- the pipeline relies on those being stable across the codec chain. """ from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.dtype import get_data_type_from_native_dtype zdtype = get_data_type_from_native_dtype(np.dtype("float64")) spec_in = ArraySpec( shape=(10,), dtype=zdtype, fill_value=zdtype.cast_scalar(0.0), config=ArrayConfig(order="C", write_empty_chunks=False), prototype=default_buffer_prototype(), ) spec_out = codec.resolve_metadata(spec_in) name = type(codec).__name__ assert spec_out.prototype is spec_in.prototype, f"{name} changed prototype" assert spec_out.dtype == spec_in.dtype, f"{name} changed dtype" assert spec_out.fill_value == spec_in.fill_value, f"{name} changed fill_value" assert spec_out.config == spec_in.config, f"{name} changed config" zarr-developers-zarr-python-a2f2415/tests/test_codecs/test_crc32c.py000066400000000000000000000021611527014137600256160ustar00rootroot00000000000000from __future__ import annotations import numpy as np from zarr.abc.codec import SupportsSyncCodec from zarr.codecs.crc32c_ import Crc32cCodec from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer import default_buffer_prototype from zarr.core.dtype import get_data_type_from_native_dtype def test_crc32c_codec_supports_sync() -> None: assert isinstance(Crc32cCodec(), SupportsSyncCodec) def test_crc32c_codec_sync_roundtrip() -> None: codec = Crc32cCodec() arr = np.arange(100, dtype="float64") zdtype = get_data_type_from_native_dtype(arr.dtype) spec = ArraySpec( shape=arr.shape, dtype=zdtype, fill_value=zdtype.cast_scalar(0), config=ArrayConfig(order="C", write_empty_chunks=True), prototype=default_buffer_prototype(), ) buf = default_buffer_prototype().buffer.from_array_like(arr.view("B")) encoded = codec._encode_sync(buf, spec) assert encoded is not None decoded = codec._decode_sync(encoded, spec) result = np.frombuffer(decoded.as_numpy_array(), dtype="float64") np.testing.assert_array_equal(arr, result) zarr-developers-zarr-python-a2f2415/tests/test_codecs/test_gzip.py000066400000000000000000000052231527014137600255120ustar00rootroot00000000000000import numpy as np import pytest import zarr from tests.conftest import gzip_streams_equal_except_mtime from zarr.abc.codec import SupportsSyncCodec from zarr.abc.store import Store from zarr.codecs import GzipCodec from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer import default_buffer_prototype from zarr.core.dtype import get_data_type_from_native_dtype from zarr.storage import StorePath @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) def test_gzip(store: Store) -> None: data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) a = zarr.create_array( StorePath(store), shape=data.shape, chunks=(16, 16), dtype=data.dtype, fill_value=0, compressors=GzipCodec(), ) a[:, :] = data assert np.array_equal(data, a[:, :]) def test_gzip_codec_supports_sync() -> None: assert isinstance(GzipCodec(), SupportsSyncCodec) def test_gzip_codec_sync_roundtrip() -> None: codec = GzipCodec(level=1) arr = np.arange(100, dtype="float64") zdtype = get_data_type_from_native_dtype(arr.dtype) spec = ArraySpec( shape=arr.shape, dtype=zdtype, fill_value=zdtype.cast_scalar(0), config=ArrayConfig(order="C", write_empty_chunks=True), prototype=default_buffer_prototype(), ) buf = default_buffer_prototype().buffer.from_array_like(arr.view("B")) encoded = codec._encode_sync(buf, spec) assert encoded is not None decoded = codec._decode_sync(encoded, spec) result = np.frombuffer(decoded.as_numpy_array(), dtype="float64") np.testing.assert_array_equal(arr, result) def test_gzip_streams_equal_except_mtime() -> None: prefix = b"\x1f\x8b\x08\x00" mtime = b"\x01\x02\x03\x04" suffix = b"\x00\xff\x10\x20" # Identical streams are equal. assert gzip_streams_equal_except_mtime( prefix + mtime + suffix, prefix + mtime + suffix, ) # Streams with different MTIME values are still equal. assert gzip_streams_equal_except_mtime( prefix + b"\x01\x02\x03\x04" + suffix, prefix + b"\x05\x06\x07\x08" + suffix, ) # Differences after MTIME are detected. assert not gzip_streams_equal_except_mtime( prefix + mtime + suffix, prefix + mtime + b"\x00\xff\x10\x21", ) # Differences before MTIME are detected. assert not gzip_streams_equal_except_mtime( prefix + mtime + suffix, b"\x1f\x8b\x09\x00" + mtime + suffix, ) # Different lengths are detected. assert not gzip_streams_equal_except_mtime( prefix + mtime + suffix, prefix + mtime + suffix + b"\x00", ) zarr-developers-zarr-python-a2f2415/tests/test_codecs/test_numcodecs.py000066400000000000000000000253321527014137600265240ustar00rootroot00000000000000from __future__ import annotations import contextlib import pickle from typing import TYPE_CHECKING, Any import numpy as np import pytest from numcodecs import GZip from zarr import config, create_array, open_array from zarr.abc.numcodec import _is_numcodec, _is_numcodec_cls from zarr.codecs import numcodecs as _numcodecs from zarr.errors import UnknownCodecError from zarr.registry import get_codec_class, get_numcodec if TYPE_CHECKING: from collections.abc import Iterator @contextlib.contextmanager def codec_conf() -> Iterator[Any]: base_conf = config.get("codecs") new_conf = { "numcodecs.bz2": "zarr.codecs.numcodecs.BZ2", "numcodecs.crc32": "zarr.codecs.numcodecs.CRC32", "numcodecs.crc32c": "zarr.codecs.numcodecs.CRC32C", "numcodecs.lz4": "zarr.codecs.numcodecs.LZ4", "numcodecs.lzma": "zarr.codecs.numcodecs.LZMA", "numcodecs.zfpy": "zarr.codecs.numcodecs.ZFPY", "numcodecs.adler32": "zarr.codecs.numcodecs.Adler32", "numcodecs.astype": "zarr.codecs.numcodecs.AsType", "numcodecs.bitround": "zarr.codecs.numcodecs.BitRound", "numcodecs.blosc": "zarr.codecs.numcodecs.Blosc", "numcodecs.delta": "zarr.codecs.numcodecs.Delta", "numcodecs.fixedscaleoffset": "zarr.codecs.numcodecs.FixedScaleOffset", "numcodecs.fletcher32": "zarr.codecs.numcodecs.Fletcher32", "numcodecs.gzip": "zarr.codecs.numcodecs.GZip", "numcodecs.jenkinslookup3": "zarr.codecs.numcodecs.JenkinsLookup3", "numcodecs.pcodec": "zarr.codecs.numcodecs.PCodec", "numcodecs.packbits": "zarr.codecs.numcodecs.PackBits", "numcodecs.shuffle": "zarr.codecs.numcodecs.Shuffle", "numcodecs.quantize": "zarr.codecs.numcodecs.Quantize", "numcodecs.zlib": "zarr.codecs.numcodecs.Zlib", "numcodecs.zstd": "zarr.codecs.numcodecs.Zstd", } yield config.set({"codecs": new_conf | base_conf}) if TYPE_CHECKING: from zarr.core.common import JSON def test_get_numcodec() -> None: assert get_numcodec({"id": "gzip", "level": 2}) == GZip(level=2) # type: ignore[typeddict-unknown-key] def test_is_numcodec() -> None: """ Test the _is_numcodec function """ assert _is_numcodec(GZip()) def test_is_numcodec_cls() -> None: """ Test the _is_numcodec_cls function """ assert _is_numcodec_cls(GZip) ALL_CODECS = tuple( filter( lambda v: issubclass(v, _numcodecs._NumcodecsCodec) and hasattr(v, "codec_name"), tuple(getattr(_numcodecs, cls_name) for cls_name in _numcodecs.__all__), ) ) @pytest.mark.parametrize("codec_cls", ALL_CODECS) def test_get_codec_class(codec_cls: type[_numcodecs._NumcodecsCodec]) -> None: assert get_codec_class(codec_cls.codec_name) == codec_cls # type: ignore[comparison-overlap,misc] @pytest.mark.parametrize("codec_class", ALL_CODECS) def test_docstring(codec_class: type[_numcodecs._NumcodecsCodec]) -> None: """ Test that the docstring for the zarr.numcodecs codecs references the wrapped numcodecs class. """ assert "See [numcodecs." in codec_class.__doc__ # type: ignore[operator] @pytest.mark.parametrize( "codec_class", [ _numcodecs.Blosc, _numcodecs.LZ4, _numcodecs.Zstd, _numcodecs.Zlib, _numcodecs.GZip, _numcodecs.BZ2, _numcodecs.LZMA, _numcodecs.Shuffle, ], ) def test_generic_compressor(codec_class: type[_numcodecs._NumcodecsBytesBytesCodec]) -> None: data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) a = create_array( {}, shape=data.shape, chunks=(16, 16), dtype=data.dtype, fill_value=0, compressors=[codec_class()], ) a[:, :] = data.copy() np.testing.assert_array_equal(data, a[:, :]) @pytest.mark.parametrize( ("codec_class", "codec_config"), [ (_numcodecs.Delta, {"dtype": "float32"}), (_numcodecs.FixedScaleOffset, {"offset": 0, "scale": 25.5}), (_numcodecs.FixedScaleOffset, {"offset": 0, "scale": 51, "astype": "uint16"}), (_numcodecs.AsType, {"encode_dtype": "float32", "decode_dtype": "float32"}), ], ids=[ "delta", "fixedscaleoffset", "fixedscaleoffset2", "astype", ], ) def test_generic_filter( codec_class: type[_numcodecs._NumcodecsArrayArrayCodec], codec_config: dict[str, JSON], ) -> None: data = np.linspace(0, 10, 256, dtype="float32").reshape((16, 16)) a = create_array( {}, shape=data.shape, chunks=(16, 16), dtype=data.dtype, fill_value=0, filters=[ codec_class(**codec_config), ], ) a[:, :] = data.copy() with codec_conf(): b = open_array(a.store, mode="r") np.testing.assert_array_equal(data, b[:, :]) @pytest.mark.parametrize( ("codec_class", "codec_config", "dtype"), [ (_numcodecs.Delta, {"dtype": "int64"}, "int64"), (_numcodecs.FixedScaleOffset, {"offset": 0, "scale": 1}, "int64"), (_numcodecs.PackBits, {}, "bool"), ], ids=["delta", "fixedscaleoffset", "packbits"], ) def test_generic_filter_f_contiguous( codec_class: type[_numcodecs._NumcodecsArrayArrayCodec], codec_config: dict[str, JSON], dtype: str, ) -> None: # gh-3558: F-contiguous chunks were handed to numcodecs filters as is, and # numcodecs flattens in memory order, so the elements came back transposed if dtype == "bool": data = np.asfortranarray(np.tril(np.ones((16, 16), dtype=bool))) else: data = np.asfortranarray(np.arange(256, dtype=dtype).reshape(16, 16)) a = create_array( {}, shape=data.shape, chunks=(16, 16), dtype=data.dtype, fill_value=0, filters=[codec_class(**codec_config)], ) a[:, :] = data np.testing.assert_array_equal(data, a[:, :]) def test_generic_filter_bitround() -> None: data = np.linspace(0, 1, 256, dtype="float32").reshape((16, 16)) a = create_array( {}, shape=data.shape, chunks=(16, 16), dtype=data.dtype, fill_value=0, filters=[_numcodecs.BitRound(keepbits=3)], ) a[:, :] = data.copy() b = open_array(a.store, mode="r") assert np.allclose(data, b[:, :], atol=0.1) def test_generic_filter_quantize() -> None: data = np.linspace(0, 10, 256, dtype="float32").reshape((16, 16)) a = create_array( {}, shape=data.shape, chunks=(16, 16), dtype=data.dtype, fill_value=0, filters=[_numcodecs.Quantize(digits=3)], ) a[:, :] = data.copy() b = open_array(a.store, mode="r") assert np.allclose(data, b[:, :], atol=0.001) def test_generic_filter_packbits() -> None: data = np.zeros((16, 16), dtype="bool") data[0:4, :] = True a = create_array( {}, shape=data.shape, chunks=(16, 16), dtype=data.dtype, fill_value=0, filters=[_numcodecs.PackBits()], ) a[:, :] = data.copy() b = open_array(a.store, mode="r") np.testing.assert_array_equal(data, b[:, :]) with pytest.raises(ValueError, match=".*requires bool dtype.*"): create_array( {}, shape=data.shape, chunks=(16, 16), dtype="uint32", fill_value=0, filters=[_numcodecs.PackBits()], ) @pytest.mark.parametrize( "codec_class", [ _numcodecs.CRC32, _numcodecs.CRC32C, _numcodecs.Adler32, _numcodecs.Fletcher32, _numcodecs.JenkinsLookup3, ], ) def test_generic_checksum(codec_class: type[_numcodecs._NumcodecsBytesBytesCodec]) -> None: # Check if the codec is available in numcodecs try: codec_class()._codec # noqa: B018 except UnknownCodecError as e: # pragma: no cover pytest.skip(f"{codec_class.codec_name} is not available in numcodecs: {e}") # type: ignore[misc] data = np.linspace(0, 10, 256, dtype="float32").reshape((16, 16)) a = create_array( {}, shape=data.shape, chunks=(16, 16), dtype=data.dtype, fill_value=0, compressors=[codec_class()], ) a[:, :] = data.copy() with codec_conf(): b = open_array(a.store, mode="r") np.testing.assert_array_equal(data, b[:, :]) @pytest.mark.parametrize("codec_class", [_numcodecs.PCodec, _numcodecs.ZFPY]) def test_generic_bytes_codec(codec_class: type[_numcodecs._NumcodecsArrayBytesCodec]) -> None: try: codec_class()._codec # noqa: B018 except (UnknownCodecError, ImportError) as e: # pragma: no cover pytest.xfail(f"{codec_class.codec_name} is not available: {e}") # type: ignore[misc] data = np.arange(0, 256, dtype="float32").reshape((16, 16)) a = create_array( {}, shape=data.shape, chunks=(16, 16), dtype=data.dtype, fill_value=0, serializer=codec_class(), ) a[:, :] = data.copy() np.testing.assert_array_equal(data, a[:, :]) def test_delta_astype() -> None: data = np.linspace(0, 10, 256, dtype="i8").reshape((16, 16)) a = create_array( {}, shape=data.shape, chunks=(16, 16), dtype=data.dtype, fill_value=0, filters=[ _numcodecs.Delta(dtype="i8", astype="i2"), ], ) a[:, :] = data.copy() with codec_conf(): b = open_array(a.store, mode="r") np.testing.assert_array_equal(data, b[:, :]) def test_repr() -> None: codec = _numcodecs.LZ4(level=5) assert repr(codec) == "LZ4(codec_name='numcodecs.lz4', codec_config={'level': 5})" def test_to_dict() -> None: codec = _numcodecs.LZ4(level=5) assert codec.to_dict() == {"name": "numcodecs.lz4", "configuration": {"level": 5}} @pytest.mark.parametrize( "codec_cls", [ _numcodecs.Blosc, _numcodecs.LZ4, _numcodecs.Zstd, _numcodecs.Zlib, _numcodecs.GZip, _numcodecs.BZ2, _numcodecs.LZMA, _numcodecs.Shuffle, _numcodecs.BitRound, _numcodecs.Delta, _numcodecs.FixedScaleOffset, _numcodecs.Quantize, _numcodecs.PackBits, _numcodecs.AsType, _numcodecs.CRC32, _numcodecs.CRC32C, _numcodecs.Adler32, _numcodecs.Fletcher32, _numcodecs.JenkinsLookup3, _numcodecs.PCodec, _numcodecs.ZFPY, ], ) def test_codecs_pickleable(codec_cls: type[_numcodecs._NumcodecsCodec]) -> None: # Check if the codec is available in numcodecs try: codec = codec_cls() except UnknownCodecError as e: # pragma: no cover pytest.skip(f"{codec_cls.codec_name} is not available in numcodecs: {e}") # type: ignore[misc] expected = codec p = pickle.dumps(codec) actual = pickle.loads(p) assert actual == expected zarr-developers-zarr-python-a2f2415/tests/test_codecs/test_scale_offset.py000066400000000000000000000406221527014137600272000ustar00rootroot00000000000000from __future__ import annotations from typing import Any import numpy as np import pytest import zarr from tests.conftest import Expect, ExpectFail from zarr.codecs.scale_offset import ( ScaleOffset, _decode, _decode_fits_natively, _encode, ) from zarr.core.buffer.core import default_buffer_prototype from zarr.storage._memory import MemoryStore # --------------------------------------------------------------------------- # Serialization # --------------------------------------------------------------------------- @pytest.mark.parametrize( "case", [ Expect(input=ScaleOffset(), output={"name": "scale_offset"}, id="default"), Expect( input=ScaleOffset(offset=5), output={"name": "scale_offset", "configuration": {"offset": 5}}, id="offset-only", ), Expect( input=ScaleOffset(scale=0.1), output={"name": "scale_offset", "configuration": {"scale": 0.1}}, id="scale-only", ), Expect( input=ScaleOffset(offset=5, scale=0.1), output={"name": "scale_offset", "configuration": {"offset": 5, "scale": 0.1}}, id="both", ), ], ids=lambda c: c.id, ) def test_to_dict(case: Expect[ScaleOffset, dict[str, Any]]) -> None: """to_dict produces the expected JSON structure.""" assert case.input.to_dict() == case.output @pytest.mark.parametrize( "case", [ Expect(input={"name": "scale_offset"}, output=(0, 1), id="no-config"), Expect( input={"name": "scale_offset", "configuration": {"offset": 3, "scale": 2}}, output=(3, 2), id="with-config", ), ], ids=lambda c: c.id, ) def test_from_dict(case: Expect[dict[str, Any], tuple[int | float, int | float]]) -> None: """from_dict deserializes configuration with correct values and defaults.""" codec = ScaleOffset.from_dict(case.input) expected_offset, expected_scale = case.output assert codec.offset == expected_offset assert codec.scale == expected_scale def test_serialization_roundtrip() -> None: """to_dict followed by from_dict produces an equal codec.""" original = ScaleOffset(offset=7, scale=0.5) restored = ScaleOffset.from_dict(original.to_dict()) assert original == restored # --------------------------------------------------------------------------- # Construction # --------------------------------------------------------------------------- @pytest.mark.parametrize( "case", [ ExpectFail( input={"offset": [1, 2]}, exception=TypeError, id="list-offset", msg="offset must be a number or string", ), ExpectFail( input={"scale": [1, 2]}, exception=TypeError, id="list-scale", msg="scale must be a number or string", ), ], ids=lambda c: c.id, ) def test_construction_rejects_non_numeric(case: ExpectFail[dict[str, Any]]) -> None: """Non-numeric offset or scale is rejected at construction time.""" with case.raises(): ScaleOffset(**case.input) @pytest.mark.parametrize( "case", [ Expect(input={"offset": 5, "scale": 2}, output=(5, 2), id="int"), Expect(input={"offset": 0.5, "scale": 0.1}, output=(0.5, 0.1), id="float"), ], ids=lambda c: c.id, ) def test_construction_accepts_numeric( case: Expect[dict[str, Any], tuple[int | float, int | float]], ) -> None: """Integer and float values are accepted for both parameters.""" codec = ScaleOffset(**case.input) assert codec.offset == case.output[0] assert codec.scale == case.output[1] # --------------------------------------------------------------------------- # Encode / decode # --------------------------------------------------------------------------- @pytest.mark.parametrize( ("dtype", "offset", "scale"), [ ("float64", 10.0, 0.1), ("float32", 5.0, 2.0), ("int32", 0, 1), (">f8", 10.0, 0.1), ("i4", -7, 3), (">u8", 2**63, 1), ("f8", "i4", ">u8", " None: """Data survives encode → decode, in either byte order.""" arr = zarr.create_array( store={}, shape=(100,), dtype=dtype, chunks=(100,), filters=[ScaleOffset(offset=offset, scale=scale)], compressors=None, fill_value=offset, ) data = np.array([offset + i for i in range(100)], dtype=dtype) arr[:] = data result = arr[:] assert result.dtype == np.dtype(dtype) # type: ignore[union-attr] np.testing.assert_array_almost_equal(result, data) # type: ignore[arg-type] def test_fill_value_transformed() -> None: """Fill value is transformed through the encode formula and read back correctly.""" arr = zarr.create_array( store={}, shape=(10,), dtype="float64", chunks=(10,), filters=[ScaleOffset(offset=5, scale=2)], compressors=None, fill_value=10.0, ) # fill_value=10.0, encode: (10 - 5) * 2 = 10.0 stored # Reading back without writing should return the original fill value np.testing.assert_array_equal(arr[:], np.full(10, 10.0)) def test_identity_is_noop() -> None: """Default codec (offset=0, scale=1) is a no-op.""" import zarr arr = zarr.create_array( store={}, shape=(50,), dtype="float64", chunks=(50,), filters=[ScaleOffset()], compressors=None, fill_value=0, ) data = np.arange(50, dtype="float64") arr[:] = data np.testing.assert_array_equal(arr[:], data) def test_rejects_complex_dtype() -> None: """Complex dtypes are rejected at array creation time.""" with pytest.raises(ValueError, match="only supports integer and floating-point"): zarr.create_array( store={}, shape=(10,), dtype="complex128", chunks=(10,), filters=[ScaleOffset(offset=1, scale=2)], compressors=None, fill_value=0, ) def test_uint64_large_value_roundtrip() -> None: """uint64 values above 2**63 must survive encode+decode (spec requires uint64 support).""" arr = zarr.create_array( store={}, shape=(3,), dtype="uint64", chunks=(3,), filters=[ScaleOffset(offset=0, scale=1)], compressors=None, fill_value=0, ) # Value above int64.max (2**63 - 1) — would wrap if we used int64 as wide dtype. data = np.array([0, 2**63, 2**64 - 1], dtype="uint64") arr[:] = data np.testing.assert_array_equal(arr[:], data) def test_float_nan_inf_preserved() -> None: """NaN and Inf are representable in float dtypes per IEEE 754 and must pass through.""" arr = np.array([1.0, np.nan, np.inf, -np.inf], dtype="float64") encoded = _encode(arr, np.float64(0.0), np.float64(2.0)) np.testing.assert_array_equal(encoded[[0]], np.array([2.0])) assert np.isnan(encoded[1]) assert encoded[2] == np.inf assert encoded[3] == -np.inf decoded = _decode(encoded, np.float64(0.0), np.float64(2.0), scale_repr=2.0) np.testing.assert_array_equal(decoded[[0]], np.array([1.0])) assert np.isnan(decoded[1]) def test_uint64_encode_rejects_underflow() -> None: """uint64 underflow during encode raises rather than silently wrapping.""" arr = zarr.create_array( store={}, shape=(3,), dtype="uint64", chunks=(3,), filters=[ScaleOffset(offset=100, scale=1)], compressors=None, fill_value=100, ) with pytest.raises(ValueError, match="outside the range of dtype uint64"): arr[:] = np.array([100, 50, 200], dtype="uint64") @pytest.mark.parametrize( ("dtype", "scale"), [ ("int32", 0), ("int32", "0"), ("float64", 0.0), ("float64", "0.0"), ("float64", "0x0000000000000000"), ], ids=["int-numeric", "int-string", "float-numeric", "float-string", "float-hex"], ) def test_rejects_zero_scale(dtype: str, scale: object) -> None: """scale=0 is rejected (destroys data and breaks decode division). A string ``scale`` is a documented input, so every spelling of zero has to be rejected the same way as the numeric one. """ with pytest.raises(ValueError, match="scale must be non-zero"): zarr.create_array( store={}, shape=(10,), dtype=dtype, chunks=(10,), filters=[ScaleOffset(offset=0, scale=scale)], compressors=None, fill_value=0, ) @pytest.mark.parametrize( "case", [ ExpectFail( input={"dtype": "int32", "offset": 1.5, "scale": 1}, exception=ValueError, id="float-offset-for-int", msg="offset value 1.5 is not representable", ), ExpectFail( input={"dtype": "int32", "offset": 0, "scale": 0.5}, exception=ValueError, id="float-scale-for-int", msg="scale value 0.5 is not representable", ), ExpectFail( input={"dtype": "int16", "offset": "NaN", "scale": 1}, exception=ValueError, id="nan-offset-for-int", msg="offset value 'NaN' is not representable", ), ], ids=lambda c: c.id, ) def test_rejects_unrepresentable_scale_offset(case: ExpectFail[dict[str, Any]]) -> None: """Scale/offset values that can't be represented in the array dtype are rejected.""" with case.raises(): zarr.create_array( store={}, shape=(10,), dtype=case.input["dtype"], chunks=(10,), filters=[ScaleOffset(offset=case.input["offset"], scale=case.input["scale"])], compressors=None, fill_value=0, ) def test_dtype_preservation() -> None: """Integer scale/offset arithmetic preserves the array dtype when division is exact.""" arr = zarr.create_array( store={}, shape=(10,), dtype="int8", chunks=(10,), filters=[ScaleOffset(offset=1, scale=2)], compressors=None, fill_value=0, ) data = np.arange(10, dtype="int8") arr[:] = data # encode=(x-1)*2 is always divisible by scale=2, so decode is exact np.testing.assert_array_equal(arr[:], data) async def test_integer_decode_rejects_non_exact_division() -> None: """Decoding an integer array raises when the stored value isn't divisible by scale.""" store = MemoryStore() arr = zarr.create_array( store=store, shape=(3,), dtype="int8", chunks=(3,), filters=[ScaleOffset(offset=0, scale=2)], compressors=None, fill_value=0, ) # Write raw encoded bytes directly so we can inject a value that isn't divisible by scale. # Array layout: int8 [2, 3, 4]; 3 % 2 != 0, so decode must fail. buf = default_buffer_prototype().buffer.from_bytes(np.array([2, 3, 4], dtype="int8").tobytes()) await arr.store_path.store.set("c/0", buf) with pytest.raises(ValueError, match="non-zero remainder"): arr[:] def test_encode_rejects_signed_integer_overflow() -> None: """Encoding raises when (value - offset) * scale exceeds the target integer range.""" arr = zarr.create_array( store={}, shape=(3,), dtype="int8", chunks=(3,), filters=[ScaleOffset(offset=0, scale=100)], compressors=None, fill_value=0, ) # 2 * 100 = 200, outside int8 range [-128, 127] with pytest.raises(ValueError, match="outside the range of dtype int8"): arr[:] = np.array([0, 1, 2], dtype="int8") def test_encode_rejects_unsigned_integer_underflow() -> None: """Encoding raises when value - offset underflows an unsigned dtype.""" arr = zarr.create_array( store={}, shape=(3,), dtype="uint8", chunks=(3,), filters=[ScaleOffset(offset=10, scale=1)], compressors=None, fill_value=10, ) # 5 - 10 = -5, outside uint8 range [0, 255] with pytest.raises(ValueError, match="outside the range of dtype uint8"): arr[:] = np.array([10, 5, 20], dtype="uint8") def test_float32_dtype_preserved() -> None: """float32 arrays survive encode+decode without being promoted to float64.""" arr = np.arange(100, dtype="float32") offset = np.float32(5.0) scale = np.float32(0.25) encoded = _encode(arr, offset, scale) assert encoded.dtype == np.dtype("float32") decoded = _decode(encoded, offset, scale, scale_repr=0.25) assert decoded.dtype == np.dtype("float32") def test_float_encode_rejects_wider_scalar() -> None: """A float64 scalar passed with a float32 array must not silently widen the result.""" arr = np.arange(10, dtype="float32") # A numpy float64 scalar (not a Python float — NEP 50 exempts those) mixed with a # float32 ndarray promotes to float64. The codec must reject that. with pytest.raises(ValueError, match="changed dtype from float32 to float64"): _encode(arr, np.float64(5.0), np.float64(0.25)) def test_float_decode_rejects_wider_scalar() -> None: """A float64 scalar passed with a float32 array must not silently widen on decode.""" arr = np.arange(10, dtype="float32") with pytest.raises(ValueError, match="changed dtype from float32 to float64"): _decode(arr, np.float64(5.0), np.float64(0.25), scale_repr=0.25) async def test_decode_rejects_integer_overflow_on_offset_add() -> None: """Decoding raises when quotient + offset overflows the target integer dtype.""" store = MemoryStore() arr = zarr.create_array( store=store, shape=(3,), dtype="int8", chunks=(3,), filters=[ScaleOffset(offset=100, scale=1)], compressors=None, fill_value=0, ) # encoded=100 → decoded = 100/1 + 100 = 200, outside int8 range buf = default_buffer_prototype().buffer.from_bytes( np.array([0, 50, 100], dtype="int8").tobytes() ) await arr.store_path.store.set("c/0", buf) with pytest.raises(ValueError, match="outside the range of dtype int8"): arr[:] def test_decode_fits_natively_negative_scale() -> None: """_decode_fits_natively handles negative scale by swapping bounds.""" # For a negative scale, x // scale flips the relationship between min/max. # The function should use info.max // scale as the lower bound and info.min // scale # as the upper bound. dtype = np.dtype("int16") # scale=-2 inverts; offset=0 means range is just q_lo..q_hi assert _decode_fits_natively(dtype, offset=0, scale=-2) is True # An offset that pushes the range out of bounds returns False assert _decode_fits_natively(dtype, offset=100000, scale=-2) is False async def test_decode_int_widened_path() -> None: """When _decode_fits_natively returns False, decode falls through to the widened path.""" # For uint32 with offset near max, q_hi + offset can exceed uint32 if computed in target dtype. # The widened path uses int64 arithmetic and range-checks the result. # We bypass encode by writing raw bytes directly to the store. store = MemoryStore() arr = zarr.create_array( store=store, shape=(3,), dtype="uint32", chunks=(3,), # offset large enough that _decode_fits_natively returns False filters=[ScaleOffset(offset=2**31, scale=1)], compressors=None, # fill_value must be >= offset to avoid uint32 underflow during encode fill_value=2**31, ) # Encoded values that, when added to offset, stay within uint32 buf = default_buffer_prototype().buffer.from_bytes( np.array([0, 100, 1000], dtype=" None: """compute_encoded_size returns the input byte length unchanged (codec is fixed-size).""" codec = ScaleOffset(offset=0, scale=1) # The chunk_spec argument is unused; pass any sentinel assert codec.compute_encoded_size(input_byte_length=100, _chunk_spec=None) == 100 # type: ignore[arg-type] assert codec.compute_encoded_size(input_byte_length=0, _chunk_spec=None) == 0 # type: ignore[arg-type] zarr-developers-zarr-python-a2f2415/tests/test_codecs/test_sharding.py000066400000000000000000001542141527014137600263450ustar00rootroot00000000000000import enum import math import pickle import warnings from typing import TYPE_CHECKING, Any, cast, get_args from unittest.mock import AsyncMock import numpy as np import numpy.typing as npt import pytest import zarr import zarr.api import zarr.api.asynchronous from zarr import Array from zarr.abc.store import Store from zarr.codecs import ( BloscCodec, BytesCodec, Crc32cCodec, GzipCodec, ShardingCodec, TransposeCodec, ) from zarr.codecs.numcodecs import CRC32 from zarr.codecs.sharding import ( INDEX_LOCATION, MAX_UINT_64, IndexLocation, ShardingCodecIndexLocation, SubchunkWriteOrder, _ShardIndex, _ShardReader, ) from zarr.core.buffer import NDArrayLike, default_buffer_prototype from zarr.core.dtype import Int32 from zarr.core.indexing import lexicographic_order_coords from zarr.core.metadata.v3 import ArrayV3Metadata from zarr.storage import MemoryStore, StorePath, ZipStore from ..conftest import ArrayRequest from .test_codecs import _AsyncArrayProxy, order_from_dim if TYPE_CHECKING: from collections.abc import Callable def _reads_are_sync(store_mock: AsyncMock) -> bool: """True when the partial-shard read for this store+pipeline goes through the synchronous methods (get_sync / get_ranges_sync). That requires BOTH the configured pipeline to be the sync (Fused) one AND the store to support sync reads — a Fused read against a non-sync store (e.g. ZipStore) falls back to the async path. Lets the partial-shard-read tests assert the same intent against whichever method family is actually exercised.""" from zarr.abc.store import SupportsGetSync from zarr.core.config import config pipeline_is_sync = "Fused" in config.get("codec_pipeline.path") # store_mock wraps the real store; check the wrapped class for sync support. wrapped = getattr(store_mock, "_mock_wraps", store_mock) return pipeline_is_sync and isinstance(wrapped, SupportsGetSync) def _index_read_count(store_mock: AsyncMock) -> int: """Number of shard-index reads, regardless of sync/async pipeline.""" method = store_mock.get_sync if _reads_are_sync(store_mock) else store_mock.get return int(method.call_count) def _range_read_count(store_mock: AsyncMock) -> int: """Number of coalesced chunk-data reads, regardless of sync/async pipeline.""" method = store_mock.get_ranges_sync if _reads_are_sync(store_mock) else store_mock.get_ranges return int(method.call_count) def _fail_index_read(store_mock: AsyncMock) -> None: """Simulate the shard-index load returning nothing, for the active path.""" if _reads_are_sync(store_mock): store_mock.get_sync.return_value = None else: store_mock.get.return_value = None def _fail_chunk_reads( store_mock: AsyncMock, key_absent_exc: type[Exception] = FileNotFoundError ) -> None: """Simulate chunk-data loads failing (key absent), for the active path. Async get_ranges raises a BaseExceptionGroup; the sync get_ranges_sync mirrors that contract, so both inject a FileNotFoundError-bearing group.""" if _reads_are_sync(store_mock): def fail_sync(key: str, byte_ranges: Any, **kwargs: Any) -> Any: raise BaseExceptionGroup("chunk read failed", [key_absent_exc(key)]) store_mock.get_ranges_sync = fail_sync else: async def fail_async(key: str, byte_ranges: Any, **kwargs: Any) -> Any: raise BaseExceptionGroup("chunk read failed", [key_absent_exc(key)]) yield # type: ignore[unreachable] # marks this as an async generator store_mock.get_ranges = fail_async @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize( "array_fixture", [ ArrayRequest(shape=(128,) * 1, dtype="uint8", order="C"), ArrayRequest(shape=(128,) * 2, dtype="uint8", order="C"), ArrayRequest(shape=(128,) * 3, dtype="uint16", order="F"), ], indirect=["array_fixture"], ) @pytest.mark.parametrize("offset", [0, 10]) def test_sharding( store: Store, array_fixture: npt.NDArray[Any], index_location: IndexLocation, offset: int, ) -> None: """ Test that we can create an array with a sharding codec, write data to that array, and get the same data out via indexing. """ data = array_fixture spath = StorePath(store) arr = zarr.create_array( spath, shape=tuple(s + offset for s in data.shape), chunks=(32,) * data.ndim, shards={"shape": (64,) * data.ndim, "index_location": index_location}, dtype=data.dtype, fill_value=6, filters=[TransposeCodec(order=order_from_dim("F", data.ndim))], compressors=BloscCodec(cname="lz4"), ) write_region = tuple(slice(offset, None) for dim in range(data.ndim)) arr[write_region] = data if offset > 0: empty_region = tuple(slice(0, offset) for dim in range(data.ndim)) assert np.all(arr[empty_region] == arr.metadata.fill_value) read_data = arr[write_region] assert isinstance(read_data, NDArrayLike) assert data.shape == read_data.shape assert np.array_equal(data, read_data) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize("offset", [0, 10]) def test_sharding_scalar( store: Store, index_location: IndexLocation, offset: int, ) -> None: """ Test that we can create an array with a sharding codec, write data to that array, and get the same data out via indexing. """ spath = StorePath(store) arr = zarr.create_array( spath, shape=(128, 128), chunks=(32, 32), shards={"shape": (64, 64), "index_location": index_location}, dtype="uint8", fill_value=6, filters=[TransposeCodec(order=order_from_dim("F", 2))], compressors=BloscCodec(cname="lz4"), ) arr[:16, :16] = 10 # intentionally write partial chunks read_data = arr[:16, :16] np.testing.assert_array_equal(read_data, 10) @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) @pytest.mark.parametrize( "array_fixture", [ ArrayRequest(shape=(128,) * 3, dtype="uint16", order="F"), ], indirect=["array_fixture"], ) def test_sharding_partial( store: Store, array_fixture: npt.NDArray[Any], index_location: IndexLocation ) -> None: data = array_fixture spath = StorePath(store) a = zarr.create_array( spath, shape=tuple(a + 10 for a in data.shape), chunks=(32, 32, 32), shards={"shape": (64, 64, 64), "index_location": index_location}, compressors=BloscCodec(cname="lz4"), filters=[TransposeCodec(order=order_from_dim("F", data.ndim))], dtype=data.dtype, fill_value=0, ) a[10:, 10:, 10:] = data read_data = a[0:10, 0:10, 0:10] assert np.all(read_data == 0) read_data = a[10:, 10:, 10:] assert isinstance(read_data, NDArrayLike) assert data.shape == read_data.shape assert np.array_equal(data, read_data) @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) @pytest.mark.parametrize( "array_fixture", [ ArrayRequest(shape=(128,) * 3, dtype="uint16", order="F"), ], indirect=["array_fixture"], ) def test_sharding_partial_readwrite( store: Store, array_fixture: npt.NDArray[Any], index_location: IndexLocation ) -> None: data = array_fixture spath = StorePath(store) a = zarr.create_array( spath, shape=data.shape, chunks=(1, data.shape[1], data.shape[2]), shards={"shape": data.shape, "index_location": index_location}, dtype=data.dtype, fill_value=0, filters=None, compressors=None, ) a[:] = data for x in range(data.shape[0]): read_data = a[x, :, :] assert np.array_equal(data[x], read_data) @pytest.mark.parametrize( "array_fixture", [ ArrayRequest(shape=(128,) * 3, dtype="uint16", order="F"), ], indirect=["array_fixture"], ) @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) def test_sharding_partial_read( store: Store, array_fixture: npt.NDArray[Any], index_location: IndexLocation ) -> None: data = array_fixture spath = StorePath(store) a = zarr.create_array( spath, shape=tuple(a + 10 for a in data.shape), chunks=(32, 32, 32), shards={"shape": (64, 64, 64), "index_location": index_location}, compressors=BloscCodec(cname="lz4"), filters=[TransposeCodec(order=order_from_dim("F", data.ndim))], dtype=data.dtype, fill_value=1, ) read_data = a[0:10, 0:10, 0:10] assert np.all(read_data == 1) @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) def test_sharding_multiple_chunks_partial_shard_read( store: Store, index_location: IndexLocation, ) -> None: array_shape = (16, 64) shard_shape = (8, 32) chunk_shape = (2, 4) data = np.arange(np.prod(array_shape), dtype="float32").reshape(array_shape) store_mock = AsyncMock(wraps=store, spec=store.__class__) a = zarr.create_array( StorePath(store_mock), shape=data.shape, chunks=chunk_shape, shards={"shape": shard_shape, "index_location": index_location}, compressors=BloscCodec(cname="lz4"), dtype=data.dtype, fill_value=1, ) a[:] = data store_mock.reset_mock() # ignore store calls during array creation # Reads 3 (2 full, 1 partial) chunks each from 2 shards (a subset of both shards) # for a total of 6 chunks accessed assert np.allclose(a[0, 22:42], np.arange(22, 42, dtype="float32")) # 2 shard index reads + 2 coalesced chunk-data reads (one per shard) assert _index_read_count(store_mock) == 2 assert _range_read_count(store_mock) == 2 store_mock.reset_mock() # Reads 4 chunks from both shards along dimension 0 for a total of 8 chunks accessed assert np.allclose(a[:, 0], np.arange(0, data.size, array_shape[1], dtype="float32")) # 2 shard index reads + 2 coalesced chunk-data reads (one per shard) assert _index_read_count(store_mock) == 2 assert _range_read_count(store_mock) == 2 @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) def test_sharding_duplicate_read_indexes( store: Store, index_location: IndexLocation, ) -> None: """ Check that duplicate index reads are handled correctly when using get_ranges for chunk data. """ array_shape = (15,) shard_shape = (8,) chunk_shape = (2,) data = np.arange(np.prod(array_shape), dtype="float32").reshape(array_shape) store_mock = AsyncMock(wraps=store, spec=store.__class__) a = zarr.create_array( StorePath(store_mock), shape=data.shape, chunks=chunk_shape, shards={"shape": shard_shape, "index_location": index_location}, compressors=BloscCodec(cname="lz4"), dtype=data.dtype, fill_value=-1, ) a[:] = data store_mock.reset_mock() # ignore store calls during array creation # Read the same index multiple times from two chunks indexer = [8, 8, 12, 12] assert np.array_equal(a[indexer], data[indexer]) # 1 shard index read + 1 coalesced chunk-data read assert _index_read_count(store_mock) == 1 assert _range_read_count(store_mock) == 1 @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) def test_sharding_read_empty_chunks_within_non_empty_shard_write_empty_false( store: Store, index_location: IndexLocation ) -> None: """ Case where - some, but not all, chunks in the last shard are empty - the last shard is not complete (array length is not a multiple of shard shape), this takes us down the partial shard read path - write_empty_chunks=False so the shard index will have fewer entries than chunks in the shard """ # array with mixed empty and non-empty chunks in second shard data = np.array([ # shard 0. full 8 elements, all chunks have some non-fill data 0, 1, 2, 3, 4, 5, 6, 7, # shard 1. 6 elements (< shard shape) 2, 0, # chunk 0, written -9, -9, # chunk 1, all fill, not written 4, 5 # chunk 2, written ], dtype="int32") # fmt: off spath = StorePath(store) a = zarr.create_array( spath, shape=(14,), chunks=(2,), shards={"shape": (8,), "index_location": index_location}, dtype="int32", fill_value=-9, filters=None, compressors=None, config={"write_empty_chunks": False}, ) a[:] = data assert np.array_equal(a[:], data) @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) def test_sharding_read_empty_chunks_within_empty_shard_write_empty_false( store: Store, index_location: IndexLocation ) -> None: """ Case where - all chunks in last shard are empty - the last shard is not complete (array length is not a multiple of shard shape), this takes us down the partial shard read path - write_empty_chunks=False so the shard index will have no entries """ fill_value = -99 shard_size = 8 data = np.arange(14, dtype="int32") data[shard_size:] = fill_value # 2nd shard is all fill value spath = StorePath(store) a = zarr.create_array( spath, shape=(14,), chunks=(2,), shards={"shape": (shard_size,), "index_location": index_location}, dtype="int32", fill_value=fill_value, filters=None, compressors=None, config={"write_empty_chunks": False}, ) a[:] = data assert np.array_equal(a[:], data) @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) def test_sharding_partial_shard_read__index_load_fails( store: Store, index_location: IndexLocation ) -> None: """Test fill value is returned when the call to the store to load the bytes of the shard's chunk index fails.""" array_shape = (16,) shard_shape = (16,) chunk_shape = (8,) data = np.arange(np.prod(array_shape), dtype="float32").reshape(array_shape) fill_value = -999 store_mock = AsyncMock(wraps=store, spec=store.__class__) a = zarr.create_array( StorePath(store_mock), shape=data.shape, chunks=chunk_shape, shards={"shape": shard_shape, "index_location": index_location}, compressors=BloscCodec(cname="lz4"), dtype=data.dtype, fill_value=fill_value, ) a[:] = data # Loading the index returns None -> simulate an index load failure, on # whichever read method the active pipeline uses (get / get_sync). _fail_index_read(store_mock) # Read from one of two chunks in a shard to test the partial shard read path assert a[0] == fill_value assert a[0] != data[0] @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) def test_sharding_partial_shard_read__index_chunk_slice_fails( store: Store, index_location: IndexLocation, monkeypatch: pytest.MonkeyPatch, ) -> None: """Test fill value is returned when looking up a chunk's byte slice within a shard fails.""" array_shape = (16,) shard_shape = (16,) chunk_shape = (8,) data = np.arange(np.prod(array_shape), dtype="float32").reshape(array_shape) fill_value = -999 monkeypatch.setattr( "zarr.codecs.sharding._ShardIndex.get_chunk_slice", lambda self, chunk_coords: None, ) a = zarr.create_array( StorePath(store), shape=data.shape, chunks=chunk_shape, shards={"shape": shard_shape, "index_location": index_location}, compressors=BloscCodec(cname="lz4"), dtype=data.dtype, fill_value=fill_value, ) a[:] = data # Read from one of two chunks in a shard to test the partial shard read path assert a[0] == fill_value assert a[0] != data[0] @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) def test_sharding_partial_shard_read__chunk_load_fails( store: Store, index_location: IndexLocation ) -> None: """Test fill value is returned when the call to the store to load a chunk's bytes fails.""" array_shape = (16,) shard_shape = (16,) chunk_shape = (8,) data = np.arange(np.prod(array_shape), dtype="float32").reshape(array_shape) fill_value = -999 store_mock = AsyncMock(wraps=store, spec=store.__class__) a = zarr.create_array( StorePath(store_mock), shape=data.shape, chunks=chunk_shape, shards={"shape": shard_shape, "index_location": index_location}, compressors=BloscCodec(cname="lz4"), dtype=data.dtype, fill_value=fill_value, ) a[:] = data # Set up store mock after array creation to simulate chunk load failure. # Index loads still succeed, but chunk-byte loads fail (the coalesced range # read raises a BaseExceptionGroup containing FileNotFoundError — the same # shape produced when a key is absent), on whichever read method the active # pipeline uses (get_ranges / get_ranges_sync). store_mock.reset_mock() _fail_chunk_reads(store_mock) # Read from one of two chunks in a shard to test the partial shard read path assert a[0] == fill_value assert a[0] != data[0] @pytest.mark.parametrize( "array_fixture", [ ArrayRequest(shape=(128,) * 3, dtype="uint16", order="F"), ], indirect=["array_fixture"], ) @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) def test_sharding_partial_overwrite( store: Store, array_fixture: npt.NDArray[Any], index_location: IndexLocation ) -> None: data = array_fixture[:10, :10, :10] spath = StorePath(store) a = zarr.create_array( spath, shape=tuple(a + 10 for a in data.shape), chunks=(32, 32, 32), shards={"shape": (64, 64, 64), "index_location": index_location}, compressors=BloscCodec(cname="lz4"), filters=[TransposeCodec(order=order_from_dim("F", data.ndim))], dtype=data.dtype, fill_value=1, ) a[:10, :10, :10] = data read_data = a[0:10, 0:10, 0:10] assert np.array_equal(data, read_data) data += 10 if isinstance(store, ZipStore): with pytest.warns(UserWarning, match="Duplicate name: "): a[:10, :10, :10] = data else: a[:10, :10, :10] = data read_data = a[0:10, 0:10, 0:10] assert np.array_equal(data, read_data) # Zip storage raises a warning about a duplicate name, which we ignore. @pytest.mark.filterwarnings("ignore:Duplicate name.*:UserWarning") @pytest.mark.parametrize( "array_fixture", [ ArrayRequest(shape=(127, 128, 129), dtype="uint16", order="F"), ], indirect=True, ) @pytest.mark.parametrize( "outer_index_location", ["start", "end"], ) @pytest.mark.parametrize( "inner_index_location", ["start", "end"], ) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) def test_nested_sharding( store: Store, array_fixture: npt.NDArray[Any], outer_index_location: IndexLocation, inner_index_location: IndexLocation, ) -> None: data = array_fixture spath = StorePath(store) # compressors=None ensures no BytesBytesCodec is added, which keeps # supports_partial_decode=True and exercises the partial decode path a = zarr.create_array( spath, data=data, chunks=(64,) * data.ndim, compressors=None, serializer=ShardingCodec( chunk_shape=(32,) * data.ndim, codecs=[ ShardingCodec(chunk_shape=(16,) * data.ndim, index_location=inner_index_location) ], index_location=outer_index_location, ), ) a[:] = data read_data = a[0 : data.shape[0], 0 : data.shape[1], 0 : data.shape[2]] assert isinstance(read_data, NDArrayLike) assert data.shape == read_data.shape assert np.array_equal(data, read_data) @pytest.mark.parametrize( "array_fixture", [ ArrayRequest(shape=(128,) * 3, dtype="uint16", order="F"), ], indirect=["array_fixture"], ) @pytest.mark.parametrize( "outer_index_location", ["start", "end"], ) @pytest.mark.parametrize( "inner_index_location", ["start", "end"], ) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) def test_nested_sharding_create_array( store: Store, array_fixture: npt.NDArray[Any], outer_index_location: IndexLocation, inner_index_location: IndexLocation, ) -> None: data = array_fixture spath = StorePath(store) a = zarr.create_array( spath, shape=data.shape, chunks=(32, 32, 32), dtype=data.dtype, fill_value=0, serializer=ShardingCodec( chunk_shape=(32, 32, 32), codecs=[ShardingCodec(chunk_shape=(16, 16, 16), index_location=inner_index_location)], index_location=outer_index_location, ), filters=None, compressors=None, ) a[:] = data read_data = a[:] assert np.array_equal(data, read_data) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) def test_open_sharding(store: Store) -> None: path = "open_sharding" spath = StorePath(store, path) a = zarr.create_array( spath, shape=(16, 16), chunks=(8, 8), shards=(16, 16), filters=[TransposeCodec(order=order_from_dim("F", 2))], compressors=BloscCodec(), dtype="int32", fill_value=0, ) b = Array.open(spath) assert a.metadata == b.metadata @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) def test_write_partial_sharded_chunks(store: Store) -> None: data = np.arange(0, 16 * 16, dtype="uint16").reshape((16, 16)) spath = StorePath(store) a = zarr.create_array( spath, shape=(40, 40), chunks=(10, 10), shards=(20, 20), dtype=data.dtype, compressors=BloscCodec(), fill_value=1, ) a[0:16, 0:16] = data assert np.array_equal(a[0:16, 0:16], data) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) async def test_delete_empty_shards(store: Store) -> None: if not store.supports_deletes: pytest.skip("store does not support deletes") path = "delete_empty_shards" spath = StorePath(store, path) a = await zarr.api.asynchronous.create_array( spath, shape=(16, 16), chunks=(8, 8), shards=(8, 16), dtype="uint16", compressors=None, fill_value=1, ) print(a.metadata.to_dict()) await _AsyncArrayProxy(a)[:, :].set(np.zeros((16, 16))) await _AsyncArrayProxy(a)[8:, :].set(np.ones((8, 16))) await _AsyncArrayProxy(a)[:, 8:].set(np.ones((16, 8))) # chunk (0, 0) is full # chunks (0, 1), (1, 0), (1, 1) are empty # shard (0, 0) is half-full # shard (1, 0) is empty data = np.ones((16, 16), dtype="uint16") data[:8, :8] = 0 assert np.array_equal(data, await _AsyncArrayProxy(a)[:, :].get()) assert await store.get(f"{path}/c/1/0", prototype=default_buffer_prototype()) is None chunk_bytes = await store.get(f"{path}/c/0/0", prototype=default_buffer_prototype()) assert chunk_bytes is not None assert len(chunk_bytes) == 16 * 2 + 8 * 8 * 2 + 4 def test_structured_dtype_fill_value() -> None: """Sharded arrays with a structured dtype are writable and readable even though the fill value is an (unhashable) ``np.void`` scalar: the sharding codec's chunk-spec caches key on ``ArraySpec``, whose hash must handle void fills (see https://github.com/zarr-developers/zarr-python/issues/3054).""" dtype = np.dtype([("a", "i4"), ("b", "f4")]) arr = zarr.create_array( MemoryStore(), shape=(8,), chunks=(2,), shards=(4,), dtype=dtype, fill_value=(1, 2.0), ) data = np.array([(i, i / 2) for i in range(8)], dtype=dtype) arr[:4] = data[:4] expected = np.zeros(8, dtype=dtype) expected[:4] = data[:4] expected[4:] = (1, 2.0) # untouched shard reads back as the fill value assert np.array_equal(arr[:], expected) @pytest.mark.parametrize( "codec", [ ShardingCodec(chunk_shape=(8, 8)), ShardingCodec(chunk_shape=(8, 8), subchunk_write_order="lexicographic"), ], ids=["default", "lexicographic"], ) def test_pickle(codec: ShardingCodec) -> None: """ShardingCodec round-trips through pickle, including the non-serialized ``subchunk_write_order`` (which ``to_dict`` omits and which must not silently revert to the ``morton`` default).""" restored = pickle.loads(pickle.dumps(codec)) assert restored == codec assert restored.chunk_shape == codec.chunk_shape assert restored.subchunk_write_order == codec.subchunk_write_order @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) @pytest.mark.parametrize("index_location", ["start", "end"]) async def test_sharding_with_empty_inner_chunk(store: Store, index_location: IndexLocation) -> None: data = np.arange(0, 16 * 16, dtype="uint32").reshape((16, 16)) fill_value = 1 path = f"sharding_with_empty_inner_chunk_{index_location}" spath = StorePath(store, path) a = await zarr.api.asynchronous.create_array( spath, shape=(16, 16), chunks=(4, 4), shards={"shape": (8, 8), "index_location": index_location}, dtype="uint32", fill_value=fill_value, ) data[:4, :4] = fill_value await a.setitem(..., data) print("read data") data_read = await a.getitem(...) assert np.array_equal(data_read, data) @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize("chunks_per_shard", [(5, 2), (2, 5), (5, 5)]) async def test_sharding_with_chunks_per_shard( store: Store, index_location: IndexLocation, chunks_per_shard: tuple[int] ) -> None: chunk_shape = (2, 1) shape = tuple(x * y for x, y in zip(chunks_per_shard, chunk_shape, strict=False)) data = np.ones(np.prod(shape), dtype="int32").reshape(shape) fill_value = 42 path = f"test_sharding_with_chunks_per_shard_{index_location}" spath = StorePath(store, path) a = zarr.create_array( spath, shape=shape, chunks=chunk_shape, shards={"shape": shape, "index_location": index_location}, dtype="int32", fill_value=fill_value, ) a[...] = data data_read = a[...] assert np.array_equal(data_read, data) @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) def test_invalid_metadata(store: Store) -> None: spath1 = StorePath(store, "invalid_inner_chunk_shape") with pytest.raises(ValueError): zarr.create_array( spath1, shape=(16, 16), shards=(16, 16), chunks=(8,), dtype=np.dtype("uint8"), fill_value=0, ) spath2 = StorePath(store, "invalid_inner_chunk_shape") with pytest.raises(ValueError): zarr.create_array( spath2, shape=(16, 16), shards=(16, 16), chunks=(8, 7), dtype=np.dtype("uint8"), fill_value=0, ) def test_invalid_shard_shape() -> None: with pytest.raises( ValueError, match=( f"Chunk edge length {16} in dimension {0} is not " f"divisible by the shard's inner chunk size {9}\\." ), ): zarr.create_array( {}, shape=(16, 16), shards=(16, 16), chunks=(9, 9), dtype=np.dtype("uint8"), fill_value=0, ) @pytest.mark.parametrize("store", ["local"], indirect=["store"]) def test_sharding_mixed_integer_list_indexing(store: Store) -> None: """Regression test for https://github.com/zarr-developers/zarr-python/issues/3691. Mixed integer/list indexing on sharded arrays should return the same shape and data as on equivalent chunked arrays. """ import numpy as np data = np.arange(200 * 100 * 10, dtype=np.uint8).reshape(200, 100, 10) chunked = zarr.create_array( store, name="chunked", shape=(200, 100, 10), dtype=np.uint8, chunks=(200, 100, 1), overwrite=True, ) chunked[:, :, :] = data sharded = zarr.create_array( store, name="sharded", shape=(200, 100, 10), dtype=np.uint8, chunks=(200, 100, 1), shards=(200, 100, 10), overwrite=True, ) sharded[:, :, :] = data # Mixed integer + list indexing c = chunked[0:10, 0, [0, 1]] # type: ignore[index] s = sharded[0:10, 0, [0, 1]] # type: ignore[index] assert c.shape == s.shape == (10, 2), ( # type: ignore[union-attr] f"Expected (10, 2), got chunked={c.shape}, sharded={s.shape}" # type: ignore[union-attr] ) np.testing.assert_array_equal(c, s) # Multiple integer axes c2 = chunked[0, 0, [0, 1, 2]] # type: ignore[index] s2 = sharded[0, 0, [0, 1, 2]] # type: ignore[index] assert c2.shape == s2.shape == (3,) # type: ignore[union-attr] np.testing.assert_array_equal(c2, s2) # Slice + integer + slice c3 = chunked[0:5, 1, 0:3] s3 = sharded[0:5, 1, 0:3] assert c3.shape == s3.shape == (5, 3) # type: ignore[union-attr] np.testing.assert_array_equal(c3, s3) async def stored_data_and_get_order( codec: ShardingCodec, chunks_per_shard: tuple[int, ...] ) -> list[tuple[int, ...]]: shard_shape = tuple(c * s for c, s in zip(chunks_per_shard, codec.chunk_shape, strict=True)) store = MemoryStore() arr = zarr.create_array( StorePath(store), shape=shard_shape, dtype="uint8", chunks=shard_shape, serializer=codec, filters=None, compressors=None, fill_value=0, ) arr[:] = np.arange(np.prod(shard_shape), dtype="uint8").reshape(shard_shape) shard_buf = await store.get("c/0/0", prototype=default_buffer_prototype()) if shard_buf is None: raise RuntimeError("data write failed") index = (await _ShardReader.from_bytes(shard_buf, codec, chunks_per_shard)).index offset_to_coord: dict[int, tuple[int, ...]] = dict( zip( index.get_chunk_slices_vectorized(np.array(list(np.ndindex(chunks_per_shard))))[ 0 ], # start list(np.ndindex(chunks_per_shard)), # coord strict=True, ) ) # The physical write order is recovered by sorting coordinates by start offset. return [coord for _, coord in sorted(offset_to_coord.items())] @pytest.mark.parametrize( "subchunk_write_order", get_args(SubchunkWriteOrder), ) async def test_encoded_subchunk_write_order(subchunk_write_order: SubchunkWriteOrder) -> None: """Subchunks must be physically laid out in the shard in the order specified by ``subchunk_write_order``. We verify this by decoding the shard index and sorting the chunk coordinates by their byte offset. ``unordered`` makes no stable-order promise, but is deterministic in this implementation, so it is checked the same way.""" # Use a non-square chunks_per_shard so all orderings are distinguishable. chunks_per_shard = (3, 2) chunk_shape = (4, 4) codec = ShardingCodec( chunk_shape=chunk_shape, codecs=[BytesCodec()], index_codecs=[BytesCodec(), Crc32cCodec()], index_location="end", subchunk_write_order=subchunk_write_order, ) actual_order = await stored_data_and_get_order(codec, chunks_per_shard) expected_order = list(codec._subchunk_order_iter(chunks_per_shard, subchunk_write_order)) assert actual_order == expected_order @pytest.mark.parametrize( "subchunk_write_order", get_args(SubchunkWriteOrder), ) @pytest.mark.parametrize("do_partial", [True, False], ids=["partial", "complete"]) def test_subchunk_write_order_roundtrip( subchunk_write_order: SubchunkWriteOrder, do_partial: bool ) -> None: """Data written with any ``subchunk_write_order`` must round-trip correctly.""" chunks_per_shard = (3, 2) chunk_shape = (4, 4) shard_shape = tuple(c * s for c, s in zip(chunks_per_shard, chunk_shape, strict=True)) data = np.arange(np.prod(shard_shape), dtype="uint16").reshape(shard_shape) arr = zarr.create_array( StorePath(MemoryStore()), shape=shard_shape, dtype=data.dtype, chunks=shard_shape, serializer=ShardingCodec( chunk_shape=chunk_shape, codecs=[BytesCodec()], subchunk_write_order=subchunk_write_order, ), filters=None, compressors=None, fill_value=0, ) if do_partial: sub_data = data[: (shard_shape[0] // 2)] arr[: (shard_shape[0] // 2)] = data[: (shard_shape[0] // 2)] data = np.vstack([sub_data, np.zeros_like(sub_data)]) else: arr[:] = data np.testing.assert_array_equal(arr[:], data) # --- Tests for ShardingCodecIndexLocation deprecation --- @pytest.mark.parametrize("location", INDEX_LOCATION) def test_sharding_codec_accepts_all_index_locations(location: IndexLocation) -> None: """ Every value in INDEX_LOCATION is accepted by ShardingCodec and round-trips to the same value on the stored attribute. Catches drift between the IndexLocation type alias and the runtime INDEX_LOCATION tuple. """ codec = ShardingCodec(chunk_shape=(1,), index_location=location) assert codec.index_location == location @pytest.mark.parametrize("location", INDEX_LOCATION) def test_sharding_codec_json_roundtrip_index_location( location: IndexLocation, ) -> None: """ ShardingCodec.to_dict writes index_location as the bare literal string, and the round-trip through from_dict preserves equality. Asserting the on-disk index_location value (not just the round-trip) catches drift between ShardingCodec's runtime representation and the V3 wire form. """ codec = ShardingCodec(chunk_shape=(1,), index_location=location) serialized = codec.to_dict() assert serialized["configuration"]["index_location"] == location # type: ignore[index, call-overload] restored = ShardingCodec.from_dict(serialized) assert restored == codec def test_sharding_codec_default_byte_order_is_little() -> None: """ The default inner and index codecs store little-endian bytes, like the default serializer, so a sharded array written with defaults is byte-identical on every host. """ serialized = ShardingCodec(chunk_shape=(1,)).to_dict() little = {"name": "bytes", "configuration": {"endian": "little"}} assert serialized["configuration"]["codecs"] == (little,) # type: ignore[index, call-overload] assert serialized["configuration"]["index_codecs"][0] == little # type: ignore[index, call-overload] @pytest.mark.parametrize( ("member", "expected"), [("start", "start"), ("end", "end")], ) def test_sharding_index_location_member_access_warns(member: str, expected: str) -> None: """ Accessing a member on the deprecated ShardingCodecIndexLocation class emits a DeprecationWarning and resolves to the equivalent literal string. """ with pytest.warns(DeprecationWarning, match=rf"ShardingCodecIndexLocation\.{member}"): value = getattr(ShardingCodecIndexLocation, member) assert value == expected def test_sharding_index_location_class_imports_silently() -> None: """ Importing the deprecated ShardingCodecIndexLocation class by name must not emit a warning; only member access does. """ with warnings.catch_warnings(): warnings.simplefilter("error") from zarr.codecs.sharding import ( # noqa: F401 ShardingCodecIndexLocation as _SCIL, ) def test_sharding_codec_init_with_enum_instance_warns() -> None: """ Passing a foreign enum.Enum instance to ShardingCodec.__init__ triggers the init-level deprecation warning (from _coerce_enum_input) and normalizes the value to the corresponding literal string. Covers the case where a downstream package defined its own enum-shaped class to bridge between zarr's old API and its own. """ class LegacyIndexLocation(enum.Enum): end = "end" with pytest.warns(DeprecationWarning, match=r"Passing an enum to ShardingCodec"): codec = ShardingCodec( chunk_shape=(1,), index_location=cast(ShardingCodecIndexLocation, LegacyIndexLocation.end), ) assert codec.index_location == "end" def test_sharding_codec_init_with_deprecated_class_member() -> None: """ The realistic legacy-upgrade idiom: ShardingCodec(index_location=ShardingCodecIndexLocation.end). Member access on ShardingCodecIndexLocation emits one DeprecationWarning (from the metaclass) and resolves to the bare string, which ShardingCodec then accepts without further warning. No second warning from _coerce_enum_input because the metaclass already produced a string. The cast is necessary because the metaclass __getattr__ is typed as returning str, which does not statically match the codec's IndexLocation parameter even though the runtime value does. """ with pytest.warns(DeprecationWarning, match=r"ShardingCodecIndexLocation\.end"): codec = ShardingCodec( chunk_shape=(1,), index_location=cast(IndexLocation, ShardingCodecIndexLocation.end), ) assert codec.index_location == "end" def test_sharding_codec_rejects_unknown_index_location() -> None: """ ShardingCodec.__init__ raises ValueError when index_location is outside INDEX_LOCATION, and the error message names the offending parameter. """ kwargs: dict[str, Any] = {"chunk_shape": (1,), "index_location": "middle"} with pytest.raises(ValueError, match="index_location must be one of"): ShardingCodec(**kwargs) @pytest.mark.parametrize( "index_codecs", [ (BytesCodec(),), (BytesCodec(), Crc32cCodec()), (TransposeCodec(order=(1, 0)), BytesCodec(), Crc32cCodec()), (BytesCodec(), CRC32()), ], ) def test_sharding_fixed_size_index_codecs_roundtrip(index_codecs: tuple[Any, ...]) -> None: """ Any chain of fixed-size codecs is accepted as `index_codecs`, and data written through it reads back unchanged. """ data = np.arange(16, dtype="uint16") arr = zarr.create_array( MemoryStore(), shape=data.shape, dtype=data.dtype, chunks=(2,), shards=(8,), compressors=None, serializer=ShardingCodec(chunk_shape=(2,), index_codecs=index_codecs), ) arr[:] = data np.testing.assert_array_equal(arr[:], data) @pytest.mark.parametrize("compressor", [GzipCodec(), BloscCodec()]) def test_sharding_codec_rejects_variable_size_index_codecs(compressor: Any) -> None: """ ShardingCodec rejects an `index_codecs` chain containing a codec whose encoded size is not fixed, as the spec requires. """ with pytest.raises(ValueError, match="must produce a fixed-size encoding"): ShardingCodec(chunk_shape=(2,), index_codecs=(BytesCodec(), compressor)) def test_sharding_index_location_attribute_error_for_unknown_member() -> None: """ Attribute access for a name that is not a known member of the deprecated ShardingCodecIndexLocation class falls through to AttributeError. """ with pytest.raises(AttributeError): getattr(ShardingCodecIndexLocation, "not_a_member") # noqa: B009 @pytest.mark.parametrize("index_location", INDEX_LOCATION) def test_create_array_with_dict_shards_index_location( index_location: IndexLocation, ) -> None: """ zarr.create_array accepts a `ShardsConfigParam`-shaped dict for `shards` with an explicit `index_location`, and the resulting sharding codec stores that value. Covers the `isinstance(shards, dict)` branch in init_array that the tuple-shaped `shards` form doesn't reach. """ arr = zarr.create_array( store={}, shape=(8,), chunks=(2,), shards={"shape": (4,), "index_location": index_location}, dtype="uint8", ) assert isinstance(arr.metadata, ArrayV3Metadata) # needed for mypy sharding = arr.metadata.codecs[0] assert isinstance(sharding, ShardingCodec) assert sharding.index_location == index_location def test_sharding_zero_dimensional() -> None: """Regression test for https://github.com/zarr-developers/zarr-python/issues/3751""" arr = zarr.create_array({}, shape=(), dtype="f4", chunks=(), shards=()) arr[()] = 42.0 assert arr[()] == pytest.approx(42.0) # Overwriting should also work arr[()] = 43.0 assert arr[()] == pytest.approx(43.0) def test_shard_index_stores_chunks_per_shard_explicitly() -> None: """_ShardIndex stores the chunk grid shape as an explicit field.""" index = _ShardIndex.create_empty((2, 3)) assert index.chunks_per_shard == (2, 3) # 0-D: chunks_per_shard is the empty tuple, distinct from the array's rank index_0d = _ShardIndex.create_empty(()) assert index_0d.chunks_per_shard == () @pytest.mark.parametrize("chunks_per_shard", [(), (3,), (2, 3)]) def test_shard_index_get_chunk_slices_vectorized(chunks_per_shard: tuple[int, ...]) -> None: """get_chunk_slices_vectorized works uniformly across chunk grid ranks, including 0-D.""" index = _ShardIndex.create_empty(chunks_per_shard) # Write the first chunk; leave the rest (if any) empty. all_coords = list(lexicographic_order_coords(chunks_per_shard)) index.set_chunk_slice(all_coords[0], slice(10, 14)) coords_array = np.array(all_coords, dtype=np.uint64).reshape( len(all_coords), len(chunks_per_shard) ) starts, ends, valid = index.get_chunk_slices_vectorized(coords_array) expected_valid = np.zeros(len(all_coords), dtype=bool) expected_valid[0] = True np.testing.assert_array_equal(valid, expected_valid) assert starts[0] == 10 assert ends[0] == 14 np.testing.assert_array_equal(starts[~expected_valid], MAX_UINT_64) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @pytest.mark.parametrize( "pipeline_path", [ "zarr.core.codec_pipeline.FusedCodecPipeline", "zarr.core.codec_pipeline.BatchedCodecPipeline", ], ) def test_sharding_vlen_inner_codec_roundtrip(pipeline_path: str) -> None: """A sharded array whose inner codec chain is a variable-length codec (VLenUTF8) must round-trip under either pipeline. Regression: the Fused pipeline's bulk-decode gate calls `c.is_fixed_size` on every inner codec. `is_fixed_size` is declared on the Codec ABC but had no default, so codecs that don't set it (VLenUTF8/VLenBytes, numcodecs wrappers) raised AttributeError on read — crashing every sharded read whose inner chain included such a codec. """ # The variable-length StringDType resolves to a VLenUTF8Codec inner chain # (a fixed-width None: """to_dict_vectorized derives its own coords and maps present chunks to buffers, empty to None. The reader is given the full per-shard chunk grid implicitly (it reads ``chunks_per_shard`` off its own index), so the result must contain every lexicographic coordinate as a key, with the stored bytes for present chunks and ``None`` for empty ones. """ all_coords = list(lexicographic_order_coords(chunks_per_shard)) # Lay two chunks back-to-back in the buffer; leave the rest (if any) empty. payload = b"abcdXY" index = _ShardIndex.create_empty(chunks_per_shard) index.set_chunk_slice(all_coords[0], slice(0, 4)) present = {all_coords[0]: payload[0:4]} if len(all_coords) > 1: index.set_chunk_slice(all_coords[1], slice(4, 6)) present[all_coords[1]] = payload[4:6] reader = _ShardReader() reader.index = index reader.buf = default_buffer_prototype().buffer.from_bytes(payload) result = reader.to_dict_vectorized() # Every lexicographic coordinate is present as a key, in order. assert list(result.keys()) == all_coords for coords in all_coords: buf = result[coords] if coords in present: assert buf is not None assert buf.to_bytes() == present[coords] else: assert buf is None @pytest.mark.parametrize( "pipeline_path", [ "zarr.core.codec_pipeline.FusedCodecPipeline", "zarr.core.codec_pipeline.BatchedCodecPipeline", ], ) @pytest.mark.parametrize("nested", [False, True], ids=["single", "nested"]) @pytest.mark.parametrize( "selection", [ pytest.param((np.array([3, 1, 2]), np.array([0, 2])), id="2d-arr-arr"), pytest.param((np.array([3, 0]), np.array([2, 0])), id="2d-arr-arr-unsorted-two-shards"), pytest.param((np.array([3, 1, 2]), 1, np.array([0, 2])), id="3d-arr-int-arr"), pytest.param((1, np.array([0, 2]), np.array([1, 3])), id="3d-int-arr-arr"), pytest.param((np.array([3, 1]), np.array([0, 2]), 2), id="3d-arr-arr-int"), pytest.param( (np.array([3, 1, 2]), np.array([0, 2]), np.array([1, 3])), id="3d-arr-arr-arr" ), pytest.param((np.array([3]), np.array([0, 2]), 1), id="3d-arr1-arr-int"), ], ) def test_sharding_orthogonal_set_multiple_array_dims( selection: tuple[int | npt.NDArray[np.intp], ...], nested: bool, pipeline_path: str ) -> None: """Orthogonal set with more than one array-indexed dimension. ``OrthogonalIndexer`` converts such a chunk selection to an ``np.ix_`` tuple of broadcastable arrays before handing it to the codec pipeline. The sharding codec re-derives an indexer from that selection and gets a ``CoordinateIndexer``, whose projections address the value buffer flat. Regression test for the resulting shape mismatch on write. An integer index alongside the arrays is the case that a shape-equality guard misses: ``OrthogonalIndexer`` drops that axis from the value but ``np.ix_`` keeps it as a length-1 axis in the chunk selection, so the value and the re-derived indexer's ``sel_shape`` differ in rank while agreeing in element count. Parametrized over both pipelines because the partial-encode path is written twice -- ``_encode_partial_single`` for ``BatchedCodecPipeline`` and ``_encode_partial_sync`` for ``FusedCodecPipeline``. """ ndim = len(selection) shape = (4,) * ndim inner = ShardingCodec(chunk_shape=(1,) * ndim, codecs=(BytesCodec(),)) serializer = ShardingCodec( chunk_shape=(2,) * ndim, codecs=((inner,) if nested else (BytesCodec(),)) ) base = np.arange(4**ndim, dtype="int32").reshape(shape) ix = np.ix_(*(np.atleast_1d(s) for s in selection)) # The value is shaped like the orthogonal result: integer axes dropped. value_shape = tuple(len(s) for s in selection if not isinstance(s, int)) value = np.arange(math.prod(value_shape), dtype="int32").reshape(value_shape) + 100 with zarr.config.set({"codec_pipeline.path": pipeline_path}): a = zarr.create_array( MemoryStore(), shape=shape, chunks=(2,) + (4,) * (ndim - 1), dtype=base.dtype, serializer=serializer, compressors=None, fill_value=0, ) a[:] = base a.oindex[selection] = value expected = base.copy() expected[ix] = value.reshape(expected[ix].shape) assert np.array_equal(a[:], expected) assert np.array_equal(a.oindex[selection], value) @pytest.mark.parametrize( "pipeline_path", [ "zarr.core.codec_pipeline.FusedCodecPipeline", "zarr.core.codec_pipeline.BatchedCodecPipeline", ], ) @pytest.mark.parametrize("store", ["memory", "local"], indirect=True) @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize("stored", [b"", b"abc"]) @pytest.mark.parametrize("op", ["full read", "partial read", "partial write"]) def test_sharding_truncated_shard_raises( pipeline_path: str, store: Store, index_location: IndexLocation, stored: bytes, op: str ) -> None: """A stored shard shorter than its index (zero-length included) is rejected by every read and partial-write path, under either pipeline, rather than read as missing by some paths and failing the checksum in others.""" with zarr.config.set({"codec_pipeline.path": pipeline_path}): arr = zarr.create_array( store, shape=(8,), chunks=(2,), shards={"shape": (4,), "index_location": index_location}, dtype="i4", compressors=None, fill_value=-1, ) zarr.core.sync.sync(store.set("c/0", default_buffer_prototype().buffer.from_bytes(stored))) ops: dict[str, Callable[[], object]] = { "full read": lambda: arr[0:4], "partial read": lambda: arr[0:2], "partial write": lambda: arr.__setitem__(slice(0, 2), 7), } with pytest.raises(ValueError, match="too short to hold its index"): ops[op]() @pytest.mark.filterwarnings( "ignore:Combining a `sharding_indexed` codec:zarr.errors.ZarrUserWarning" ) @pytest.mark.parametrize( "inner_codecs", [ (ShardingCodec(chunk_shape=(3, 10)),), (ShardingCodec(chunk_shape=(6, 20)),), (TransposeCodec(order=(1, 0)), ShardingCodec(chunk_shape=(10, 3))), (ShardingCodec(chunk_shape=(3, 10), codecs=(ShardingCodec(chunk_shape=(1, 5)),)),), ], ids=["divides", "equal", "after-transpose", "three-levels"], ) def test_nested_sharding_roundtrip(inner_codecs: tuple[Any, ...]) -> None: """A sharding codec nested in a sharding codec is accepted when its inner chunk shape divides the chunk it encodes (the outer codec's inner chunk, as the codecs before it leave it, so permuted after a transpose), and data round-trips.""" data = np.arange(1, 121, dtype="int32").reshape(6, 20) store = MemoryStore() arr = zarr.create_array( store, shape=data.shape, dtype=data.dtype, chunks=(6, 20), serializer=ShardingCodec(chunk_shape=(6, 20), codecs=inner_codecs), ) arr[:] = data np.testing.assert_array_equal(zarr.open_array(store, mode="r")[:], data) @pytest.mark.filterwarnings( "ignore:Combining a `sharding_indexed` codec:zarr.errors.ZarrUserWarning" ) @pytest.mark.parametrize("nested_chunk_shape", [(4, 7), (12, 40), (3, 20, 1)]) def test_nested_sharding_rejects_indivisible_inner_chunk_shape( nested_chunk_shape: tuple[int, ...], ) -> None: """A sharding codec nested in a sharding codec is checked as the outer one is: an inner chunk shape that does not divide the chunk it encodes, or has another number of dimensions, is rejected when the array is created, and nothing is stored. Such arrays used to be created and to read back wrong data.""" store = MemoryStore() with pytest.raises(ValueError, match="not divisible|same number of dimensions"): zarr.create_array( store, shape=(6, 20), dtype="int32", chunks=(6, 20), serializer=ShardingCodec( chunk_shape=(6, 20), codecs=(ShardingCodec(chunk_shape=nested_chunk_shape),) ), ) assert store._store_dict == {} @pytest.mark.parametrize("chunk_shape", [(0, 5), (5, 0), (False, 5), [0], 0]) def test_sharding_codec_rejects_inner_chunk_size_zero(chunk_shape: Any) -> None: """`ShardingCodec` rejects an inner chunk size of 0 (or `False`) when it is constructed, with a `ValueError` naming the dimension, so no array metadata, nested codec or stored document can hold one.""" with pytest.raises(ValueError, match=r"Dimension \d: chunk edge length must be >= 1, got"): ShardingCodec(chunk_shape=chunk_shape) @pytest.mark.filterwarnings( "ignore:Combining a `sharding_indexed` codec:zarr.errors.ZarrUserWarning" ) def test_open_rejects_stored_nested_sharding_with_indivisible_inner_chunk_shape() -> None: """A stored array document whose nested sharding codec has an inner chunk shape that does not divide the chunk it encodes is rejected when the metadata is parsed, not when chunks are read or written.""" valid = ArrayV3Metadata( shape=(6, 20), data_type=Int32(), chunk_grid={"name": "regular", "configuration": {"chunk_shape": (6, 20)}}, chunk_key_encoding={"name": "default"}, fill_value=0, codecs=(ShardingCodec(chunk_shape=(6, 20), codecs=(ShardingCodec(chunk_shape=(3, 10)),)),), attributes={}, dimension_names=None, ) doc: Any = valid.to_dict() doc["codecs"][0]["configuration"]["codecs"][0]["configuration"]["chunk_shape"] = (4, 7) with pytest.raises(ValueError, match="not divisible"): ArrayV3Metadata.from_dict(doc) zarr-developers-zarr-python-a2f2415/tests/test_codecs/test_sharding_unit.py000066400000000000000000001251521527014137600274030ustar00rootroot00000000000000from __future__ import annotations import asyncio from dataclasses import dataclass, replace from typing import TYPE_CHECKING, Any, Literal, cast from unittest.mock import AsyncMock import numpy as np import numpy.typing as npt import pytest import zarr from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec from zarr.codecs.bytes import BytesCodec from zarr.codecs.crc32c_ import Crc32cCodec from zarr.codecs.gzip import GzipCodec from zarr.codecs.sharding import ( MAX_UINT_64, ShardingCodec, _ShardIndex, _ShardingByteGetter, _ShardReader, ) from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer import Buffer as ABCBuffer from zarr.core.buffer import NDBuffer, default_buffer_prototype from zarr.core.buffer.cpu import Buffer from zarr.core.buffer.cpu import NDBuffer as CPUNDBuffer from zarr.core.chunk_grids import ChunkGrid from zarr.core.config import config from zarr.core.dtype import get_data_type_from_native_dtype from zarr.core.indexing import BasicIndexer from zarr.storage._common import StorePath from zarr.storage._memory import MemoryStore if TYPE_CHECKING: from zarr.core.array import ShardsConfigParam from zarr.core.array_spec import ArrayConfigParams # ============================================================================ # _ShardIndex tests # ============================================================================ def test_shard_index_create_empty() -> None: """Test that create_empty creates an index filled with MAX_UINT_64.""" chunks_per_shard = (2, 3) index = _ShardIndex.create_empty(chunks_per_shard) assert index.chunks_per_shard == chunks_per_shard assert index.offsets_and_lengths.shape == (2, 3, 2) assert index.offsets_and_lengths.dtype == np.dtype(" None: """Test create_empty with 1D chunks_per_shard.""" chunks_per_shard = (4,) index = _ShardIndex.create_empty(chunks_per_shard) assert index.chunks_per_shard == chunks_per_shard assert index.offsets_and_lengths.shape == (4, 2) def test_shard_index_is_all_empty_true() -> None: """Test is_all_empty returns True for a freshly created empty index.""" index = _ShardIndex.create_empty((2, 2)) assert index.is_all_empty() is True def test_shard_index_is_all_empty_false() -> None: """Test is_all_empty returns False when at least one chunk is set.""" index = _ShardIndex.create_empty((2, 2)) index.set_chunk_slice((0, 0), slice(0, 100)) assert index.is_all_empty() is False def test_shard_index_get_chunk_slice_empty() -> None: """Test get_chunk_slice returns None for empty chunks.""" index = _ShardIndex.create_empty((2, 2)) assert index.get_chunk_slice((0, 0)) is None assert index.get_chunk_slice((1, 1)) is None def test_shard_index_get_chunk_slice_set() -> None: """Test get_chunk_slice returns correct (start, end) tuple after setting.""" index = _ShardIndex.create_empty((2, 2)) index.set_chunk_slice((0, 1), slice(100, 200)) result = index.get_chunk_slice((0, 1)) assert result == (100, 200) def test_shard_index_set_chunk_slice() -> None: """Test set_chunk_slice correctly sets offset and length.""" index = _ShardIndex.create_empty((3, 3)) # Set a chunk slice index.set_chunk_slice((1, 2), slice(50, 150)) # Verify the underlying array assert index.offsets_and_lengths[1, 2, 0] == 50 # offset assert index.offsets_and_lengths[1, 2, 1] == 100 # length (150 - 50) def test_shard_index_set_chunk_slice_none() -> None: """Test set_chunk_slice with None marks chunk as empty.""" index = _ShardIndex.create_empty((2, 2)) # First set a value index.set_chunk_slice((0, 0), slice(0, 100)) assert index.get_chunk_slice((0, 0)) == (0, 100) # Then clear it index.set_chunk_slice((0, 0), None) assert index.get_chunk_slice((0, 0)) is None assert index.offsets_and_lengths[0, 0, 0] == MAX_UINT_64 assert index.offsets_and_lengths[0, 0, 1] == MAX_UINT_64 def test_shard_index_get_full_chunk_map() -> None: """Test get_full_chunk_map returns correct boolean array.""" index = _ShardIndex.create_empty((2, 3)) # Set some chunks index.set_chunk_slice((0, 0), slice(0, 10)) index.set_chunk_slice((1, 2), slice(10, 20)) chunk_map = index.get_full_chunk_map() assert chunk_map.shape == (2, 3) assert chunk_map.dtype == np.bool_ assert chunk_map[0, 0] is np.True_ assert chunk_map[0, 1] is np.False_ assert chunk_map[0, 2] is np.False_ assert chunk_map[1, 0] is np.False_ assert chunk_map[1, 1] is np.False_ assert chunk_map[1, 2] is np.True_ def test_shard_index_localize_chunk() -> None: """Test _localize_chunk maps global coords to local shard coords via modulo.""" index = _ShardIndex.create_empty((2, 3)) # Within bounds - should return same coords assert index._localize_chunk((0, 0)) == (0, 0) assert index._localize_chunk((1, 2)) == (1, 2) # Out of bounds - should wrap via modulo assert index._localize_chunk((2, 0)) == (0, 0) # 2 % 2 = 0 assert index._localize_chunk((3, 5)) == (1, 2) # 3 % 2 = 1, 5 % 3 = 2 assert index._localize_chunk((4, 6)) == (0, 0) # 4 % 2 = 0, 6 % 3 = 0 # ============================================================================ # _load_partial_shard_maybe tests # # These exercise the partial-shard read path against a real MemoryStore wrapped # in a StorePath (the external-store branch in `_load_partial_shard_maybe`), # plus one test against a real `_ShardingByteGetter` (the in-memory branch used # by nested sharding). # ============================================================================ async def _store_path_with_blob(key: str, blob: bytes) -> StorePath: """Build a `StorePath` over a fresh `MemoryStore` containing `blob` at `key`.""" store = MemoryStore() await store.set(key, Buffer.from_bytes(blob)) return StorePath(store, key) async def test_load_partial_shard_maybe_index_load_fails() -> None: """Returns None when the shard key is absent (index load fails).""" codec = ShardingCodec(chunk_shape=(8,)) byte_getter = StorePath(MemoryStore(), "missing") result = await codec._load_partial_shard_maybe( byte_getter=byte_getter, prototype=default_buffer_prototype(), chunks_per_shard=(2,), all_chunk_coords={(0,)}, max_gap_bytes=1 << 20, max_coalesced_bytes=16 << 20, ) assert result is None async def test_load_partial_shard_maybe_with_empty_chunks( monkeypatch: pytest.MonkeyPatch, ) -> None: """Chunks whose index entry is empty are silently skipped.""" codec = ShardingCodec(chunk_shape=(8,)) chunks_per_shard = (4,) # Index where chunk (1,) is empty; the others point into the stored blob. index = _ShardIndex.create_empty(chunks_per_shard) index.set_chunk_slice((0,), slice(0, 100)) index.set_chunk_slice((2,), slice(100, 200)) index.set_chunk_slice((3,), slice(200, 300)) async def mock_load_index( self: ShardingCodec, byte_getter: StorePath, cps: tuple[int, ...] ) -> _ShardIndex: return index monkeypatch.setattr(ShardingCodec, "_load_shard_index_maybe", mock_load_index) byte_getter = await _store_path_with_blob("shard", b"x" * 300) result = await codec._load_partial_shard_maybe( byte_getter=byte_getter, prototype=default_buffer_prototype(), chunks_per_shard=chunks_per_shard, all_chunk_coords={(0,), (1,), (2,)}, max_gap_bytes=1 << 20, max_coalesced_bytes=16 << 20, ) assert result is not None assert (0,) in result assert (1,) not in result # empty in index assert (2,) in result async def test_load_partial_shard_maybe_all_chunks_empty( monkeypatch: pytest.MonkeyPatch, ) -> None: """Returns an empty dict when all requested chunks are empty (no I/O issued).""" codec = ShardingCodec(chunk_shape=(8,)) chunks_per_shard = (4,) # Fully-empty index — `get_chunk_slice` returns None for every coord. index = _ShardIndex.create_empty(chunks_per_shard) async def mock_load_index( self: ShardingCodec, byte_getter: StorePath, cps: tuple[int, ...] ) -> _ShardIndex: return index monkeypatch.setattr(ShardingCodec, "_load_shard_index_maybe", mock_load_index) # Empty store is fine — we never reach the chunk-read path when all are empty. byte_getter = StorePath(MemoryStore(), "shard") result = await codec._load_partial_shard_maybe( byte_getter=byte_getter, prototype=default_buffer_prototype(), chunks_per_shard=chunks_per_shard, all_chunk_coords={(0,), (1,), (2,)}, max_gap_bytes=1 << 20, max_coalesced_bytes=16 << 20, ) assert result == {} async def test_load_partial_shard_returns_chunk_contents( monkeypatch: pytest.MonkeyPatch, ) -> None: """Returns the correct bytes for each requested chunk.""" codec = ShardingCodec(chunk_shape=(8,)) chunks_per_shard = (4,) index = _ShardIndex.create_empty(chunks_per_shard) index.set_chunk_slice((0,), slice(0, 100)) index.set_chunk_slice((1,), slice(100, 200)) async def mock_load_index( self: ShardingCodec, byte_getter: StorePath, cps: tuple[int, ...] ) -> _ShardIndex: return index monkeypatch.setattr(ShardingCodec, "_load_shard_index_maybe", mock_load_index) blob = b"A" * 100 + b"B" * 100 byte_getter = await _store_path_with_blob("shard", blob) result = await codec._load_partial_shard_maybe( byte_getter=byte_getter, prototype=default_buffer_prototype(), chunks_per_shard=chunks_per_shard, all_chunk_coords={(0,), (1,)}, max_gap_bytes=1 << 20, max_coalesced_bytes=16 << 20, ) assert result is not None buf_0, buf_1 = result[(0,)], result[(1,)] assert buf_0 is not None assert buf_1 is not None assert buf_0.to_bytes() == b"A" * 100 assert buf_1.to_bytes() == b"B" * 100 async def test_load_partial_shard_shard_disappears_returns_none( monkeypatch: pytest.MonkeyPatch, ) -> None: """If the shard key is missing when chunk reads run, returns None. This models a race: the index loaded successfully, but the shard was deleted before the chunk-byte fetches landed. `Store.get_ranges` surfaces this as a `BaseExceptionGroup` containing `FileNotFoundError`, which the codec catches and converts to None to match the index-missing branch's behavior. """ codec = ShardingCodec(chunk_shape=(8,)) chunks_per_shard = (4,) index = _ShardIndex.create_empty(chunks_per_shard) index.set_chunk_slice((0,), slice(0, 100)) async def mock_load_index( self: ShardingCodec, byte_getter: StorePath, cps: tuple[int, ...] ) -> _ShardIndex: return index monkeypatch.setattr(ShardingCodec, "_load_shard_index_maybe", mock_load_index) # Store has no value for "shard" — `get_ranges` will raise FileNotFoundError. byte_getter = StorePath(MemoryStore(), "shard") result = await codec._load_partial_shard_maybe( byte_getter=byte_getter, prototype=default_buffer_prototype(), chunks_per_shard=chunks_per_shard, all_chunk_coords={(0,)}, max_gap_bytes=1 << 20, max_coalesced_bytes=16 << 20, ) assert result is None async def test_load_partial_shard_non_fnf_error_propagates( monkeypatch: pytest.MonkeyPatch, ) -> None: """Non-FileNotFoundError errors from get_ranges are re-raised, not swallowed. Our `BaseExceptionGroup.split(FileNotFoundError)` keeps the "shard gone" behavior for FNF only; anything else (e.g. an OSError from the underlying fetch) must bubble up. """ codec = ShardingCodec(chunk_shape=(8,)) chunks_per_shard = (4,) index = _ShardIndex.create_empty(chunks_per_shard) index.set_chunk_slice((0,), slice(0, 100)) async def mock_load_index( self: ShardingCodec, byte_getter: StorePath, cps: tuple[int, ...] ) -> _ShardIndex: return index monkeypatch.setattr(ShardingCodec, "_load_shard_index_maybe", mock_load_index) # Make the underlying store.get raise OSError. The default Store.get_ranges # impl routes through self.get; coalesced_get wraps the failure in a # BaseExceptionGroup, which our code re-raises (minus FNF leaves, of which # there are none here). async def boom(*args: object, **kwargs: object) -> Buffer | None: raise OSError("injected disk error") store = MemoryStore() monkeypatch.setattr(store, "get", boom) byte_getter = StorePath(store, "shard") with pytest.RaisesGroup(pytest.RaisesExc(OSError, match="injected disk error")): await codec._load_partial_shard_maybe( byte_getter=byte_getter, prototype=default_buffer_prototype(), chunks_per_shard=chunks_per_shard, all_chunk_coords={(0,)}, max_gap_bytes=1 << 20, max_coalesced_bytes=16 << 20, ) async def test_load_partial_shard_nested_sharding_path( monkeypatch: pytest.MonkeyPatch, ) -> None: """Nested sharding: byte_getter is a `_ShardingByteGetter` over an in-memory dict.""" codec = ShardingCodec(chunk_shape=(8,)) chunks_per_shard = (4,) index = _ShardIndex.create_empty(chunks_per_shard) index.set_chunk_slice((0,), slice(0, 100)) index.set_chunk_slice((1,), slice(100, 200)) async def mock_load_index( self: ShardingCodec, byte_getter: _ShardingByteGetter, cps: tuple[int, ...] ) -> _ShardIndex: return index monkeypatch.setattr(ShardingCodec, "_load_shard_index_maybe", mock_load_index) # The "store" for an inner shard is a dict keyed by outer-chunk coords; the # byte_getter reads ranges out of one entry of that dict. blob = b"A" * 100 + b"B" * 100 shard_dict: dict[tuple[int, ...], Buffer | None] = {(0,): Buffer.from_bytes(blob)} byte_getter = _ShardingByteGetter(shard_dict, (0,)) result = await codec._load_partial_shard_maybe( byte_getter=byte_getter, prototype=default_buffer_prototype(), chunks_per_shard=chunks_per_shard, all_chunk_coords={(0,), (1,)}, max_gap_bytes=1 << 20, max_coalesced_bytes=16 << 20, ) assert result is not None buf_0, buf_1 = result[(0,)], result[(1,)] assert buf_0 is not None assert buf_1 is not None assert buf_0.to_bytes() == b"A" * 100 assert buf_1.to_bytes() == b"B" * 100 async def test_load_partial_shard_nested_sharding_missing_outer_chunk( monkeypatch: pytest.MonkeyPatch, ) -> None: """Nested sharding: outer chunk absent → `_ShardingByteGetter.get` returns None → chunks are silently skipped, yielding an empty shard_dict.""" codec = ShardingCodec(chunk_shape=(8,)) chunks_per_shard = (4,) index = _ShardIndex.create_empty(chunks_per_shard) index.set_chunk_slice((0,), slice(0, 100)) async def mock_load_index( self: ShardingCodec, byte_getter: _ShardingByteGetter, cps: tuple[int, ...] ) -> _ShardIndex: return index monkeypatch.setattr(ShardingCodec, "_load_shard_index_maybe", mock_load_index) # Empty outer dict — _ShardingByteGetter.get(...) returns None for any range. shard_dict: dict[tuple[int, ...], Buffer | None] = {} byte_getter = _ShardingByteGetter(shard_dict, (0,)) result = await codec._load_partial_shard_maybe( byte_getter=byte_getter, prototype=default_buffer_prototype(), chunks_per_shard=chunks_per_shard, all_chunk_coords={(0,)}, max_gap_bytes=1 << 20, max_coalesced_bytes=16 << 20, ) assert result == {} # ============================================================================ # Supporting class tests (_ShardReader, _is_total_shard) # ============================================================================ def test_shard_reader_create_empty() -> None: """Test _ShardReader.create_empty creates reader with empty index.""" chunks_per_shard = (2, 3) reader = _ShardReader.create_empty(chunks_per_shard) assert reader.index.is_all_empty() assert len(reader.buf) == 0 assert len(reader) == 2 * 3 def test_shard_reader_iteration() -> None: """Test _ShardReader iteration yields all chunk coordinates.""" chunks_per_shard = (2, 2) reader = _ShardReader.create_empty(chunks_per_shard) coords = list(reader) assert len(coords) == 4 assert (0, 0) in coords assert (0, 1) in coords assert (1, 0) in coords assert (1, 1) in coords def test_shard_reader_getitem_raises_for_empty() -> None: """Test _ShardReader.__getitem__ raises KeyError for empty chunks.""" chunks_per_shard = (2,) reader = _ShardReader.create_empty(chunks_per_shard) with pytest.raises(KeyError): _ = reader[(0,)] def test_is_total_shard_full() -> None: """Test _is_total_shard returns True when all chunk coords are present.""" codec = ShardingCodec(chunk_shape=(8,)) chunks_per_shard = (2, 2) all_chunk_coords: set[tuple[int, ...]] = {(0, 0), (0, 1), (1, 0), (1, 1)} assert codec._is_total_shard(all_chunk_coords, chunks_per_shard) is True def test_is_total_shard_partial() -> None: """Test _is_total_shard returns False for partial chunk coords.""" codec = ShardingCodec(chunk_shape=(8,)) chunks_per_shard = (2, 2) all_chunk_coords: set[tuple[int, ...]] = {(0, 0), (1, 1)} # Missing (0, 1) and (1, 0) assert codec._is_total_shard(all_chunk_coords, chunks_per_shard) is False def test_is_total_shard_empty() -> None: """Test _is_total_shard returns False for empty chunk coords.""" codec = ShardingCodec(chunk_shape=(8,)) chunks_per_shard = (2, 2) all_chunk_coords: set[tuple[int, ...]] = set() assert codec._is_total_shard(all_chunk_coords, chunks_per_shard) is False def test_is_total_shard_1d() -> None: """Test _is_total_shard works with 1D shards.""" codec = ShardingCodec(chunk_shape=(8,)) chunks_per_shard = (4,) all_chunk_coords: set[tuple[int, ...]] = {(0,), (1,), (2,), (3,)} assert codec._is_total_shard(all_chunk_coords, chunks_per_shard) is True # Partial partial_coords: set[tuple[int, ...]] = {(0,), (2,)} assert codec._is_total_shard(partial_coords, chunks_per_shard) is False # ============================================================================ # _inner_codecs_fixed_size tests # ============================================================================ def test_inner_codecs_fixed_size_no_compression() -> None: """Inner codecs without compression should be fixed-size.""" codec = ShardingCodec(chunk_shape=(10,), codecs=[BytesCodec()]) assert codec._inner_codecs_fixed_size is True def test_inner_codecs_fixed_size_with_compression() -> None: """Inner codecs with compression should NOT be fixed-size.""" codec = ShardingCodec(chunk_shape=(10,), codecs=[BytesCodec(), GzipCodec()]) assert codec._inner_codecs_fixed_size is False # ============================================================================ # inner-chain spec threading # ============================================================================ @dataclass(frozen=True) class _WidenToInt16(ArrayArrayCodec): """Test-only sync-capable AA codec that reports its output dtype as int16.""" is_fixed_size = True def to_dict(self) -> dict[str, Any]: return {"name": "_widen_to_int16"} @classmethod def from_dict(cls, data: dict[str, Any]) -> _WidenToInt16: return cls() def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: return replace(chunk_spec, dtype=get_data_type_from_native_dtype(np.dtype("int16"))) def compute_encoded_size(self, input_byte_length: int, _spec: ArraySpec) -> int: return input_byte_length def _encode_sync(self, chunk_array: Any, chunk_spec: ArraySpec) -> Any: return chunk_array # pragma: no cover def _decode_sync(self, chunk_array: Any, chunk_spec: ArraySpec) -> Any: return chunk_array # pragma: no cover async def _encode_single(self, chunk_array: Any, chunk_spec: ArraySpec) -> Any: return chunk_array # pragma: no cover async def _decode_single(self, chunk_array: Any, chunk_spec: ArraySpec) -> Any: return chunk_array # pragma: no cover def _int8_spec(shape: tuple[int, ...]) -> ArraySpec: zdtype = get_data_type_from_native_dtype(np.dtype("int8")) # single-byte source return ArraySpec( shape=shape, dtype=zdtype, fill_value=zdtype.cast_scalar(0), config=ArrayConfig(order="C", write_empty_chunks=False), prototype=default_buffer_prototype(), ) def test_inner_chunk_transform_threads_spec() -> None: """The inner codec chain must be evolved with the spec threaded forward. A dtype-widening inner array->array codec means the BytesCodec serializer is evolved against the WIDENED dtype, not the single-byte source — otherwise it strips its `endian` to None and fails to decode multi-byte inner chunks. Same contract as the pipeline-level `evolve_codecs` regression test, applied to `_get_inner_chunk_transform`. """ codec = ShardingCodec(chunk_shape=(4,), codecs=[_WidenToInt16(), BytesCodec(endian="little")]) shard_spec = _int8_spec((8,)) transform = codec._get_inner_chunk_transform(shard_spec) serializer = transform._ab_codec assert isinstance(serializer, BytesCodec) assert serializer.endian is not None, ( "inner BytesCodec lost its `endian` — _get_inner_chunk_transform did not " "thread the dtype-widening codec's spec into the serializer" ) def test_evolve_from_array_spec_threads_spec() -> None: """`ShardingCodec.evolve_from_array_spec` must thread the spec through the inner chain, like `_get_inner_chunk_transform` does. This method runs EARLIER, on the real array-creation path (the outer pipeline evolves the sharding codec itself), so an unthreaded evolve here bakes an endian-stripped BytesCodec into the evolved instance's `codecs` before the transform builders ever run — and the later threaded evolve then raises instead of recovering. Calling `_get_inner_chunk_transform` on the EVOLVED instance pins the full real path. """ codec = ShardingCodec(chunk_shape=(4,), codecs=[_WidenToInt16(), BytesCodec(endian="little")]) # the array spec the OUTER pipeline evolves the sharding codec against array_spec = _int8_spec((8,)) evolved = codec.evolve_from_array_spec(array_spec) inner_serializer = next(c for c in evolved.codecs if isinstance(c, BytesCodec)) assert inner_serializer.endian is not None, ( "evolve_from_array_spec evolved the inner BytesCodec against the " "un-widened spec, stripping its `endian`" ) # and the evolved instance must still build a working inner transform transform = evolved._get_inner_chunk_transform(array_spec) serializer = transform._ab_codec assert isinstance(serializer, BytesCodec) assert serializer.endian is not None # ============================================================================ # async whole-shard codec methods # # `ShardingCodec` advertises partial decode/encode, so the codec pipeline # always routes sharded reads/writes through `_decode_partial_single` / # `_encode_partial_single`. The whole-shard async methods `_decode_single` / # `_encode_single` are reached only via the direct `ArrayBytesCodec` API (e.g. # a consumer that calls the codec outside a pipeline), so they get no coverage # from end-to-end array tests. Pin them with a direct round-trip. # ============================================================================ @pytest.mark.parametrize("write_empty_chunks", [True, False]) def test_decode_single_encode_single_roundtrip(write_empty_chunks: bool) -> None: """`ShardingCodec._encode_single` then `_decode_single` round-trips a whole shard. Covers the async whole-shard path the pipeline bypasses in favor of the partial methods.""" zdtype = get_data_type_from_native_dtype(np.dtype("float64")) spec = ArraySpec( shape=(50,), dtype=zdtype, fill_value=zdtype.cast_scalar(0), config=ArrayConfig(order="C", write_empty_chunks=write_empty_chunks), prototype=default_buffer_prototype(), ) codec = ShardingCodec(chunk_shape=(10,), codecs=[BytesCodec()]) data = np.arange(50, dtype="float64") value = CPUNDBuffer.from_numpy_array(data) encoded = asyncio.run(codec._encode_single(value, spec)) assert encoded is not None # data is non-empty -> a shard is always produced decoded = asyncio.run(codec._decode_single(encoded, spec)) np.testing.assert_array_equal(decoded.as_numpy_array(), data) def test_encode_single_all_empty_returns_none() -> None: """`_encode_single` of an all-fill shard under write_empty_chunks=False elides every inner chunk and returns None (the all-empty branch).""" zdtype = get_data_type_from_native_dtype(np.dtype("float64")) spec = ArraySpec( shape=(50,), dtype=zdtype, fill_value=zdtype.cast_scalar(0), config=ArrayConfig(order="C", write_empty_chunks=False), prototype=default_buffer_prototype(), ) codec = ShardingCodec(chunk_shape=(10,), codecs=[BytesCodec()]) fill = CPUNDBuffer.from_numpy_array(np.zeros(50, dtype="float64")) assert asyncio.run(codec._encode_single(fill, spec)) is None def test_decode_single_all_empty_fills() -> None: """`_decode_single` of a shard whose index is all-empty fills the output with the fill value (the is_all_empty fast path).""" zdtype = get_data_type_from_native_dtype(np.dtype("float64")) spec = ArraySpec( shape=(50,), dtype=zdtype, fill_value=zdtype.cast_scalar(-1.0), config=ArrayConfig(order="C", write_empty_chunks=False), prototype=default_buffer_prototype(), ) codec = ShardingCodec(chunk_shape=(10,), codecs=[BytesCodec()]) # an empty shard is just the encoded empty index empty_index = asyncio.run(codec._encode_shard_index(_ShardIndex.create_empty((5,)))) decoded = asyncio.run(codec._decode_single(empty_index, spec)) np.testing.assert_array_equal(decoded.as_numpy_array(), np.full(50, -1.0)) # ============================================================================ # async-only index codec fallback (#269) # # `_decode_shard_index` / `_encode_shard_index` delegate to their sync twins # when every index codec is sync-capable, and otherwise fall back to the async # pipeline. The default index chain (bytes + crc32c) is sync-capable, so the # fallback is exercised only by an async-only index codec. # ============================================================================ class _AsyncOnlyBytesCodec(ArrayBytesCodec): """An array<->bytes codec that implements ONLY the async per-chunk methods. Wraps a real `BytesCodec` for the actual conversion but deliberately omits `_encode_sync`/`_decode_sync`, so it is NOT a `SupportsSyncCodec`. Used as an index codec to force the async-pipeline fallback in `_decode_shard_index`/`_encode_shard_index`. """ is_fixed_size = True _inner = BytesCodec() def to_dict(self) -> dict[str, Any]: return {"name": "_async_only_bytes"} @classmethod def from_dict(cls, data: dict[str, Any]) -> _AsyncOnlyBytesCodec: return cls() def evolve_from_array_spec(self, array_spec: ArraySpec) -> _AsyncOnlyBytesCodec: return self def compute_encoded_size(self, input_byte_length: int, _spec: ArraySpec) -> int: return input_byte_length async def _decode_single(self, chunk_bytes: ABCBuffer, chunk_spec: ArraySpec) -> NDBuffer: return await self._inner._decode_single(chunk_bytes, chunk_spec) async def _encode_single(self, chunk_array: NDBuffer, chunk_spec: ArraySpec) -> ABCBuffer: result = await self._inner._encode_single(chunk_array, chunk_spec) assert result is not None return result def test_shard_index_async_fallback_for_async_only_index_codec() -> None: """An async-only index codec is not sync-capable, so `_encode_shard_index` and `_decode_shard_index` must take the async-pipeline fallback (#269) instead of the sync twins — and still round-trip.""" from zarr.abc.codec import SupportsSyncCodec codec = ShardingCodec( chunk_shape=(10,), codecs=[BytesCodec()], index_codecs=[_AsyncOnlyBytesCodec(), Crc32cCodec()], ) assert not codec._index_codecs_sync_capable() assert not isinstance(_AsyncOnlyBytesCodec(), SupportsSyncCodec) chunks_per_shard = (5,) index = _ShardIndex.create_empty(chunks_per_shard) index.set_chunk_slice((0,), slice(0, 42)) encoded = asyncio.run(codec._encode_shard_index(index)) decoded = asyncio.run(codec._decode_shard_index(encoded, chunks_per_shard)) np.testing.assert_array_equal(decoded.offsets_and_lengths, index.offsets_and_lengths) # ============================================================================ # Coalescing config option tests # # Assert that the `array.sharding_coalesce_max_gap_bytes` and # `array.sharding_coalesce_max_bytes` global config keys flow through # `ArrayConfig` to `Store.get_ranges` as `max_gap_bytes` / # `max_coalesced_bytes` kwargs, and that per-array `config={...}` overrides # the global default. # ============================================================================ def _trigger_partial_shard_read(array_config: ArrayConfigParams | None = None) -> AsyncMock: """Build a sharded array on a mocked `MemoryStore`, trigger a partial-shard read via the public read path, and return the `get_ranges` mock. """ import zarr chunk_shape = (2,) shard_shape = (8,) data = np.arange(8, dtype="int32") store = MemoryStore() store_mock = AsyncMock(wraps=store, spec=store.__class__) shards: ShardsConfigParam = { "shape": shard_shape, "index_location": "end", } a = zarr.create_array( StorePath(store_mock), shape=(8,), chunks=chunk_shape, shards=shards, dtype=data.dtype, fill_value=-1, config=array_config, ) a[:] = data store_mock.reset_mock() # Read a strict subset of chunks to take the partial-shard read path. _ = a[0:4] return cast(AsyncMock, store_mock.get_ranges) def test_load_partial_shard_forwards_global_config_to_get_ranges() -> None: """Global `array.sharding_coalesce_*` values flow into ArrayConfig at array-creation time and are forwarded to `Store.get_ranges`.""" with config.set( { "array.sharding_coalesce_max_gap_bytes": 4242, "array.sharding_coalesce_max_bytes": 424242, } ): get_ranges_mock = _trigger_partial_shard_read() assert get_ranges_mock.call_count >= 1 for call in get_ranges_mock.call_args_list: kwargs = call.kwargs assert kwargs["max_gap_bytes"] == 4242 assert kwargs["max_coalesced_bytes"] == 424242 def test_load_partial_shard_per_array_config_overrides_global() -> None: """Per-array `config={...}` passed to `create_array` takes precedence over the global config and is forwarded to `Store.get_ranges`.""" with config.set( { "array.sharding_coalesce_max_gap_bytes": 4242, "array.sharding_coalesce_max_bytes": 424242, } ): get_ranges_mock = _trigger_partial_shard_read( array_config={ "sharding_coalesce_max_gap_bytes": 99, "sharding_coalesce_max_bytes": 9999, }, ) assert get_ranges_mock.call_count >= 1 for call in get_ranges_mock.call_args_list: kwargs = call.kwargs assert kwargs["max_gap_bytes"] == 99 assert kwargs["max_coalesced_bytes"] == 9999 def test_load_partial_shard_uses_config_defaults() -> None: """Without explicit config, defaults from `zarr.config` are forwarded.""" get_ranges_mock = _trigger_partial_shard_read() assert get_ranges_mock.call_count >= 1 for call in get_ranges_mock.call_args_list: kwargs = call.kwargs assert kwargs["max_gap_bytes"] == config.get("array.sharding_coalesce_max_gap_bytes") assert kwargs["max_coalesced_bytes"] == config.get("array.sharding_coalesce_max_bytes") async def test_load_partial_shard_explicit_kwargs_passthrough( monkeypatch: pytest.MonkeyPatch, ) -> None: """`_load_partial_shard_maybe` forwards its explicit kwargs to `get_ranges`.""" codec = ShardingCodec(chunk_shape=(2,)) chunks_per_shard = (4,) index = _ShardIndex.create_empty(chunks_per_shard) index.set_chunk_slice((0,), slice(0, 100)) index.set_chunk_slice((2,), slice(200, 300)) store = MemoryStore() await store.set("shard", Buffer.from_bytes(b"x" * 300)) store_mock = AsyncMock(wraps=store, spec=store.__class__) byte_getter = StorePath(store_mock, "shard") async def mock_load_index( self: ShardingCodec, byte_getter: StorePath, cps: tuple[int, ...] ) -> _ShardIndex: return index monkeypatch.setattr(ShardingCodec, "_load_shard_index_maybe", mock_load_index) await codec._load_partial_shard_maybe( byte_getter=byte_getter, prototype=default_buffer_prototype(), chunks_per_shard=chunks_per_shard, all_chunk_coords={(0,), (2,)}, max_gap_bytes=12345, max_coalesced_bytes=67890, ) store_mock.get_ranges.assert_called_once() kwargs = store_mock.get_ranges.call_args.kwargs assert kwargs["max_gap_bytes"] == 12345 assert kwargs["max_coalesced_bytes"] == 67890 # ============================================================================ # Bulk full-shard decode: identity-read gating and dtype gating # ============================================================================ _FUSED_PIPELINE = "zarr.core.codec_pipeline.FusedCodecPipeline" def _fused_uncompressed_array( index_location: Literal["start", "end"], ) -> tuple[Any, npt.NDArray[np.int32]]: """Sharded `(8, 8)` array whose inner chain is a bare BytesCodec (no crc), one shard covering the whole array, filled with `arange` data. Callers must be inside a config context selecting the fused pipeline.""" shards: ShardsConfigParam = {"shape": (8, 8), "index_location": index_location} arr = zarr.create_array( store=MemoryStore(), shape=(8, 8), chunks=(2, 2), shards=shards, dtype="int32", compressors=None, filters=None, fill_value=0, config={"write_empty_chunks": True}, ) ref = np.arange(64, dtype="int32").reshape(8, 8) arr[:] = ref return arr, ref def _spy_on_bulk_decode(monkeypatch: pytest.MonkeyPatch) -> list[bool]: """Record, per call, whether `_decode_full_shard_bulk_if_uncompressed` engaged (returned non-None).""" engaged: list[bool] = [] orig = ShardingCodec._decode_full_shard_bulk_if_uncompressed def spy(self: ShardingCodec, shard_bytes: Any, shard_spec: Any, indexer: Any) -> Any: result = orig(self, shard_bytes, shard_spec, indexer) engaged.append(result is not None) return result monkeypatch.setattr(ShardingCodec, "_decode_full_shard_bulk_if_uncompressed", spy) return engaged _PERM_8 = np.array([7, 2, 5, 0, 3, 6, 1, 4]) _DUP_8 = np.array([0, 0, 1, 2, 3, 4, 5, 6]) @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize( ("read", "expected", "expect_bulk"), [ pytest.param(lambda a: a[:], lambda r: r, True, id="full-slice"), pytest.param(lambda a: a[...], lambda r: r, True, id="ellipsis"), pytest.param( lambda a: a[_PERM_8, :], lambda r: r[_PERM_8, :], False, id="fancy-permutation" ), pytest.param(lambda a: a[_DUP_8, :], lambda r: r[_DUP_8, :], False, id="fancy-duplicates"), pytest.param( lambda a: a.oindex[_PERM_8, :], lambda r: r[_PERM_8, :], False, id="oindex-permutation", ), pytest.param( lambda a: a.oindex[_DUP_8, :], lambda r: r[_DUP_8, :], False, id="oindex-duplicates" ), pytest.param(lambda a: a[1:7, :], lambda r: r[1:7, :], False, id="subset-slice"), ], ) def test_bulk_decode_engagement_and_correctness( monkeypatch: pytest.MonkeyPatch, index_location: Literal["start", "end"], read: Any, expected: Any, expect_bulk: bool, ) -> None: """Under the fused pipeline on an uncompressed crc-free shard, the bulk whole-shard decode fires exactly for identity full reads — and every read returns what numpy returns. The engagement assertions keep the correctness half non-vacuous: a gate that simply disabled the fast path would pass the value checks but fail here.""" engaged = _spy_on_bulk_decode(monkeypatch) with config.set({"codec_pipeline.path": _FUSED_PIPELINE}): arr, ref = _fused_uncompressed_array(index_location) engaged.clear() np.testing.assert_array_equal(read(arr), expected(ref)) if expect_bulk: assert len(engaged) > 0, "bulk fast path was never reached for a full read" assert all(engaged), "bulk fast path did not engage for a full read" else: assert not any(engaged), "bulk fast path engaged for a non-identity selection" @pytest.mark.parametrize("index_location", ["start", "end"]) def test_multi_shard_permutation_read(index_location: Literal["start", "end"]) -> None: """A row permutation crossing shard boundaries must return permuted data under the fused pipeline (each shard sees a gather selection whose shape coincides with the shard shape).""" shards: ShardsConfigParam = {"shape": (8, 8), "index_location": index_location} with config.set({"codec_pipeline.path": _FUSED_PIPELINE}): arr = zarr.create_array( store=MemoryStore(), shape=(16, 16), chunks=(2, 2), shards=shards, dtype="int32", compressors=None, filters=None, fill_value=0, config={"write_empty_chunks": True}, ) ref = np.arange(256, dtype="int32").reshape(16, 16) arr[:] = ref perm = np.array([9, 3, 12, 0, 15, 6, 10, 1, 14, 5, 8, 2, 13, 7, 11, 4]) np.testing.assert_array_equal(arr[perm, :8], ref[perm, :8]) np.testing.assert_array_equal(arr.oindex[perm, :8], ref[perm, :8]) def _dense_shard_blob( codec: ShardingCodec, data: np.ndarray[Any, np.dtype[Any]], chunk_len: int ) -> Buffer: """Hand-assemble a dense `index_location="end"` shard blob for 1-D `data`: natural-order chunk payloads followed by the encoded index.""" n_chunks = data.shape[0] // chunk_len chunk_nbytes = chunk_len * data.dtype.itemsize index = _ShardIndex.create_empty((n_chunks,)) for i in range(n_chunks): index.set_chunk_slice((i,), slice(i * chunk_nbytes, (i + 1) * chunk_nbytes)) index_bytes = codec._encode_shard_index_sync(index) return Buffer.from_bytes(data.tobytes() + index_bytes.to_bytes()) def _identity_indexer(shape: tuple[int, ...], chunk_shape: tuple[int, ...]) -> BasicIndexer: return BasicIndexer( tuple(slice(0, s) for s in shape), shape=shape, chunk_grid=ChunkGrid.from_sizes(shape, chunk_shape), ) def _spec_for(data: np.ndarray[Any, np.dtype[Any]]) -> ArraySpec: zdt = get_data_type_from_native_dtype(data.dtype) return ArraySpec( shape=data.shape, dtype=zdt, fill_value=zdt.cast_scalar(0), config=ArrayConfig(order="C", write_empty_chunks=True), prototype=default_buffer_prototype(), ) def test_bulk_decode_declines_structured_dtype() -> None: """The bulk path has no structured-dtype byte-order handling (the `Struct` branch of `BytesCodec._decode_sync`), so it must decline structured specs. The plain-dtype control on an identically constructed blob proves the decline comes from the dtype gate, not from a malformed blob.""" codec = ShardingCodec(chunk_shape=(2,), codecs=[BytesCodec(endian="little")]) # control: same construction with a plain dtype engages the bulk path plain = np.arange(4, dtype=" None: """`is_dense` accepts every layout whose fixed-size payloads exactly tile the data section, wherever that section starts and in whatever order the chunks were laid out.""" chunk_len = 24 n = int(np.prod(chunks_per_shard)) slots = np.arange(n) if layout_order == "reversed": slots = slots[::-1] offsets = data_section_start + slots * chunk_len index = _ShardIndex.create_empty(chunks_per_shard) for coord, off in zip(np.ndindex(chunks_per_shard), offsets, strict=True): index.set_chunk_slice(tuple(coord), slice(int(off), int(off) + chunk_len)) assert index.is_dense(chunk_len, data_section_start=data_section_start) is True def test_shard_index_is_dense_rejects_overlapping_offsets() -> None: """Unique but overlapping offsets (second payload starts inside the first) are not dense.""" chunk_len = 24 index = _ShardIndex.create_empty((2,)) index.set_chunk_slice((0,), slice(0, chunk_len)) index.set_chunk_slice((1,), slice(12, 12 + chunk_len)) assert index.is_dense(chunk_len, data_section_start=0) is False def test_shard_index_is_dense_rejects_out_of_range_offsets() -> None: """An offset outside the data section (here: chunk 0 pointing into an `index_location="start"` index region) is not dense, even though offsets are unique and non-overlapping.""" chunk_len = 24 data_section_start = 16 index = _ShardIndex.create_empty((2,)) index.set_chunk_slice((0,), slice(0, chunk_len)) index.set_chunk_slice( (1,), slice(data_section_start + chunk_len, data_section_start + 2 * chunk_len) ) assert index.is_dense(chunk_len, data_section_start=data_section_start) is False zarr-developers-zarr-python-a2f2415/tests/test_codecs/test_transpose.py000066400000000000000000000106611527014137600265610ustar00rootroot00000000000000import numpy as np import pytest import zarr from zarr import AsyncArray, config from zarr.abc.codec import SupportsSyncCodec from zarr.abc.store import Store from zarr.codecs import TransposeCodec from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer import NDBuffer, default_buffer_prototype from zarr.core.common import MemoryOrder from zarr.core.dtype import get_data_type_from_native_dtype from zarr.storage import StorePath from .test_codecs import _AsyncArrayProxy @pytest.mark.parametrize("input_order", ["F", "C"]) @pytest.mark.parametrize("runtime_write_order", ["F", "C"]) @pytest.mark.parametrize("runtime_read_order", ["F", "C"]) @pytest.mark.parametrize("with_sharding", [True, False]) @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) async def test_transpose( store: Store, input_order: MemoryOrder, runtime_write_order: MemoryOrder, runtime_read_order: MemoryOrder, with_sharding: bool, ) -> None: data = np.arange(0, 256, dtype="uint16").reshape((1, 32, 8), order=input_order) spath = StorePath(store, path="transpose") with config.set({"array.order": runtime_write_order}): a = await zarr.api.asynchronous.create_array( spath, shape=data.shape, chunks=(1, 16, 8) if with_sharding else (1, 32, 8), shards=(1, 32, 8) if with_sharding else None, dtype=data.dtype, fill_value=0, chunk_key_encoding={"name": "v2", "separator": "."}, filters=[TransposeCodec(order=(2, 1, 0))], ) await _AsyncArrayProxy(a)[:, :].set(data) read_data = await _AsyncArrayProxy(a)[:, :].get() assert np.array_equal(data, read_data) with config.set({"array.order": runtime_read_order}): a = await AsyncArray.open( spath, ) read_data = await _AsyncArrayProxy(a)[:, :].get() assert np.array_equal(data, read_data) assert isinstance(read_data, np.ndarray) if runtime_read_order == "F": assert read_data.flags["F_CONTIGUOUS"] assert not read_data.flags["C_CONTIGUOUS"] else: assert not read_data.flags["F_CONTIGUOUS"] assert read_data.flags["C_CONTIGUOUS"] @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) @pytest.mark.parametrize("order", [[1, 2, 0], [1, 2, 3, 0], [3, 2, 4, 0, 1]]) def test_transpose_non_self_inverse(store: Store, order: list[int]) -> None: shape = [i + 3 for i in range(len(order))] data = np.arange(0, np.prod(shape), dtype="uint16").reshape(shape) spath = StorePath(store, "transpose_non_self_inverse") a = zarr.create_array( spath, shape=data.shape, chunks=data.shape, dtype=data.dtype, fill_value=0, filters=[TransposeCodec(order=order)], ) a[:, :] = data read_data = a[:, :] assert np.array_equal(data, read_data) @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) def test_transpose_invalid( store: Store, ) -> None: data = np.arange(0, 256, dtype="uint16").reshape((1, 32, 8)) spath = StorePath(store, "transpose_invalid") for order in [(1, 0), (3, 2, 1), (3, 3, 1), "F", "C"]: with pytest.raises((ValueError, TypeError)): zarr.create_array( spath, shape=data.shape, chunks=(1, 32, 8), dtype=data.dtype, fill_value=0, chunk_key_encoding={"name": "v2", "separator": "."}, filters=[TransposeCodec(order=order)], # type: ignore[arg-type] ) def test_transpose_codec_supports_sync() -> None: assert isinstance(TransposeCodec(order=(0, 1)), SupportsSyncCodec) def test_transpose_codec_sync_roundtrip() -> None: codec = TransposeCodec(order=(1, 0)) arr = np.arange(12, dtype="float64").reshape(3, 4) zdtype = get_data_type_from_native_dtype(arr.dtype) spec = ArraySpec( shape=arr.shape, dtype=zdtype, fill_value=zdtype.cast_scalar(0), config=ArrayConfig(order="C", write_empty_chunks=True), prototype=default_buffer_prototype(), ) nd_buf: NDBuffer = default_buffer_prototype().nd_buffer.from_numpy_array(arr) encoded = codec._encode_sync(nd_buf, spec) assert encoded is not None resolved_spec = codec.resolve_metadata(spec) decoded = codec._decode_sync(encoded, resolved_spec) np.testing.assert_array_equal(arr, decoded.as_numpy_array()) zarr-developers-zarr-python-a2f2415/tests/test_codecs/test_vlen.py000066400000000000000000000076351527014137600255160ustar00rootroot00000000000000from typing import Any import numpy as np import pytest import zarr from zarr import Array from zarr.abc.codec import Codec, SupportsSyncCodec from zarr.abc.store import Store from zarr.codecs import ZstdCodec from zarr.codecs.vlen_utf8 import VLenBytesCodec, VLenUTF8Codec from zarr.core.dtype import get_data_type_from_native_dtype from zarr.core.metadata.v3 import ArrayV3Metadata from zarr.storage import StorePath # The explicit id for the "str" literal avoids colliding with the auto-generated id for # the `str` builtin. numpy_str_dtypes: list[Any] = [ None, str, pytest.param("str", id="str-literal"), np.dtypes.StrDType, "S", "U", np.dtypes.StringDType, ] expected_array_string_dtype: np.dtype[Any] = np.dtypes.StringDType() @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @pytest.mark.parametrize("store", ["memory", "local"], indirect=["store"]) @pytest.mark.parametrize("dtype", numpy_str_dtypes) @pytest.mark.parametrize("as_object_array", [False, True]) @pytest.mark.parametrize("compressor", [None, ZstdCodec()]) def test_vlen_string( store: Store, dtype: np.dtype[Any] | None, as_object_array: bool, compressor: Codec | None ) -> None: strings = ["hello", "world", "this", "is", "a", "test"] data = np.array(strings, dtype=dtype).reshape((2, 3)) sp = StorePath(store, path="string") a = zarr.create_array( sp, shape=data.shape, chunks=data.shape, dtype=data.dtype, fill_value="", compressors=compressor, # type: ignore[arg-type] ) assert isinstance(a.metadata, ArrayV3Metadata) # needed for mypy # should also work if input array is an object array, provided we explicitly specified # a stringlike dtype when creating the Array if as_object_array: data_obj = data.astype("O") a[:, :] = data_obj else: a[:, :] = data assert np.array_equal(data, a[:, :]) assert a.metadata.data_type == get_data_type_from_native_dtype(data.dtype) assert a.dtype == data.dtype # test round trip b = Array.open(sp) assert isinstance(b.metadata, ArrayV3Metadata) # needed for mypy assert np.array_equal(data, b[:, :]) assert b.metadata.data_type == get_data_type_from_native_dtype(data.dtype) assert a.dtype == data.dtype @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @pytest.mark.parametrize("store", ["memory"], indirect=["store"]) @pytest.mark.parametrize( ("dtype", "fill_value", "elements"), [ pytest.param("string", "", [f"S{i:05}" for i in range(9)], id="string"), pytest.param("variable_length_bytes", b"", [b"%05d" % i for i in range(9)], id="bytes"), ], ) def test_vlen_f_contiguous( store: Store, dtype: str, fill_value: str | bytes, elements: list[str] | list[bytes] ) -> None: """An F-contiguous chunk written through a vlen codec round-trips in the original element order rather than the transposed memory order (gh-3558).""" # reshape(order="F") gives an F-contiguous view directly, so the whole-array write # below hands an F-contiguous chunk to the codec; assert it to guard the precondition. data = np.array(elements, dtype=object).reshape((3, 3), order="F") assert data.flags.f_contiguous sp = StorePath(store, path="vlen-f-contiguous") # chunks == shape so the write is a single complete chunk, which the codec pipeline # forwards to the codec untouched; a partial-chunk layout would be recopied to C order. a = zarr.create_array( sp, shape=data.shape, chunks=data.shape, dtype=dtype, fill_value=fill_value ) a[:, :] = data assert np.array_equal(data, np.asarray(a[:, :], dtype=object)) def test_vlen_utf8_codec_supports_sync() -> None: assert isinstance(VLenUTF8Codec(), SupportsSyncCodec) def test_vlen_bytes_codec_supports_sync() -> None: assert isinstance(VLenBytesCodec(), SupportsSyncCodec) zarr-developers-zarr-python-a2f2415/tests/test_codecs/test_zstd.py000066400000000000000000000032711527014137600255260ustar00rootroot00000000000000import numpy as np import pytest import zarr from zarr.abc.codec import SupportsSyncCodec from zarr.abc.store import Store from zarr.codecs import ZstdCodec from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer import default_buffer_prototype from zarr.core.dtype import get_data_type_from_native_dtype from zarr.storage import StorePath @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) @pytest.mark.parametrize("checksum", [True, False]) def test_zstd(store: Store, checksum: bool) -> None: data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) a = zarr.create_array( StorePath(store, path="zstd"), shape=data.shape, chunks=(16, 16), dtype=data.dtype, fill_value=0, compressors=ZstdCodec(level=0, checksum=checksum), ) a[:, :] = data assert np.array_equal(data, a[:, :]) def test_zstd_codec_supports_sync() -> None: assert isinstance(ZstdCodec(), SupportsSyncCodec) def test_zstd_codec_sync_roundtrip() -> None: codec = ZstdCodec(level=1) arr = np.arange(100, dtype="float64") zdtype = get_data_type_from_native_dtype(arr.dtype) spec = ArraySpec( shape=arr.shape, dtype=zdtype, fill_value=zdtype.cast_scalar(0), config=ArrayConfig(order="C", write_empty_chunks=True), prototype=default_buffer_prototype(), ) buf = default_buffer_prototype().buffer.from_array_like(arr.view("B")) encoded = codec._encode_sync(buf, spec) assert encoded is not None decoded = codec._decode_sync(encoded, spec) result = np.frombuffer(decoded.as_numpy_array(), dtype="float64") np.testing.assert_array_equal(arr, result) zarr-developers-zarr-python-a2f2415/tests/test_common.py000066400000000000000000000260771527014137600235440ustar00rootroot00000000000000from __future__ import annotations import asyncio from collections.abc import Iterable from typing import TYPE_CHECKING, get_args import numpy as np import pytest from tests.conftest import Expect from zarr.core.common import ( ANY_ACCESS_MODE, AccessModeLiteral, ceildiv, concurrent_foreach, concurrent_iter, parse_bool, parse_int, parse_name, parse_order, parse_shapelike, product, ) from zarr.core.config import parse_indexing_order if TYPE_CHECKING: from typing import Any, Literal @pytest.mark.parametrize( ("a", "b", "expected"), [(0, 0, 0), (7, 3, 3), (7.5, 2, 4), (2**62 - 1, 1, 2**62)], ) def test_ceildiv(a: float, b: float, expected: int) -> None: """The original helper retains its floating-point division behavior.""" assert ceildiv(a, b) == expected @pytest.mark.parametrize( "case", [ Expect(input=(0, 3), output=0, id="zero"), Expect(input=(7, 3), output=3, id="round-up"), Expect(input=(9, 3), output=3, id="exact"), Expect(input=(-7, 3), output=-2, id="negative-numerator"), Expect(input=(7, -3), output=-2, id="negative-divisor"), Expect(input=(-7, -3), output=3, id="both-negative"), Expect(input=(2**62 - 1, 1), output=2**62 - 1, id="large-exact"), Expect(input=(2**62 + 1, 2), output=2**61 + 1, id="large-round-up"), Expect(input=(2**60 + 3, 1), output=2**60 + 3, id="large-low-bits"), Expect( input=(np.int64(2**62 - 1), np.int64(1)), output=2**62 - 1, id="numpy-signed", ), Expect( input=(np.int64(-(2**63)), np.int64(-1)), output=2**63, id="numpy-signed-minimum", ), Expect( input=(np.uint64(2**64 - 1), np.uint64(2)), output=2**63, id="numpy-unsigned-maximum", ), ], ids=lambda case: case.id, ) def test_ceildiv_int(case: Expect[tuple[int, int], int]) -> None: from zarr.core.common import ceildiv_int result = ceildiv_int(*case.input) assert result == case.output assert isinstance(result, int) @pytest.mark.parametrize("numerator", [0, 1]) def test_ceildiv_int_zero_divisor(numerator: int) -> None: from zarr.core.common import ceildiv_int with pytest.raises(ZeroDivisionError): ceildiv_int(numerator, 0) @pytest.mark.parametrize("data", [(0, 0, 0, 0), (1, 3, 4, 5, 6), (2, 4)]) def test_product(data: tuple[int, ...]) -> None: assert product(data) == np.prod(data) def test_access_modes() -> None: """ Test that the access modes type and variable for run-time checking are equivalent. """ assert set(ANY_ACCESS_MODE) == set(get_args(AccessModeLiteral)) async def test_concurrent_iter_schedules_eagerly() -> None: """`concurrent_iter` must return already-scheduled tasks, not a lazy generator. Its docstring promises `func(*item)` is launched concurrently for every item up front; a caller that awaits the returned tasks one at a time (rather than via `gather`/`as_completed`, which force iteration) relies on that eager scheduling to get any overlap at all. """ started = [False, False, False] async def mark(i: int) -> int: started[i] = True return i tasks = concurrent_iter([(0,), (1,), (2,)], mark) # Give the event loop one chance to run before awaiting anything # individually. If `concurrent_iter` were lazy, nothing would have been # scheduled yet and `started` would still be all-False here. await asyncio.sleep(0) assert started == [True, True, True] results = [await t for t in tasks] assert results == [0, 1, 2] # todo: test def test_concurrent_map() -> None: ... class _Abort(BaseException): """A BaseException that is neither KeyboardInterrupt nor SystemExit.""" @pytest.mark.parametrize(("n_items", "limit"), [(0, 1), (1, 4), (7, 1), (7, 3), (7, 7), (7, 50)]) async def test_concurrent_foreach(n_items: int, limit: int) -> None: """Every item is processed exactly once, lazily, with at most `limit` in flight.""" seen: list[int] = [] in_flight = 0 peak = 0 pulled = 0 def items() -> Iterable[tuple[int]]: nonlocal pulled for i in range(n_items): pulled += 1 yield (i,) async def work(i: int) -> None: nonlocal in_flight, peak # Laziness: an item is pulled only when a worker is free to take it. assert pulled <= i + limit in_flight += 1 peak = max(peak, in_flight) await asyncio.sleep(0) in_flight -= 1 seen.append(i) await concurrent_foreach(items(), work, limit) assert sorted(seen) == list(range(n_items)) assert peak == min(n_items, limit) async def test_concurrent_foreach_rejects_non_positive_limit() -> None: with pytest.raises(ValueError, match="limit must be at least 1"): await concurrent_foreach([(0,)], asyncio.sleep, 0) async def test_concurrent_foreach_single_failure_is_raised_bare() -> None: """One failing call surfaces as itself and stops the rest of the stream.""" started: list[int] = [] async def work(i: int) -> None: started.append(i) await asyncio.sleep(0) if i == 2: raise OSError("boom") with pytest.raises(OSError, match="boom") as info: await concurrent_foreach(((i,) for i in range(100)), work, 2) assert info.value.__cause__ is None assert info.value.__suppress_context__ assert len(started) < 100 async def test_concurrent_foreach_fast_failure_is_not_repeated() -> None: """A call that fails before its first await stops the other workers from taking items, so a read-only store raises once, not once per worker.""" calls = 0 async def work(i: int) -> None: nonlocal calls calls += 1 raise PermissionError("read only") with pytest.raises(PermissionError, match="read only") as info: await concurrent_foreach(((i,) for i in range(100)), work, 8) assert calls == 1 assert info.value.__suppress_context__ async def test_concurrent_foreach_simultaneous_failures_keep_the_group() -> None: """Failures from the same loop iteration are all kept on the raised exception.""" async def work(i: int) -> None: await asyncio.sleep(0) if i == 0: raise OSError("first") raise ValueError("second") with pytest.raises(OSError, match="first") as info: await concurrent_foreach([(0,), (1,)], work, 2) group = info.value.__cause__ assert isinstance(group, ExceptionGroup) assert {type(e) for e in group.exceptions} == {OSError, ValueError} async def test_concurrent_foreach_base_exception_is_raised_bare() -> None: async def work(i: int) -> None: raise _Abort with pytest.raises(_Abort): await concurrent_foreach([(0,), (1,)], work, 1) # todo: test def test_to_thread() -> None: ... # todo: test def test_enum_names() -> None: ... # todo: test def test_parse_enum() -> None: ... @pytest.mark.parametrize("data", [("foo", "bar"), (10, 11)]) def test_parse_name_invalid(data: tuple[Any, Any]) -> None: observed, expected = data if isinstance(observed, str): with pytest.raises(ValueError, match=f"Expected '{expected}'. Got {observed} instead."): parse_name(observed, expected) else: with pytest.raises( TypeError, match=f"Expected a string, got an instance of {type(observed)}." ): parse_name(observed, expected) @pytest.mark.parametrize("data", [("foo", "foo"), ("10", "10")]) def test_parse_name_valid(data: tuple[Any, Any]) -> None: observed, expected = data assert parse_name(observed, expected) == observed @pytest.mark.parametrize("data", [0, 1, "hello", "f"]) def test_parse_indexing_order_invalid(data: Any) -> None: with pytest.raises(ValueError, match="Failed to parse input for 'order'"): parse_indexing_order(data) @pytest.mark.parametrize("data", [0, 1, "hello", "f"]) def test_parse_order_invalid(data: Any) -> None: with pytest.raises(ValueError, match="Failed to parse input for 'order'"): parse_order(data) @pytest.mark.parametrize("data", [0, 1, "true", None, [True]]) def test_parse_bool_invalid(data: Any) -> None: """Non-bool values are rejected with a ValueError.""" with pytest.raises(ValueError, match="Expected instance of bool"): parse_bool(data) @pytest.mark.parametrize("data", [True, False]) def test_parse_bool_valid(data: bool) -> None: assert parse_bool(data) is data @pytest.mark.parametrize("data", ["1", 1.0, True, False, None, [1], (1,)]) def test_parse_int_invalid(data: Any) -> None: """Non-int values (including bools, which are int subclasses) are rejected.""" with pytest.raises(ValueError, match="Expected int"): parse_int(data) @pytest.mark.parametrize("data", [0, 1, -1, 2**63]) def test_parse_int_valid(data: int) -> None: assert parse_int(data) == data @pytest.mark.parametrize("data", ["C", "F"]) def parse_indexing_order_valid(data: Literal["C", "F"]) -> None: assert parse_indexing_order(data) == data @pytest.mark.parametrize("data", [lambda v: v, slice(None)]) def test_parse_shapelike_invalid_single_type(data: Any) -> None: """ Test that we get the expected error message when passing in a value that is not an integer or an iterable of integers. """ with pytest.raises(TypeError, match="Expected an integer or an iterable of integers."): parse_shapelike(data) def test_parse_shapelike_invalid_single_value() -> None: """ Test that we get the expected error message when passing in a negative integer. """ with pytest.raises(ValueError, match="Expected a non-negative integer."): parse_shapelike(-1) @pytest.mark.parametrize("data", ["shape", ("0", 1, 2, 3), {"0": "0"}, ((1, 2), (2, 2)), (4.0, 2)]) def test_parse_shapelike_invalid_iterable_types(data: Any) -> None: """ Test that we get the expected error message when passing in an iterable containing non-integer elements """ with pytest.raises(TypeError, match="Expected an iterable of integers"): parse_shapelike(data) @pytest.mark.parametrize("data", [(1, 2, 3, -1), (-10,)]) def test_parse_shapelike_invalid_iterable_values(data: Any) -> None: """ Test that we get the expected error message when passing in an iterable containing negative integers """ with pytest.raises(ValueError, match="Expected all values to be non-negative."): parse_shapelike(data) @pytest.mark.parametrize( "data", [range(10), [0, 1, 2, np.uint64(3)], (3, 4, 5), (), 1, np.uint8(1)] ) def test_parse_shapelike_valid(data: Iterable[int] | int) -> None: if isinstance(data, Iterable): expected = tuple(data) else: expected = (data,) assert parse_shapelike(data) == expected # todo: more dtypes @pytest.mark.parametrize("data", [("uint8", np.uint8), ("float64", np.float64)]) def parse_dtype(data: tuple[str, np.dtype[Any]]) -> None: unparsed, parsed = data assert parse_dtype(unparsed) == parsed # todo: figure out what it means to test this def test_parse_fill_value() -> None: ... zarr-developers-zarr-python-a2f2415/tests/test_config.py000066400000000000000000000425401527014137600235120ustar00rootroot00000000000000import inspect import os from collections.abc import Iterable from typing import Any from unittest import mock from unittest.mock import Mock import numpy as np import pytest import zarr from zarr import zeros from zarr.abc.codec import CodecPipeline from zarr.abc.store import ByteSetter, Store from zarr.codecs import ( BloscCodec, BytesCodec, Crc32cCodec, ShardingCodec, ) from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer import NDBuffer from zarr.core.buffer.core import Buffer from zarr.core.codec_pipeline import BatchedCodecPipeline from zarr.core.config import BadConfigError, config from zarr.core.indexing import SelectorTuple from zarr.errors import ChunkNotFoundError, UnknownCodecError, ZarrUserWarning from zarr.registry import ( fully_qualified_name, get_buffer_class, get_codec_class, get_ndbuffer_class, get_pipeline_class, register_buffer, register_codec, register_ndbuffer, register_pipeline, ) from zarr.testing.buffer import ( NDBufferUsingTestNDArrayLike, StoreExpectingTestBuffer, TestBuffer, TestNDArrayLike, ) def test_config_defaults_set() -> None: # regression test for available defaults assert ( config.defaults == [ { "default_zarr_format": 3, "array": { "order": "C", "write_empty_chunks": False, "read_missing_chunks": True, "target_shard_size_bytes": None, "rectilinear_chunks": False, "sharding_coalesce_max_gap_bytes": 1 << 20, "sharding_coalesce_max_bytes": 16 << 20, "allow_unsafe_pickle_decode": False, }, "async": {"concurrency": 10, "timeout": None}, "threading": {"max_workers": None}, "json_indent": 2, "codec_pipeline": { "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", "batch_size": 1, "max_workers": None, }, "codecs": { "blosc": "zarr.codecs.blosc.BloscCodec", "gzip": "zarr.codecs.gzip.GzipCodec", "zstd": "zarr.codecs.zstd.ZstdCodec", "bytes": "zarr.codecs.bytes.BytesCodec", "endian": "zarr.codecs.bytes.BytesCodec", # compatibility with earlier versions of ZEP1 "crc32c": "zarr.codecs.crc32c_.Crc32cCodec", "sharding_indexed": "zarr.codecs.sharding.ShardingCodec", "transpose": "zarr.codecs.transpose.TransposeCodec", "vlen-utf8": "zarr.codecs.vlen_utf8.VLenUTF8Codec", "vlen-bytes": "zarr.codecs.vlen_utf8.VLenBytesCodec", "numcodecs.bz2": "zarr.codecs.numcodecs.BZ2", "numcodecs.crc32": "zarr.codecs.numcodecs.CRC32", "numcodecs.crc32c": "zarr.codecs.numcodecs.CRC32C", "numcodecs.lz4": "zarr.codecs.numcodecs.LZ4", "numcodecs.lzma": "zarr.codecs.numcodecs.LZMA", "numcodecs.zfpy": "zarr.codecs.numcodecs.ZFPY", "numcodecs.adler32": "zarr.codecs.numcodecs.Adler32", "numcodecs.astype": "zarr.codecs.numcodecs.AsType", "numcodecs.bitround": "zarr.codecs.numcodecs.BitRound", "numcodecs.blosc": "zarr.codecs.numcodecs.Blosc", "numcodecs.delta": "zarr.codecs.numcodecs.Delta", "numcodecs.fixedscaleoffset": "zarr.codecs.numcodecs.FixedScaleOffset", "numcodecs.fletcher32": "zarr.codecs.numcodecs.Fletcher32", "numcodecs.gzip": "zarr.codecs.numcodecs.GZip", "numcodecs.jenkins_lookup3": "zarr.codecs.numcodecs.JenkinsLookup3", "numcodecs.pcodec": "zarr.codecs.numcodecs.PCodec", "numcodecs.packbits": "zarr.codecs.numcodecs.PackBits", "numcodecs.shuffle": "zarr.codecs.numcodecs.Shuffle", "numcodecs.quantize": "zarr.codecs.numcodecs.Quantize", "numcodecs.zlib": "zarr.codecs.numcodecs.Zlib", "numcodecs.zstd": "zarr.codecs.numcodecs.Zstd", }, "buffer": "zarr.buffer.cpu.Buffer", "ndbuffer": "zarr.buffer.cpu.NDBuffer", } ] ) assert config.get("array.order") == "C" assert config.get("async.concurrency") == 10 assert config.get("async.timeout") is None assert config.get("codec_pipeline.batch_size") == 1 assert config.get("json_indent") == 2 def test_array_config_init_defaults_match_global_config() -> None: """Each `ArrayConfig.__init__` parameter that has a default must match the value of `array.` in the global config. Catches drift between the two sources of truth.""" params = inspect.signature(ArrayConfig.__init__).parameters has_defaults = { name: p.default for name, p in params.items() if name != "self" and p.default is not inspect.Parameter.empty } assert has_defaults, "expected at least one default to check" for name, default in has_defaults.items(): assert default == config.get(f"array.{name}"), ( f"ArrayConfig.__init__ default for {name!r} ({default!r}) does not " f"match global config value for 'array.{name}' " f"({config.get(f'array.{name}')!r})" ) @pytest.mark.parametrize( ("key", "old_val", "new_val"), [("array.order", "C", "F"), ("async.concurrency", 10, 128), ("json_indent", 2, 0)], ) def test_config_defaults_can_be_overridden(key: str, old_val: Any, new_val: Any) -> None: assert config.get(key) == old_val with config.set({key: new_val}): assert config.get(key) == new_val def test_fully_qualified_name() -> None: class MockClass: pass assert ( fully_qualified_name(MockClass) == "tests.test_config.test_fully_qualified_name..MockClass" ) @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) def test_config_codec_pipeline_class(store: Store) -> None: # has default value assert get_pipeline_class().__name__ != "" config.set({"codec_pipeline.path": "zarr.core.codec_pipeline.BatchedCodecPipeline"}) assert get_pipeline_class() == zarr.core.codec_pipeline.BatchedCodecPipeline _mock = Mock() class MockCodecPipeline(BatchedCodecPipeline): async def write( self, batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], value: NDBuffer, drop_axes: tuple[int, ...] = (), ) -> None: _mock.call() register_pipeline(MockCodecPipeline) config.set({"codec_pipeline.path": fully_qualified_name(MockCodecPipeline)}) assert get_pipeline_class() == MockCodecPipeline # test if codec is used arr = zarr.create_array( store=store, shape=(100,), chunks=(10,), zarr_format=3, dtype="i4", ) arr[:] = range(100) _mock.call.assert_called() config.set({"codec_pipeline.path": "wrong_name"}) with pytest.raises(BadConfigError): get_pipeline_class() class MockEnvCodecPipeline(CodecPipeline): pass register_pipeline(MockEnvCodecPipeline) # type: ignore[type-abstract] with mock.patch.dict( os.environ, {"ZARR_CODEC_PIPELINE__PATH": fully_qualified_name(MockEnvCodecPipeline)} ): assert get_pipeline_class(reload_config=True) == MockEnvCodecPipeline @pytest.mark.filterwarnings("error") @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) def test_config_codec_implementation(store: Store) -> None: # has default value assert fully_qualified_name(get_codec_class("blosc")) == config.defaults[0]["codecs"]["blosc"] _mock = Mock() class MockBloscCodec(BloscCodec): # Record a call from whichever encode entry point the active codec # pipeline uses: the async `_encode_single` (BatchedCodecPipeline, the # default) or the synchronous `_encode_sync` (FusedCodecPipeline). # Overriding both keeps this test ("the configured codec is actually # used") independent of which pipeline is the default. async def _encode_single(self, chunk_bytes: Buffer, chunk_spec: ArraySpec) -> Buffer | None: _mock.call() return None def _encode_sync(self, chunk_bytes: Buffer, chunk_spec: ArraySpec) -> Buffer | None: _mock.call() return None register_codec("blosc", MockBloscCodec) with config.set({"codecs.blosc": fully_qualified_name(MockBloscCodec)}): assert get_codec_class("blosc") == MockBloscCodec # test if codec is used arr = zarr.create_array( store=store, shape=(100,), chunks=(10,), zarr_format=3, dtype="i4", compressors=[{"name": "blosc", "configuration": {}}], ) arr[:] = range(100) _mock.call.assert_called() # test set codec with environment variable class NewBloscCodec(BloscCodec): pass register_codec("blosc", NewBloscCodec) with mock.patch.dict(os.environ, {"ZARR_CODECS__BLOSC": fully_qualified_name(NewBloscCodec)}): assert get_codec_class("blosc", reload_config=True) == NewBloscCodec @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) def test_config_ndbuffer_implementation(store: Store) -> None: # set custom ndbuffer with TestNDArrayLike implementation register_ndbuffer(NDBufferUsingTestNDArrayLike) with config.set({"ndbuffer": fully_qualified_name(NDBufferUsingTestNDArrayLike)}): assert get_ndbuffer_class() == NDBufferUsingTestNDArrayLike arr = zarr.create_array( store=store, shape=(100,), chunks=(10,), zarr_format=3, dtype="i4", ) got = arr[:] assert isinstance(got, TestNDArrayLike) def test_config_buffer_implementation() -> None: # has default value assert config.defaults[0]["buffer"] == "zarr.buffer.cpu.Buffer" arr = zeros(shape=(100,), store=StoreExpectingTestBuffer()) # AssertionError of StoreExpectingTestBuffer when not using my buffer with pytest.raises(AssertionError): arr[:] = np.arange(100) register_buffer(TestBuffer) with config.set({"buffer": fully_qualified_name(TestBuffer)}): assert get_buffer_class() == TestBuffer # no error using TestBuffer data = np.arange(100) arr[:] = np.arange(100) assert np.array_equal(arr[:], data) data2d = np.arange(1000).reshape(100, 10) arr_sharding = zeros( shape=(100, 10), store=StoreExpectingTestBuffer(), codecs=[ShardingCodec(chunk_shape=(10, 10))], ) arr_sharding[:] = data2d assert np.array_equal(arr_sharding[:], data2d) arr_Crc32c = zeros( shape=(100, 10), store=StoreExpectingTestBuffer(), codecs=[BytesCodec(), Crc32cCodec()], ) arr_Crc32c[:] = data2d assert np.array_equal(arr_Crc32c[:], data2d) def test_config_buffer_backwards_compatibility() -> None: # This should warn once zarr.core is private # https://github.com/zarr-developers/zarr-python/issues/2621 with zarr.config.set( {"buffer": "zarr.core.buffer.cpu.Buffer", "ndbuffer": "zarr.core.buffer.cpu.NDBuffer"} ): get_buffer_class() get_ndbuffer_class() @pytest.mark.gpu def test_config_buffer_backwards_compatibility_gpu() -> None: # This should warn once zarr.core is private # https://github.com/zarr-developers/zarr-python/issues/2621 with zarr.config.set( {"buffer": "zarr.core.buffer.gpu.Buffer", "ndbuffer": "zarr.core.buffer.gpu.NDBuffer"} ): get_buffer_class() get_ndbuffer_class() @pytest.mark.filterwarnings("error") def test_warning_on_missing_codec_config() -> None: class NewCodec(BytesCodec): pass class NewCodec2(BytesCodec): pass # error if codec is not registered with pytest.raises(UnknownCodecError): get_codec_class("missing_codec") # no warning if only one implementation is available register_codec("new_codec", NewCodec) get_codec_class("new_codec") # warning because multiple implementations are available but none is selected in the config register_codec("new_codec", NewCodec2) with pytest.warns( ZarrUserWarning, match="not configured in config. Selecting any implementation" ): get_codec_class("new_codec") # no warning if multiple implementations are available and one is selected in the config with config.set({"codecs.new_codec": fully_qualified_name(NewCodec)}): get_codec_class("new_codec") @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) @pytest.mark.parametrize( "kwargs", [ {"shards": (4, 4)}, {"compressors": None}, ], ids=["partial_decode", "full_decode"], ) def test_config_read_missing_chunks(store: Store, kwargs: dict[str, Any]) -> None: arr = zarr.create_array( store=store, shape=(4, 4), chunks=(2, 2), dtype="int32", fill_value=42, **kwargs, ) # default behavior: missing chunks are filled with the fill value result = zarr.open_array(store)[:] assert np.array_equal(result, np.full((4, 4), 42, dtype="int32")) # with read_missing_chunks=False, reading missing chunks raises an error with config.set({"array.read_missing_chunks": False}): with pytest.raises(ChunkNotFoundError): zarr.open_array(store)[:] # after writing data, all chunks exist and no error is raised arr[:] = np.arange(16, dtype="int32").reshape(4, 4) with config.set({"array.read_missing_chunks": False}): result = zarr.open_array(store)[:] assert np.array_equal(result, np.arange(16, dtype="int32").reshape(4, 4)) @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) def test_config_read_missing_chunks_sharded_inner(store: Store) -> None: """Because the shard index and inner chunks should be stored together in a single storage object (read: a file or blob), we delegate to the shard index the responsibility of determining what chunks should be present. Thus, `read_missing_chunks` raises an error only if the entire *shard* is missing. Missing inner chunks are filled with the array's fill value and do not raise an error, even if `read_missing_chunks=False` at the array level. """ arr = zarr.create_array( store=store, shape=(8, 4), chunks=(2, 2), shards=(4, 4), dtype="int32", fill_value=42, ) # write only one inner chunk in the first shard, leaving the second shard empty arr[0:2, 0:2] = np.ones((2, 2), dtype="int32") with config.set({"array.read_missing_chunks": False}): a = zarr.open_array(store) # first shard exists: missing inner chunks are filled, no error result = a[:4] expected = np.full((4, 4), 42, dtype="int32") expected[0:2, 0:2] = 1 assert np.array_equal(result, expected) # second shard is entirely missing: raises an error with pytest.raises(ChunkNotFoundError): a[4:] @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) def test_config_read_missing_chunks_write_empty_chunks(store: Store) -> None: """write_empty_chunks=False drops chunks equal to fill_value, which then appear missing to read_missing_chunks=False.""" arr = zarr.create_array( store=store, shape=(4,), chunks=(2,), dtype="int32", fill_value=0, config={"write_empty_chunks": False, "read_missing_chunks": False}, ) # write non-fill-value data: chunks are stored arr[:] = [1, 2, 3, 4] assert np.array_equal(arr[:], [1, 2, 3, 4]) # overwrite with fill_value: chunks are dropped by write_empty_chunks=False arr[:] = 0 with pytest.raises(ChunkNotFoundError): arr[:] # with write_empty_chunks=True, chunks are kept and no error is raised with config.set({"array.write_empty_chunks": True}): arr = zarr.open_array(store) arr[:] = 0 assert np.array_equal(arr[:], [0, 0, 0, 0]) @pytest.mark.parametrize( "key", [ "array.v2_default_compressor.numeric", "array.v2_default_compressor.string", "array.v2_default_compressor.bytes", "array.v2_default_filters.string", "array.v2_default_filters.bytes", "array.v3_default_filters.numeric", "array.v3_default_filters.raw", "array.v3_default_filters.bytes", "array.v3_default_serializer.numeric", "array.v3_default_serializer.string", "array.v3_default_serializer.bytes", "array.v3_default_compressors.string", "array.v3_default_compressors.bytes", "array.v3_default_compressors", ], ) def test_deprecated_config(key: str) -> None: """ Test that a valuerror is raised when setting the default chunk encoding for a given data type category """ with pytest.raises(ValueError): with zarr.config.set({key: "foo"}): pass zarr-developers-zarr-python-a2f2415/tests/test_docs.py000066400000000000000000000324631527014137600232000ustar00rootroot00000000000000""" Tests for executable code blocks in markdown documentation. This module uses pytest-examples to validate Python code examples in the docs. A block is validated if it renders output at build (exec="true") or is explicitly marked for testing (test="true"). The two flags are separate on purpose: exec= drives markdown-exec's build-time rendering, while test= lets a block be validated without being run at build (e.g. gpu/s3 examples the build environment cannot run). The test_no_unvalidated_blocks guard ensures every python block declares one of those, or an explicit exec="false" opt-out with a reason, so a block can never silently skip validation. """ from __future__ import annotations import re from collections import defaultdict from pathlib import Path from typing import TYPE_CHECKING, Any import pytest pytest.importorskip("pytest_examples") from pytest_examples import CodeExample, EvalExample, find_examples if TYPE_CHECKING: from collections.abc import Generator DOCS_ROOT = Path(__file__).parent.parent / "docs" SOURCES_ROOT = Path(__file__).parent.parent / "src" / "zarr" def name_example(path: str, session: str) -> str: """Generate a readable name for a test case from file path and session.""" file = Path(path) try: file = file.relative_to(DOCS_ROOT) except ValueError: # Path is outside DOCS_ROOT (e.g. a tmp_path fixture in unit tests); use the # bare file name rather than an absolute path for a stable, readable id. file = Path(file.name) return f"{file}:{session}" def _marker_names(settings: dict[str, str]) -> list[str]: """Parse a block's markers="a b" attribute into a list of marker names.""" return [name for name in settings.get("markers", "").split() if name] def _is_tested(settings: dict[str, str]) -> bool: """A block is validated by our pytest harness if it is run at build to render output (exec="true") OR explicitly marked for testing (test="true"). The two flags are separate on purpose: exec= drives markdown-exec's build-time rendering, while test= lets a block be validated without being run at build (e.g. gpu/s3 blocks, which the build environment cannot run).""" return settings.get("exec") == "true" or settings.get("test") == "true" def _session_params(root: Path) -> list[Any]: """Group tested examples (exec="true" or test="true") by (file, session) and emit one pytest.param per session, carrying the union of markers declared by that session's blocks.""" sessions: defaultdict[tuple[str, str], list[CodeExample]] = defaultdict(list) marks_by_session: defaultdict[tuple[str, str], set[str]] = defaultdict(set) for example in find_examples(str(root)): settings = example.prefix_settings() if not _is_tested(settings): continue session_name = settings.get("session", "_default") key = (str(example.path), session_name) sessions[key].append(example) marks_by_session[key].update(_marker_names(settings)) params = [] for key in sorted(sessions.keys(), key=lambda x: (x[0], x[1])): marks = tuple(getattr(pytest.mark, name) for name in sorted(marks_by_session[key])) params.append(pytest.param(key, marks=marks, id=name_example(key[0], key[1]))) return params S3_BUCKET = "example-bucket" @pytest.fixture def docs_s3_backend( moto_server: str, monkeypatch: pytest.MonkeyPatch ) -> Generator[None, None, None]: """Point docs S3 examples at the shared moto server (tests/conftest.py) via a process-wide AWS_ENDPOINT_URL, so a block can use a bare s3:// URL with no storage_options (see spike in the design notes). The server lifecycle belongs to the session-scoped `moto_server` fixture; this fixture only adds the docs-specific endpoint env var and a fresh bucket, and restores both on teardown.""" s3fs = pytest.importorskip("s3fs") botocore = pytest.importorskip("botocore") requests = pytest.importorskip("requests") monkeypatch.setenv("AWS_ENDPOINT_URL", moto_server) session = botocore.session.Session() client = session.create_client("s3", endpoint_url=moto_server, region_name="us-east-1") client.create_bucket(Bucket=S3_BUCKET) client.close() s3fs.S3FileSystem.clear_instance_cache() try: yield finally: # Reset moto state; AWS_ENDPOINT_URL is restored automatically by monkeypatch. # The shared server keeps running (the moto_server fixture stops it at session end). requests.post(f"{moto_server}moto-api/reset") def test_markers_attribute_is_parsed(tmp_path: Path) -> None: """A test="true" block tagged markers="s3" must surface that marker on its parametrized case, so pytest can gate/bind it (e.g. attach the moto fixture). Uses test="true" (not exec="true") because marker-bound blocks are validated by the harness without being run at build time.""" md = tmp_path / "ex.md" md.write_text( '```python test="true" session="demo" markers="s3"\nimport zarr\n```\n', encoding="utf-8", ) params = _session_params(md.parent) assert len(params) == 1 marks = params[0].marks assert any(m.name == "s3" for m in marks) def test_no_unvalidated_blocks() -> None: """Every python code block in docs/ must declare its validation state: exec="true" (run at build to render output), test="true" (validated by this harness without being run at build), or exec="false" with a reason (explicit, documented opt-out). A bare or mistyped fence (e.g. exec="on") fails here, so a block can never silently opt out of validation -- the gap that hid the invalid create_array(mode="w") example in #4016. A separate placement constraint is enforced by test_test_only_blocks_come_last.""" offenders: list[str] = [] for example in find_examples(str(DOCS_ROOT)): rel = Path(example.path).relative_to(DOCS_ROOT) settings = example.prefix_settings() exec_val = settings.get("exec") loc = f"{rel}:{example.start_line}" # Validated either by build-render (exec="true") or by the test harness # (test="true"). if _is_tested(settings): continue # Explicit, documented opt-out from execution. if exec_val == "false" and settings.get("reason", "").strip(): continue offenders.append( f"{loc} (exec={exec_val!r}, test={settings.get('test')!r}, " f"reason={settings.get('reason')!r})" ) assert not offenders, ( 'Docs python blocks must be exec="true", test="true", or exec="false" with a ' "reason:\n" + "\n".join(offenders) ) _DESTRUCTIVE_FS_CALL = re.compile( r"shutil\.rmtree|os\.(?:remove|unlink|rmdir|removedirs)\b|\.unlink\(|\.rmdir\(|rm -rf" ) def test_no_destructive_filesystem_calls() -> None: """Executed docs blocks must not delete files or directories. Every python block that runs at build (exec="true") or under this harness (test="true") executes in the *process* working directory: markdown-exec runs it inside the mkdocs process and pytest-examples inside the pytest process, and neither changes directory to the docs tree. A relative path in a deletion call therefore resolves against wherever `mkdocs build` or `pytest` was started. Two sessions used to open with `shutil.rmtree('data', ignore_errors=True)` to make their examples re-runnable; started from any directory that happened to contain a `data/` folder -- a project checkout, or `/` -- the docs build silently emptied it. The sdist ships `docs/` and `tests/` and `testpaths` collects `docs/user-guide`, so that reached users running the shipped test suite, not just contributors. Make examples re-runnable by creating with `overwrite=True` (or `mode="w"`) instead, which is also what a reader copy-pasting the example a second time needs.""" offenders: list[str] = [] for example in find_examples(str(DOCS_ROOT)): if not _is_tested(example.prefix_settings()): continue rel = Path(example.path).relative_to(DOCS_ROOT) for offset, line in enumerate(example.source.splitlines()): if _DESTRUCTIVE_FS_CALL.search(line): offenders.append(f"{rel}:{example.start_line + offset}: {line.strip()}") assert not offenders, ( "Executed docs blocks must not delete files or directories (they run in the " "caller's working directory); create with overwrite=True instead:\n" + "\n".join(offenders) ) def test_test_only_blocks_come_last() -> None: """A conservative placement convention: a test="true"-only block must come after every exec="true" block in the same file. Mechanism (established by experiment + markdown-exec's SuperFences integration): a python fence that markdown-exec does not execute -- i.e. one lacking exec="true", whether test="true" or exec="false" -- placed before an exec="true" block disrupts markdown-exec's build-time execution of a *later, state-dependent* block. Observed: a non-exec python fence inserted before the quickstart ZipStore write/read pair made the read block fail with FileNotFoundError (the write never took effect), aborting `mkdocs build --strict`. The effect needs a cross-block dependency to surface, so it does not affect the standalone exec="true" blocks in e.g. data_types.md/performance.md that already have exec="false" opt-out blocks above them. Because we cannot statically tell which later blocks are state-dependent, this guard enforces the simple, safe convention only for the blocks we author this way (test="true" marker-bound examples like s3/gpu). It is NOT a complete build-hazard check -- the authoritative check is `mkdocs build --strict` (`just docs-build` in CI), which catches the exec="false" case too. This guard just turns the common test-only case into a fast, local failure.""" # Collect, per published-docs file, the start lines of test-only and exec blocks. test_only: defaultdict[str, list[int]] = defaultdict(list) exec_lines: defaultdict[str, list[int]] = defaultdict(list) for example in find_examples(str(DOCS_ROOT)): settings = example.prefix_settings() path = str(example.path) if settings.get("exec") == "true": exec_lines[path].append(example.start_line) elif settings.get("test") == "true": test_only[path].append(example.start_line) offenders: list[str] = [] for path, only_lines in test_only.items(): rel = Path(path).relative_to(DOCS_ROOT) last_exec = max(exec_lines.get(path, [0])) offenders.extend( f'{rel}:{line} (test="true" block precedes an exec="true" block at line {last_exec})' for line in only_lines if line < last_exec ) assert not offenders, ( 'A test="true"-only block must come after every exec="true" block in the same ' 'file: a non-executed python fence before an exec="true" block can disrupt ' "markdown-exec's build-time execution of a later state-dependent block (see this " "test's docstring):\n" + "\n".join(offenders) ) # Get all example sessions @pytest.mark.parametrize("session_key", _session_params(DOCS_ROOT)) def test_documentation_examples( session_key: tuple[str, str], eval_example: EvalExample, request: pytest.FixtureRequest, ) -> None: """ Test that all validated code examples (exec="true" or test="true") in documentation execute successfully. This test groups examples by session (file + session name) and runs them sequentially in the same execution context, allowing code to build on previous examples. This test uses pytest-examples to: - Find all code examples marked exec="true" or test="true" in markdown files - Group them by session - Execute them in order within the same context - Verify no exceptions are raised """ if request.node.get_closest_marker("gpu") is not None: pytest.importorskip("cupy") if request.node.get_closest_marker("s3") is not None: request.getfixturevalue("docs_s3_backend") file_path, session_name = session_key # Get examples for this session all_examples = list(find_examples(DOCS_ROOT)) examples = [] for example in all_examples: settings = example.prefix_settings() if not _is_tested(settings): continue if str(example.path) == file_path and settings.get("session", "_default") == session_name: examples.append(example) # Run all examples in this session sequentially, preserving state module_globals: dict[str, object] = {} for example in examples: # TODO: uncomment this line when we are ready to fix output checks # result = eval_example.run_print_check(example, module_globals=module_globals) result = eval_example.run(example, module_globals=module_globals) # Update globals with the results from this execution module_globals.update(result) @pytest.mark.parametrize("example", list(find_examples(str(SOURCES_ROOT))), ids=str) def test_docstrings(example: CodeExample, eval_example: EvalExample) -> None: """Test our docstring examples.""" if example.path.name == "config.py" and "your.module" in example.source: pytest.skip("Skip testing docstring example that assumes nonexistent module.") eval_example.run_print_check(example) zarr-developers-zarr-python-a2f2415/tests/test_dtype/000077500000000000000000000000001527014137600230135ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/test_dtype/__init__.py000066400000000000000000000000001527014137600251120ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/test_dtype/conftest.py000066400000000000000000000063761527014137600252260ustar00rootroot00000000000000# Generate a collection of zdtype instances for use in testing. import warnings from collections import Counter from typing import Any import numpy as np from zarr.core.dtype import data_type_registry from zarr.core.dtype.common import HasLength from zarr.core.dtype.npy.structured import Struct from zarr.core.dtype.npy.time import DateTime64, TimeDelta64 from zarr.core.dtype.wrapper import ZDType zdtype_examples: tuple[ZDType[Any, Any], ...] = () for wrapper_cls in data_type_registry.contents.values(): if wrapper_cls is Struct: with warnings.catch_warnings(): warnings.simplefilter("ignore") zdtype_examples += ( wrapper_cls.from_native_dtype(np.dtype([("a", np.float64), ("b", np.int8)])), ) elif issubclass(wrapper_cls, HasLength): zdtype_examples += (wrapper_cls(length=1),) elif issubclass(wrapper_cls, DateTime64 | TimeDelta64): zdtype_examples += (wrapper_cls(unit="s", scale_factor=10),) else: zdtype_examples += (wrapper_cls(),) def pytest_generate_tests(metafunc: Any) -> None: """ This is a pytest hook to parametrize class-scoped fixtures. This hook allows us to define class-scoped fixtures as class attributes and then generate the parametrize calls for pytest. This allows the fixtures to be reused across multiple tests within the same class. For example, if you had a regular pytest class like this: class TestClass: @pytest.mark.parametrize("param_a", [1, 2, 3]) def test_method(self, param_a): ... Child classes inheriting from ``TestClass`` would not be able to override the ``param_a`` fixture this implementation of ``pytest_generate_tests`` allows you to define class-scoped fixtures as class attributes, which allows the following to work: class TestExample: param_a = [1, 2, 3] def test_example(self, param_a): ... # this class will have its test_example method parametrized with the values of TestB.param_a class TestB(TestExample): param_a = [1, 2, 100, 10] """ # Iterate over all the fixtures defined in the class # and parametrize them with the values defined in the class # This allows us to define class-scoped fixtures as class attributes # and then generate the parametrize calls for pytest for fixture_name in metafunc.fixturenames: if hasattr(metafunc.cls, fixture_name): params = getattr(metafunc.cls, fixture_name) metafunc.parametrize( fixture_name, params, scope="class", ids=_unique_ids([str(p) for p in params]) ) def _unique_ids(ids: list[str]) -> list[str]: """Suffix repeated ids with their positional index so every id is unique. Distinct parameters can stringify identically: for example `np.dtype("i")` and `np.dtype(" 1 else id_ for idx, id_ in enumerate(ids)] zarr-developers-zarr-python-a2f2415/tests/test_dtype/test_common.py000066400000000000000000000015111527014137600257120ustar00rootroot00000000000000from __future__ import annotations import pytest from zarr.dtype import check_dtype_spec_no_object_codec_v2 @pytest.mark.parametrize( ("data", "expected"), [ ({"name": "|u1", "object_codec_id": None}, True), ({"name": [["a", " None: assert check_dtype_spec_no_object_codec_v2(data) is expected zarr-developers-zarr-python-a2f2415/tests/test_dtype/test_npy/000077500000000000000000000000001527014137600246605ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/test_dtype/test_npy/test_bool.py000066400000000000000000000022301527014137600272210ustar00rootroot00000000000000from __future__ import annotations import numpy as np from tests.test_dtype.test_wrapper import BaseTestZDType from zarr.core.dtype.npy.bool import Bool class TestBool(BaseTestZDType): test_cls = Bool valid_dtype = (np.dtype(np.bool_),) invalid_dtype = ( np.dtype(np.int8), np.dtype(np.float64), np.dtype(np.uint16), ) valid_json_v2 = ({"name": "|b1", "object_codec_id": None},) valid_json_v3 = ("bool",) invalid_json_v2 = ( {"name": "|b1", "object_codec_id": "vlen-utf8"}, {"name": "bool", "object_codec_id": None}, {"name": "|f8", "object_codec_id": None}, ) invalid_json_v3 = ( "|b1", "|f8", {"name": "bool", "configuration": {"endianness": "little"}}, ) scalar_v2_params = ((Bool(), True), (Bool(), False)) scalar_v3_params = ((Bool(), True), (Bool(), False)) cast_value_params = ( (Bool(), "true", np.True_), (Bool(), True, np.True_), (Bool(), False, np.False_), (Bool(), np.True_, np.True_), (Bool(), np.False_, np.False_), ) invalid_scalar_params = (None,) item_size_params = (Bool(),) zarr-developers-zarr-python-a2f2415/tests/test_dtype/test_npy/test_bytes.py000066400000000000000000000133761527014137600274310ustar00rootroot00000000000000import numpy as np import pytest from tests.test_dtype.test_wrapper import BaseTestZDType from zarr.core.dtype.npy.bytes import NullTerminatedBytes, RawBytes, VariableLengthBytes from zarr.errors import UnstableSpecificationWarning class TestNullTerminatedBytes(BaseTestZDType): test_cls = NullTerminatedBytes valid_dtype = (np.dtype("|S10"), np.dtype("|S4")) invalid_dtype = ( np.dtype(np.int8), np.dtype(np.float64), np.dtype("|U10"), ) valid_json_v2 = ( {"name": "|S1", "object_codec_id": None}, {"name": "|S2", "object_codec_id": None}, {"name": "|S4", "object_codec_id": None}, ) valid_json_v3 = ({"name": "null_terminated_bytes", "configuration": {"length_bytes": 10}},) invalid_json_v2 = ( {"name": "|S", "object_codec_id": None}, {"name": "|U10", "object_codec_id": None}, {"name": "|f8", "object_codec_id": None}, {"name": "|S4", "object_codec_id": "vlen-bytes"}, ) invalid_json_v3 = ( {"name": "fixed_length_ascii", "configuration": {"length_bits": 0}}, {"name": "numpy.fixed_length_ascii", "configuration": {"length_bits": "invalid"}}, ) scalar_v2_params = ( (NullTerminatedBytes(length=1), "MA=="), (NullTerminatedBytes(length=2), "YWI="), (NullTerminatedBytes(length=4), "YWJjZA=="), ) scalar_v3_params = ( (NullTerminatedBytes(length=1), "MA=="), (NullTerminatedBytes(length=2), "YWI="), (NullTerminatedBytes(length=4), "YWJjZA=="), ) cast_value_params = ( (NullTerminatedBytes(length=1), "", np.bytes_("")), (NullTerminatedBytes(length=2), "ab", np.bytes_("ab")), (NullTerminatedBytes(length=4), "abcdefg", np.bytes_("abcd")), ) invalid_scalar_params = ((NullTerminatedBytes(length=1), 1.0),) item_size_params = ( NullTerminatedBytes(length=1), NullTerminatedBytes(length=4), NullTerminatedBytes(length=10), ) class TestRawBytes(BaseTestZDType): test_cls = RawBytes valid_dtype = (np.dtype("|V10"),) invalid_dtype = ( np.dtype(np.int8), np.dtype(np.float64), np.dtype("|S10"), ) valid_json_v2 = ({"name": "|V10", "object_codec_id": None},) valid_json_v3 = ( {"name": "raw_bytes", "configuration": {"length_bytes": 1}}, {"name": "raw_bytes", "configuration": {"length_bytes": 8}}, ) invalid_json_v2 = ( {"name": "|V", "object_codec_id": None}, {"name": "|S10", "object_codec_id": None}, {"name": "|f8", "object_codec_id": None}, ) invalid_json_v3 = ( {"name": "r10"}, {"name": "r-80"}, ) scalar_v2_params = ( (RawBytes(length=1), "AA=="), (RawBytes(length=2), "YWI="), (RawBytes(length=4), "YWJjZA=="), ) scalar_v3_params = ( (RawBytes(length=1), "AA=="), (RawBytes(length=2), "YWI="), (RawBytes(length=4), "YWJjZA=="), ) cast_value_params = ( (RawBytes(length=1), b"\x00", np.void(b"\x00")), (RawBytes(length=2), b"ab", np.void(b"ab")), (RawBytes(length=4), b"abcd", np.void(b"abcd")), ) invalid_scalar_params = ((RawBytes(length=1), 1.0),) item_size_params = ( RawBytes(length=1), RawBytes(length=4), RawBytes(length=10), ) class TestVariableLengthBytes(BaseTestZDType): test_cls = VariableLengthBytes valid_dtype = (np.dtype("|O"),) invalid_dtype = ( np.dtype(np.int8), np.dtype(np.float64), np.dtype("|U10"), ) valid_json_v2 = ({"name": "|O", "object_codec_id": "vlen-bytes"},) valid_json_v3 = ("variable_length_bytes",) invalid_json_v2 = ( {"name": "|S", "object_codec_id": None}, {"name": "|U10", "object_codec_id": None}, {"name": "|f8", "object_codec_id": None}, {"name": "|O", "object_codec_id": "vlen-utf8"}, ) invalid_json_v3 = ( {"name": "fixed_length_ascii", "configuration": {"length_bits": 0}}, {"name": "numpy.fixed_length_ascii", "configuration": {"length_bits": "invalid"}}, ) scalar_v2_params = ( (VariableLengthBytes(), ""), (VariableLengthBytes(), "YWI="), (VariableLengthBytes(), "YWJjZA=="), ) scalar_v3_params = ( (VariableLengthBytes(), ""), (VariableLengthBytes(), "YWI="), (VariableLengthBytes(), "YWJjZA=="), ) cast_value_params = ( (VariableLengthBytes(), "", b""), (VariableLengthBytes(), "ab", b"ab"), (VariableLengthBytes(), "abcdefg", b"abcdefg"), ) invalid_scalar_params = ((VariableLengthBytes(), 1.0),) item_size_params = (VariableLengthBytes(),) def test_vlen_bytes_alias() -> None: """Test that "bytes" is an accepted alias for "variable_length_bytes" in JSON metadata""" a = VariableLengthBytes.from_json("bytes", zarr_format=3) b = VariableLengthBytes.from_json("variable_length_bytes", zarr_format=3) assert a == b @pytest.mark.parametrize( "zdtype", [NullTerminatedBytes(length=10), RawBytes(length=10), VariableLengthBytes()] ) def test_unstable_dtype_warning( zdtype: NullTerminatedBytes | RawBytes | VariableLengthBytes, ) -> None: """ Test that we get a warning when serializing a dtype without a zarr v3 spec to json when zarr_format is 3 """ with pytest.warns(UnstableSpecificationWarning): zdtype.to_json(zarr_format=3) @pytest.mark.parametrize("zdtype_cls", [NullTerminatedBytes, RawBytes]) def test_invalid_size(zdtype_cls: type[NullTerminatedBytes] | type[RawBytes]) -> None: """ Test that it's impossible to create a data type that has no length """ length = 0 msg = f"length must be >= 1, got {length}." with pytest.raises(ValueError, match=msg): zdtype_cls(length=length) zarr-developers-zarr-python-a2f2415/tests/test_dtype/test_npy/test_common.py000066400000000000000000000304421527014137600275640ustar00rootroot00000000000000from __future__ import annotations import base64 import re import sys from typing import TYPE_CHECKING, Any, get_args import numpy as np import pytest from tests.conftest import nan_equal from zarr.core.dtype.common import ENDIANNESS_STR, JSONFloatV2, SpecialFloatStrings from zarr.core.dtype.npy.common import ( NumpyEndiannessStr, bytes_from_json, bytes_to_json, check_json_bool, check_json_complex_float_v2, check_json_complex_float_v3, check_json_float_v2, check_json_float_v3, check_json_int, check_json_intish_float, check_json_str, complex_float_to_json_v2, complex_float_to_json_v3, endianness_from_numpy_str, endianness_to_numpy_str, float_from_json_v2, float_from_json_v3, float_to_json_v2, float_to_json_v3, ) if TYPE_CHECKING: from zarr.core.common import JSON, ZarrFormat # Each special value is tested as both a Python float and a numpy scalar. The explicit # ids are load-bearing: np.float64("inf") stringifies identically to float("inf"), so # without them these parameter sets would produce duplicate test ids. json_float_v2_roundtrip_cases: tuple[Any, ...] = ( pytest.param("Infinity", float("inf"), id="Infinity-float"), pytest.param("Infinity", np.float64("inf"), id="Infinity-float64"), pytest.param("-Infinity", float("-inf"), id="-Infinity-float"), pytest.param("-Infinity", np.float64("-inf"), id="-Infinity-float64"), pytest.param("NaN", float("nan"), id="NaN-float"), pytest.param("NaN", np.float64("nan"), id="NaN-float64"), pytest.param(1.0, 1.0, id="1.0-1.0"), ) json_float_v3_cases = json_float_v2_roundtrip_cases @pytest.mark.parametrize( ("data", "expected"), [(">", "big"), ("<", "little"), ("=", sys.byteorder), ("|", None), ("err", "")], ) def test_endianness_from_numpy_str(data: str, expected: str | None) -> None: """ Test that endianness_from_numpy_str correctly converts a numpy str literal to a human-readable literal value. This test also checks that an invalid string input raises a ``ValueError`` """ if data in get_args(NumpyEndiannessStr): assert endianness_from_numpy_str(data) == expected # type: ignore[arg-type] else: msg = f"Invalid endianness: {data!r}. Expected one of {get_args(NumpyEndiannessStr)}" with pytest.raises(ValueError, match=re.escape(msg)): endianness_from_numpy_str(data) # type: ignore[arg-type] @pytest.mark.parametrize( ("data", "expected"), [("big", ">"), ("little", "<"), (None, "|"), ("err", "")], ) def test_endianness_to_numpy_str(data: str | None, expected: str) -> None: """ Test that endianness_to_numpy_str correctly converts a human-readable literal value to a numpy str literal. This test also checks that an invalid string input raises a ``ValueError`` """ if data in ENDIANNESS_STR: assert endianness_to_numpy_str(data) == expected # type: ignore[arg-type] else: msg = f"Invalid endianness: {data!r}. Expected one of {ENDIANNESS_STR}" with pytest.raises(ValueError, match=re.escape(msg)): endianness_to_numpy_str(data) # type: ignore[arg-type] @pytest.mark.parametrize( ("data", "expected"), json_float_v2_roundtrip_cases + (("SHOULD_ERR", ""),) ) def test_float_from_json_v2(data: JSONFloatV2 | str, expected: float | str) -> None: """ Test that float_from_json_v2 correctly converts a JSON string representation of a float to a float. This test also checks that an invalid string input raises a ``ValueError`` """ if data != "SHOULD_ERR": assert nan_equal(float_from_json_v2(data), expected) # type: ignore[arg-type] else: msg = f"could not convert string to float: {data!r}" with pytest.raises(ValueError, match=msg): float_from_json_v2(data) # type: ignore[arg-type] @pytest.mark.parametrize( ("data", "expected"), json_float_v3_cases + (("SHOULD_ERR", ""), ("0x", "")) ) def test_float_from_json_v3(data: JSONFloatV2 | str, expected: float | str) -> None: """ Test that float_from_json_v3 correctly converts a JSON string representation of a float to a float. This test also checks that an invalid string input raises a ``ValueError`` """ if data == "SHOULD_ERR": msg = ( f"Invalid float value: {data!r}. Expected a string starting with the hex prefix" " '0x', or one of 'NaN', 'Infinity', or '-Infinity'." ) with pytest.raises(ValueError, match=msg): float_from_json_v3(data) elif data == "0x": msg = ( f"Invalid hexadecimal float value: {data!r}. " "Expected the '0x' prefix to be followed by 4, 8, or 16 numeral characters" ) with pytest.raises(ValueError, match=msg): float_from_json_v3(data) else: assert nan_equal(float_from_json_v3(data), expected) # note the order of parameters relative to the order of the parametrized variable. @pytest.mark.parametrize(("expected", "data"), json_float_v2_roundtrip_cases) def test_float_to_json_v2(data: float | np.floating[Any], expected: JSONFloatV2) -> None: """ Test that floats are JSON-encoded properly for zarr v2 """ observed = float_to_json_v2(data) assert observed == expected # note the order of parameters relative to the order of the parametrized variable. @pytest.mark.parametrize(("expected", "data"), json_float_v3_cases) def test_float_to_json_v3(data: float | np.floating[Any], expected: JSONFloatV2) -> None: """ Test that floats are JSON-encoded properly for zarr v3 """ observed = float_to_json_v3(data) assert observed == expected def test_bytes_from_json(zarr_format: ZarrFormat) -> None: """ Test that a string is interpreted as base64-encoded bytes using the ascii alphabet. This test takes zarr_format as a parameter but doesn't actually do anything with it, because at present there is no zarr-format-specific logic in the code being tested, but such logic may exist in the future. """ data = "\00" assert bytes_from_json(data, zarr_format=zarr_format) == base64.b64decode(data.encode("ascii")) def test_bytes_to_json(zarr_format: ZarrFormat) -> None: """ Test that bytes are encoded with base64 using the ascii alphabet. This test takes zarr_format as a parameter but doesn't actually do anything with it, because at present there is no zarr-format-specific logic in the code being tested, but such logic may exist in the future. """ data = b"asdas" assert bytes_to_json(data, zarr_format=zarr_format) == base64.b64encode(data).decode("ascii") # note the order of parameters relative to the order of the parametrized variable. @pytest.mark.parametrize(("json_expected", "float_data"), json_float_v2_roundtrip_cases) def test_complex_to_json_v2( float_data: float | np.floating[Any], json_expected: JSONFloatV2 ) -> None: """ Test that complex numbers are correctly converted to JSON in v2 format. This use the same test input as the float tests, but the conversion is tested for complex numbers with real and imaginary parts equal to the float values provided in the test cases. """ cplx = complex(float_data, float_data) cplx_npy = np.complex128(cplx) assert complex_float_to_json_v2(cplx) == (json_expected, json_expected) assert complex_float_to_json_v2(cplx_npy) == (json_expected, json_expected) # note the order of parameters relative to the order of the parametrized variable. @pytest.mark.parametrize(("json_expected", "float_data"), json_float_v3_cases) def test_complex_to_json_v3( float_data: float | np.floating[Any], json_expected: JSONFloatV2 ) -> None: """ Test that complex numbers are correctly converted to JSON in v3 format. This use the same test input as the float tests, but the conversion is tested for complex numbers with real and imaginary parts equal to the float values provided in the test cases. """ cplx = complex(float_data, float_data) cplx_npy = np.complex128(cplx) assert complex_float_to_json_v3(cplx) == (json_expected, json_expected) assert complex_float_to_json_v3(cplx_npy) == (json_expected, json_expected) @pytest.mark.parametrize(("json_expected", "float_data"), json_float_v3_cases) def test_complex_float_to_json( float_data: float | np.floating[Any], json_expected: JSONFloatV2, zarr_format: ZarrFormat ) -> None: """ Test that complex numbers are correctly converted to JSON in v2 or v3 formats, depending on the ``zarr_format`` keyword argument. This use the same test input as the float tests, but the conversion is tested for complex numbers with real and imaginary parts equal to the float values provided in the test cases. """ cplx = complex(float_data, float_data) cplx_npy = np.complex128(cplx) if zarr_format == 2: assert complex_float_to_json_v2(cplx) == (json_expected, json_expected) assert complex_float_to_json_v2(cplx_npy) == ( json_expected, json_expected, ) elif zarr_format == 3: assert complex_float_to_json_v3(cplx) == (json_expected, json_expected) assert complex_float_to_json_v3(cplx_npy) == ( json_expected, json_expected, ) else: raise ValueError("zarr_format must be 2 or 3") # pragma: no cover check_json_float_cases = get_args(SpecialFloatStrings) + (1.0, 2) @pytest.mark.parametrize("data", check_json_float_cases) def test_check_json_float_v2_valid(data: JSONFloatV2 | int) -> None: assert check_json_float_v2(data) def test_check_json_float_v2_invalid() -> None: assert not check_json_float_v2("invalid") @pytest.mark.parametrize("data", check_json_float_cases) def test_check_json_float_v3_valid(data: JSONFloatV2 | int) -> None: assert check_json_float_v3(data) def test_check_json_float_v3_invalid() -> None: assert not check_json_float_v3("invalid") check_json_complex_float_true_cases: tuple[list[JSONFloatV2], ...] = ( [0.0, 1.0], [0.0, 1.0], [-1.0, "NaN"], ["Infinity", 1.0], ["Infinity", "NaN"], ) check_json_complex_float_false_cases: tuple[object, ...] = ( 0.0, "foo", [0.0], [1.0, 2.0, 3.0], [1.0, "_infinity_"], {"hello": 1.0}, ) @pytest.mark.parametrize("data", check_json_complex_float_true_cases) def test_check_json_complex_float_v2_true(data: JSON) -> None: assert check_json_complex_float_v2(data) @pytest.mark.parametrize("data", check_json_complex_float_false_cases) def test_check_json_complex_float_v2_false(data: JSON) -> None: assert not check_json_complex_float_v2(data) @pytest.mark.parametrize("data", check_json_complex_float_true_cases) def test_check_json_complex_float_v3_true(data: JSON) -> None: assert check_json_complex_float_v3(data) @pytest.mark.parametrize("data", check_json_complex_float_false_cases) def test_check_json_complex_float_v3_false(data: JSON) -> None: assert not check_json_complex_float_v3(data) @pytest.mark.parametrize("data", check_json_complex_float_true_cases) def test_check_json_complex_float_true(data: JSON, zarr_format: ZarrFormat) -> None: if zarr_format == 2: assert check_json_complex_float_v2(data) elif zarr_format == 3: assert check_json_complex_float_v3(data) else: raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover @pytest.mark.parametrize("data", check_json_complex_float_false_cases) def test_check_json_complex_float_false(data: JSON, zarr_format: ZarrFormat) -> None: if zarr_format == 2: assert not check_json_complex_float_v2(data) elif zarr_format == 3: assert not check_json_complex_float_v3(data) else: raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover def test_check_json_int() -> None: assert check_json_int(0) assert not check_json_int(1.0) def test_check_json_intish_float() -> None: assert check_json_intish_float(0.0) assert check_json_intish_float(1.0) assert not check_json_intish_float("0") assert not check_json_intish_float(1.1) def test_check_json_str() -> None: assert check_json_str("0") assert not check_json_str(1.0) def test_check_json_bool() -> None: assert check_json_bool(True) assert check_json_bool(False) assert not check_json_bool(1.0) assert not check_json_bool("True") zarr-developers-zarr-python-a2f2415/tests/test_dtype/test_npy/test_complex.py000066400000000000000000000064461527014137600277520ustar00rootroot00000000000000from __future__ import annotations import math import numpy as np from tests.test_dtype.test_wrapper import BaseTestZDType from zarr.core.dtype.npy.complex import Complex64, Complex128 class _BaseTestFloat(BaseTestZDType): def scalar_equals(self, scalar1: object, scalar2: object) -> bool: if np.isnan(scalar1) and np.isnan(scalar2): # type: ignore[call-overload] return True return super().scalar_equals(scalar1, scalar2) class TestComplex64(_BaseTestFloat): test_cls = Complex64 valid_dtype = (np.dtype(">c8"), np.dtype("c8", "object_codec_id": None}, {"name": "c16"), np.dtype("c16", "object_codec_id": None}, {"name": " bool: if np.isnan(scalar1) and np.isnan(scalar2): # type: ignore[call-overload] return True return super().scalar_equals(scalar1, scalar2) hex_string_params: tuple[tuple[str, float], ...] = () def test_hex_encoding(self, hex_string_params: tuple[str, float]) -> None: """ Test that hexadecimal strings can be read as NaN values """ hex_string, expected = hex_string_params zdtype = self.test_cls() observed = zdtype.from_json_scalar(hex_string, zarr_format=3) assert self.scalar_equals(observed, expected) class TestFloat16(_BaseTestFloat): test_cls = Float16 valid_dtype = (np.dtype(">f2"), np.dtype("f2", "object_codec_id": None}, {"name": "f4"), np.dtype("f4", "object_codec_id": None}, {"name": "f8"), np.dtype("f8", "object_codec_id": None}, {"name": " None: """Test the check_json_floatish_str function.""" from zarr.core.dtype.npy.common import check_json_floatish_str # Test valid string floats assert check_json_floatish_str("3.14") assert check_json_floatish_str("0.0") assert check_json_floatish_str("-2.5") assert check_json_floatish_str("1.0") # Test invalid cases assert not check_json_floatish_str("not_a_number") assert not check_json_floatish_str("") assert not check_json_floatish_str(3.14) # actual float, not string assert not check_json_floatish_str(42) # int assert not check_json_floatish_str(None) # Test that special cases still work via float() conversion # (these will be handled by existing functions first in practice) assert check_json_floatish_str("NaN") assert check_json_floatish_str("Infinity") assert check_json_floatish_str("-Infinity") def test_string_float_from_json_scalar() -> None: """Test that string representations of floats can be parsed by from_json_scalar.""" # Test with Float32 dtype_instance = Float32() result = dtype_instance.from_json_scalar("3.14", zarr_format=3) assert abs(result - np.float32(3.14)) < 1e-6 assert isinstance(result, np.float32) # Test other cases result = dtype_instance.from_json_scalar("0.0", zarr_format=3) assert result == np.float32(0.0) result = dtype_instance.from_json_scalar("-2.5", zarr_format=3) assert result == np.float32(-2.5) # Test that it works for v2 format too result = dtype_instance.from_json_scalar("1.5", zarr_format=2) assert result == np.float32(1.5) zarr-developers-zarr-python-a2f2415/tests/test_dtype/test_npy/test_int.py000066400000000000000000000255601527014137600270730ustar00rootroot00000000000000from __future__ import annotations import numpy as np from tests.test_dtype.test_wrapper import BaseTestZDType from zarr.core.dtype.npy.int import Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64 class TestInt8(BaseTestZDType): test_cls = Int8 scalar_type = np.int8 valid_dtype = (np.dtype(np.int8),) invalid_dtype = ( np.dtype(np.int16), np.dtype(np.uint16), np.dtype(np.float64), ) valid_json_v2 = ({"name": "|i1", "object_codec_id": None},) valid_json_v3 = ("int8",) invalid_json_v2 = ( {"name": "|u1", "object_codec_id": None}, {"name": "int8", "object_codec_id": None}, {"name": "|f8", "object_codec_id": None}, {"name": "|i1", "object_codec_id": "vlen-utf8"}, ) invalid_json_v3 = ( "|i1", "|f8", {"name": "int8", "configuration": {"endianness": "little"}}, ) scalar_v2_params = ((Int8(), 1), (Int8(), -1), (Int8(), 1.0)) scalar_v3_params = ((Int8(), 1), (Int8(), -1)) cast_value_params = ( (Int8(), 1, np.int8(1)), (Int8(), -1, np.int8(-1)), ) invalid_scalar_params = ((Int8(), {"set!"}), (Int8(), ("tuple",))) item_size_params = (Int8(),) class TestInt16(BaseTestZDType): test_cls = Int16 scalar_type = np.int16 valid_dtype = (np.dtype(">i2"), np.dtype("i2", "object_codec_id": None}, {"name": "i4"), np.dtype("i4", "object_codec_id": None}, {"name": "i8"), np.dtype("i8", "object_codec_id": None}, {"name": "u2"), np.dtype("u2", "object_codec_id": None}, {"name": "u4"), np.dtype("u4", "object_codec_id": None}, {"name": "u8"), np.dtype("u8", "object_codec_id": None}, {"name": " None: """Test the check_json_intish_str function.""" from zarr.core.dtype.npy.common import check_json_intish_str # Test valid string integers assert check_json_intish_str("0") assert check_json_intish_str("42") assert check_json_intish_str("-5") assert check_json_intish_str("123") # Test invalid cases assert not check_json_intish_str("3.14") assert not check_json_intish_str("not_a_number") assert not check_json_intish_str("") assert not check_json_intish_str(42) # actual int, not string assert not check_json_intish_str(3.14) # float assert not check_json_intish_str(None) def test_string_integer_from_json_scalar() -> None: """Test that string representations of integers can be parsed by from_json_scalar.""" # Test the specific reproducer case dtype_instance = Int32() result = dtype_instance.from_json_scalar("0", zarr_format=3) assert result == np.int32(0) assert isinstance(result, np.int32) # Test other cases result = dtype_instance.from_json_scalar("42", zarr_format=3) assert result == np.int32(42) result = dtype_instance.from_json_scalar("-5", zarr_format=3) assert result == np.int32(-5) # Test that it works for v2 format too result = dtype_instance.from_json_scalar("123", zarr_format=2) assert result == np.int32(123) zarr-developers-zarr-python-a2f2415/tests/test_dtype/test_npy/test_string.py000066400000000000000000000063631527014137600276070ustar00rootroot00000000000000from __future__ import annotations import numpy as np import pytest from tests.test_dtype.test_wrapper import BaseTestZDType from zarr.core.dtype import FixedLengthUTF32 from zarr.core.dtype.npy.string import VariableLengthUTF8 class TestVariableLengthString(BaseTestZDType): test_cls = VariableLengthUTF8 # type: ignore[assignment] valid_dtype = (np.dtypes.StringDType(),) # type: ignore[assignment] invalid_dtype = ( np.dtype(np.int8), np.dtype(np.float64), np.dtype("|S10"), ) valid_json_v2 = ({"name": "|O", "object_codec_id": "vlen-utf8"},) valid_json_v3 = ("string",) invalid_json_v2 = ( {"name": "|S10", "object_codec_id": None}, {"name": "|f8", "object_codec_id": None}, {"name": "invalid", "object_codec_id": None}, {"name": "|O", "object_codec_id": "vlen-bytes"}, ) invalid_json_v3 = ( {"name": "variable_length_utf8", "configuration": {"invalid_key": "value"}}, {"name": "invalid_name"}, ) scalar_v2_params = ((VariableLengthUTF8(), ""), (VariableLengthUTF8(), "hi")) scalar_v3_params = ( (VariableLengthUTF8(), ""), (VariableLengthUTF8(), "hi"), ) cast_value_params = ( (VariableLengthUTF8(), "", np.str_("")), (VariableLengthUTF8(), "hi", np.str_("hi")), ) # anything can become a string invalid_scalar_params = (None,) item_size_params = (VariableLengthUTF8(),) class TestFixedLengthUTF32(BaseTestZDType): test_cls = FixedLengthUTF32 valid_dtype = (np.dtype(">U10"), np.dtype("U10", "object_codec_id": None}, {"name": " None: """ Test that it's impossible to create a data type that has no length """ length = 0 msg = f"length must be >= 1, got {length}." with pytest.raises(ValueError, match=msg): FixedLengthUTF32(length=length) zarr-developers-zarr-python-a2f2415/tests/test_dtype/test_npy/test_structured.py000066400000000000000000000355731527014137600305120ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING, Any, Literal import numpy as np import pytest from numpy.lib.recfunctions import repack_fields import zarr from tests.test_dtype.test_wrapper import BaseTestZDType from zarr.core.dtype import ( Float16, Float64, Int32, Int64, Struct, Structured, UInt8, get_data_type_from_json, ) from zarr.errors import ZarrUserWarning if TYPE_CHECKING: from zarr.core.common import ZarrFormat class TestStruct(BaseTestZDType): """Test the canonical 'struct' dtype format.""" test_cls = Struct valid_dtype = ( np.dtype([("field1", np.int32), ("field2", np.float64)]), np.dtype([("field1", np.int64), ("field2", np.int32)]), ) invalid_dtype = ( np.dtype(np.int8), np.dtype(np.float64), np.dtype("|S10"), ) valid_json_v2 = ( {"name": [["field1", ">i4"], ["field2", ">f8"]], "object_codec_id": None}, {"name": [["field1", ">i8"], ["field2", ">i4"]], "object_codec_id": None}, ) valid_json_v3 = ( { "name": "struct", "configuration": { "fields": [ {"name": "field1", "data_type": "int32"}, {"name": "field2", "data_type": "float64"}, ] }, }, { "name": "struct", "configuration": { "fields": [ { "name": "field1", "data_type": { "name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}, }, }, { "name": "field2", "data_type": { "name": "fixed_length_utf32", "configuration": {"length_bytes": 32}, }, }, ] }, }, ) invalid_json_v2 = ( {"name": "|f8", "object_codec_id": None}, {"name": [["field1", "|i9"], ["field2", " bool: if hasattr(scalar1, "shape") and hasattr(scalar2, "shape"): return np.array_equal(scalar1, scalar2) return super().scalar_equals(scalar1, scalar2) class TestStructured: """Test the legacy 'structured' dtype format.""" def test_invalid_size(self) -> None: """Test that it's impossible to create a data type that has no fields.""" fields = () msg = f"must have at least one field. Got {fields!r}" with pytest.raises(ValueError, match=msg): Structured(fields=fields) def test_structured_legacy_name_with_tuple_format(self) -> None: """Test that the legacy 'structured' name with tuple field format is accepted.""" json_v3 = { "name": "structured", "configuration": { "fields": [ ["field1", "int32"], ["field2", "float64"], ] }, } dtype = Structured.from_json(json_v3, zarr_format=3) assert dtype.fields[0][0] == "field1" assert dtype.fields[1][0] == "field2" @pytest.mark.filterwarnings("ignore::zarr.errors.UnstableSpecificationWarning") def test_structured_writes_tuple_format(self) -> None: """Test that 'structured' writes the tuple field format.""" dtype = Structured(fields=(("field1", Int32()), ("field2", Float64()))) json_v3 = dtype.to_json(zarr_format=3) assert json_v3["name"] == "structured" assert json_v3["configuration"]["fields"][0] == ["field1", "int32"] def test_invalid_size() -> None: """Test that it's impossible to create a data type that has no fields.""" fields = () msg = f"must have at least one field. Got {fields!r}" with pytest.raises(ValueError, match=msg): Struct(fields=fields) @pytest.mark.filterwarnings("ignore::zarr.errors.UnstableSpecificationWarning") def test_struct_name_is_primary() -> None: """Test that 'struct' is the primary name written to JSON.""" dtype = Struct(fields=(("field1", Int32()), ("field2", Float64()))) json_v3 = dtype.to_json(zarr_format=3) assert json_v3["name"] == "struct" def test_struct_reads_legacy_tuple_format() -> None: """Test that 'struct' dtype reads the legacy tuple field format.""" json_v3 = { "name": "struct", "configuration": { "fields": [ ["field1", "int32"], ["field2", "float64"], ] }, } dtype = Struct.from_json(json_v3, zarr_format=3) assert isinstance(dtype, Struct) assert dtype.fields[0][0] == "field1" assert dtype.fields[1][0] == "field2" def test_struct_reads_canonical_object_format() -> None: """Test that 'struct' dtype reads the new object field format.""" json_v3 = { "name": "struct", "configuration": { "fields": [ {"name": "field1", "data_type": "int32"}, {"name": "field2", "data_type": "float64"}, ] }, } dtype = Struct.from_json(json_v3, zarr_format=3) assert isinstance(dtype, Struct) assert dtype.fields[0][0] == "field1" assert dtype.fields[1][0] == "field2" def test_fill_value_dict_form() -> None: """Test that dict form fill values are properly parsed.""" dtype = Struct(fields=(("x", Int32()), ("y", Float64()))) fill_value = dtype.from_json_scalar({"x": 42, "y": 3.14}, zarr_format=3) assert fill_value["x"] == 42 assert fill_value["y"] == 3.14 def test_fill_value_dict_form_missing_fields() -> None: """Test that missing fields in dict form fill values use defaults.""" dtype = Struct(fields=(("x", Int32()), ("y", Float64()))) fill_value = dtype.from_json_scalar({"x": 42}, zarr_format=3) assert fill_value["x"] == 42 assert fill_value["y"] == 0.0 def test_fill_value_legacy_base64() -> None: """Test that legacy base64-encoded fill values are still readable.""" dtype = Struct(fields=(("field1", Int32()), ("field2", Float64()))) fill_value = dtype.from_json_scalar("AQAAAAAAAAAAAPA/", zarr_format=3) assert fill_value["field1"] == 1 assert fill_value["field2"] == 1.0 @pytest.mark.parametrize("endianness", ["little", "big"]) def test_v3_base64_fill_value_is_little_endian(endianness: Literal["little", "big"]) -> None: """Zarr V3 data type metadata has no byte order, so the bytes of a base64 fill value are little-endian whatever byte order the fields have in memory.""" fields = (("field1", Int32(endianness=endianness)), ("field2", Float64(endianness=endianness))) encoded = "AQAAAAAAAAAAAPA/" for cls in (Structured, Struct): fill_value = cls(fields=fields).from_json_scalar(encoded, zarr_format=3) assert (fill_value["field1"], fill_value["field2"]) == (1, 1.0) assert Structured(fields=fields).to_json_scalar((1, 1.0), zarr_format=3) == encoded def test_fill_value_to_json_dict_form() -> None: """Test that fill values are serialized as dict form.""" dtype = Struct(fields=(("x", Int32()), ("y", Float64()))) scalar = np.array((42, 3.14), dtype=[("x", np.int32), ("y", np.float64)])[()] json_val = dtype.to_json_scalar(scalar, zarr_format=3) assert isinstance(json_val, dict) assert json_val["x"] == 42 assert json_val["y"] == 3.14 def test_has_multi_byte_fields_true() -> None: """Test that has_multi_byte_fields returns True for dtypes with multi-byte fields.""" dtype = Struct(fields=(("field1", Int32()), ("field2", Float64()))) assert dtype.has_multi_byte_fields() is True def test_has_multi_byte_fields_false() -> None: """Test that has_multi_byte_fields returns False for dtypes with only single-byte fields.""" dtype = Struct(fields=(("field1", UInt8()), ("field2", UInt8()))) assert dtype.has_multi_byte_fields() is False def test_struct_from_native_dtype() -> None: """Test that Struct can be created from native numpy dtype.""" dtype = np.dtype([("field1", np.int32), ("field2", np.float64)]) struct = Struct.from_native_dtype(dtype) assert struct.fields[0][0] == "field1" assert struct.fields[1][0] == "field2" @pytest.mark.filterwarnings("ignore::zarr.errors.UnstableSpecificationWarning") @pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.parametrize( "dtype", [ # flat np.dtype([("a", "i1"), ("b", "i8")]), # one level of nesting np.dtype([("a", "i4"), ("nested", [("x", "f4"), ("y", "f8")])]), # two levels of nesting np.dtype( [ ("a", "i4"), ("lvl1", [("b", "i2"), ("lvl2", [("x", "f4"), ("y", "f8")])]), ] ), ], ) def test_packed_structured_dtype_round_trips( dtype: np.dtype[np.void], zarr_format: ZarrFormat ) -> None: """ A packed (default-layout) structured dtype, flat or nested, round-trips unchanged through both the JSON form and the native form. Nested dtypes are the regression case: `Struct.to_json(zarr_format=2)` emits a nested field as `[name, [[sub, dt], ...]]`, and the inner V2 type guard used to reject that form, so Zarr wrote V2 metadata it could not read back. """ zdtype = Struct.from_native_dtype(dtype) recovered = get_data_type_from_json( zdtype.to_json(zarr_format=zarr_format), zarr_format=zarr_format ) assert recovered == zdtype assert recovered.to_native_dtype() == dtype assert recovered.to_native_dtype().itemsize == dtype.itemsize @pytest.mark.filterwarnings("ignore::zarr.errors.UnstableSpecificationWarning") def test_nested_structured_v2_array_round_trip() -> None: """End-to-end test: write and read a Zarr V2 array with a nested structured dtype.""" dtype = np.dtype([("a", " None: """ Preserve the existing conversion of padded records to packed records, with a warning. The layout changes, but field values must survive writing and reopening the array. The warning is emitted exactly once, even for padding inside a nested field. """ data = np.ones(3, dtype=dtype) data["a"] = [1, 2, 3] expected = repack_fields(data, recurse=True) store = zarr.storage.MemoryStore() with pytest.warns(ZarrUserWarning, match="packed.*layout") as record: zarr.create_array(store, data=data, chunks=(2,), zarr_format=zarr_format) layout_warnings = [w for w in record if issubclass(w.category, ZarrUserWarning)] assert len(layout_warnings) == 1 reopened = zarr.open_array(store) assert reopened.dtype == expected.dtype assert reopened.dtype.itemsize == expected.dtype.itemsize np.testing.assert_array_equal(reopened[:], expected) def test_titled_structured_dtype_raises() -> None: """ The current conversion rejects titles rather than treating their aliases as extra fields. NumPy's `fields` mapping lists a string title as an extra key. """ dtype = np.dtype([(("title", "f0"), "i4"), ("g", "f8")]) with pytest.raises(ValueError, match="field 'f0' has a title"): Struct.from_native_dtype(dtype) def test_subarray_structured_dtype_raises() -> None: """ The current conversion rejects subarrays rather than resolving them as raw bytes and dropping their shape and element type. V2's support for subarray metadata is separate. """ dtype = np.dtype([("f0", "i4", (2,))]) with pytest.raises(ValueError, match="field 'f0' is a subarray"): Struct.from_native_dtype(dtype) @pytest.mark.parametrize("field_dtype", ["S4", "U4", "V4"]) def test_default_scalar_bytes_field_is_zero(field_dtype: str) -> None: """ The default scalar of a string or bytes field is zero bytes, as for the same dtype outside a struct. """ dtype = np.dtype([("a", field_dtype), ("b", "") def _native_dtypes(kind: str) -> tuple[np.dtype[np.generic], ...]: return tuple( np.dtype(f"{byteorder}{kind}[{scale}{unit}]") for unit in _UNITS for scale in _SCALES for byteorder in _BYTEORDERS ) def _v2_names(kind: str) -> tuple[DTypeSpec_V2, ...]: # NumPy's spelling, except that a generic scale is written explicitly. def suffix(unit: str, scale: int) -> str: if unit == "generic" and scale == 1: return "" return f"[{unit}]" if scale == 1 else f"[{scale}{unit}]" return tuple( {"name": f"{byteorder}{kind}{suffix(unit, scale)}", "object_codec_id": None} for unit in _UNITS if unit != "μs" for scale in _SCALES for byteorder in _BYTEORDERS ) def _v3_configurations(name: str) -> tuple[DTypeSpec_V3, ...]: return tuple( {"name": name, "configuration": {"unit": unit, "scale_factor": scale}} for unit in _UNITS if unit != "μs" for scale in _SCALES ) class _TestTimeBase(BaseTestZDType): def json_scalar_equals(self, scalar1: object, scalar2: object) -> bool: # This method gets overridden here to support the equivalency between NaT and # -9223372036854775808 fill values nat_scalars = (-9223372036854775808, "NaT") if scalar1 in nat_scalars and scalar2 in nat_scalars: return True return scalar1 == scalar2 def scalar_equals(self, scalar1: object, scalar2: object) -> bool: if np.isnan(scalar1) and np.isnan(scalar2): # type: ignore[call-overload] return True return super().scalar_equals(scalar1, scalar2) class TestDateTime64(_TestTimeBase): test_cls = DateTime64 valid_dtype = (np.dtype("datetime64"), *_native_dtypes("M8")) invalid_dtype = ( np.dtype(np.int8), np.dtype(np.float64), np.dtype("timedelta64[ns]"), ) valid_json_v2 = _v2_names("M8") valid_json_v3 = _v3_configurations("numpy.datetime64") invalid_json_v2 = ( {"name": "datetime64", "object_codec_id": None}, {"name": "|f8", "object_codec_id": None}, {"name": "timedelta64[ns]", "object_codec_id": None}, {"name": " None: """ Test that an invalid unit raises a ValueError. """ unit = "invalid" msg = f"unit must be one of ('Y', 'M', 'W', 'D', 'h', 'm', 's', 'ms', 'us', 'μs', 'ns', 'ps', 'fs', 'as', 'generic'), got {unit!r}." with pytest.raises(ValueError, match=re.escape(msg)): DateTime64(unit=unit) # type: ignore[arg-type] with pytest.raises(ValueError, match=re.escape(msg)): TimeDelta64(unit=unit) # type: ignore[arg-type] def test_time_scale_factor_too_low() -> None: """ Test that an invalid unit raises a ValueError. """ scale_factor = 0 msg = f"scale_factor must be > 0, got {scale_factor}." with pytest.raises(ValueError, match=msg): DateTime64(scale_factor=scale_factor) with pytest.raises(ValueError, match=msg): TimeDelta64(scale_factor=scale_factor) @pytest.mark.parametrize("cls", [DateTime64, TimeDelta64]) @pytest.mark.parametrize("unit", _UNITS) @pytest.mark.parametrize("scale_factor", _SCALES) @pytest.mark.filterwarnings( "ignore:The 'generic' unit for NumPy timedelta is deprecated:DeprecationWarning" ) def test_nat_scalars_carry_scale( cls: type[DateTime64 | TimeDelta64], unit: DateTimeUnit, scale_factor: int ) -> None: """ The default scalar and a cast NaT are NaT in this dtype's unit and scale. NumPy < 2.2 turns a timedelta NaT into a count when casting it between scales. """ zdtype = cls(unit=unit, scale_factor=scale_factor) nat = np.timedelta64("NaT", "ns") if cls is TimeDelta64 else np.datetime64("NaT", "ns") expected_unit = "us" if unit == "μs" else unit for scalar in (zdtype.default_scalar(), zdtype.cast_scalar(nat)): assert np.isnat(scalar) assert np.datetime_data(scalar.dtype) == (expected_unit, scale_factor) def test_time_scale_factor_too_high() -> None: """ Test that an invalid unit raises a ValueError. """ scale_factor = 2**31 msg = f"scale_factor must be < 2147483648, got {scale_factor}." with pytest.raises(ValueError, match=msg): DateTime64(scale_factor=scale_factor) with pytest.raises(ValueError, match=msg): TimeDelta64(scale_factor=scale_factor) @pytest.mark.parametrize("cls", [DateTime64, TimeDelta64]) def test_time_microsecond_alias_normalized(cls: type[DateTime64 | TimeDelta64]) -> None: """ Test that the 'μs' unit is stored as NumPy's 'us' spelling. The two spellings are equivalent, but NumPy only ever reports 'us', so an instance that kept 'μs' would compare unequal to itself after a trip through NumPy. Stored metadata may still spell the unit 'μs' and reads back as the normalized instance. """ zdtype = cls(unit="μs", scale_factor=3) assert zdtype.unit == "us" assert zdtype == cls(unit="us", scale_factor=3) assert cls.from_native_dtype(zdtype.to_native_dtype()) == zdtype json_v3 = {"name": cls._zarr_v3_name, "configuration": {"unit": "μs", "scale_factor": 3}} assert cls.from_json(json_v3, zarr_format=3) == zdtype assert zdtype.to_json(zarr_format=3)["configuration"]["unit"] == "us" @pytest.mark.parametrize("unit", get_args(DateTimeUnit)) @pytest.mark.parametrize("scale_factor", [1, 10]) @pytest.mark.parametrize("value", [0, 1, 10]) def test_datetime_from_int(unit: DateTimeUnit, scale_factor: int, value: int) -> None: """ Test datetime_from_int. """ expected = np.int64(value).view(f"datetime64[{scale_factor}{unit}]") assert datetime_from_int(value, unit=unit, scale_factor=scale_factor) == expected @pytest.mark.parametrize("unit", ["generic", "us"]) @pytest.mark.parametrize("kind", ["M8", "m8"]) @pytest.mark.parametrize("byteorder", ["<", ">"]) @pytest.mark.parametrize("scale_factor", [1, 2, 2**31 - 1]) @pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.parametrize("structured", [False, True]) @pytest.mark.filterwarnings("ignore::zarr.errors.UnstableSpecificationWarning") @pytest.mark.filterwarnings( "ignore:The 'generic' unit for NumPy timedelta is deprecated:DeprecationWarning" ) def test_generic_time_array_roundtrip( unit: str, kind: str, byteorder: str, scale_factor: int, zarr_format: ZarrFormat, structured: bool, ) -> None: """Persist counts and generic scale through metadata, chunk IO, and output allocation.""" leaf = np.dtype(f"{byteorder}{kind}[{scale_factor}{unit}]") dtype = np.dtype([("time", leaf)]) if structured else leaf counts = np.array([0, 1, -2, 100], dtype=f"{byteorder}i8") data = counts.view(dtype) array = zarr.create_array( store={}, data=data, chunks=2, zarr_format=zarr_format, compressors=None ) array.resize((6,)) reopened = zarr.open_array(array.store, mode="r") result = np.asarray(reopened[:]) values = result["time"] if structured else result assert np.datetime_data(values.dtype) == (unit, scale_factor) np.testing.assert_array_equal(values[:4].view(values.dtype.byteorder + "i8"), counts) expected_fill = 0 if structured else -(2**63) np.testing.assert_array_equal( values[4:].view(values.dtype.byteorder + "i8"), [expected_fill, expected_fill] ) zarr-developers-zarr-python-a2f2415/tests/test_dtype/test_wrapper.py000066400000000000000000000207531527014137600261130ustar00rootroot00000000000000from __future__ import annotations import re from typing import TYPE_CHECKING, Any, ClassVar import pytest from zarr.core.dtype.common import DTypeSpec_V2, DTypeSpec_V3, HasItemSize from zarr.errors import DataTypeValidationError if TYPE_CHECKING: from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType """ class _TestZDTypeSchema: # subclasses define the URL for the schema, if available schema_url: ClassVar[str] = "" @pytest.fixture(scope="class") def get_schema(self) -> object: response = requests.get(self.schema_url) response.raise_for_status() return json_schema.loads(response.text) def test_schema(self, schema: json_schema.Schema) -> None: assert schema.is_valid(self.test_cls.to_json(zarr_format=2)) """ class BaseTestZDType: """ A base class for testing ZDType subclasses. This class works in conjunction with the custom pytest collection function ``pytest_generate_tests`` defined in conftest.py, which applies the following procedure when generating tests: At test generation time, for each test fixture referenced by a method on this class pytest will look for an attribute with the same name as that fixture. Pytest will assume that this class attribute is a tuple of values to be used for generating a parametrized test fixture. This means that child classes can, by using different values for these class attributes, have customized test parametrization. Attributes ---------- test_cls : type[ZDType[TBaseDType, TBaseScalar]] The ZDType subclass being tested. scalar_type : ClassVar[type[TBaseScalar]] The expected scalar type for the ZDType. valid_dtype : ClassVar[tuple[TBaseDType, ...]] A tuple of valid numpy dtypes for the ZDType. invalid_dtype : ClassVar[tuple[TBaseDType, ...]] A tuple of invalid numpy dtypes for the ZDType. valid_json_v2 : ClassVar[tuple[str | dict[str, object] | list[object], ...]] A tuple of valid JSON representations for Zarr format version 2. invalid_json_v2 : ClassVar[tuple[DTypeSpec_V2, ...]] A tuple of invalid JSON representations for Zarr format version 2. Each has the ``{"name": ..., "object_codec_id": ...}`` shape of a Zarr V2 data type, so that it is rejected for its contents rather than its shape. valid_json_v3 : ClassVar[tuple[str | dict[str, object], ...]] A tuple of valid JSON representations for Zarr format version 3. invalid_json_v3 : ClassVar[tuple[str | dict[str, object], ...]] A tuple of invalid JSON representations for Zarr format version 3. cast_value_params : ClassVar[tuple[tuple[Any, Any, Any], ...]] A tuple of (dtype, value, expected) tuples for testing ZDType.cast_value. scalar_v2_params : ClassVar[tuple[Any, ...]] A tuple of (dtype, scalar json) tuples for testing ZDType.from_json_scalar / ZDType.to_json_scalar for zarr v2 scalar_v3_params : ClassVar[tuple[Any, ...]] A tuple of (dtype, scalar json) tuples for testing ZDType.from_json_scalar / ZDType.to_json_scalar for zarr v3 invalid_scalar_params : ClassVar[tuple[Any, ...]] A tuple of (dtype, value) tuples, where each value is expected to fail ZDType.cast_value. item_size_params : ClassVar[tuple[Any, ...]] A tuple of (dtype, expected) tuples for testing ZDType.item_size """ test_cls: type[ZDType[TBaseDType, TBaseScalar]] scalar_type: ClassVar[type[TBaseScalar]] valid_dtype: ClassVar[tuple[TBaseDType, ...]] = () invalid_dtype: ClassVar[tuple[TBaseDType, ...]] = () valid_json_v2: ClassVar[tuple[DTypeSpec_V2, ...]] = () invalid_json_v2: ClassVar[tuple[DTypeSpec_V2, ...]] = () valid_json_v3: ClassVar[tuple[DTypeSpec_V3, ...]] = () invalid_json_v3: ClassVar[tuple[str | dict[str, object], ...]] = () # for testing scalar round-trip serialization, we need a tuple of (data type json, scalar json) # pairs. the first element of the pair is used to create a dtype instance, and the second # element is the json serialization of the scalar that we want to round-trip. scalar_v2_params: ClassVar[tuple[tuple[ZDType[Any, Any], Any], ...]] = () scalar_v3_params: ClassVar[tuple[tuple[Any, Any], ...]] = () cast_value_params: ClassVar[tuple[tuple[ZDType[Any, Any], Any, Any], ...]] = () # Some data types, like bool and string, can consume any python object as a scalar. # So we allow passing None in to this test to indicate that it should be skipped. invalid_scalar_params: ClassVar[tuple[tuple[ZDType[Any, Any], Any], ...] | tuple[None]] = () item_size_params: ClassVar[tuple[ZDType[Any, Any], ...]] = () def json_scalar_equals(self, scalar1: object, scalar2: object) -> bool: # An equality check for json-encoded scalars. This defaults to regular equality, # but some classes may need to override this for special cases return scalar1 == scalar2 def scalar_equals(self, scalar1: object, scalar2: object) -> bool: # An equality check for scalars. This defaults to regular equality, # but some classes may need to override this for special cases return scalar1 == scalar2 def test_check_dtype_valid(self, valid_dtype: TBaseDType) -> None: assert self.test_cls._check_native_dtype(valid_dtype) def test_check_dtype_invalid(self, invalid_dtype: object) -> None: assert not self.test_cls._check_native_dtype(invalid_dtype) # type: ignore[arg-type] def test_from_dtype_roundtrip(self, valid_dtype: Any) -> None: zdtype = self.test_cls.from_native_dtype(valid_dtype) assert zdtype.to_native_dtype() == valid_dtype def test_from_json_roundtrip_v2(self, valid_json_v2: DTypeSpec_V2) -> None: zdtype = self.test_cls.from_json(valid_json_v2, zarr_format=2) assert zdtype.to_json(zarr_format=2) == valid_json_v2 @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_from_json_roundtrip_v3(self, valid_json_v3: DTypeSpec_V3) -> None: zdtype = self.test_cls.from_json(valid_json_v3, zarr_format=3) assert zdtype.to_json(zarr_format=3) == valid_json_v3 def test_from_json_invalid_v2(self, invalid_json_v2: DTypeSpec_V2) -> None: with pytest.raises(DataTypeValidationError): self.test_cls.from_json(invalid_json_v2, zarr_format=2) def test_from_json_invalid_v3(self, invalid_json_v3: DTypeSpec_V3) -> None: with pytest.raises(DataTypeValidationError): self.test_cls.from_json(invalid_json_v3, zarr_format=3) def test_scalar_roundtrip_v2(self, scalar_v2_params: tuple[ZDType[Any, Any], Any]) -> None: zdtype, scalar_json = scalar_v2_params scalar = zdtype.from_json_scalar(scalar_json, zarr_format=2) assert self.json_scalar_equals(scalar_json, zdtype.to_json_scalar(scalar, zarr_format=2)) def test_scalar_roundtrip_v3(self, scalar_v3_params: tuple[ZDType[Any, Any], Any]) -> None: zdtype, scalar_json = scalar_v3_params scalar = zdtype.from_json_scalar(scalar_json, zarr_format=3) assert self.json_scalar_equals(scalar_json, zdtype.to_json_scalar(scalar, zarr_format=3)) def test_cast_value(self, cast_value_params: tuple[ZDType[Any, Any], Any, Any]) -> None: zdtype, value, expected = cast_value_params observed = zdtype.cast_scalar(value) assert self.scalar_equals(expected, observed) # check that casting is idempotent assert self.scalar_equals(zdtype.cast_scalar(observed), observed) def test_invalid_scalar( self, invalid_scalar_params: tuple[ZDType[Any, Any], Any] | None ) -> None: if invalid_scalar_params is None: pytest.skip(f"No test data provided for {self}.{__name__}") zdtype, data = invalid_scalar_params msg = ( f"Cannot convert object {data!r} with type {type(data)} to a scalar compatible with the " f"data type {zdtype}." ) with pytest.raises(TypeError, match=re.escape(msg)): zdtype.cast_scalar(data) def test_item_size(self, item_size_params: ZDType[Any, Any]) -> None: """ Test that the item_size attribute matches the numpy dtype itemsize attribute, for dtypes with a fixed scalar size. """ if isinstance(item_size_params, HasItemSize): assert item_size_params.item_size == item_size_params.to_native_dtype().itemsize else: pytest.skip(f"Data type {item_size_params} does not implement HasItemSize") zarr-developers-zarr-python-a2f2415/tests/test_dtype_registry.py000066400000000000000000000405151527014137600253220ustar00rootroot00000000000000from __future__ import annotations import re from typing import TYPE_CHECKING, Any, Literal, Self, get_args import numpy as np import pytest from tests.conftest import skip_object_dtype from zarr.core.dtype import ( AnyDType, DataTypeRegistry, TBaseDType, TBaseScalar, get_data_type_from_json, ) from zarr.core.dtype.common import unpack_dtype_json from zarr.dtype import ( # type: ignore[attr-defined] Bool, FixedLengthUTF32, Int8, Int16, RawBytes, Struct, UInt8, VariableLengthUTF8, ZDType, data_type_registry, parse_data_type, parse_dtype, ) from zarr.errors import DataTypeValidationError if TYPE_CHECKING: from zarr.core.common import ZarrFormat from zarr.core.dtype.common import DTypeJSON, DTypeName_V2 from .test_dtype.conftest import zdtype_examples @pytest.fixture def data_type_registry_fixture() -> DataTypeRegistry: return DataTypeRegistry() class TestRegistry: @staticmethod def test_register(data_type_registry_fixture: DataTypeRegistry) -> None: """ Test that registering a dtype in a data type registry works. """ data_type_registry_fixture.register(Bool._zarr_v3_name, Bool) assert data_type_registry_fixture.get(Bool._zarr_v3_name) == Bool assert isinstance(data_type_registry_fixture.match_dtype(np.dtype("bool")), Bool) @staticmethod def test_override(data_type_registry_fixture: DataTypeRegistry) -> None: """ Test that registering a new dtype with the same name works (overriding the previous one). """ data_type_registry_fixture.register(Bool._zarr_v3_name, Bool) class NewBool(Bool): def default_scalar(self) -> np.bool_: return np.True_ data_type_registry_fixture.register(NewBool._zarr_v3_name, NewBool) assert isinstance(data_type_registry_fixture.match_dtype(np.dtype("bool")), NewBool) @staticmethod @pytest.mark.parametrize( ("wrapper_cls", "dtype_str"), [(Bool, "bool"), (FixedLengthUTF32, "|U4")] ) def test_match_dtype( data_type_registry_fixture: DataTypeRegistry, wrapper_cls: type[ZDType[TBaseDType, TBaseScalar]], dtype_str: str, ) -> None: """ Test that match_dtype resolves a numpy dtype into an instance of the correspond wrapper for that dtype. """ data_type_registry_fixture.register(wrapper_cls._zarr_v3_name, wrapper_cls) assert isinstance(data_type_registry_fixture.match_dtype(np.dtype(dtype_str)), wrapper_cls) @staticmethod def test_match_dtype_string_na_object_error( data_type_registry_fixture: DataTypeRegistry, ) -> None: data_type_registry_fixture.register(VariableLengthUTF8._zarr_v3_name, VariableLengthUTF8) # type: ignore[arg-type] dtype: np.dtype[Any] = np.dtypes.StringDType(na_object=None) with pytest.raises(ValueError, match=r"Zarr data type resolution from StringDType.*failed"): data_type_registry_fixture.match_dtype(dtype) @staticmethod def test_unregistered_dtype(data_type_registry_fixture: DataTypeRegistry) -> None: """ Test that match_dtype raises an error if the dtype is not registered. """ outside_dtype_name = "int8" outside_dtype = np.dtype(outside_dtype_name) msg = f"No Zarr data type found that matches dtype '{outside_dtype!r}'" with pytest.raises(ValueError, match=re.escape(msg)): data_type_registry_fixture.match_dtype(outside_dtype) with pytest.raises(KeyError): data_type_registry_fixture.get(outside_dtype_name) @staticmethod @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @pytest.mark.parametrize("zdtype", zdtype_examples) def test_registered_dtypes_match_dtype(zdtype: ZDType[TBaseDType, TBaseScalar]) -> None: """ Test that the registered dtypes can be retrieved from the registry. """ skip_object_dtype(zdtype) assert data_type_registry.match_dtype(zdtype.to_native_dtype()) == zdtype @staticmethod @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @pytest.mark.parametrize("zdtype", zdtype_examples) def test_registered_dtypes_match_json( zdtype: ZDType[TBaseDType, TBaseScalar], zarr_format: ZarrFormat ) -> None: assert ( data_type_registry.match_json( zdtype.to_json(zarr_format=zarr_format), zarr_format=zarr_format ) == zdtype ) @staticmethod @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @pytest.mark.parametrize("zdtype", zdtype_examples) def test_match_dtype_unique( zdtype: ZDType[Any, Any], data_type_registry_fixture: DataTypeRegistry, zarr_format: ZarrFormat, ) -> None: """ Test that the match_dtype method uniquely specifies a registered data type. We create a local registry that excludes the data type class being tested, and ensure that an instance of the wrapped data type fails to match anything in the registry """ skip_object_dtype(zdtype) for _cls in get_args(AnyDType): if _cls is not type(zdtype): data_type_registry_fixture.register(_cls._zarr_v3_name, _cls) dtype_instance = zdtype.to_native_dtype() msg = f"No Zarr data type found that matches dtype '{dtype_instance!r}'" with pytest.raises(ValueError, match=re.escape(msg)): data_type_registry_fixture.match_dtype(dtype_instance) instance_dict = zdtype.to_json(zarr_format=zarr_format) msg = f"No Zarr data type found that matches {instance_dict!r}" with pytest.raises(ValueError, match=re.escape(msg)): data_type_registry_fixture.match_json(instance_dict, zarr_format=zarr_format) @pytest.mark.usefixtures("set_path") def test_entrypoint_dtype(zarr_format: ZarrFormat) -> None: from package_with_entrypoint import TestDataType # the registry loads entry point data types when it is first used instance = TestDataType() dtype_json = instance.to_json(zarr_format=zarr_format) assert get_data_type_from_json(dtype_json, zarr_format=zarr_format) == instance data_type_registry.unregister(TestDataType._zarr_v3_name) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @pytest.mark.parametrize("data_type", zdtype_examples, ids=str) @pytest.mark.parametrize("json_style", [(2, "internal"), (2, "metadata"), (3, None)], ids=str) @pytest.mark.parametrize( "dtype_parser_func", [parse_dtype, parse_data_type], ids=["parse_dtype", "parse_data_type"] ) def test_parse_data_type( data_type: ZDType[Any, Any], json_style: tuple[ZarrFormat, Literal["internal", "metadata"] | None], dtype_parser_func: Any, ) -> None: """ Test the parsing of data types into ZDType instances. This function tests the ability of `dtype_parser_func` to correctly interpret and parse data type specifications into `ZDType` instances according to the specified Zarr format and JSON style. Parameters ---------- data_type : ZDType[Any, Any] The data type to be tested for parsing. json_style : tuple[ZarrFormat, None or Literal["internal", "metadata"]] A tuple specifying the Zarr format version and the JSON style for Zarr V2 2. For Zarr V2 there are 2 JSON styles: "internal", and "metadata". The internal style takes the form {"name": , "object_codec_id": }, while the metadata style is just . dtype_parser_func : Any The function to be tested for parsing the data type. This is necessary for compatibility reasons, as we support multiple functions that perform the same data type parsing operation. """ zarr_format, style = json_style dtype_spec: Any if zarr_format == 2: dtype_spec = data_type.to_json(zarr_format=zarr_format) if style == "internal": pass elif style == "metadata": dtype_spec = unpack_dtype_json(dtype_spec) else: raise ValueError(f"Invalid zarr v2 json style: {style}") else: dtype_spec = data_type.to_json(zarr_format=zarr_format) if dtype_spec == "|O": # The object data type on its own is ambiguous and should fail to resolve. msg = "Zarr data type resolution from object failed." with pytest.raises(ValueError, match=msg): dtype_parser_func(dtype_spec, zarr_format=zarr_format) else: observed = dtype_parser_func(dtype_spec, zarr_format=zarr_format) assert observed == data_type @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @pytest.mark.parametrize("data_type", zdtype_examples, ids=str) def test_v3_data_types_parse_in_host_byte_order(data_type: ZDType[TBaseDType, TBaseScalar]) -> None: """Zarr V3 data type metadata has no byte order, so a data type parsed from it holds values in the host byte order, like the NumPy dtype of the same name.""" parsed = get_data_type_from_json(data_type.to_json(zarr_format=3), zarr_format=3) assert parsed.to_native_dtype().isnative class _LittleEndianUInt8(UInt8): """A data type that declares the non-canonical spelling "b1")), *((name, Int8()) for name in ("|i1", "i1")), *((name, UInt8()) for name in ("|u1", "u1")), *((name, RawBytes(length=4)) for name in ("|V4", "V4")), ("i2", Int16(endianness="big")), ([["a", "b1"]], Struct(fields=(("a", Int8()), ("b", Bool())))), ], ids=str, ) def test_match_json_v2_byte_order(name: DTypeName_V2, expected: ZDType[Any, Any]) -> None: """ A Zarr V2 data type name whose byte order is not relevant matches the same data type for any byte order character, and serializes with the canonical "|". """ observed = data_type_registry.match_json({"name": name, "object_codec_id": None}, zarr_format=2) assert observed == expected assert observed.to_json(zarr_format=2) == expected.to_json(zarr_format=2) def test_match_json_v2_declared_spelling_takes_precedence( data_type_registry_fixture: DataTypeRegistry, ) -> None: """ A data type that declares a non-canonical spelling matches that spelling, and the other spellings still match the canonical data type. """ data_type_registry_fixture.register(UInt8._zarr_v3_name, UInt8) data_type_registry_fixture.register(_LittleEndianUInt8._zarr_v3_name, _LittleEndianUInt8) for name, expected in (("u1", UInt8()), ("|u1", UInt8())): observed = data_type_registry_fixture.match_json( {"name": name, "object_codec_id": None}, zarr_format=2 ) assert type(observed) is type(expected) assert observed.to_json(zarr_format=2)["name"] == (" None: """The byte order of a multi-byte data type is part of its name, so "|" is rejected.""" with pytest.raises(ValueError, match="No Zarr data type found"): data_type_registry.match_json({"name": name, "object_codec_id": None}, zarr_format=2) @pytest.mark.parametrize("name", ["=u1", "=i2"]) def test_match_json_v2_native_byte_order(name: str) -> None: """The Zarr V2 byte order character is one of "<", ">", or "|"; NumPy's "=" is not allowed.""" with pytest.raises(ValueError, match="No Zarr data type found"): data_type_registry.match_json({"name": name, "object_codec_id": None}, zarr_format=2) def test_match_json_v2_byte_order_alias_object_codec() -> None: """Normalizing the byte order does not relax the other members of the data type.""" with pytest.raises(ValueError, match="No Zarr data type found"): data_type_registry.match_json( {"name": "V", " None: """A fixed-length bytes name needs a length of ASCII digits to have a canonical alias.""" with pytest.raises(ValueError, match="No Zarr data type found"): data_type_registry.match_json({"name": name, "object_codec_id": None}, zarr_format=2) class _Byte(UInt8): """A data type that is not in the default registry, spelled " DataTypeRegistry: registry = DataTypeRegistry() for cls in classes: registry.register(cls._zarr_v3_name, cls) return registry @pytest.mark.parametrize("zarr_format", [2, 3]) def test_resolve_struct_fields_with_registry(zarr_format: ZarrFormat) -> None: """ The fields of a structured data type are resolved from the registry that resolves the structured data type. """ registry = _registry_with(Struct, _Byte, Int8) expected = Struct(fields=(("a", _Byte()), ("b", Int8()))) observed = registry.match_json( expected.to_json(zarr_format=zarr_format), zarr_format=zarr_format ) assert observed == expected assert type(observed.fields[0][1]) is _Byte def test_resolve_struct_fields_default_registry() -> None: """Resolved from the default registry, which lacks _Byte, the field matches nothing.""" data = Struct(fields=(("a", _Byte()),)).to_json(zarr_format=3) with pytest.raises(DataTypeValidationError, match="at /configuration/fields/0/data_type$"): get_data_type_from_json(data, zarr_format=3) def test_resolve_struct_fields_missing_from_registry() -> None: """A registry that lacks a field's data type rejects the struct, although the default has it.""" data = Struct(fields=(("a", UInt8()),)).to_json(zarr_format=3) with pytest.raises(DataTypeValidationError, match="at /configuration/fields/0/data_type$"): _registry_with(Struct).match_json(data, zarr_format=3) class _OverridesFromJSON(Int8): """A data type that overrides `from_json` itself, as data types that contain none may.""" _zarr_v3_name = "test.overrides_from_json" # type: ignore[assignment] @classmethod def from_json(cls, data: DTypeJSON, *, zarr_format: ZarrFormat) -> Self: if zarr_format == 3 and data == cls._zarr_v3_name: return cls() raise DataTypeValidationError(f"Invalid JSON representation of {cls.__name__}: {data!r}") def test_resolve_data_type_overriding_from_json() -> None: """A registry creates a data type that contains no other data types with its `from_json`.""" registry = _registry_with(Struct, _OverridesFromJSON) expected = Struct(fields=(("a", _OverridesFromJSON()),)) observed = registry.match_json(expected.to_json(zarr_format=3), zarr_format=3) assert observed == expected @pytest.mark.parametrize( ("zarr_format", "zdtype", "pointer"), [ (2, Struct(fields=(("x", Int8()), ("y", _Byte()))), "/1/1"), # Zarr V2 structured data types nested in structured data types do not round trip ( 3, Struct(fields=(("outer", Struct(fields=(("x", Int8()), ("y", _Byte())))),)), "/configuration/fields/0/data_type/configuration/fields/1/data_type", ), ], ids=str, ) def test_resolve_struct_field_missing_location( zarr_format: ZarrFormat, zdtype: Struct, pointer: str ) -> None: """ A field data type that matches no data type is reported with its location in the JSON, through the structured data types that contain it, rather than as a mismatch of the outermost data type. """ registry = _registry_with(Struct, Int8) with pytest.raises(DataTypeValidationError, match=f"at {re.escape(pointer)}$"): registry.match_json(zdtype.to_json(zarr_format=zarr_format), zarr_format=zarr_format) zarr-developers-zarr-python-a2f2415/tests/test_errors.py000066400000000000000000000061321527014137600235560ustar00rootroot00000000000000"""Test errors""" import pytest from zarr.errors import ( ArrayNotFoundError, ContainsArrayAndGroupError, ContainsArrayError, ContainsGroupError, DataTypeValidationError, GroupNotFoundError, MetadataValidationError, NodeTypeValidationError, ZarrDeprecationWarning, ) def test_group_not_found_error() -> None: """ Test that calling GroupNotFoundError with multiple arguments returns a formatted string. This is deprecated behavior. """ err = GroupNotFoundError("store", "path") assert str(err) == "No group found in store 'store' at path 'path'" def test_array_not_found_error() -> None: """ Test that calling ArrayNotFoundError with multiple arguments returns a formatted string. This is deprecated behavior. """ err = ArrayNotFoundError("store", "path") assert str(err) == "No array found in store 'store' at path 'path'" def test_metadata_validation_error() -> None: """ Test that calling MetadataValidationError with multiple arguments returns a formatted string. This is deprecated behavior. """ err = MetadataValidationError("a", "b", "c") assert str(err) == "Invalid value for 'a'. Expected 'b'. Got 'c'." def test_contains_group_error() -> None: """ Test that calling ContainsGroupError with multiple arguments returns a formatted string. This is deprecated behavior. """ err = ContainsGroupError("store", "path") assert str(err) == "A group exists in store 'store' at path 'path'." def test_contains_array_error() -> None: """ Test that calling ContainsArrayError with multiple arguments returns a formatted string. This is deprecated behavior. """ err = ContainsArrayError("store", "path") assert str(err) == "An array exists in store 'store' at path 'path'." def test_contains_array_and_group_error() -> None: """ Test that calling ContainsArrayAndGroupError with multiple arguments returns a formatted string. This is deprecated behavior. """ err = ContainsArrayAndGroupError("store", "path") assert str(err) == ( "Array and group metadata documents (.zarray and .zgroup) were both found in store 'store' " "at path 'path'. Only one of these files may be present in a given directory / prefix. " "Remove the .zarray file, or the .zgroup file, or both." ) def test_node_type_validation_error() -> None: """ Test that calling NodeTypeValidationError with multiple arguments returns a formatted string. This is deprecated behavior. """ err = NodeTypeValidationError("a", "b", "c") assert str(err) == "Invalid value for 'a'. Expected 'b'. Got 'c'." @pytest.mark.parametrize( "module_name", [ "zarr.core.dtype.common", "zarr.core.dtype", "zarr.dtype", ], ) def test_data_type_validation_error_deprecated_import(module_name: str) -> None: import importlib module = importlib.import_module(module_name) with pytest.warns(ZarrDeprecationWarning, match=f"{module_name}"): cls = module.DataTypeValidationError assert cls is DataTypeValidationError zarr-developers-zarr-python-a2f2415/tests/test_examples.py000066400000000000000000000057751527014137600240740ustar00rootroot00000000000000from __future__ import annotations import re import subprocess import sys from pathlib import Path from typing import Final import pytest import tomlkit from packaging.requirements import Requirement examples_dir = "examples" script_paths = tuple(Path(examples_dir).rglob("*.py")) PEP_723_REGEX: Final = r"(?m)^# /// (?P[a-zA-Z0-9-]+)$\s(?P(^#(| .*)$\s)+)^# ///$" # This is the absolute path to the local Zarr installation. Moving this test to a different directory will break it. ZARR_PROJECT_PATH = Path(".").absolute() def set_dep(script: str, dependency: str) -> str: """ Set a dependency in a PEP-723 script header. If the package is already in the list, it will be replaced. If the package is not already in the list, it will be added. Source code modified from https://packaging.python.org/en/latest/specifications/inline-script-metadata/#reference-implementation """ match = re.search(PEP_723_REGEX, script) if match is None: raise ValueError(f"PEP-723 header not found in {script}") content = "".join( line[2:] if line.startswith("# ") else line[1:] for line in match.group("content").splitlines(keepends=True) ) config = tomlkit.parse(content) for idx, dep in enumerate(tuple(config["dependencies"])): if Requirement(dep).name == Requirement(dependency).name: config["dependencies"][idx] = dependency new_content = "".join( f"# {line}" if line.strip() else f"#{line}" for line in tomlkit.dumps(config).splitlines(keepends=True) ) start, end = match.span("content") return script[:start] + new_content + script[end:] def resave_script(source_path: Path, dest_path: Path) -> None: """ Read a script from source_path and save it to dest_path after inserting the absolute path to the local Zarr project directory in the PEP-723 header. """ source_text = source_path.read_text() dest_text = set_dep(source_text, f"zarr @ file:///{ZARR_PROJECT_PATH}") dest_path.write_text(dest_text) def test_script_paths() -> None: """ Test that our test fixture is working properly and collecting script paths. """ assert len(script_paths) > 0 @pytest.mark.skipif( sys.platform == "win32", reason="This test fails for unknown reasons on Windows in CI." ) @pytest.mark.parametrize("script_path", script_paths) def test_scripts_can_run(script_path: Path, tmp_path: Path) -> None: dest_path = tmp_path / script_path.name # We resave the script after inserting the absolute path to the local Zarr project directory, # and then test its behavior. # This allows the example to be useful to users who don't have Zarr installed, but also testable. resave_script(script_path, dest_path) result = subprocess.run( ["uv", "run", "--refresh", str(dest_path)], capture_output=True, text=True, check=False ) assert result.returncode == 0, ( f"Script at {script_path} failed to run. Output: {result.stdout} Error: {result.stderr}" ) zarr-developers-zarr-python-a2f2415/tests/test_experimental/000077500000000000000000000000001527014137600243635ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/test_experimental/test_cache_store.py000066400000000000000000001245121527014137600302600ustar00rootroot00000000000000""" Tests for the dual-store cache implementation. """ import asyncio import time import pytest from zarr.abc.store import RangeByteRequest, Store, SuffixByteRequest from zarr.core.buffer.core import default_buffer_prototype from zarr.core.buffer.cpu import Buffer as CPUBuffer from zarr.experimental.cache_store import CacheStore from zarr.storage import MemoryStore class TestCacheStore: """Test the dual-store cache implementation.""" @pytest.fixture def source_store(self) -> MemoryStore: """Create a source store with some test data.""" return MemoryStore() @pytest.fixture def cache_store(self) -> MemoryStore: """Create an empty cache store.""" return MemoryStore() @pytest.fixture def cached_store(self, source_store: Store, cache_store: Store) -> CacheStore: """Create a cached store instance.""" return CacheStore(source_store, cache_store=cache_store) async def test_with_read_only_round_trip(self) -> None: """ Ensure that CacheStore.with_read_only returns another CacheStore with the requested read_only state, shares cache state, and does not change the original store's read_only flag. """ source = MemoryStore() cache = MemoryStore() # Start from a read-only underlying store source_ro = source.with_read_only(read_only=True) cached_ro = CacheStore(store=source_ro, cache_store=cache) assert cached_ro.read_only buf = CPUBuffer.from_bytes(b"0123") # Cannot write through the read-only cache store with pytest.raises( ValueError, match="store was opened in read-only mode and does not support writing" ): await cached_ro.set("foo", buf) # Create a writable cache store from the read-only one writer = cached_ro.with_read_only(read_only=False) assert isinstance(writer, CacheStore) assert not writer.read_only # Cache configuration and state are shared assert writer._cache is cached_ro._cache assert writer._state is cached_ro._state assert writer._state.key_insert_times is cached_ro._state.key_insert_times # Writes via the writable cache store succeed and are cached await writer.set("foo", buf) out = await writer.get("foo", default_buffer_prototype()) assert out is not None assert out.to_bytes() == buf.to_bytes() # The original cache store remains read-only assert cached_ro.read_only with pytest.raises( ValueError, match="store was opened in read-only mode and does not support writing" ): await cached_ro.set("bar", buf) # Creating a read-only copy from the writable cache store works and is enforced reader = writer.with_read_only(read_only=True) assert isinstance(reader, CacheStore) assert reader.read_only with pytest.raises( ValueError, match="store was opened in read-only mode and does not support writing" ): await reader.set("baz", buf) async def test_basic_caching(self, cached_store: CacheStore, source_store: Store) -> None: """Test basic cache functionality.""" # Store some data test_data = CPUBuffer.from_bytes(b"test data") await cached_store.set("test_key", test_data) # Verify it's in both stores assert await source_store.exists("test_key") assert await cached_store._cache.exists("test_key") # Retrieve and verify caching works result = await cached_store.get("test_key", default_buffer_prototype()) assert result is not None assert result.to_bytes() == b"test data" async def test_cache_miss_and_population( self, cached_store: CacheStore, source_store: Store ) -> None: """Test cache miss and subsequent population.""" # Put data directly in source store (bypassing cache) test_data = CPUBuffer.from_bytes(b"source data") await source_store.set("source_key", test_data) # First access should miss cache but populate it result = await cached_store.get("source_key", default_buffer_prototype()) assert result is not None assert result.to_bytes() == b"source data" # Verify data is now in cache assert await cached_store._cache.exists("source_key") async def test_cache_expiration(self) -> None: """Test cache expiration based on max_age_seconds.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore( source_store, cache_store=cache_store, max_age_seconds=1, # 1 second expiration ) # Store data test_data = CPUBuffer.from_bytes(b"expiring data") await cached_store.set("expire_key", test_data) # Should be fresh initially assert cached_store._is_key_fresh("expire_key") # Wait for expiration await asyncio.sleep(1.1) # Should now be stale assert not cached_store._is_key_fresh("expire_key") async def test_cache_set_data_false(self, source_store: Store, cache_store: Store) -> None: """Test behavior when cache_set_data=False.""" cached_store = CacheStore(source_store, cache_store=cache_store, cache_set_data=False) test_data = CPUBuffer.from_bytes(b"no cache data") await cached_store.set("no_cache_key", test_data) # Data should be in source but not cache assert await source_store.exists("no_cache_key") assert not await cache_store.exists("no_cache_key") async def test_delete_removes_from_both_stores(self, cached_store: CacheStore) -> None: """Test that delete removes from both source and cache.""" test_data = CPUBuffer.from_bytes(b"delete me") await cached_store.set("delete_key", test_data) # Verify in both stores assert await cached_store._store.exists("delete_key") assert await cached_store._cache.exists("delete_key") # Delete await cached_store.delete("delete_key") # Verify removed from both assert not await cached_store._store.exists("delete_key") assert not await cached_store._cache.exists("delete_key") async def test_exists_checks_source_store( self, cached_store: CacheStore, source_store: Store ) -> None: """Test that exists() checks the source store (source of truth).""" # Put data directly in source test_data = CPUBuffer.from_bytes(b"exists test") await source_store.set("exists_key", test_data) # Should exist even though not in cache assert await cached_store.exists("exists_key") async def test_list_operations(self, cached_store: CacheStore, source_store: Store) -> None: """Test listing operations delegate to source store.""" # Add some test data test_data = CPUBuffer.from_bytes(b"list test") await cached_store.set("list/item1", test_data) await cached_store.set("list/item2", test_data) await cached_store.set("other/item3", test_data) # Test list_dir list_items = [key async for key in cached_store.list_dir("list/")] assert len(list_items) >= 2 # Should include our items # Test list_prefix prefix_items = [key async for key in cached_store.list_prefix("list/")] assert len(prefix_items) >= 2 async def test_stale_cache_refresh(self) -> None: """Test that stale cache entries are refreshed from source.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(source_store, cache_store=cache_store, max_age_seconds=1) # Store initial data old_data = CPUBuffer.from_bytes(b"old data") await cached_store.set("refresh_key", old_data) # Wait for expiration await asyncio.sleep(1.1) # Update source store directly (simulating external update) new_data = CPUBuffer.from_bytes(b"new data") await source_store.set("refresh_key", new_data) # Access should refresh from source when cache is stale result = await cached_store.get("refresh_key", default_buffer_prototype()) assert result is not None assert result.to_bytes() == b"new data" async def test_infinity_max_age(self, cached_store: CacheStore) -> None: """Test that 'infinity' max_age means cache never expires.""" test_data = CPUBuffer.from_bytes(b"eternal data") await cached_store.set("eternal_key", test_data) # Should always be fresh assert cached_store._is_key_fresh("eternal_key") # Even after time passes await asyncio.sleep(0.1) assert cached_store._is_key_fresh("eternal_key") async def test_cache_returns_cached_data_for_performance( self, cached_store: CacheStore, source_store: Store ) -> None: """Test that cache returns cached data for performance, even if not in source.""" # Put data in cache but not source (simulates orphaned cache entry) test_data = CPUBuffer.from_bytes(b"orphaned data") await cached_store._cache.set("orphan_key", test_data) cached_store._state.key_insert_times["orphan_key"] = time.monotonic() # Cache should return data for performance (no source verification) result = await cached_store.get("orphan_key", default_buffer_prototype()) assert result is not None assert result.to_bytes() == b"orphaned data" # Cache entry should remain (performance optimization) assert await cached_store._cache.exists("orphan_key") assert "orphan_key" in cached_store._state.key_insert_times async def test_cache_coherency_through_expiration(self) -> None: """Test that cache coherency is managed through cache expiration, not source verification.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore( source_store, cache_store=cache_store, max_age_seconds=1, # Short expiration for coherency ) # Add data to both stores test_data = CPUBuffer.from_bytes(b"original data") await cached_store.set("coherency_key", test_data) # Remove from source (simulating external deletion) await source_store.delete("coherency_key") # Cache should still return cached data (performance optimization) result = await cached_store.get("coherency_key", default_buffer_prototype()) assert result is not None assert result.to_bytes() == b"original data" # Wait for cache expiration await asyncio.sleep(1.1) # Now stale cache should be refreshed from source result = await cached_store.get("coherency_key", default_buffer_prototype()) assert result is None # Key no longer exists in source async def test_cache_info(self, cached_store: CacheStore) -> None: """Test cache_info method returns correct information.""" # Test initial state info = cached_store.cache_info() # Check all expected keys are present expected_keys = { "cache_store_type", "max_age_seconds", "max_size", "current_size", "cache_set_data", "tracked_keys", "cached_keys", } assert set(info.keys()) == expected_keys # Check initial values assert info["cache_store_type"] == "MemoryStore" assert info["max_age_seconds"] == "infinity" assert info["max_size"] is None # Default unlimited assert info["current_size"] == 0 assert info["cache_set_data"] is True assert info["tracked_keys"] == 0 assert info["cached_keys"] == 0 # Add some data and verify tracking test_data = CPUBuffer.from_bytes(b"test data for cache info") await cached_store.set("info_test_key", test_data) # Check updated info updated_info = cached_store.cache_info() assert updated_info["tracked_keys"] == 1 assert updated_info["cached_keys"] == 1 assert updated_info["current_size"] > 0 # Should have some size now async def test_cache_info_with_max_size(self) -> None: """Test cache_info with max_size configuration.""" source_store = MemoryStore() cache_store = MemoryStore() # Create cache with specific max_size and max_age cached_store = CacheStore( source_store, cache_store=cache_store, max_size=1024, max_age_seconds=300, ) info = cached_store.cache_info() assert info["max_size"] == 1024 assert info["max_age_seconds"] == 300 assert info["current_size"] == 0 async def test_clear_cache(self, cached_store: CacheStore) -> None: """Test clear_cache method clears all cache data and tracking.""" # Add some test data test_data1 = CPUBuffer.from_bytes(b"test data 1") test_data2 = CPUBuffer.from_bytes(b"test data 2") await cached_store.set("clear_test_1", test_data1) await cached_store.set("clear_test_2", test_data2) # Verify data is cached info_before = cached_store.cache_info() assert info_before["tracked_keys"] == 2 assert info_before["cached_keys"] == 2 assert info_before["current_size"] > 0 # Verify data exists in cache assert await cached_store._cache.exists("clear_test_1") assert await cached_store._cache.exists("clear_test_2") # Clear the cache await cached_store.clear_cache() # Verify cache is cleared info_after = cached_store.cache_info() assert info_after["tracked_keys"] == 0 assert info_after["cached_keys"] == 0 assert info_after["current_size"] == 0 # Verify data is removed from cache store (if it supports clear) if hasattr(cached_store._cache, "clear"): # If cache store supports clear, all data should be gone assert not await cached_store._cache.exists("clear_test_1") assert not await cached_store._cache.exists("clear_test_2") # Verify data still exists in source store assert await cached_store._store.exists("clear_test_1") assert await cached_store._store.exists("clear_test_2") async def test_max_age_infinity(self) -> None: """Test cache with infinite max age.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(source_store, cache_store=cache_store, max_age_seconds="infinity") # Add data and verify it never expires test_data = CPUBuffer.from_bytes(b"test data") await cached_store.set("test_key", test_data) # Even after time passes, key should be fresh assert cached_store._is_key_fresh("test_key") async def test_max_age_numeric(self) -> None: """Test cache with numeric max age.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore( source_store, cache_store=cache_store, max_age_seconds=1, # 1 second ) # Add data test_data = CPUBuffer.from_bytes(b"test data") await cached_store.set("test_key", test_data) # Key should be fresh initially assert cached_store._is_key_fresh("test_key") # Manually set old timestamp to test expiration cached_store._state.key_insert_times["test_key"] = time.monotonic() - 2 # 2 seconds ago # Key should now be stale assert not cached_store._is_key_fresh("test_key") async def test_cache_set_data_disabled(self) -> None: """Test cache behavior when cache_set_data is False.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(source_store, cache_store=cache_store, cache_set_data=False) # Set data test_data = CPUBuffer.from_bytes(b"test data") await cached_store.set("test_key", test_data) # Data should be in source but not in cache assert await source_store.exists("test_key") assert not await cache_store.exists("test_key") # Cache info should show no cached data info = cached_store.cache_info() assert info["cache_set_data"] is False assert info["cached_keys"] == 0 async def test_eviction_with_max_size(self) -> None: """Test LRU eviction when max_size is exceeded.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore( source_store, cache_store=cache_store, max_size=100, # Small cache size ) # Add data that exceeds cache size small_data = CPUBuffer.from_bytes(b"a" * 40) # 40 bytes medium_data = CPUBuffer.from_bytes(b"b" * 40) # 40 bytes large_data = CPUBuffer.from_bytes(b"c" * 40) # 40 bytes (would exceed 100 byte limit) # Set first two items await cached_store.set("key1", small_data) await cached_store.set("key2", medium_data) # Cache should have 2 items info = cached_store.cache_info() assert info["cached_keys"] == 2 assert info["current_size"] == 80 # Add third item - should trigger eviction of first item await cached_store.set("key3", large_data) # Cache should still have items but first one may be evicted info = cached_store.cache_info() assert info["current_size"] <= 100 async def test_value_exceeds_max_size(self) -> None: """Test behavior when a single value exceeds max_size.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore( source_store, cache_store=cache_store, max_size=50, # Small cache size ) # Try to cache data larger than max_size large_data = CPUBuffer.from_bytes(b"x" * 100) # 100 bytes > 50 byte limit await cached_store.set("large_key", large_data) # Data should be in source but not cached assert await source_store.exists("large_key") info = cached_store.cache_info() assert info["cached_keys"] == 0 assert info["current_size"] == 0 async def test_get_nonexistent_key(self) -> None: """Test getting a key that doesn't exist in either store.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(source_store, cache_store=cache_store) # Try to get nonexistent key result = await cached_store.get("nonexistent", default_buffer_prototype()) assert result is None # Should not create any cache entries info = cached_store.cache_info() assert info["cached_keys"] == 0 async def test_delete_both_stores(self) -> None: """Test that delete removes from both source and cache stores.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(source_store, cache_store=cache_store) # Add data test_data = CPUBuffer.from_bytes(b"test data") await cached_store.set("test_key", test_data) # Verify it's in both stores assert await source_store.exists("test_key") assert await cache_store.exists("test_key") # Delete await cached_store.delete("test_key") # Verify it's removed from both assert not await source_store.exists("test_key") assert not await cache_store.exists("test_key") # Verify tracking is updated info = cached_store.cache_info() assert info["cached_keys"] == 0 async def test_invalid_max_age_seconds(self) -> None: """Test that invalid max_age_seconds values raise ValueError.""" source_store = MemoryStore() cache_store = MemoryStore() with pytest.raises(ValueError, match="max_age_seconds string value must be 'infinity'"): CacheStore(source_store, cache_store=cache_store, max_age_seconds="invalid") async def test_unlimited_cache_size(self) -> None: """Test behavior when max_size is None (unlimited).""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore( source_store, cache_store=cache_store, max_size=None, # Unlimited cache ) # Add large amounts of data for i in range(10): large_data = CPUBuffer.from_bytes(b"x" * 1000) # 1KB each await cached_store.set(f"large_key_{i}", large_data) # All should be cached since there's no size limit info = cached_store.cache_info() assert info["cached_keys"] == 10 assert info["current_size"] == 10000 # 10 * 1000 bytes async def test_evict_key_exception_handling(self) -> None: """Test exception handling in _evict_key method.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(source_store, cache_store=cache_store, max_size=100) # Add some data test_data = CPUBuffer.from_bytes(b"test data") await cached_store.set("test_key", test_data) # Manually corrupt the tracking to trigger exception # Remove from one structure but not others to create inconsistency del cached_store._state.cache_order["test_key"] # Try to evict - should handle the KeyError gracefully await cached_store._evict_key("test_key") # Should still work and not crash info = cached_store.cache_info() assert isinstance(info, dict) async def test_get_no_cache_delete_tracking(self) -> None: """Test _get_no_cache when key doesn't exist and needs cleanup.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(source_store, cache_store=cache_store) # First, add key to cache tracking but not to source test_data = CPUBuffer.from_bytes(b"test data") await cache_store.set("phantom_key", test_data) await cached_store._track_entry("phantom_key", test_data) # Verify it's in tracking assert "phantom_key" in cached_store._state.cache_order assert "phantom_key" in cached_store._state.key_insert_times # Now try to get it - since it's not in source, should clean up tracking result = await cached_store._get_no_cache("phantom_key", default_buffer_prototype()) assert result is None # Should have cleaned up tracking assert "phantom_key" not in cached_store._state.cache_order assert "phantom_key" not in cached_store._state.key_insert_times async def test_accommodate_value_no_max_size(self) -> None: """Test _accommodate_value early return when max_size is None.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore( source_store, cache_store=cache_store, max_size=None, # No size limit ) # This should return early without doing anything await cached_store._accommodate_value(1000000) # Large value # Should not affect anything since max_size is None info = cached_store.cache_info() assert info["current_size"] == 0 async def test_concurrent_set_operations(self) -> None: """Test that concurrent set operations don't corrupt cache size tracking.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(source_store, cache_store=cache_store, max_size=1000) # Create 10 concurrent set operations async def set_data(key: str) -> None: data = CPUBuffer.from_bytes(b"x" * 50) await cached_store.set(key, data) # Run concurrently await asyncio.gather(*[set_data(f"key_{i}") for i in range(10)]) info = cached_store.cache_info() # Expected: 10 keys * 50 bytes = 500 bytes assert info["cached_keys"] == 10 assert info["current_size"] == 500 # WOULD FAIL due to race condition async def test_concurrent_eviction_race(self) -> None: """Test concurrent evictions don't corrupt size tracking.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(source_store, cache_store=cache_store, max_size=200) # Fill cache to near capacity data = CPUBuffer.from_bytes(b"x" * 80) await cached_store.set("key1", data) await cached_store.set("key2", data) # Now trigger two concurrent sets that both need to evict async def set_large(key: str) -> None: large_data = CPUBuffer.from_bytes(b"y" * 100) await cached_store.set(key, large_data) await asyncio.gather(set_large("key3"), set_large("key4")) info = cached_store.cache_info() # Size should be consistent with tracked keys assert info["current_size"] <= 200 # Might pass # But verify actual cache store size matches tracking total_size = sum( cached_store._state.key_sizes.get(k, 0) for k in cached_store._state.cache_order ) assert total_size == info["current_size"] # WOULD FAIL async def test_concurrent_get_and_evict(self) -> None: """Test get operations during eviction don't cause corruption.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(source_store, cache_store=cache_store, max_size=100) # Setup data = CPUBuffer.from_bytes(b"x" * 40) await cached_store.set("key1", data) await cached_store.set("key2", data) # Concurrent: read key1 while adding key3 (triggers eviction) async def read_key() -> None: for _ in range(100): await cached_store.get("key1", default_buffer_prototype()) async def write_key() -> None: for i in range(10): new_data = CPUBuffer.from_bytes(b"y" * 40) await cached_store.set(f"new_{i}", new_data) await asyncio.gather(read_key(), write_key()) # Verify consistency info = cached_store.cache_info() assert info["current_size"] <= 100 assert len(cached_store._state.cache_order) == len(cached_store._state.key_sizes) async def test_eviction_actually_deletes_from_cache_store(self) -> None: """Test that eviction removes keys from cache_store, not just tracking.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(source_store, cache_store=cache_store, max_size=100) # Add data that will be evicted data1 = CPUBuffer.from_bytes(b"x" * 60) data2 = CPUBuffer.from_bytes(b"y" * 60) await cached_store.set("key1", data1) # Verify key1 is in cache_store assert await cache_store.exists("key1") # Add key2, which should evict key1 await cached_store.set("key2", data2) # Check tracking - key1 should be removed assert "key1" not in cached_store._state.cache_order assert "key1" not in cached_store._state.key_sizes # CRITICAL: key1 should also be removed from cache_store assert not await cache_store.exists("key1"), ( "Evicted key still exists in cache_store! _evict_key doesn't actually delete." ) # But key1 should still exist in source store assert await source_store.exists("key1") async def test_eviction_no_orphaned_keys(self) -> None: """Test that eviction doesn't leave orphaned keys in cache_store.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(source_store, cache_store=cache_store, max_size=150) # Add multiple keys that will cause evictions for i in range(10): data = CPUBuffer.from_bytes(b"x" * 60) await cached_store.set(f"key_{i}", data) # Check tracking info = cached_store.cache_info() tracked_keys = info["cached_keys"] # Count actual keys in cache_store actual_keys = 0 async for _ in cache_store.list(): actual_keys += 1 # Cache store should have same number of keys as tracking assert actual_keys == tracked_keys, ( f"Cache store has {actual_keys} keys but tracking shows {tracked_keys}. " f"Eviction doesn't delete from cache_store!" ) async def test_size_accounting_with_key_updates(self) -> None: """Test that updating the same key replaces size instead of accumulating.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(source_store, cache_store=cache_store, max_size=500) # Set initial value data1 = CPUBuffer.from_bytes(b"x" * 100) await cached_store.set("same_key", data1) info1 = cached_store.cache_info() assert info1["current_size"] == 100 # Update with different size data2 = CPUBuffer.from_bytes(b"y" * 200) await cached_store.set("same_key", data2) info2 = cached_store.cache_info() # Should be 200, not 300 (update replaces, doesn't accumulate) assert info2["current_size"] == 200, ( f"Expected size 200 but got {info2['current_size']}. " "Updating same key should replace, not accumulate." ) async def test_all_tracked_keys_exist_in_cache_store(self) -> None: """Test invariant: all keys in tracking should exist in cache_store.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(source_store, cache_store=cache_store, max_size=500) # Add some data for i in range(5): data = CPUBuffer.from_bytes(b"x" * 50) await cached_store.set(f"key_{i}", data) # Every str key in tracking should exist in cache_store # (tuple keys are byte-range entries stored in-memory, not in the Store) for entry_key in cached_store._state.cache_order: if isinstance(entry_key, str): assert await cache_store.exists(entry_key), ( f"Key '{entry_key}' is tracked but doesn't exist in cache_store" ) # Every str key in _key_sizes should exist in cache_store for entry_key in cached_store._state.key_sizes: if isinstance(entry_key, str): assert await cache_store.exists(entry_key), ( f"Key '{entry_key}' has size tracked but doesn't exist in cache_store" ) # Additional coverage tests for 100% coverage async def test_cache_store_requires_delete_support(self) -> None: """Test that CacheStore validates cache_store supports deletes.""" from unittest.mock import MagicMock # Create a mock store that doesn't support deletes source_store = MemoryStore() cache_store = MagicMock() cache_store.supports_deletes = False with pytest.raises(ValueError, match="does not support deletes"): CacheStore(store=source_store, cache_store=cache_store) async def test_evict_key_exception_handling_with_real_error( self, monkeypatch: pytest.MonkeyPatch ) -> None: """Test _evict_key exception handling when deletion fails.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(store=source_store, cache_store=cache_store, max_size=100) # Set up a key in tracking buffer = CPUBuffer.from_bytes(b"test data") await cached_store.set("test_key", buffer) # Mock the cache delete to raise an exception async def failing_delete(key: str) -> None: raise RuntimeError("Simulated cache deletion failure") monkeypatch.setattr(cache_store, "delete", failing_delete) # Attempt to evict should raise the exception with pytest.raises(RuntimeError, match="Simulated cache deletion failure"): async with cached_store._state.lock: await cached_store._evict_key("test_key") async def test_cache_stats_method(self) -> None: """Test cache_stats method returns correct statistics.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(store=source_store, cache_store=cache_store, max_size=1000) # Initially, stats should be zero stats = cached_store.cache_stats() assert stats["hits"] == 0 assert stats["misses"] == 0 assert stats["evictions"] == 0 assert stats["total_requests"] == 0 assert stats["hit_rate"] == 0.0 # Perform some operations buffer = CPUBuffer.from_bytes(b"x" * 100) # Write to source store directly to avoid affecting stats await source_store.set("key1", buffer) # First get is a miss (not in cache yet) result1 = await cached_store.get("key1", default_buffer_prototype()) assert result1 is not None # Second get is a hit (now in cache) result2 = await cached_store.get("key1", default_buffer_prototype()) assert result2 is not None stats = cached_store.cache_stats() assert stats["hits"] == 1 assert stats["misses"] == 1 assert stats["total_requests"] == 2 assert stats["hit_rate"] == 0.5 async def test_cache_stats_with_evictions(self) -> None: """Test cache_stats tracks evictions correctly.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore( store=source_store, cache_store=cache_store, max_size=150, # Small size to force eviction ) # Add items that will trigger eviction buffer1 = CPUBuffer.from_bytes(b"x" * 100) buffer2 = CPUBuffer.from_bytes(b"y" * 100) await cached_store.set("key1", buffer1) await cached_store.set("key2", buffer2) # Should evict key1 stats = cached_store.cache_stats() assert stats["evictions"] == 1 def test_repr_method(self) -> None: """Test __repr__ returns useful string representation.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore( store=source_store, cache_store=cache_store, max_age_seconds=60, max_size=1024 ) repr_str = repr(cached_store) # Check that repr contains key information assert "CacheStore" in repr_str assert "max_age_seconds=60" in repr_str assert "max_size=1024" in repr_str assert "current_size=0" in repr_str assert "cached_keys=0" in repr_str async def test_cache_stats_zero_division_protection(self) -> None: """Test cache_stats handles zero requests correctly.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(store=source_store, cache_store=cache_store) # With no requests, hit_rate should be 0.0 (not NaN or error) stats = cached_store.cache_stats() assert stats["hit_rate"] == 0.0 assert stats["total_requests"] == 0 async def test_byte_range_does_not_corrupt_cache(self) -> None: """Test that fetching a byte range does not store partial data under the full key. Reproduces https://github.com/zarr-developers/zarr-python/issues/3690: when a byte-range read populates the cache, subsequent reads of different ranges (or the full key) return wrong data. """ source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(store=source_store, cache_store=cache_store) full_data = b"bar baz" await source_store.set("foo", CPUBuffer.from_bytes(full_data)) proto = default_buffer_prototype() # First read: byte range [0, 3) -> b"bar" bar = await cached_store.get("foo", proto, byte_range=RangeByteRequest(0, 3)) assert bar is not None assert bar.to_bytes() == b"bar" # Second read: different byte range [4, 7) -> b"baz" baz = await cached_store.get("foo", proto, byte_range=RangeByteRequest(4, 7)) assert baz is not None assert baz.to_bytes() == b"baz" # Third read: full key -> full data full = await cached_store.get("foo", proto) assert full is not None assert full.to_bytes() == full_data async def test_full_read_then_byte_range(self) -> None: """Test that a cached full read correctly serves subsequent byte-range requests.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(store=source_store, cache_store=cache_store) full_data = b"hello world" await source_store.set("key", CPUBuffer.from_bytes(full_data)) proto = default_buffer_prototype() # Full read populates cache full = await cached_store.get("key", proto) assert full is not None assert full.to_bytes() == full_data # Byte-range reads should return the correct slices part = await cached_store.get("key", proto, byte_range=RangeByteRequest(0, 5)) assert part is not None assert part.to_bytes() == b"hello" part2 = await cached_store.get("key", proto, byte_range=RangeByteRequest(6, 11)) assert part2 is not None assert part2.to_bytes() == b"world" suffix = await cached_store.get("key", proto, byte_range=SuffixByteRequest(5)) assert suffix is not None assert suffix.to_bytes() == b"world" async def test_byte_range_set_then_read(self) -> None: """Test that data written via set() can be read back with byte ranges.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(store=source_store, cache_store=cache_store) full_data = b"abcdefghij" await cached_store.set("key", CPUBuffer.from_bytes(full_data)) proto = default_buffer_prototype() # Byte-range reads from the cached data mid = await cached_store.get("key", proto, byte_range=RangeByteRequest(3, 7)) assert mid is not None assert mid.to_bytes() == b"defg" # Full read should still work full = await cached_store.get("key", proto) assert full is not None assert full.to_bytes() == full_data async def test_set_invalidates_cached_byte_ranges(self) -> None: """Test that set() invalidates previously cached byte-range entries.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(store=source_store, cache_store=cache_store) proto = default_buffer_prototype() # Populate source and cache some byte ranges await source_store.set("key", CPUBuffer.from_bytes(b"old data!!")) r1 = await cached_store.get("key", proto, byte_range=RangeByteRequest(0, 3)) assert r1 is not None assert r1.to_bytes() == b"old" # Byte-range entry should be in range_cache assert ("key", RangeByteRequest(0, 3)) in cached_store._state.cache_order # Overwrite via set() — range entries must be invalidated await cached_store.set("key", CPUBuffer.from_bytes(b"NEW DATA!!")) # The old range entry should be gone from tracking and range_cache assert ("key", RangeByteRequest(0, 3)) not in cached_store._state.cache_order assert "key" not in cached_store._state.range_cache # A fresh byte-range read should return the new data r2 = await cached_store.get("key", proto, byte_range=RangeByteRequest(0, 3)) assert r2 is not None assert r2.to_bytes() == b"NEW" async def test_delete_invalidates_cached_byte_ranges(self) -> None: """Test that delete() removes previously cached byte-range entries.""" source_store = MemoryStore() cache_store = MemoryStore() cached_store = CacheStore(store=source_store, cache_store=cache_store) proto = default_buffer_prototype() # Populate and cache a byte range await source_store.set("key", CPUBuffer.from_bytes(b"hello world")) r = await cached_store.get("key", proto, byte_range=RangeByteRequest(0, 5)) assert r is not None assert r.to_bytes() == b"hello" assert ("key", RangeByteRequest(0, 5)) in cached_store._state.cache_order # Delete the key — range entries must be cleaned up await cached_store.delete("key") assert ("key", RangeByteRequest(0, 5)) not in cached_store._state.cache_order assert "key" not in cached_store._state.range_cache # Key is gone from source result = await cached_store.get("key", proto) assert result is None def test_cache_store_opts_out_of_sync_io() -> None: """`CacheStore` must not advertise sync IO capability. Its caching logic lives only in the async `get`/`set`/`delete` overrides, while the inherited `WrapperStore` sync methods delegate straight to the source store. If the fused codec pipeline took the sync fast path, writes and deletes would bypass the cache and later async reads would serve stale entries. The opt-out forces sync-capable consumers onto the async path, which keeps the cache coherent. """ from zarr.abc.store import _store_supports_sync_io from zarr.storage import MemoryStore cached = CacheStore(MemoryStore(), cache_store=MemoryStore()) assert _store_supports_sync_io(cached) is False async def test_cache_coherent_after_fused_pipeline_write() -> None: """Writing through the fused pipeline must not leave stale cache entries.""" import numpy as np import zarr from zarr.core.config import config as zarr_config from zarr.storage import MemoryStore source = MemoryStore() cached = CacheStore(source, cache_store=MemoryStore()) with zarr_config.set({"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"}): arr = zarr.create_array(cached, shape=(8,), chunks=(8,), dtype="int32", fill_value=0) arr[:] = np.arange(8, dtype="int32") np.testing.assert_array_equal(arr[:], np.arange(8)) # Overwrite, then read back through the same cached handle: the read # must observe the overwrite, not a cached copy of the first write. arr[:] = np.arange(100, 108, dtype="int32") np.testing.assert_array_equal(arr[:], np.arange(100, 108)) zarr-developers-zarr-python-a2f2415/tests/test_fastpath_equivalence.py000066400000000000000000000425721527014137600264450ustar00rootroot00000000000000"""Property tests: every fast path must equal the general path. The codec pipelines contain fast paths that skip work whose result is known — the complete-chunk merge view, the vectorized whole-shard bulk decode, the scalar-broadcast write memoization, byte-range coalescing. Each is only safe if it produces results identical to the general path it bypasses. These tests pin that equivalence on randomized inputs, so a fast path that silently diverges (the bug class behind the bulk-decode endianness fix) fails here instead of corrupting data downstream. Convention for new fast paths: a fast path is "skip work whose result is known", never "a different algorithm" — and it ships with a property test in this module asserting equality with the general path. """ from __future__ import annotations from typing import Any import hypothesis.extra.numpy as npst import hypothesis.strategies as st import numpy as np from hypothesis import given, settings import zarr from zarr.abc.store import OffsetByteRequest, RangeByteRequest, SuffixByteRequest from zarr.codecs.bytes import BytesCodec from zarr.codecs.sharding import ShardingCodec from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer import default_buffer_prototype from zarr.core.buffer.cpu import Buffer as CPUBuffer from zarr.core.buffer.cpu import NDBuffer as CPUNDBuffer from zarr.core.chunk_grids import ChunkGrid from zarr.core.chunk_utils import _merge_chunk_array from zarr.core.dtype import get_data_type_from_native_dtype from zarr.core.indexing import BasicIndexer from zarr.storage import MemoryStore _DTYPES = st.sampled_from(["uint8", "int16", "float32"]) def _spec(shape: tuple[int, ...], dtype: str, *, write_empty_chunks: bool = True) -> ArraySpec: zdtype = get_data_type_from_native_dtype(np.dtype(dtype)) return ArraySpec( shape=shape, dtype=zdtype, fill_value=zdtype.cast_scalar(0), config=ArrayConfig(order="C", write_empty_chunks=write_empty_chunks), prototype=default_buffer_prototype(), ) # --------------------------------------------------------------------------- # _merge_chunk_array: the complete-chunk early return (a view of # value[out_selection]) must equal the general create/copy + setitem path. # --------------------------------------------------------------------------- @st.composite def _merge_cases(draw: st.DrawFn) -> tuple[np.ndarray, tuple[int, ...], tuple[slice, ...]]: ndim = draw(st.integers(1, 3)) chunk_shape = tuple(draw(st.integers(1, 5)) for _ in range(ndim)) n_blocks = draw(st.integers(1, 3)) dtype = draw(_DTYPES) # value spans n_blocks chunk-sized blocks along axis 0; out_selection picks one value_shape = (chunk_shape[0] * n_blocks, *chunk_shape[1:]) value = draw(npst.arrays(dtype=np.dtype(dtype), shape=value_shape)) block = draw(st.integers(0, n_blocks - 1)) out_selection = ( slice(block * chunk_shape[0], (block + 1) * chunk_shape[0]), *(slice(0, s) for s in chunk_shape[1:]), ) return value, chunk_shape, out_selection @settings(max_examples=200, deadline=None) @given(case=_merge_cases(), with_existing=st.booleans()) def test_merge_complete_chunk_equals_general_path( case: tuple[np.ndarray, tuple[int, ...], tuple[slice, ...]], with_existing: bool ) -> None: """The is_complete_chunk fast path (return a view of value[out_selection]) must produce exactly what the general merge path produces — and a complete write must be independent of any existing chunk content.""" value_np, chunk_shape, out_selection = case spec = _spec(chunk_shape, str(value_np.dtype)) value = CPUNDBuffer.from_numpy_array(value_np) chunk_selection = tuple(slice(0, s) for s in chunk_shape) existing = None if with_existing: existing = CPUNDBuffer.from_numpy_array(np.full(chunk_shape, 7, dtype=value_np.dtype)) fast = _merge_chunk_array(existing, value, out_selection, spec, chunk_selection, True, ()) general = _merge_chunk_array(existing, value, out_selection, spec, chunk_selection, False, ()) np.testing.assert_array_equal(fast.as_numpy_array(), general.as_numpy_array()) np.testing.assert_array_equal(fast.as_numpy_array(), value_np[out_selection]) # --------------------------------------------------------------------------- # _decode_full_shard_bulk_if_uncompressed: the vectorized dense-shard decode must equal the # general per-chunk decode (_decode_sync), across dtypes, endianness, write # orders, and index locations. This is the bug class of the historical # bulk-decode endianness fix. # --------------------------------------------------------------------------- @st.composite def _shard_cases(draw: st.DrawFn) -> dict[str, Any]: ndim = draw(st.integers(1, 2)) chunk_shape = tuple(draw(st.integers(1, 4)) for _ in range(ndim)) grid = tuple(draw(st.integers(1, 3)) for _ in range(ndim)) shard_shape = tuple(c * g for c, g in zip(chunk_shape, grid, strict=True)) dtype = draw(_DTYPES) data = draw(npst.arrays(dtype=np.dtype(dtype), shape=shard_shape)) return { "chunk_shape": chunk_shape, "shard_shape": shard_shape, "data": data, "endian": draw(st.sampled_from(["little", "big"])), "index_location": draw(st.sampled_from(["start", "end"])), "subchunk_write_order": draw( st.sampled_from(["morton", "lexicographic", "colexicographic", "unordered"]) ), } @settings(max_examples=100, deadline=None) @given(case=_shard_cases()) def test_bulk_shard_decode_equals_general_decode(case: dict[str, Any]) -> None: """For dense fixed-size uncompressed shards, the vectorized bulk decode must reproduce the general per-chunk decode exactly, whatever the endianness, subchunk write order, or index location.""" codec = ShardingCodec( chunk_shape=case["chunk_shape"], codecs=[BytesCodec(endian=case["endian"])], index_location=case["index_location"], subchunk_write_order=case["subchunk_write_order"], ) spec = _spec(case["shard_shape"], str(case["data"].dtype), write_empty_chunks=True) blob = codec._encode_sync(CPUNDBuffer.from_numpy_array(case["data"]), spec) assert blob is not None # write_empty_chunks=True -> dense, never elided general = codec._decode_sync(blob, spec) indexer = BasicIndexer( tuple(slice(0, s) for s in case["shard_shape"]), shape=case["shard_shape"], chunk_grid=ChunkGrid.from_sizes(case["shard_shape"], case["chunk_shape"]), ) bulk = codec._decode_full_shard_bulk_if_uncompressed(blob, spec, indexer) # the fast path must APPLY for this dense uncompressed configuration — # a vacuous None would silently stop testing the equivalence assert bulk is not None np.testing.assert_array_equal(bulk.as_numpy_array(), general.as_numpy_array()) np.testing.assert_array_equal(general.as_numpy_array(), case["data"]) def test_merge_complete_chunk_returns_view_and_write_does_not_mutate_source() -> None: """The complete-chunk merge fast path returns a VIEW of the caller's value (no copy — that is the perf win), and a multi-chunk write through either pipeline leaves the user's source array untouched. Pins both halves of the aliasing contract: a future "defensive copy" refactor that silently reintroduces the per-chunk create/fill/copy breaks the first assertion, and an in-place-mutating codec that corrupts the user's array through the shared view breaks the second. """ # the fast path must return a view aliasing `value`, not a copy value_np = np.arange(30, dtype="uint16") spec = _spec((10,), "uint16") value = CPUNDBuffer.from_numpy_array(value_np) merged = _merge_chunk_array(None, value, (slice(10, 20),), spec, (slice(0, 10),), True, ()) assert np.shares_memory(merged.as_numpy_array(), value_np), ( "complete-chunk merge no longer returns a view of the caller's value" ) # end-to-end: the source array is byte-identical after a multi-chunk write for pipeline_path in ( "zarr.core.codec_pipeline.FusedCodecPipeline", "zarr.core.codec_pipeline.BatchedCodecPipeline", ): with zarr.config.set({"codec_pipeline.path": pipeline_path}): arr = zarr.create_array( store=MemoryStore(), shape=(30,), chunks=(10,), dtype="uint16", compressors=None, fill_value=0, ) source = np.arange(30, dtype="uint16") snapshot = source.copy() arr[:] = source np.testing.assert_array_equal(source, snapshot, err_msg=pipeline_path) np.testing.assert_array_equal(arr[:], snapshot, err_msg=pipeline_path) # --------------------------------------------------------------------------- # Whole-shard bulk decode under arbitrary indexing: the bulk decode only fires # for an *identity full-shard* read (every dimension a whole-dim step-1 slice), # but it is reached through the partial-read path (`_decode_partial_sync`) for # any indexer. A reordering or duplicating coordinate/orthogonal selection can # have an output shape equal to the shard shape — trivially in 1-D (any # selection of `shard_len` points), and in >=2-D whenever an axis-0 index array # has exactly `shard_shape[0]` entries — and must NOT be served by the bulk # path in natural order; it must honor the selection. This pins the END-TO-END # read, which `test_bulk_shard_decode_equals_general_decode` (identity # BasicIndexer only) cannot reach. See the vindex- and # oindex-on-uncompressed-shard corruption bugs. # --------------------------------------------------------------------------- @st.composite def _uncompressed_shard_index_cases(draw: st.DrawFn) -> dict[str, Any]: ndim = draw(st.integers(1, 2)) chunk_shape = tuple(draw(st.integers(1, 4)) for _ in range(ndim)) grid = tuple(draw(st.integers(1, 4)) for _ in range(ndim)) shard_shape = tuple(c * g for c, g in zip(chunk_shape, grid, strict=True)) dtype = draw(_DTYPES) data = draw(npst.arrays(dtype=np.dtype(dtype), shape=shard_shape)) dim0 = shard_shape[0] # axis-0 index array sized to the dimension: either a permutation # (reordering, no duplicates) or an arbitrary list (duplicates likely) — # both keep the output shape equal to the shard shape. if draw(st.booleans()): idx = np.array(draw(st.permutations(list(range(dim0)))), dtype=np.intp) else: idx = np.array( draw(st.lists(st.integers(0, dim0 - 1), min_size=dim0, max_size=dim0)), dtype=np.intp, ) return { "chunk_shape": chunk_shape, "shard_shape": shard_shape, "data": data, "idx": idx, "endian": draw(st.sampled_from(["little", "big"])), "index_location": draw(st.sampled_from(["start", "end"])), "subchunk_write_order": draw( st.sampled_from(["morton", "lexicographic", "colexicographic", "unordered"]) ), } @settings(max_examples=200, deadline=None) @given(case=_uncompressed_shard_index_cases()) def test_reordering_read_on_uncompressed_shard_honors_selection(case: dict[str, Any]) -> None: """A reordering or duplicating fancy/vindex/oindex read over a full uncompressed shard must return the selected data, not the shard in natural order — under the Fused pipeline (where the bulk-decode fast path engages) exactly as under numpy.""" idx = case["idx"] data = case["data"] ndim = data.ndim serializer = BytesCodec(endian=case["endian"]) with zarr.config.set({"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"}): arr = zarr.create_array( store=MemoryStore(), shape=case["shard_shape"], chunks=case["chunk_shape"], shards={"shape": case["shard_shape"], "index_location": case["index_location"]}, dtype=data.dtype, serializer=serializer, compressors=None, filters=None, fill_value=0, ) arr[...] = data # axis-0 index array (rest full slices): an OrthogonalIndexer whose # output shape equals the shard shape but which reorders/duplicates rows. rest = (slice(None),) * (ndim - 1) np.testing.assert_array_equal(arr[(idx, *rest)], data[idx]) np.testing.assert_array_equal(arr.oindex[(idx, *rest)], data[idx]) if ndim == 1: # coordinate selection: flattened point count == shard shape. np.testing.assert_array_equal(arr.vindex[idx], data[idx]) else: # 2-D broadcast index arrays: full-coverage coordinate selection # whose shape equals the shard shape. cols = np.arange(case["shard_shape"][1]) np.testing.assert_array_equal( arr.vindex[idx[:, None], cols[None, :]], data[idx[:, None], cols[None, :]] ) # --------------------------------------------------------------------------- # Scalar-broadcast write memoization: writing a scalar must produce the same # STORED BYTES as writing the equivalent broadcast array. # --------------------------------------------------------------------------- @st.composite def _scalar_cases(draw: st.DrawFn) -> dict[str, Any]: n_chunks = draw(st.integers(2, 6)) chunk = draw(st.integers(2, 6)) shape = n_chunks * chunk start = draw(st.integers(0, shape - 1)) stop = draw(st.integers(start + 1, shape)) return { "shape": shape, "chunk": chunk, "sel": slice(start, stop), "scalar": draw(st.integers(0, 255)), "write_empty_chunks": draw(st.booleans()), "sharded": draw(st.booleans()), } @settings(max_examples=100, deadline=None) @given(case=_scalar_cases()) def test_scalar_write_equals_broadcast_write(case: dict[str, Any]) -> None: """arr[sel] = scalar and arr[sel] = full(sel_shape, scalar) must leave the store byte-identical (pins the scalar-broadcast memoization in the sharded partial-write path, incl. its empty-chunk normalization).""" def build() -> tuple[MemoryStore, zarr.Array[Any]]: store = MemoryStore() arr = zarr.create_array( store=store, shape=(case["shape"],), chunks=(case["chunk"],), shards=(case["shape"],) if case["sharded"] else None, dtype="uint8", compressors=None, fill_value=0, config={"write_empty_chunks": case["write_empty_chunks"]}, ) return store, arr # The scalar-broadcast memoization lives in the Fused sync write path; pin it # explicitly since Fused is no longer the default pipeline. with zarr.config.set({"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"}): store_a, arr_a = build() arr_a[case["sel"]] = case["scalar"] store_b, arr_b = build() n = case["sel"].stop - case["sel"].start arr_b[case["sel"]] = np.full(n, case["scalar"], dtype="uint8") keys_a = {k: bytes(v.to_bytes()) for k, v in store_a._store_dict.items()} keys_b = {k: bytes(v.to_bytes()) for k, v in store_b._store_dict.items()} assert keys_a == keys_b # --------------------------------------------------------------------------- # get_ranges_sync coalescing: merged fetches must return exactly what # individual per-range gets return, for any gap/coalesce limits. # --------------------------------------------------------------------------- @st.composite def _range_cases(draw: st.DrawFn) -> dict[str, Any]: blob_len = draw(st.integers(1, 200)) n = draw(st.integers(1, 8)) ranges: list[RangeByteRequest | OffsetByteRequest | SuffixByteRequest | None] = [] for _ in range(n): kind = draw(st.sampled_from(["range", "offset", "suffix", "none"])) if kind == "range": start = draw(st.integers(0, blob_len - 1)) end = draw(st.integers(start + 1, blob_len)) ranges.append(RangeByteRequest(start, end)) elif kind == "offset": ranges.append(OffsetByteRequest(draw(st.integers(0, blob_len - 1)))) elif kind == "suffix": ranges.append(SuffixByteRequest(draw(st.integers(1, blob_len)))) else: ranges.append(None) return { "blob": draw(st.binary(min_size=blob_len, max_size=blob_len)), "ranges": ranges, "max_gap": draw(st.integers(0, 64)), "max_coalesced": draw(st.integers(1, 512)), } @settings(max_examples=200, deadline=None) @given(case=_range_cases()) def test_get_ranges_sync_equals_individual_gets(case: dict[str, Any]) -> None: """Coalesced byte-range reads must return exactly what one get_sync per range returns — for any gap/coalesce limits (the offset re-slicing math is where a coalescing bug would corrupt data).""" store = MemoryStore() store._is_open = True proto = default_buffer_prototype() store._store_dict["k"] = CPUBuffer.from_bytes(case["blob"]) expected = [store.get_sync("k", prototype=proto, byte_range=r) for r in case["ranges"]] got: dict[int, bytes | None] = {} for idx, buf in store.get_ranges_sync( "k", case["ranges"], prototype=proto, max_gap_bytes=case["max_gap"], max_coalesced_bytes=case["max_coalesced"], ): got[idx] = None if buf is None else bytes(buf.to_bytes()) for i, exp in enumerate(expected): exp_bytes = None if exp is None else bytes(exp.to_bytes()) assert got.get(i) == exp_bytes, f"range {i} ({case['ranges'][i]!r}) mismatch" zarr-developers-zarr-python-a2f2415/tests/test_fused_pipeline.py000066400000000000000000001325131527014137600252400ustar00rootroot00000000000000"""Tests for FusedCodecPipeline -- the per-chunk-fused codec pipeline.""" from __future__ import annotations import asyncio from dataclasses import dataclass, field, replace from typing import TYPE_CHECKING, Any from unittest.mock import patch import numpy as np import pytest import zarr from tests.conftest import Expect, gzip_streams_equal_except_mtime from zarr.abc.codec import ( ArrayBytesCodec, ArrayBytesCodecPartialDecodeMixin, ArrayBytesCodecPartialEncodeMixin, BytesBytesCodec, ) from zarr.abc.store import Store, _store_supports_sync_io from zarr.codecs.bytes import BytesCodec from zarr.codecs.gzip import GzipCodec from zarr.codecs.transpose import TransposeCodec from zarr.codecs.zstd import ZstdCodec from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer import BufferPrototype, default_buffer_prototype from zarr.core.buffer.cpu import NDBuffer as CPUNDBuffer from zarr.core.chunk_utils import ChunkTransform, evolve_codecs from zarr.core.codec_pipeline import AsyncChunkTransform, FusedCodecPipeline from zarr.core.config import config as zarr_config from zarr.core.dtype import Float32, Float64, Int32, UInt8, ZDType from zarr.registry import register_codec from zarr.storage import MemoryStore, StorePath, WrapperStore from zarr.storage._utils import _normalize_byte_range_index from zarr.testing.store import LatencyStore if TYPE_CHECKING: from collections.abc import AsyncIterator, Callable, Iterable from zarr.abc.store import ByteRequest from zarr.core.buffer import Buffer, NDBuffer _FLOAT64 = Float64() def _make_spec( shape: tuple[int, ...], zdtype: ZDType[Any, Any] = _FLOAT64, fill_value: float = 0, *, write_empty_chunks: bool = True, prototype: BufferPrototype | None = None, ) -> ArraySpec: """An ArraySpec with the C-order defaults shared by the tests in this file.""" return ArraySpec( shape=shape, dtype=zdtype, fill_value=zdtype.cast_scalar(fill_value), config=ArrayConfig(order="C", write_empty_chunks=write_empty_chunks), prototype=prototype if prototype is not None else default_buffer_prototype(), ) @pytest.mark.parametrize( "codecs", [ (BytesCodec(),), (BytesCodec(), GzipCodec(level=1)), (BytesCodec(), ZstdCodec(level=1)), (TransposeCodec(order=(1, 0)), BytesCodec()), (TransposeCodec(order=(1, 0)), BytesCodec(), ZstdCodec(level=1)), ], ids=["bytes-only", "gzip", "zstd", "transpose", "transpose+zstd"], ) def test_construction(codecs: tuple[Any, ...]) -> None: """FusedCodecPipeline can be constructed from valid codec combinations.""" pipeline = FusedCodecPipeline.from_codecs(codecs) assert pipeline.codecs == codecs def test_sync_api_compute_off_event_loop(monkeypatch: pytest.MonkeyPatch) -> None: """Codec compute must never run on the thread driving the event loop. Every sync-API call, from every user thread, is serviced by the one global `zarr_io` event loop. Running decode/encode inline on that loop's thread turns it into a mutex around codec compute: concurrent readers serialize, and the penalty grows with codec cost (observed as "fused pipeline is slower for zstd data" under dask-style multi-threaded single-chunk reads). """ compute_on_loop = {"decode": False, "encode": False} calls = {"decode": 0, "encode": 0} real_decode = ChunkTransform.decode_chunk real_encode = ChunkTransform.encode_chunk def _running_loop() -> bool: try: asyncio.get_running_loop() except RuntimeError: return False return True def traced_decode(self: ChunkTransform, chunk_bytes: Any, chunk_spec: Any) -> Any: calls["decode"] += 1 compute_on_loop["decode"] = compute_on_loop["decode"] or _running_loop() return real_decode(self, chunk_bytes, chunk_spec) def traced_encode(self: ChunkTransform, chunk_array: Any, chunk_spec: Any) -> Any: calls["encode"] += 1 compute_on_loop["encode"] = compute_on_loop["encode"] or _running_loop() return real_encode(self, chunk_array, chunk_spec) monkeypatch.setattr(ChunkTransform, "decode_chunk", traced_decode) monkeypatch.setattr(ChunkTransform, "encode_chunk", traced_encode) with zarr_config.set({"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"}): arr = zarr.create_array( MemoryStore(), shape=(8, 8), chunks=(4, 4), dtype="float64", compressors=ZstdCodec(level=1), ) data = np.arange(64, dtype="float64").reshape(8, 8) arr[:4, :4] = data[:4, :4] # single-chunk write (batch of 1) arr[:] = data # multi-chunk write # Guard against vacuity: if the sync fast path stops triggering, the # traced ChunkTransform methods are never called (the async fallback # uses AsyncChunkTransform) and the on-loop flags stay trivially False. assert calls["encode"] > 0 assert compute_on_loop["encode"] is False np.testing.assert_array_equal(arr[:4, :4], data[:4, :4]) # single-chunk read np.testing.assert_array_equal(arr[:], data) # multi-chunk read assert calls["decode"] > 0 assert compute_on_loop["decode"] is False def test_evolve_from_array_spec() -> None: """evolve_from_array_spec creates a sync transform.""" pipeline = FusedCodecPipeline.from_codecs((BytesCodec(),)) assert pipeline.sync_transform is None evolved = pipeline.evolve_from_array_spec(_make_spec((100,))) assert evolved.sync_transform is not None # --------------------------------------------------------------------------- # Sync path tests # # These exercise FusedCodecPipeline's synchronous API (write_sync / read_sync / # sync_transform), which has no equivalent on BatchedCodecPipeline -- so they # cannot live in the pipeline-agnostic CodecPipelineTests suite. The async # roundtrip / fill-value behaviour is covered there (test_scenario) across both # pipelines and sync/async stores. # --------------------------------------------------------------------------- @pytest.mark.parametrize( ("zdtype", "shape"), [ (Float64(), (100,)), (Float32(), (50,)), (Int32(), (200,)), (Float64(), (10, 10)), ], ids=["f64-1d", "f32-1d", "i32-1d", "f64-2d"], ) def test_read_write_sync_roundtrip(zdtype: ZDType[Any, Any], shape: tuple[int, ...]) -> None: """Data written via write_sync can be read back via read_sync.""" store = MemoryStore() dtype = zdtype.to_native_dtype() spec = _make_spec(shape, zdtype) pipeline = FusedCodecPipeline.from_codecs((BytesCodec(),)) pipeline = pipeline.evolve_from_array_spec(spec) data = np.arange(int(np.prod(shape)), dtype=dtype).reshape(shape) value = CPUNDBuffer.from_numpy_array(data) chunk_selection = tuple(slice(0, s) for s in shape) out_selection = chunk_selection store_path = StorePath(store, "c/0") # Write sync pipeline.write_sync( [(store_path, spec, chunk_selection, out_selection, True)], value, ) # Read sync out = CPUNDBuffer.from_numpy_array(np.zeros(shape, dtype=dtype)) pipeline.read_sync( [(store_path, spec, chunk_selection, out_selection, True)], out, ) np.testing.assert_array_equal(data, out.as_numpy_array()) def test_read_sync_missing_chunk_fills() -> None: """Sync read of a missing chunk fills with the fill value.""" store = MemoryStore() spec = _make_spec((10,), fill_value=42.0) pipeline = FusedCodecPipeline.from_codecs((BytesCodec(),)) pipeline = pipeline.evolve_from_array_spec(spec) out = CPUNDBuffer.from_numpy_array(np.zeros(10, dtype="float64")) store_path = StorePath(store, "c/0") chunk_sel = (slice(0, 10),) pipeline.read_sync( [(store_path, spec, chunk_sel, chunk_sel, True)], out, ) np.testing.assert_array_equal(out.as_numpy_array(), np.full(10, 42.0)) def test_sync_write_async_read_roundtrip() -> None: """Data written via write_sync can be read back via async read.""" from zarr.core.sync import sync store = MemoryStore() spec = _make_spec((100,)) pipeline = FusedCodecPipeline.from_codecs((BytesCodec(),)) pipeline = pipeline.evolve_from_array_spec(spec) data = np.arange(100, dtype="float64") value = CPUNDBuffer.from_numpy_array(data) chunk_sel = (slice(0, 100),) store_path = StorePath(store, "c/0") # Write sync pipeline.write_sync( [(store_path, spec, chunk_sel, chunk_sel, True)], value, ) # Read async out = CPUNDBuffer.from_numpy_array(np.zeros(100, dtype="float64")) sync( pipeline.read( [(store_path, spec, chunk_sel, chunk_sel, True)], out, ) ) np.testing.assert_array_equal(data, out.as_numpy_array()) def test_chunk_transform_uses_runtime_prototype() -> None: """ChunkTransform must pass each codec the prototype from the runtime chunk_spec, not one captured at evolve time. Constructs ChunkTransform directly (a Fused-internal data structure with no BatchedCodecPipeline equivalent). """ class _PrototypeRecordingCodec(BytesBytesCodec): # type: ignore[misc,unused-ignore] """A no-op BB codec that records the prototype it was called with.""" is_fixed_size = True seen_prototypes: list[object] def __init__(self) -> None: object.__setattr__(self, "seen_prototypes", []) def to_dict(self) -> dict[str, Any]: return {"name": "_prototype_recording", "configuration": {}} @classmethod def from_dict(cls, data: dict[str, Any]) -> _PrototypeRecordingCodec: return cls() def compute_encoded_size(self, input_byte_length: int, _spec: ArraySpec) -> int: return input_byte_length def _encode_sync(self, chunk_bytes: Buffer, chunk_spec: ArraySpec) -> Buffer | None: self.seen_prototypes.append(chunk_spec.prototype) return chunk_bytes def _decode_sync(self, chunk_bytes: Buffer, chunk_spec: ArraySpec) -> Buffer: self.seen_prototypes.append(chunk_spec.prototype) return chunk_bytes async def _encode_single(self, chunk_bytes: Buffer, chunk_spec: ArraySpec) -> Buffer | None: return self._encode_sync(chunk_bytes, chunk_spec) async def _decode_single(self, chunk_bytes: Buffer, chunk_spec: ArraySpec) -> Buffer: return self._decode_sync(chunk_bytes, chunk_spec) recording = _PrototypeRecordingCodec() transform = ChunkTransform(codecs=(BytesCodec(), recording)) def _spec(prototype: BufferPrototype) -> ArraySpec: return _make_spec((10,), write_empty_chunks=False, prototype=prototype) proto_default = default_buffer_prototype() # A distinct BufferPrototype instance with the same buffer/nd_buffer types -- # fails an identity check but works at runtime. proto_other = BufferPrototype(buffer=proto_default.buffer, nd_buffer=proto_default.nd_buffer) assert proto_other is not proto_default arr = proto_default.nd_buffer.from_numpy_array(np.arange(10, dtype="float64")) transform.encode_chunk(arr, _spec(proto_default)) transform.encode_chunk(arr, _spec(proto_other)) assert recording.seen_prototypes[0] is proto_default assert recording.seen_prototypes[1] is proto_other, ( "ChunkTransform did not pass the runtime prototype to the codec" ) # --------------------------------------------------------------------------- # Thread-pool (max_workers > 1) tests # # The pool dispatch in read_sync/write_sync is Fused-only and off by default # (codec_pipeline.max_workers defaults to 1 == sequential). These tests opt in # and exercise the pool path end-to-end, exception propagation from workers, # and concurrent decode through the shared ChunkTransform. # --------------------------------------------------------------------------- _FUSED_POOL_CONFIG = { "codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline", "codec_pipeline.max_workers": 4, } def test_read_write_with_thread_pool() -> None: """With max_workers > 1, multi-chunk reads and writes dispatch through the thread pool (pool.map in read_sync/write_sync) and produce the same results as sequential execution. The `_get_pool` spy pins that the pool branch actually fires: without it, a config-resolution regression (renamed key, `_resolve_max_workers` returning 1) would silently degrade all the pool tests into re-testing the sequential branch while staying green. """ import zarr.core.codec_pipeline as cp_mod with zarr_config.set(_FUSED_POOL_CONFIG): assert cp_mod._resolve_max_workers() == 4, ( "codec_pipeline.max_workers config did not reach _resolve_max_workers" ) store = MemoryStore() arr = zarr.create_array( store=store, shape=(100,), chunks=(10,), dtype="float64", compressors=None, fill_value=0.0, ) assert isinstance(arr._async_array.codec_pipeline, FusedCodecPipeline) data = np.arange(100, dtype="float64") with patch.object(cp_mod, "_get_pool", wraps=cp_mod._get_pool) as pool_spy: arr[:] = data # 10 chunks -> pool dispatch in write_sync assert pool_spy.call_count >= 1, "multi-chunk write did not take the pool branch" writes = pool_spy.call_count np.testing.assert_array_equal(arr[:], data) # pool dispatch in read_sync assert pool_spy.call_count > writes, "multi-chunk read did not take the pool branch" arr[5:25] = 7.0 # partial write: merge path through the pool data[5:25] = 7.0 np.testing.assert_array_equal(arr[:], data) def test_thread_pool_write_worker_exception_propagates() -> None: """A store error raised inside a pool worker during write_sync surfaces to the caller (write_sync consumes pool.map, so worker exceptions re-raise).""" with zarr_config.set(_FUSED_POOL_CONFIG): store = MemoryStore() arr = zarr.create_array( store=store, shape=(100,), chunks=(10,), dtype="float64", compressors=None, fill_value=0.0, ) with ( patch.object(store, "set_sync", side_effect=RuntimeError("simulated store error")), pytest.raises(RuntimeError, match="simulated store error"), ): arr[:] = np.arange(100, dtype="float64") def test_thread_pool_read_worker_exception_propagates() -> None: """A store error raised inside a pool worker during read_sync surfaces to the caller (read_sync consumes pool.map into a tuple).""" with zarr_config.set(_FUSED_POOL_CONFIG): store = MemoryStore() arr = zarr.create_array( store=store, shape=(100,), chunks=(10,), dtype="float64", compressors=None, fill_value=0.0, ) arr[:] = np.arange(100, dtype="float64") with ( patch.object(store, "get_sync", side_effect=RuntimeError("simulated store error")), pytest.raises(RuntimeError, match="simulated store error"), ): arr[:] def test_resolve_max_workers_warns_and_falls_back_on_invalid_config() -> None: """`codec_pipeline.max_workers` arrives via the config/env layer (e.g. `ZARR_CODEC_PIPELINE__MAX_WORKERS`), so garbage input should warn and fall back to the default rather than raising mid-read. """ import os import zarr.core.codec_pipeline as cp_mod from zarr.errors import ZarrUserWarning default = os.cpu_count() or 1 with zarr_config.set({"codec_pipeline.max_workers": "fast"}): with pytest.warns(ZarrUserWarning, match="max_workers"): result = cp_mod._resolve_max_workers() assert result == default async def test_encode_and_write_as_completed_cancels_stray_writes_on_failure() -> None: """A failing write must not leave sibling writes running in the background. `_encode_and_write_as_completed` fires one write task per chunk as soon as its encode completes, then `gather`s them. Plain `gather` (without `return_exceptions=True`) re-raises the first exception without cancelling the other in-flight tasks, so a still-running write would keep going after the caller has already seen the exception -- and its eventual outcome is never retrieved (an unraisable "Task exception was never retrieved" warning if it later fails). The failing write raises only once the slow write is in flight. Encodes land in any order, and if the failing chunk's write has already raised when `gather` is called, `gather` fails its result immediately (it runs done-callbacks of finished children eagerly), so the slow write is cancelled before its first step. That is also correct, but it is not the in-flight cancellation this test checks. """ from zarr.core.codec_pipeline import _encode_and_write_as_completed write_started = asyncio.Event() write_finished = False class _SlowByteSetter: async def get( self, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: return None async def set(self, value: Buffer) -> None: nonlocal write_finished write_started.set() await asyncio.sleep(0.2) write_finished = True async def delete(self) -> None: pass async def set_if_not_exists(self, default: Buffer) -> None: pass class _FailingByteSetter: async def get( self, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: return None async def set(self, value: Buffer) -> None: # Bounded so a regression that never starts the slow write fails # with a TimeoutError instead of hanging. await asyncio.wait_for(write_started.wait(), timeout=5) raise RuntimeError("simulated write failure") async def delete(self) -> None: pass async def set_if_not_exists(self, default: Buffer) -> None: pass chunk_spec = _make_spec((1,), UInt8()) chunk_array = CPUNDBuffer.from_numpy_array(np.zeros(1, dtype="uint8")) transform = ChunkTransform(codecs=(BytesCodec(),)) batch = [ (_SlowByteSetter(), chunk_array, chunk_spec), (_FailingByteSetter(), chunk_array, chunk_spec), ] with pytest.raises(RuntimeError, match="simulated write failure"): await _encode_and_write_as_completed(batch, transform) # type: ignore[arg-type] assert write_started.is_set() # Give the slow write's sleep long enough to finish if it were left # running unattended in the background instead of being cancelled. await asyncio.sleep(0.3) assert not write_finished, "the slow write should have been cancelled, not left running" def test_concurrent_reads_shared_transform_with_pool() -> None: """Concurrent decode through the shared ChunkTransform produces correct data. The transform's `_resolve_specs` cache is shared mutable state. With no array->array codecs the cache is bypassed entirely, so this uses a transpose filter to force cache traffic, max_workers=4 so pool workers decode chunks concurrently, and an outer thread pool so multiple reads are in flight at once. Each round RE-OPENS the array so the shared transform starts with a cold spec cache and the concurrent readers race the non-atomic first fill — reading through a single pre-warmed handle would only ever exercise cache hits. This pins correctness under concurrency (it cannot prove the absence of a race, but a torn cache would corrupt results here). """ from concurrent.futures import ThreadPoolExecutor with zarr_config.set(_FUSED_POOL_CONFIG): store = MemoryStore() arr = zarr.create_array( store=store, shape=(40, 40), chunks=(5, 5), dtype="int32", filters=[TransposeCodec(order=(1, 0))], serializer=BytesCodec(), compressors=None, fill_value=-1, ) data = np.arange(1600, dtype="int32").reshape(40, 40) arr[:] = data for _ in range(5): # several rounds, each racing a cold cache fresh = zarr.open_array(store=store, mode="r") def read_row_block(i: int, handle: zarr.Array[Any] = fresh) -> np.ndarray: return np.asarray(handle[i * 4 : (i + 1) * 4, :]) with ThreadPoolExecutor(max_workers=8) as ex: futures = {ex.submit(read_row_block, i): i for i in range(10)} for fut, i in futures.items(): np.testing.assert_array_equal(fut.result(), data[i * 4 : (i + 1) * 4, :]) def test_shared_transform_decode_alternating_specs() -> None: """A single ChunkTransform must decode chunks of DIFFERENT specs correctly when calls alternate, exercising eviction/refill of its single-entry `_resolve_specs` cache. The two specs differ in shape, so each call evicts the other's cached entry. A transpose filter forces the cache to be used (with no AA codec the cache is bypassed). The cache entry is stored as one atomic tuple precisely so a concurrent reader can never observe a key paired with another spec's resolved chain; this test pins the sequential eviction/refill correctness that underpins that guarantee. (The concurrent counterpart is `test_concurrent_reads_shared_transform_with_pool`.) """ transform = ChunkTransform(codecs=(TransposeCodec(order=(1, 0)), BytesCodec())) # two distinct specs (different shapes) sharing the one transform + cache slot cases = [] for shape in [(5, 7), (3, 11)]: spec = _make_spec(shape, Int32()) arr = np.arange(int(np.prod(shape)), dtype="int32").reshape(shape) encoded = transform.encode_chunk(CPUNDBuffer.from_numpy_array(arr), spec) assert encoded is not None cases.append((spec, encoded, arr)) # Alternate specs so every call evicts and refills the single cache slot. for i in range(20): spec, encoded, expected = cases[i % len(cases)] got = transform.decode_chunk(encoded, spec).as_numpy_array() np.testing.assert_array_equal(got, expected) def test_sharded_fallback_inner_chunks_avoid_async_transform() -> None: """Inner chunks of a shard on a NON-sync store decode through the sync ChunkTransform, not per-chunk AsyncChunkTransform coroutines. The sharding byte getters are in-memory dict wrappers; they implement SyncByteGetter/SyncByteSetter, and the nested inner pipeline is evolved (so its sync transform exists), letting the nested read/write take the sync fast path. Without this, every inner chunk pays a coroutine for a dict lookup plus an async per-chunk transform — measured at 1.5x (raw) to 3.6x (gzip) of sharded fallback read time. """ calls = {"decode": 0, "encode": 0} orig_decode = AsyncChunkTransform.decode_chunk orig_encode = AsyncChunkTransform.encode_chunk async def spy_decode(self: Any, *args: Any, **kwargs: Any) -> Any: calls["decode"] += 1 return await orig_decode(self, *args, **kwargs) async def spy_encode(self: Any, *args: Any, **kwargs: Any) -> Any: calls["encode"] += 1 return await orig_encode(self, *args, **kwargs) # LatencyStore is not sync-capable -> the OUTER pipeline takes the async # fallback; the INNER chunks go over the sharding byte getters. store = LatencyStore(MemoryStore(), get_latency=0.0, set_latency=0.0) arr = zarr.create_array( store=store, shape=(100,), chunks=(10,), shards=(50,), dtype="uint8", compressors=None, fill_value=0, ) if not isinstance(arr._async_array.codec_pipeline, FusedCodecPipeline): pytest.skip("sync fast path for inner chunks is specific to FusedCodecPipeline") data = np.arange(100, dtype="uint8") sync_calls = {"read_sync": 0} orig_read_sync = FusedCodecPipeline.read_sync def spy_read_sync(self: Any, *args: Any, **kwargs: Any) -> Any: sync_calls["read_sync"] += 1 return orig_read_sync(self, *args, **kwargs) with ( patch.object(AsyncChunkTransform, "decode_chunk", spy_decode), patch.object(AsyncChunkTransform, "encode_chunk", spy_encode), patch.object(FusedCodecPipeline, "read_sync", spy_read_sync), ): arr[:] = data out = np.asarray(arr[:]) np.testing.assert_array_equal(out, data) assert calls == {"decode": 0, "encode": 0}, ( f"inner chunks went through per-chunk AsyncChunkTransform coroutines: {calls}" ) # The outer store is not sync-capable, so any read_sync calls are the # NESTED pipeline taking the sync fast path over the sharding byte getters # (the SyncByteGetter gate). Without the gate, inner chunks go through # concurrent_map with one coroutine per chunk. (Writes don't appear here: # the fallback write encodes whole shards through the outer sync transform # -> ShardingCodec._encode_sync, never touching the nested byte setters.) assert sync_calls["read_sync"] >= 1, "nested read did not take the sync fast path" def test_write_over_sync_byte_setter_takes_sync_path() -> None: """`FusedCodecPipeline.write` routes a non-StorePath `SyncByteSetter` (the sharding codec's `_ShardingByteSetter`) through `write_sync`. This is the write-side twin of the SyncByteGetter gate: the read test above cannot guard it because fallback whole-array writes encode shards via `_encode_sync` and never touch the nested byte setters. The nested `write` over `_ShardingByteSetter` is reached from the async shard encode paths (`_encode_single`/`_encode_partial_single`), so pin the gate directly: without it, this write degrades to the async fallback (one coroutine per inner chunk for an in-memory dict store). """ from zarr.codecs.sharding import _ShardingByteSetter spec = _make_spec((10,), UInt8()) pipeline = FusedCodecPipeline.from_codecs([BytesCodec()]).evolve_from_array_spec(spec) assert pipeline.sync_transform is not None shard_dict: dict[tuple[int, ...], Any] = {} setter = _ShardingByteSetter(shard_dict, (0,)) value = default_buffer_prototype().nd_buffer.from_numpy_array(np.arange(10, dtype="uint8")) sync_calls = {"write_sync": 0} orig_write_sync = FusedCodecPipeline.write_sync def spy_write_sync(self: Any, *args: Any, **kwargs: Any) -> Any: sync_calls["write_sync"] += 1 return orig_write_sync(self, *args, **kwargs) sel = (slice(0, 10),) with patch.object(FusedCodecPipeline, "write_sync", spy_write_sync): asyncio.run(pipeline.write([(setter, spec, sel, sel, True)], value)) assert sync_calls["write_sync"] >= 1, ( "write over a SyncByteSetter did not take the sync fast path" ) written = shard_dict[(0,)] np.testing.assert_array_equal( np.frombuffer(written.to_bytes(), dtype="uint8"), np.arange(10, dtype="uint8") ) # --------------------------------------------------------------------------- # Async-only codecs inside a shard's inner codec chain # --------------------------------------------------------------------------- class _AsyncOnlyNoopCodec(BytesBytesCodec): # type: ignore[misc,unused-ignore] """A no-op BB codec implementing ONLY the async codec interface. Deliberately does NOT satisfy `SupportsSyncCodec` (no `_decode_sync` / `_encode_sync`), modelling a third-party codec that predates the sync protocol. Class-level counters prove the codec actually ran. """ is_fixed_size = True encode_calls = 0 decode_calls = 0 def to_dict(self) -> dict[str, Any]: return {"name": "test-async-only-noop", "configuration": {}} @classmethod def from_dict(cls, data: dict[str, Any]) -> _AsyncOnlyNoopCodec: return cls() def compute_encoded_size(self, input_byte_length: int, _spec: Any) -> int: return input_byte_length async def _encode_single(self, chunk_bytes: Any, chunk_spec: Any) -> Any: type(self).encode_calls += 1 return chunk_bytes async def _decode_single(self, chunk_bytes: Any, chunk_spec: Any) -> Any: type(self).decode_calls += 1 return chunk_bytes def test_sharded_roundtrip_with_async_only_inner_codec() -> None: """A sharded array whose INNER codec chain contains an async-only codec round-trips under FusedCodecPipeline (full write, partial write, full read, partial read). Regression: the pipeline's top-level guard (evolve_from_array_spec -> sync_transform=None) only inspected the top-level chain. ShardingCodec structurally satisfies SupportsSyncCodec, so a sync transform was built and the sync fast path dove into ShardingCodec's sync shard paths, which raised TypeError from the inner ChunkTransform. The pipeline must instead decline the sync fast path and fall back to the async inner pipeline, like BatchedCodecPipeline. """ _AsyncOnlyNoopCodec.encode_calls = 0 _AsyncOnlyNoopCodec.decode_calls = 0 with zarr_config.set({"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"}): store = MemoryStore() arr = zarr.create_array( store=store, shape=(16, 16), shards=(8, 8), chunks=(4, 4), dtype="int32", compressors=[_AsyncOnlyNoopCodec()], fill_value=-1, ) assert isinstance(arr._async_array.codec_pipeline, FusedCodecPipeline) data = np.arange(256, dtype="int32").reshape(16, 16) arr[:] = data # full write np.testing.assert_array_equal(arr[:], data) # full read np.testing.assert_array_equal(arr[2:11, 3:14], data[2:11, 3:14]) # partial read arr[5:7, 5:13] = 0 # partial write (read-merge-write of existing shards) data[5:7, 5:13] = 0 np.testing.assert_array_equal(arr[:], data) assert _AsyncOnlyNoopCodec.encode_calls > 0, "async-only inner codec never encoded" assert _AsyncOnlyNoopCodec.decode_calls > 0, "async-only inner codec never decoded" # The stored bytes are valid for the default pipeline too: read them back # under BatchedCodecPipeline (default codec_pipeline.path). Opening from # metadata needs the codec name in the registry. register_codec("test-async-only-noop", _AsyncOnlyNoopCodec) reread = zarr.open_array(store=store, mode="r") np.testing.assert_array_equal(reread[:], data) # --------------------------------------------------------------------------- # AsyncChunkTransform: the async per-chunk codec chain used on the async # fallback path. It is the async mirror of ChunkTransform, so it must produce # identical bytes/arrays — except for gzip's embedded 4-byte MTIME header # field, which the gzip case's comparator deliberately ignores. # The default (Fused, sync-store) path never uses it; # these tests drive it directly over multi-codec chains so the aa/bb loops and # the all-fill drop branch are exercised. # --------------------------------------------------------------------------- def _bytes_identical(a: bytes, b: bytes) -> bool: return a == b # Each case pairs a codec chain with the expected relationship between the # async and sync encoders' output: byte-identical, except for a chain # containing gzip, whose embedded 4-byte MTIME header field varies with the # encode time. _ASYNC_SYNC_PARITY_CASES: list[Expect[tuple[Any, ...], Callable[[bytes, bytes], bool]]] = [ Expect( input=(BytesCodec(),), output=_bytes_identical, id="bytes-only", ), Expect( input=(BytesCodec(), GzipCodec(level=1)), output=gzip_streams_equal_except_mtime, id="bb", ), Expect( input=(TransposeCodec(order=(1, 0)), BytesCodec()), output=_bytes_identical, id="aa", ), Expect( input=(TransposeCodec(order=(1, 0)), BytesCodec(), ZstdCodec(level=1)), output=_bytes_identical, id="aa+ab+bb", ), ] @pytest.mark.parametrize("case", _ASYNC_SYNC_PARITY_CASES, ids=lambda c: c.id) def test_async_chunk_transform_matches_sync( case: Expect[tuple[Any, ...], Callable[[bytes, bytes], bool]], ) -> None: """`AsyncChunkTransform.decode_chunk`/`encode_chunk` must round-trip and produce what the synchronous `ChunkTransform` produces, across array->array, array->bytes, and bytes->bytes codec combinations. Each case's expected output is a byte-comparison function: exact equality, except for gzip, whose embedded 4-byte MTIME header field is deliberately ignored. This is the async mirror of the codecs the default pipeline runs synchronously; a divergence here corrupts data only on the async fallback path (remote stores), which no end-to-end test of the default pipeline touches. """ shape = (4, 4) spec = _make_spec(shape, Int32()) evolved = evolve_codecs(case.input, spec) sync_t = ChunkTransform(codecs=evolved) async_t = AsyncChunkTransform(codecs=evolved) data = np.arange(16, dtype="int32").reshape(shape) value = CPUNDBuffer.from_numpy_array(data) sync_bytes = sync_t.encode_chunk(value, spec) async_bytes = asyncio.run(async_t.encode_chunk(value, spec)) assert sync_bytes is not None assert async_bytes is not None assert case.output(async_bytes.to_bytes(), sync_bytes.to_bytes()) sync_arr = sync_t.decode_chunk(async_bytes, spec) async_arr = asyncio.run(async_t.decode_chunk(async_bytes, spec)) np.testing.assert_array_equal(async_arr.as_numpy_array(), sync_arr.as_numpy_array()) np.testing.assert_array_equal(async_arr.as_numpy_array(), data) def test_async_decode_encode_passes_through_none_chunks() -> None: """`FusedCodecPipeline.decode`/`encode` (the async batch entry points used on the fallback path) map a None chunk to None and leave real chunks untouched — pins the None-passthrough branch the default sync path skips.""" spec = _make_spec((4,), Int32()) pipeline = FusedCodecPipeline.from_codecs([BytesCodec()]).evolve_from_array_spec(spec) data = np.arange(4, dtype="int32") value = CPUNDBuffer.from_numpy_array(data) # encode a real chunk and a None chunk together encoded = list(asyncio.run(pipeline.encode([(value, spec), (None, spec)]))) assert encoded[1] is None assert encoded[0] is not None # decode the real chunk and a None chunk together decoded = list(asyncio.run(pipeline.decode([(encoded[0], spec), (None, spec)]))) assert decoded[1] is None assert decoded[0] is not None np.testing.assert_array_equal(decoded[0].as_numpy_array(), data) # --------------------------------------------------------------------------- # Graceful fallback for partial-mixin codecs without private sync-partial hooks # # The public partial-decode/encode contract (`ArrayBytesCodecPartialDecodeMixin` # / `ArrayBytesCodecPartialEncodeMixin`) only requires the async # `_decode_partial_single` / `_encode_partial_single`. The fused pipeline must # route such codecs through its full-chunk sync path instead of asserting on # the private `_decode_partial_sync` / `_encode_partial_sync` hooks. The double # below is a minimal conforming implementer of that contract; it guards the # public extension API, so it must not grow the private sync-partial methods. # --------------------------------------------------------------------------- @dataclass(frozen=True) class PartialMixinCodec( ArrayBytesCodec, ArrayBytesCodecPartialDecodeMixin, ArrayBytesCodecPartialEncodeMixin ): """Serializer with sync whole-chunk methods plus ONLY async partial methods. This is the pre-fused public contract for partial-capable codecs: the mixins' `_decode_partial_single` / `_encode_partial_single`. It must not implement `_decode_partial_sync` / `_encode_partial_sync`. """ inner: BytesCodec = field(default_factory=BytesCodec) @classmethod def from_dict(cls, data: dict[str, Any]) -> PartialMixinCodec: return cls() def to_dict(self) -> dict[str, Any]: return {"name": "test-partial-mixin"} def evolve_from_array_spec(self, array_spec: ArraySpec) -> PartialMixinCodec: return replace(self, inner=self.inner.evolve_from_array_spec(array_spec)) def compute_encoded_size(self, input_byte_length: int, chunk_spec: ArraySpec) -> int: return self.inner.compute_encoded_size(input_byte_length, chunk_spec) def _decode_sync(self, chunk_bytes: Buffer, chunk_spec: ArraySpec) -> NDBuffer: return self.inner._decode_sync(chunk_bytes, chunk_spec) def _encode_sync(self, chunk_array: NDBuffer, chunk_spec: ArraySpec) -> Buffer | None: return self.inner._encode_sync(chunk_array, chunk_spec) async def _decode_single(self, chunk_bytes: Buffer, chunk_spec: ArraySpec) -> NDBuffer: return self._decode_sync(chunk_bytes, chunk_spec) async def _encode_single(self, chunk_array: NDBuffer, chunk_spec: ArraySpec) -> Buffer | None: return self._encode_sync(chunk_array, chunk_spec) async def _decode_partial_single( self, byte_getter: Any, selection: Any, chunk_spec: ArraySpec ) -> NDBuffer | None: chunk_bytes = await byte_getter.get(prototype=chunk_spec.prototype) if chunk_bytes is None: return None return self._decode_sync(chunk_bytes, chunk_spec)[selection] async def _encode_partial_single( self, byte_setter: Any, chunk_array: NDBuffer, selection: Any, chunk_spec: ArraySpec ) -> None: existing = await byte_setter.get(prototype=chunk_spec.prototype) if existing is None: full = chunk_spec.prototype.nd_buffer.create( shape=chunk_spec.shape, dtype=chunk_spec.dtype.to_native_dtype(), fill_value=chunk_spec.fill_value, ) else: full = self._decode_sync(existing, chunk_spec) full[selection] = chunk_array encoded = self._encode_sync(full, chunk_spec) assert encoded is not None await byte_setter.set(encoded) register_codec("test-partial-mixin", PartialMixinCodec) _FUSED = {"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"} _BATCHED = {"codec_pipeline.path": "zarr.core.codec_pipeline.BatchedCodecPipeline"} @pytest.mark.filterwarnings("ignore::zarr.errors.UnstableSpecificationWarning") @pytest.mark.parametrize("dtype", ["uint8", "float64"]) def test_partial_mixin_codec_async_partial_only_round_trip(dtype: str) -> None: """A serializer advertising the partial mixins with only async partial methods must round-trip under the fused pipeline: full write, full read, partial read, partial write, plus cross-pipeline parity with BatchedCodecPipeline.""" data = np.arange(64, dtype=dtype).reshape(8, 8) with zarr_config.set(_FUSED): store = MemoryStore() arr = zarr.create_array( store, shape=(8, 8), chunks=(4, 4), dtype=dtype, serializer=PartialMixinCodec(), compressors=None, filters=None, fill_value=0, ) pipeline = arr._async_array.codec_pipeline assert isinstance(pipeline, FusedCodecPipeline) assert pipeline.supports_partial_decode assert pipeline.supports_partial_encode assert pipeline.sync_transform is not None arr[:] = data np.testing.assert_array_equal(arr[:], data) np.testing.assert_array_equal(arr[1:5, 2:7], data[1:5, 2:7]) expected = data.copy() expected[2:6, 1:3] = 7 arr[2:6, 1:3] = expected[2:6, 1:3] np.testing.assert_array_equal(arr[:], expected) with zarr_config.set(_BATCHED): np.testing.assert_array_equal(zarr.open_array(store, mode="r")[:], expected) # Sync-IO capability gating (`zarr.abc.store._store_supports_sync_io`) # # The fused read/write fast paths must engage iff the store advertises the # FULL synchronous IO surface (get_sync + set_sync + delete_sync): write_sync # needs get_sync for partial-chunk read-modify-write and delete_sync for # all-fill chunk cleanup, so gating on any single protocol can crash # mid-batch. Wrappers must forward the capability of the wrapped store. # --------------------------------------------------------------------------- class AsyncOnlyStore(Store): """Dict-backed store implementing only the async `Store` surface (no `*_sync`).""" def __init__(self) -> None: super().__init__(read_only=False) self._data: dict[str, Buffer] = {} def __eq__(self, other: object) -> bool: return other is self @property def supports_writes(self) -> bool: return True @property def supports_deletes(self) -> bool: return True @property def supports_listing(self) -> bool: return True async def get( self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> Buffer | None: try: value = self._data[key] except KeyError: return None start, stop = _normalize_byte_range_index(value, byte_range) return prototype.buffer.from_buffer(value[start:stop]) async def get_partial_values( self, prototype: BufferPrototype, key_ranges: Iterable[tuple[str, ByteRequest | None]], ) -> list[Buffer | None]: return [await self.get(key, prototype, byte_range) for key, byte_range in key_ranges] async def exists(self, key: str) -> bool: return key in self._data async def set(self, key: str, value: Buffer) -> None: self._check_writable() self._data[key] = value async def delete(self, key: str) -> None: self._check_writable() self._data.pop(key, None) async def list(self) -> AsyncIterator[str]: for key in list(self._data): yield key async def list_prefix(self, prefix: str) -> AsyncIterator[str]: for key in list(self._data): if key.startswith(prefix): yield key async def list_dir(self, prefix: str) -> AsyncIterator[str]: if prefix and not prefix.endswith("/"): prefix += "/" seen: set[str] = set() for key in list(self._data): if key.startswith(prefix): head = key.removeprefix(prefix).split("/")[0] if head not in seen: seen.add(head) yield head class SetOnlySyncStore(AsyncOnlyStore): """Implements `set_sync` but not `get_sync`/`delete_sync` (partial sync surface).""" def set_sync(self, key: str, value: Buffer) -> None: self._check_writable() self._data[key] = value @pytest.mark.parametrize( ("store_factory", "expect_sync_path"), [ (MemoryStore, True), (lambda: WrapperStore(MemoryStore()), True), (lambda: LatencyStore(MemoryStore()), True), (SetOnlySyncStore, False), (lambda: WrapperStore(AsyncOnlyStore()), False), ], ids=[ "full-sync", "wrapper-of-sync", "latency-wrapper-of-sync", "set-sync-only", "wrapper-of-async-only", ], ) def test_sync_io_capability_gates_fused_paths( store_factory: Callable[[], Store], expect_sync_path: bool ) -> None: """The fused pipeline takes the sync fast path iff the store satisfies `_store_supports_sync_io`, and every store round-trips correctly through full writes, partial (read-modify-write) writes, and all-fill (delete) writes — a store with a partial sync surface must get a clean async fallback, never a mid-batch error.""" store = store_factory() assert _store_supports_sync_io(store) is expect_sync_path calls = {"read_sync": 0, "write_sync": 0} orig_read_sync = FusedCodecPipeline.read_sync orig_write_sync = FusedCodecPipeline.write_sync def spy_read_sync(self: FusedCodecPipeline, *args: Any, **kwargs: Any) -> Any: calls["read_sync"] += 1 return orig_read_sync(self, *args, **kwargs) def spy_write_sync(self: FusedCodecPipeline, *args: Any, **kwargs: Any) -> Any: calls["write_sync"] += 1 return orig_write_sync(self, *args, **kwargs) with zarr_config.set({"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"}): arr = zarr.create_array( store=store, shape=(8,), chunks=(4,), dtype="uint8", compressors=None, fill_value=0, ) assert isinstance(arr._async_array.codec_pipeline, FusedCodecPipeline) with ( patch.object(FusedCodecPipeline, "read_sync", spy_read_sync), patch.object(FusedCodecPipeline, "write_sync", spy_write_sync), ): data = np.arange(8, dtype="uint8") arr[:] = data # complete-chunk writes arr[:3] = 7 # partial write -> read-modify-write needs get data[:3] = 7 np.testing.assert_array_equal(arr[:], data) arr[4:8] = 0 # all-fill chunk -> delete needed data[4:8] = 0 np.testing.assert_array_equal(arr[:], data) if expect_sync_path: assert calls["write_sync"] > 0, "sync-capable store did not take the sync write path" assert calls["read_sync"] > 0, "sync-capable store did not take the sync read path" else: assert calls["write_sync"] == 0, "non-sync store took the sync write path" assert calls["read_sync"] == 0, "non-sync store took the sync read path" zarr-developers-zarr-python-a2f2415/tests/test_group.py000066400000000000000000003036621527014137600234060ustar00rootroot00000000000000from __future__ import annotations import asyncio import contextlib import inspect import json import operator import pickle import re import time import warnings from typing import TYPE_CHECKING, Any, Literal, get_args import numpy as np import pytest from numcodecs import Blosc import zarr import zarr.api.asynchronous import zarr.api.synchronous import zarr.storage from zarr import Array, AsyncArray, AsyncGroup, Group from zarr.abc.store import Store from zarr.core import sync_group from zarr.core._info import GroupInfo from zarr.core.array_spec import ArrayConfig from zarr.core.buffer import default_buffer_prototype from zarr.core.config import config as zarr_config from zarr.core.dtype import Float64, Int32 from zarr.core.dtype.common import unpack_dtype_json from zarr.core.dtype.npy.int import UInt8 from zarr.core.group import ( GROUP_METADATA_KEYS, ConsolidatedMetadata, GroupMetadata, ImplicitGroupMarker, _build_metadata_v3, _get_roots, _parse_hierarchy_dict, create_hierarchy, create_nodes, create_rooted_hierarchy, get_node, ) from zarr.core.metadata.v3 import ArrayV3Metadata from zarr.core.sync import _collect_aiterator, sync from zarr.errors import ( ArrayNotFoundError, ContainsArrayError, ContainsGroupError, GroupNotFoundError, MetadataValidationError, NodeTypeValidationError, ZarrUserWarning, ) from zarr.storage import LocalStore, MemoryStore, StorePath, ZipStore from zarr.storage._common import make_store_path from zarr.storage._utils import _join_paths, normalize_path from zarr.testing.store import LatencyStore from .conftest import meta_from_array, parse_store if TYPE_CHECKING: import pathlib from collections.abc import Callable from zarr.core.array import ShardsLike from zarr.core.array_spec import ArrayConfigLike from zarr.core.buffer.core import Buffer from zarr.core.common import JSON, ChunksLike, ZarrFormat from zarr.core.dtype import ZDType, ZDTypeLike @pytest.fixture(params=["local", "memory", "zip"]) async def store(request: pytest.FixtureRequest, tmp_path: pathlib.Path) -> Store: result = await parse_store(request.param, str(tmp_path)) if not isinstance(result, Store): raise TypeError(f"Wrong store class returned by test fixture! got {result} instead") return result @pytest.fixture(params=[True, False]) def overwrite(request: pytest.FixtureRequest) -> bool: result = request.param if not isinstance(result, bool): raise TypeError("Wrong type returned by test fixture.") return result def test_group_init(store: Store, zarr_format: ZarrFormat) -> None: """ Test that initializing a group from an asyncgroup works. """ agroup = sync(AsyncGroup.from_store(store=store, zarr_format=zarr_format)) group = Group(agroup) assert group._async_group == agroup async def test_create_creates_parents(store: Store, zarr_format: ZarrFormat) -> None: # prepare a root node, with some data set await zarr.api.asynchronous.open_group( store=store, path="a", zarr_format=zarr_format, attributes={"key": "value"} ) objs = {x async for x in store.list()} if zarr_format == 2: assert objs == {".zgroup", ".zattrs", "a/.zgroup", "a/.zattrs"} else: assert objs == {"zarr.json", "a/zarr.json"} # test that root group node was created root = await zarr.api.asynchronous.open_group( store=store, ) agroup = await root.get_group("a") assert agroup.attrs == {"key": "value"} # create a child node with a couple intermediates await zarr.api.asynchronous.open_group(store=store, path="a/b/c/d", zarr_format=zarr_format) parts = ["a", "a/b", "a/b/c"] if zarr_format == 2: files = [".zattrs", ".zgroup"] else: files = ["zarr.json"] expected = [f"{part}/{file}" for file in files for part in parts] if zarr_format == 2: expected.extend([".zgroup", ".zattrs", "a/b/c/d/.zgroup", "a/b/c/d/.zattrs"]) else: expected.extend(["zarr.json", "a/b/c/d/zarr.json"]) expected = sorted(expected) result = sorted([x async for x in store.list_prefix("")]) assert result == expected paths = ["a", "a/b", "a/b/c"] for path in paths: g = await zarr.api.asynchronous.open_group(store=store, path=path) assert isinstance(g, AsyncGroup) if path == "a": # ensure we didn't overwrite the root attributes assert g.attrs == {"key": "value"} else: assert g.attrs == {} @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize("root_name", ["", "/", "a", "/a"]) @pytest.mark.parametrize("branch_name", ["foo", "/foo", "foo/bar", "/foo/bar"]) def test_group_name_properties( store: Store, zarr_format: ZarrFormat, root_name: str, branch_name: str ) -> None: """ Test that the path, name, and basename attributes of a group and its subgroups are consistent """ root = Group.from_store(store=StorePath(store=store, path=root_name), zarr_format=zarr_format) assert root.path == normalize_path(root_name) assert root.name == f"/{root.path}" assert root.basename == root.path branch = root.create_group(branch_name) if root.path == "": assert branch.path == normalize_path(branch_name) else: assert branch.path == "/".join([root.path, normalize_path(branch_name)]) assert branch.name == f"/{branch.path}" assert branch.basename == branch_name.split("/")[-1] @pytest.mark.parametrize("consolidated_metadata", [True, False]) def test_group_members(store: Store, zarr_format: ZarrFormat, consolidated_metadata: bool) -> None: """ Test that `Group.members` returns correct values, i.e. the arrays and groups (explicit and implicit) contained in that group. """ # group/ # subgroup/ # subsubgroup/ # subsubsubgroup # subarray path = "group" group = Group.from_store( store=store, zarr_format=zarr_format, ) members_expected: dict[str, Array | Group] = {} members_expected["subgroup"] = group.create_group("subgroup") # make a sub-sub-subgroup, to ensure that the children calculation doesn't go # too deep in the hierarchy subsubgroup = members_expected["subgroup"].create_group("subsubgroup") subsubsubgroup = subsubgroup.create_group("subsubsubgroup") members_expected["subarray"] = group.create_array( "subarray", shape=(100,), dtype="uint8", chunks=(10,), overwrite=True ) # add an extra object to the domain of the group. # the list of children should ignore this object. sync( store.set( f"{path}/extra_object-1", default_buffer_prototype().buffer.from_bytes(b"000000"), ) ) # add an extra object under a directory-like prefix in the domain of the group. # this creates a directory with a random key in it # this should not show up as a member sync( store.set( f"{path}/extra_directory/extra_object-2", default_buffer_prototype().buffer.from_bytes(b"000000"), ) ) # this warning shows up when extra objects show up in the hierarchy warn_context = pytest.warns( ZarrUserWarning, match=r"(?:Object at .* is not recognized as a component of a Zarr hierarchy.)|(?:Consolidated metadata is currently not part in the Zarr format 3 specification.)", ) if consolidated_metadata: if isinstance(store, ZipStore): with warn_context: with pytest.warns(UserWarning, match="Duplicate name: "): zarr.consolidate_metadata(store=store, zarr_format=zarr_format) else: with warn_context: zarr.consolidate_metadata(store=store, zarr_format=zarr_format) # now that we've consolidated the store, we shouldn't get the warnings from the unrecognized objects anymore # we use a nullcontext to handle these cases warn_context = contextlib.nullcontext() group = zarr.open_consolidated(store=store, zarr_format=zarr_format) with warn_context: members_observed = group.members() # members are not guaranteed to be ordered, so sort before comparing assert sorted(dict(members_observed)) == sorted(members_expected) # partial with warn_context: members_observed = group.members(max_depth=1) members_expected["subgroup/subsubgroup"] = subsubgroup # members are not guaranteed to be ordered, so sort before comparing assert sorted(dict(members_observed)) == sorted(members_expected) # total with warn_context: members_observed = group.members(max_depth=None) members_expected["subgroup/subsubgroup/subsubsubgroup"] = subsubsubgroup # members are not guaranteed to be ordered, so sort before comparing assert sorted(dict(members_observed)) == sorted(members_expected) with pytest.raises(ValueError, match="max_depth"): members_observed = group.members(max_depth=-1) def test_group(store: Store, zarr_format: ZarrFormat) -> None: """ Test basic Group routines. """ store_path = StorePath(store) agroup = AsyncGroup(metadata=GroupMetadata(zarr_format=zarr_format), store_path=store_path) group = Group(agroup) assert agroup.metadata is group.metadata assert agroup.store_path == group.store_path == store_path # create two groups foo = group.create_group("foo") bar = foo.create_group("bar", attributes={"baz": "qux"}) # create an array from the "bar" group data = np.arange(0, 4 * 4, dtype="uint16").reshape((4, 4)) arr = bar.create_array("baz", shape=data.shape, dtype=data.dtype, chunks=(2, 2), overwrite=True) arr[:] = data # check the array assert arr == bar["baz"] assert arr.shape == data.shape assert arr.dtype == data.dtype # TODO: update this once the array api settles down assert arr.chunks == (2, 2) bar2 = foo["bar"] assert dict(bar2.attrs) == {"baz": "qux"} # update a group's attributes if isinstance(store, ZipStore): with pytest.warns(UserWarning, match="Duplicate name: "): bar2.attrs.update({"name": "bar"}) else: bar2.attrs.update({"name": "bar"}) # bar.attrs was modified in-place assert dict(bar2.attrs) == {"baz": "qux", "name": "bar"} # and the attrs were modified in the store bar3 = foo["bar"] assert dict(bar3.attrs) == {"baz": "qux", "name": "bar"} def test_group_create(store: Store, overwrite: bool, zarr_format: ZarrFormat) -> None: """ Test that `Group.from_store` works as expected. """ attributes = {"foo": 100} group = Group.from_store( store, attributes=attributes, zarr_format=zarr_format, overwrite=overwrite ) assert group.attrs == attributes if not overwrite: with pytest.raises(ContainsGroupError): _ = Group.from_store(store, overwrite=overwrite, zarr_format=zarr_format) def test_group_open(store: Store, zarr_format: ZarrFormat, overwrite: bool) -> None: """ Test the `Group.open` method. """ spath = StorePath(store) # attempt to open a group that does not exist with pytest.raises(FileNotFoundError): Group.open(store) # create the group attrs = {"path": "foo"} group_created = Group.from_store( store, attributes=attrs, zarr_format=zarr_format, overwrite=overwrite ) assert group_created.attrs == attrs assert group_created.metadata.zarr_format == zarr_format assert group_created.store_path == spath # attempt to create a new group in place, to test overwrite new_attrs = {"path": "bar"} if not overwrite: with pytest.raises(ContainsGroupError): Group.from_store(store, attributes=attrs, zarr_format=zarr_format, overwrite=overwrite) else: if not store.supports_deletes: pytest.skip( "Store does not support deletes but `overwrite` is True, requiring deletes to override a group" ) group_created_again = Group.from_store( store, attributes=new_attrs, zarr_format=zarr_format, overwrite=overwrite ) assert group_created_again.attrs == new_attrs assert group_created_again.metadata.zarr_format == zarr_format assert group_created_again.store_path == spath @pytest.mark.parametrize("consolidated", [True, False]) def test_group_getitem(store: Store, zarr_format: ZarrFormat, consolidated: bool) -> None: """ Test the `Group.__getitem__` method. """ group = Group.from_store(store, zarr_format=zarr_format) subgroup = group.create_group(name="subgroup") subarray = group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") subsubarray = subgroup.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") if consolidated: if zarr_format == 3: with pytest.warns( # noqa: PT031 ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): if isinstance(store, ZipStore): with pytest.warns(UserWarning, match="Duplicate name: "): group = zarr.api.synchronous.consolidate_metadata( store=store, zarr_format=zarr_format ) else: group = zarr.api.synchronous.consolidate_metadata( store=store, zarr_format=zarr_format ) else: if isinstance(store, ZipStore): with pytest.warns(UserWarning, match="Duplicate name: "): group = zarr.api.synchronous.consolidate_metadata( store=store, zarr_format=zarr_format ) else: group = zarr.api.synchronous.consolidate_metadata( store=store, zarr_format=zarr_format ) # we're going to assume that `group.metadata` is correct, and reuse that to focus # on indexing in this test. Other tests verify the correctness of group.metadata object.__setattr__( subgroup.metadata, "consolidated_metadata", ConsolidatedMetadata( metadata={"subarray": group.metadata.consolidated_metadata.metadata["subarray"]} ), ) assert group["subgroup"] == subgroup assert group["subarray"] == subarray assert group["subgroup"]["subarray"] == subsubarray assert group["subgroup/subarray"] == subsubarray with pytest.raises(KeyError, match="nope"): group["nope"] with pytest.raises(KeyError, match="subarray/subsubarray"): group["subarray/subsubarray"] # Now test the mixed case if consolidated: object.__setattr__( group.metadata.consolidated_metadata.metadata["subgroup"], "consolidated_metadata", None, ) # test the implementation directly with pytest.raises(KeyError): group._async_group._getitem_consolidated( group.store_path, "subgroup/subarray", prefix="/" ) with pytest.raises(KeyError): # We've chosen to trust the consolidated metadata, which doesn't # contain this array group["subgroup/subarray"] with pytest.raises(KeyError, match="subarray/subsubarray"): group["subarray/subsubarray"] def test_group_get_with_default(store: Store, zarr_format: ZarrFormat) -> None: group = Group.from_store(store, zarr_format=zarr_format) # default behavior result = group.get("subgroup") assert result is None # custom default result = group.get("subgroup", 8) assert result == 8 # now with a group subgroup = group.require_group("subgroup") if isinstance(store, ZipStore): with pytest.warns(UserWarning, match="Duplicate name: "): subgroup.attrs["foo"] = "bar" else: subgroup.attrs["foo"] = "bar" result = group.get("subgroup", 8) assert result.attrs["foo"] == "bar" def test_group_get_array(store: Store, zarr_format: ZarrFormat) -> None: """ `Group.get_array` returns the array at the given path, for both direct child names and nested paths, and the result is statically typed as an Array. """ group = Group.from_store(store, zarr_format=zarr_format) subgroup = group.create_group(name="subgroup") subarray = group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") subsubarray = subgroup.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") observed = group.get_array("subarray") assert isinstance(observed, Array) assert observed == subarray assert group.get_array("subgroup/subarray") == subsubarray def test_group_get_array_missing(store: Store, zarr_format: ZarrFormat) -> None: """ `Group.get_array` raises `ArrayNotFoundError` when no node exists at the given path. """ group = Group.from_store(store, zarr_format=zarr_format) with pytest.raises(ArrayNotFoundError, match="No array found in store"): group.get_array("missing") def test_group_get_array_wrong_node_type(store: Store, zarr_format: ZarrFormat) -> None: """ `Group.get_array` raises `ContainsGroupError` when the node at the given path is a group rather than an array. """ group = Group.from_store(store, zarr_format=zarr_format) group.create_group(name="subgroup") with pytest.raises(ContainsGroupError, match="A group exists in store"): group.get_array("subgroup") def test_group_get_group(store: Store, zarr_format: ZarrFormat) -> None: """ `Group.get_group` returns the group at the given path, for both direct child names and nested paths, and the result is statically typed as a Group. """ group = Group.from_store(store, zarr_format=zarr_format) subgroup = group.create_group(name="subgroup") subsubgroup = subgroup.create_group(name="subsubgroup") observed = group.get_group("subgroup") assert isinstance(observed, Group) assert observed == subgroup assert group.get_group("subgroup/subsubgroup") == subsubgroup def test_group_get_group_missing(store: Store, zarr_format: ZarrFormat) -> None: """ `Group.get_group` raises `GroupNotFoundError` when no node exists at the given path. """ group = Group.from_store(store, zarr_format=zarr_format) with pytest.raises(GroupNotFoundError, match="No group found in store"): group.get_group("missing") def test_group_get_group_wrong_node_type(store: Store, zarr_format: ZarrFormat) -> None: """ `Group.get_group` raises `ContainsArrayError` when the node at the given path is an array rather than a group. """ group = Group.from_store(store, zarr_format=zarr_format) group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") with pytest.raises(ContainsArrayError, match="An array exists in store"): group.get_group("subarray") @pytest.mark.parametrize("consolidated", [True, False]) def test_group_delitem(store: Store, zarr_format: ZarrFormat, consolidated: bool) -> None: """ Test the `Group.__delitem__` method. """ if not store.supports_deletes: pytest.skip("store does not support deletes") group = Group.from_store(store, zarr_format=zarr_format) subgroup = group.create_group(name="subgroup") subarray = group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") if consolidated: if zarr_format == 3: with pytest.warns( # noqa: PT031 ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): if isinstance(store, ZipStore): with pytest.warns(UserWarning, match="Duplicate name: "): group = zarr.api.synchronous.consolidate_metadata( store=store, zarr_format=zarr_format ) else: group = zarr.api.synchronous.consolidate_metadata( store=store, zarr_format=zarr_format ) else: group = zarr.api.synchronous.consolidate_metadata(store=store, zarr_format=zarr_format) object.__setattr__( subgroup.metadata, "consolidated_metadata", ConsolidatedMetadata(metadata={}) ) assert group["subgroup"] == subgroup assert group["subarray"] == subarray del group["subgroup"] with pytest.raises(KeyError, match="subgroup"): group["subgroup"] del group["subarray"] with pytest.raises(KeyError, match="subarray"): group["subarray"] def test_group_iter(store: Store, zarr_format: ZarrFormat) -> None: """ Test the `Group.__iter__` method. """ group = Group.from_store(store, zarr_format=zarr_format) assert list(group) == [] def test_group_len(store: Store, zarr_format: ZarrFormat) -> None: """ Test the `Group.__len__` method. """ group = Group.from_store(store, zarr_format=zarr_format) assert len(group) == 0 def test_group_setitem(store: Store, zarr_format: ZarrFormat) -> None: """ Test the `Group.__setitem__` method. """ group = Group.from_store(store, zarr_format=zarr_format) arr = np.ones((2, 4)) group["key"] = arr assert list(group.array_keys()) == ["key"] assert group["key"].shape == (2, 4) np.testing.assert_array_equal(group["key"][:], arr) if store.supports_deletes: key = "key" else: # overwriting with another array requires deletes # for stores that don't support this, we just use a new key key = "key2" # overwrite with another array arr = np.zeros((3, 5)) group[key] = arr assert key in list(group.array_keys()) assert group[key].shape == (3, 5) np.testing.assert_array_equal(group[key], arr) def test_group_contains(store: Store, zarr_format: ZarrFormat) -> None: """ Test the `Group.__contains__` method """ group = Group.from_store(store, zarr_format=zarr_format) assert "foo" not in group _ = group.create_group(name="foo") assert "foo" in group @pytest.mark.parametrize("consolidate", [True, False]) def test_group_child_iterators(store: Store, zarr_format: ZarrFormat, consolidate: bool): group = Group.from_store(store, zarr_format=zarr_format) expected_group_keys = ["g0", "g1"] expected_group_values = [group.create_group(name=name) for name in expected_group_keys] expected_groups = list(zip(expected_group_keys, expected_group_values, strict=False)) fill_value = 3 dtype = UInt8() expected_group_values[0].create_group("subgroup") expected_group_values[0].create_array( "subarray", shape=(1,), dtype=dtype, fill_value=fill_value ) expected_array_keys = ["a0", "a1"] expected_array_values = [ group.create_array(name=name, shape=(1,), dtype=dtype, fill_value=fill_value) for name in expected_array_keys ] expected_arrays = list(zip(expected_array_keys, expected_array_values, strict=False)) if consolidate: if zarr_format == 3: with pytest.warns( # noqa: PT031 ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): if isinstance(store, ZipStore): with pytest.warns(UserWarning, match="Duplicate name: "): group = zarr.consolidate_metadata(store) else: group = zarr.consolidate_metadata(store) else: if isinstance(store, ZipStore): with pytest.warns(UserWarning, match="Duplicate name: "): group = zarr.consolidate_metadata(store) else: group = zarr.consolidate_metadata(store) if zarr_format == 2: metadata = { "subarray": { "attributes": {}, "dtype": unpack_dtype_json(dtype.to_json(zarr_format=zarr_format)), "fill_value": fill_value, "shape": (1,), "chunks": (1,), "order": "C", "filters": None, "compressor": Blosc(), "zarr_format": zarr_format, }, "subgroup": { "attributes": {}, "consolidated_metadata": { "metadata": {}, "kind": "inline", "must_understand": False, }, "node_type": "group", "zarr_format": zarr_format, }, } else: metadata = { "subarray": { "attributes": {}, "chunk_grid": { "configuration": {"chunk_shape": (1,)}, "name": "regular", }, "chunk_key_encoding": { "configuration": {"separator": "/"}, "name": "default", }, "codecs": ( {"configuration": {"endian": "little"}, "name": "bytes"}, {"configuration": {}, "name": "zstd"}, ), "data_type": unpack_dtype_json(dtype.to_json(zarr_format=zarr_format)), "fill_value": fill_value, "node_type": "array", "shape": (1,), "zarr_format": zarr_format, }, "subgroup": { "attributes": {}, "consolidated_metadata": { "metadata": {}, "kind": "inline", "must_understand": False, }, "node_type": "group", "zarr_format": zarr_format, }, } object.__setattr__( expected_group_values[0].metadata, "consolidated_metadata", ConsolidatedMetadata.from_dict( { "kind": "inline", "metadata": metadata, "must_understand": False, } ), ) object.__setattr__( expected_group_values[1].metadata, "consolidated_metadata", ConsolidatedMetadata(metadata={}), ) result = sorted(group.groups(), key=operator.itemgetter(0)) assert result == expected_groups assert sorted(group.groups(), key=operator.itemgetter(0)) == expected_groups assert sorted(group.group_keys()) == expected_group_keys assert sorted(group.group_values(), key=lambda x: x.name) == expected_group_values assert sorted(group.arrays(), key=operator.itemgetter(0)) == expected_arrays assert sorted(group.array_keys()) == expected_array_keys assert sorted(group.array_values(), key=lambda x: x.name) == expected_array_values def test_group_update_attributes(store: Store, zarr_format: ZarrFormat) -> None: """ Test the behavior of `Group.update_attributes` """ attrs = {"foo": 100} group = Group.from_store(store, zarr_format=zarr_format, attributes=attrs) assert group.attrs == attrs new_attrs = {"bar": 100} if isinstance(store, ZipStore): with pytest.warns(UserWarning, match="Duplicate name: "): new_group = group.update_attributes(new_attrs) else: new_group = group.update_attributes(new_attrs) updated_attrs = attrs.copy() updated_attrs.update(new_attrs) assert new_group.attrs == updated_attrs async def test_group_update_attributes_async(store: Store, zarr_format: ZarrFormat) -> None: """ Test the behavior of `Group.update_attributes_async` """ attrs = {"foo": 100} group = Group.from_store(store, zarr_format=zarr_format, attributes=attrs) assert group.attrs == attrs new_attrs = {"bar": 100} if isinstance(store, ZipStore): with pytest.warns(UserWarning, match="Duplicate name: "): new_group = await group.update_attributes_async(new_attrs) else: new_group = await group.update_attributes_async(new_attrs) assert new_group.attrs == new_attrs @pytest.mark.parametrize("name", ["a", "/a"]) @pytest.mark.parametrize( "chunks", [(2, 2), [2, 2], np.array([2, 2]), (np.int64(2), np.int64(2))], ids=["tuple", "list", "array", "numpy-scalars"], ) @pytest.mark.parametrize( "shards", [None, (4, 4), [4, 4], np.array([4, 4]), (np.int64(4), np.int64(4))], ids=["none", "tuple", "list", "array", "numpy-scalars"], ) def test_group_create_array( store: Store, zarr_format: ZarrFormat, overwrite: bool, name: str, chunks: ChunksLike, shards: ShardsLike | None, ) -> None: """ Test `Group.create_array` """ if zarr_format == 2 and shards is not None: pytest.skip("Zarr format 2 does not support sharding") group = Group.from_store(store, zarr_format=zarr_format) shape = (10, 10) dtype = "uint8" data = np.arange(np.prod(shape)).reshape(shape).astype(dtype) array = group.create_array(name=name, shape=shape, dtype=dtype, chunks=chunks, shards=shards) array[:] = data if not overwrite: with pytest.raises(ContainsArrayError): # noqa: PT012 a = group.create_array(name=name, shape=shape, dtype=dtype) a[:] = data assert array.path == normalize_path(name) assert array.name == f"/{array.path}" assert array.chunks == (2, 2) np.testing.assert_array_equal(array.shards, shards) assert array.shape == shape assert array.dtype == np.dtype(dtype) assert np.array_equal(array[:], data) @pytest.mark.parametrize("method", ["create_array", "create_group"]) def test_create_with_parent_array(store: Store, zarr_format: ZarrFormat, method: str): """Test that groups/arrays cannot be created under a parent array.""" # create a group with a child array group = Group.from_store(store, zarr_format=zarr_format) group.create_array(name="arr_1", shape=(10, 10), dtype="uint8") error_msg = r"A parent of .* is an array - only groups may have child nodes." if method == "create_array": with pytest.raises(ValueError, match=error_msg): group.create_array("arr_1/group_1/group_2/arr_2", shape=(10, 10), dtype="uint8") else: with pytest.raises(ValueError, match=error_msg): group.create_group("arr_1/group_1/group_2/group_3") LikeMethodName = Literal["zeros_like", "ones_like", "empty_like", "full_like"] @pytest.mark.parametrize("method_name", get_args(LikeMethodName)) @pytest.mark.parametrize("out_shape", ["keep", (10, 10)]) @pytest.mark.parametrize("out_chunks", ["keep", (10, 10)]) @pytest.mark.parametrize("out_dtype", ["keep", "int8"]) def test_group_array_like_creation( zarr_format: ZarrFormat, method_name: LikeMethodName, out_shape: Literal["keep"] | tuple[int, ...], out_chunks: Literal["keep"] | tuple[int, ...], out_dtype: str, ) -> None: """ Test Group.{zeros_like, ones_like, empty_like, full_like}, ensuring that we can override the shape, chunks, and dtype of the array-like object provided to these functions with appropriate keyword arguments """ ref_arr = zarr.ones(store={}, shape=(11, 12), dtype="uint8", chunks=(11, 12)) group = Group.from_store({}, zarr_format=zarr_format) kwargs = {} if method_name == "full_like": expect_fill = 4 kwargs["fill_value"] = expect_fill meth = group.full_like elif method_name == "zeros_like": expect_fill = 0 meth = group.zeros_like elif method_name == "ones_like": expect_fill = 1 meth = group.ones_like elif method_name == "empty_like": expect_fill = ref_arr.fill_value meth = group.empty_like else: raise AssertionError if out_shape != "keep": kwargs["shape"] = out_shape expect_shape = out_shape else: expect_shape = ref_arr.shape if out_chunks != "keep": kwargs["chunks"] = out_chunks expect_chunks = out_chunks else: expect_chunks = ref_arr.chunks if out_dtype != "keep": kwargs["dtype"] = out_dtype expect_dtype = out_dtype else: expect_dtype = ref_arr.dtype new_arr = meth(name="foo", data=ref_arr, **kwargs) assert new_arr.shape == expect_shape assert new_arr.chunks == expect_chunks assert new_arr.dtype == expect_dtype assert np.all(new_arr[:] == expect_fill) @pytest.mark.parametrize("group_format", [2, 3]) @pytest.mark.parametrize( "method_name", ["empty", "zeros", "ones", "full", "empty_like", "zeros_like", "ones_like", "full_like"], ) @pytest.mark.parametrize("source_format", [2, 3]) @pytest.mark.parametrize("pass_zarr_format", [False, True]) def test_group_array_helpers_use_group_format( group_format: ZarrFormat, method_name: str, source_format: ZarrFormat, pass_zarr_format: bool, ) -> None: """ Group.{empty, zeros, ones, full} and their *_like versions create an array in the zarr format of the group, which makes it a member of the group. For the *_like versions this holds whatever the format of the source array. """ group = Group.from_store(MemoryStore(), zarr_format=group_format) kwargs: dict[str, Any] = {"zarr_format": group_format} if pass_zarr_format else {} if method_name == "full": kwargs["fill_value"] = 3 if method_name.endswith("_like"): source = zarr.zeros(store={}, shape=(4,), dtype="int32", zarr_format=source_format) arr = getattr(group, method_name)(name="a", data=source, **kwargs) else: arr = getattr(group, method_name)(name="a", shape=(4,), **kwargs) assert arr.metadata.zarr_format == group_format assert list(group.array_keys()) == ["a"] def test_group_array_helpers_other_format() -> None: """ Asking a group helper for an array in a zarr format other than the group's raises. """ group = Group.from_store(MemoryStore(), zarr_format=2) with pytest.raises(ValueError, match="zarr_format=3 array in a zarr_format=2 group"): group.zeros(name="a", shape=(4,), zarr_format=3) def test_group_array_creation( store: Store, zarr_format: ZarrFormat, ): group = Group.from_store(store, zarr_format=zarr_format) shape = (10, 10) empty_array = group.empty(name="empty", shape=shape) assert isinstance(empty_array, Array) assert empty_array.fill_value == 0 assert empty_array.shape == shape assert empty_array.store_path.store == store assert empty_array.store_path.path == "empty" empty_like_array = group.empty_like(name="empty_like", data=empty_array) assert isinstance(empty_like_array, Array) assert empty_like_array.fill_value == 0 assert empty_like_array.shape == shape assert empty_like_array.store_path.store == store empty_array_bool = group.empty(name="empty_bool", shape=shape, dtype=np.dtype("bool")) assert isinstance(empty_array_bool, Array) assert not empty_array_bool.fill_value assert empty_array_bool.shape == shape assert empty_array_bool.store_path.store == store empty_like_array_bool = group.empty_like(name="empty_like_bool", data=empty_array_bool) assert isinstance(empty_like_array_bool, Array) assert not empty_like_array_bool.fill_value assert empty_like_array_bool.shape == shape assert empty_like_array_bool.store_path.store == store zeros_array = group.zeros(name="zeros", shape=shape) assert isinstance(zeros_array, Array) assert zeros_array.fill_value == 0 assert zeros_array.shape == shape assert zeros_array.store_path.store == store zeros_like_array = group.zeros_like(name="zeros_like", data=zeros_array) assert isinstance(zeros_like_array, Array) assert zeros_like_array.fill_value == 0 assert zeros_like_array.shape == shape assert zeros_like_array.store_path.store == store ones_array = group.ones(name="ones", shape=shape) assert isinstance(ones_array, Array) assert ones_array.fill_value == 1 assert ones_array.shape == shape assert ones_array.store_path.store == store ones_like_array = group.ones_like(name="ones_like", data=ones_array) assert isinstance(ones_like_array, Array) assert ones_like_array.fill_value == 1 assert ones_like_array.shape == shape assert ones_like_array.store_path.store == store full_array = group.full(name="full", shape=shape, fill_value=42) assert isinstance(full_array, Array) assert full_array.fill_value == 42 assert full_array.shape == shape assert full_array.store_path.store == store full_like_array = group.full_like(name="full_like", data=full_array, fill_value=43) assert isinstance(full_like_array, Array) assert full_like_array.fill_value == 43 assert full_like_array.shape == shape assert full_like_array.store_path.store == store @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) @pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.parametrize("overwrite", [True, False]) @pytest.mark.parametrize("extant_node", ["array", "group"]) def test_group_creation_existing_node( store: Store, zarr_format: ZarrFormat, overwrite: bool, extant_node: Literal["array", "group"], ) -> None: """ Check that an existing array or group is handled as expected during group creation. """ spath = StorePath(store) group = Group.from_store(spath, zarr_format=zarr_format) expected_exception: type[ContainsArrayError | ContainsGroupError] attributes: dict[str, JSON] = {"old": True} if extant_node == "array": expected_exception = ContainsArrayError _ = group.create_array("extant", shape=(10,), dtype="uint8", attributes=attributes) elif extant_node == "group": expected_exception = ContainsGroupError _ = group.create_group("extant", attributes=attributes) else: raise AssertionError new_attributes = {"new": True} if overwrite: if not store.supports_deletes: pytest.skip("store does not support deletes but overwrite is True") node_new = Group.from_store( spath / "extant", attributes=new_attributes, zarr_format=zarr_format, overwrite=overwrite, ) assert node_new.attrs == new_attributes else: with pytest.raises(expected_exception): node_new = Group.from_store( spath / "extant", attributes=new_attributes, zarr_format=zarr_format, overwrite=overwrite, ) async def test_asyncgroup_create( store: Store, overwrite: bool, zarr_format: ZarrFormat, ) -> None: """ Test that `AsyncGroup.from_store` works as expected. """ spath = StorePath(store=store) attributes = {"foo": 100} agroup = await AsyncGroup.from_store( store, attributes=attributes, overwrite=overwrite, zarr_format=zarr_format, ) assert agroup.metadata == GroupMetadata(zarr_format=zarr_format, attributes=attributes) assert agroup.store_path == await make_store_path(store) if not overwrite: with pytest.raises(ContainsGroupError): agroup = await AsyncGroup.from_store( spath, attributes=attributes, overwrite=overwrite, zarr_format=zarr_format, ) # create an array at our target path collision_name = "foo" _ = await zarr.api.asynchronous.create_array( spath / collision_name, shape=(10,), dtype="uint8", zarr_format=zarr_format ) with pytest.raises(ContainsArrayError): _ = await AsyncGroup.from_store( StorePath(store=store) / collision_name, attributes=attributes, overwrite=overwrite, zarr_format=zarr_format, ) async def test_asyncgroup_attrs(store: Store, zarr_format: ZarrFormat) -> None: attributes = {"foo": 100} agroup = await AsyncGroup.from_store(store, zarr_format=zarr_format, attributes=attributes) assert agroup.attrs == agroup.metadata.attributes == attributes async def test_asyncgroup_open( store: Store, zarr_format: ZarrFormat, ) -> None: """ Create an `AsyncGroup`, then ensure that we can open it using `AsyncGroup.open` """ attributes = {"foo": 100} group_w = await AsyncGroup.from_store( store=store, attributes=attributes, overwrite=False, zarr_format=zarr_format, ) group_r = await AsyncGroup.open(store=store, zarr_format=zarr_format) assert group_w.attrs == group_w.attrs == attributes assert group_w == group_r async def test_asyncgroup_open_wrong_format( store: Store, zarr_format: ZarrFormat, ) -> None: _ = await AsyncGroup.from_store(store=store, overwrite=False, zarr_format=zarr_format) zarr_format_wrong: ZarrFormat # try opening with the wrong zarr format if zarr_format == 3: zarr_format_wrong = 2 elif zarr_format == 2: zarr_format_wrong = 3 else: raise AssertionError with pytest.raises(FileNotFoundError): await AsyncGroup.open(store=store, zarr_format=zarr_format_wrong) def test_group_metadata_from_dict_wrong_node_type_raises() -> None: """ A metadata document whose node_type is not 'group' cannot become GroupMetadata. """ with pytest.raises(NodeTypeValidationError, match="node_type"): GroupMetadata.from_dict({"zarr_format": 3, "node_type": "array"}) # todo: replace the dict[str, Any] type with something a bit more specific # should this be async? @pytest.mark.parametrize( "data", [ {"zarr_format": 3, "node_type": "group", "attributes": {"foo": 100}}, {"zarr_format": 2, "attributes": {"foo": 100}}, ], ) def test_asyncgroup_from_dict(store: Store, data: dict[str, Any]) -> None: """ Test that we can create an AsyncGroup from a dict """ path = "test" store_path = StorePath(store=store, path=path) group = AsyncGroup.from_dict(store_path, data=data) assert group.metadata.zarr_format == data["zarr_format"] assert group.metadata.attributes == data["attributes"] # todo: replace this with a declarative API where we model a full hierarchy async def test_asyncgroup_getitem(store: Store, zarr_format: ZarrFormat) -> None: """ Create an `AsyncGroup`, then create members of that group, and ensure that we can access those members via the `AsyncGroup.getitem` method. """ agroup = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) array_name = "sub_array" sub_array = await agroup.create_array(name=array_name, shape=(10,), dtype="uint8", chunks=(2,)) assert await agroup.getitem(array_name) == sub_array sub_group_path = "sub_group" sub_group = await agroup.create_group(sub_group_path, attributes={"foo": 100}) assert await agroup.getitem(sub_group_path) == sub_group # check that asking for a nonexistent key raises KeyError with pytest.raises(KeyError, match="foo"): await agroup.getitem("foo") async def test_asyncgroup_delitem(store: Store, zarr_format: ZarrFormat) -> None: if not store.supports_deletes: pytest.skip("store does not support deletes") agroup = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) array_name = "sub_array" _ = await agroup.create_array( name=array_name, shape=(10,), dtype="uint8", chunks=(2,), attributes={"foo": 100}, ) await agroup.delitem(array_name) # todo: clean up the code duplication here if zarr_format == 2: assert not await agroup.store_path.store.exists(f"{array_name}/.zarray") assert not await agroup.store_path.store.exists(f"{array_name}/.zattrs") elif zarr_format == 3: assert not await agroup.store_path.store.exists(f"{array_name}/zarr.json") else: raise AssertionError sub_group_path = "sub_group" _ = await agroup.create_group(sub_group_path, attributes={"foo": 100}) await agroup.delitem(sub_group_path) if zarr_format == 2: assert not await agroup.store_path.store.exists(f"{array_name}/.zgroup") assert not await agroup.store_path.store.exists(f"{array_name}/.zattrs") elif zarr_format == 3: assert not await agroup.store_path.store.exists(f"{array_name}/zarr.json") else: raise AssertionError @pytest.mark.parametrize("name", ["a", "/a"]) async def test_asyncgroup_create_group( store: Store, name: str, zarr_format: ZarrFormat, ) -> None: agroup = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) attributes = {"foo": 999} subgroup = await agroup.create_group(name=name, attributes=attributes) assert isinstance(subgroup, AsyncGroup) assert subgroup.path == normalize_path(name) assert subgroup.name == f"/{subgroup.path}" assert subgroup.attrs == attributes assert subgroup.store_path.path == subgroup.path assert subgroup.store_path.store == store assert subgroup.metadata.zarr_format == zarr_format async def test_asyncgroup_create_array( store: Store, zarr_format: ZarrFormat, overwrite: bool ) -> None: """ Test that the AsyncGroup.create_array method works correctly. We ensure that array properties specified in create_array are present on the resulting array. """ agroup = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) if not overwrite: with pytest.raises(ContainsGroupError): agroup = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) shape = (10,) dtype = "uint8" chunk_shape = (4,) attributes: dict[str, JSON] = {"foo": 100} sub_node_path = "sub_array" subnode = await agroup.create_array( name=sub_node_path, shape=shape, dtype=dtype, chunks=chunk_shape, attributes=attributes, ) assert isinstance(subnode, AsyncArray) assert subnode.attrs == attributes assert subnode.store_path.path == sub_node_path assert subnode.store_path.store == store assert subnode.shape == shape assert subnode.dtype == dtype assert subnode._chunk_grid.chunk_shape == chunk_shape assert subnode.metadata.zarr_format == zarr_format async def test_asyncgroup_update_attributes(store: Store, zarr_format: ZarrFormat) -> None: """ Test that the AsyncGroup.update_attributes method works correctly. """ attributes_old = {"foo": 10} attributes_new = {"baz": "new"} agroup = await AsyncGroup.from_store( store=store, zarr_format=zarr_format, attributes=attributes_old ) if isinstance(store, ZipStore): with pytest.warns(UserWarning, match="Duplicate name"): agroup_new_attributes = await agroup.update_attributes(attributes_new) else: agroup_new_attributes = await agroup.update_attributes(attributes_new) attributes_updated = attributes_old.copy() attributes_updated.update(attributes_new) assert agroup_new_attributes.attrs == attributes_updated @pytest.mark.parametrize("store", ["local"], indirect=["store"]) @pytest.mark.parametrize("zarr_format", [2, 3]) async def test_serializable_async_group(store: LocalStore, zarr_format: ZarrFormat) -> None: expected = await AsyncGroup.from_store( store=store, attributes={"foo": 999}, zarr_format=zarr_format ) p = pickle.dumps(expected) actual = pickle.loads(p) assert actual == expected @pytest.mark.parametrize("store", ["local"], indirect=["store"]) @pytest.mark.parametrize("zarr_format", [2, 3]) def test_serializable_sync_group(store: LocalStore, zarr_format: ZarrFormat) -> None: expected = Group.from_store(store=store, attributes={"foo": 999}, zarr_format=zarr_format) p = pickle.dumps(expected) actual = pickle.loads(p) assert actual == expected @pytest.mark.parametrize("consolidated_metadata", [True, False]) async def test_group_members_async(store: Store, consolidated_metadata: bool) -> None: group = await AsyncGroup.from_store( store=store, ) a0 = await group.create_array("a0", shape=(1,), dtype="uint8") g0 = await group.create_group("g0") a1 = await g0.create_array("a1", shape=(1,), dtype="uint8") g1 = await g0.create_group("g1") a2 = await g1.create_array("a2", shape=(1,), dtype="uint8") g2 = await g1.create_group("g2") # immediate children children = sorted([x async for x in group.members()], key=operator.itemgetter(0)) assert children == [ ("a0", a0), ("g0", g0), ] nmembers = await group.nmembers() assert nmembers == 2 # partial children = sorted([x async for x in group.members(max_depth=1)], key=operator.itemgetter(0)) expected = [ ("a0", a0), ("g0", g0), ("g0/a1", a1), ("g0/g1", g1), ] assert children == expected nmembers = await group.nmembers(max_depth=1) assert nmembers == 4 # all children all_children = sorted( [x async for x in group.members(max_depth=None)], key=operator.itemgetter(0) ) expected = [ ("a0", a0), ("g0", g0), ("g0/a1", a1), ("g0/g1", g1), ("g0/g1/a2", a2), ("g0/g1/g2", g2), ] assert all_children == expected if consolidated_metadata: with pytest.warns( # noqa: PT031 ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): if isinstance(store, ZipStore): with pytest.warns(UserWarning, match="Duplicate name"): await zarr.api.asynchronous.consolidate_metadata(store=store) else: await zarr.api.asynchronous.consolidate_metadata(store=store) group = await zarr.api.asynchronous.open_group(store=store) nmembers = await group.nmembers(max_depth=None) assert nmembers == 6 with pytest.raises(ValueError, match="max_depth"): [x async for x in group.members(max_depth=-1)] if consolidated_metadata: # test for mixed known and unknown metadata. # For now, we trust the consolidated metadata. object.__setattr__( group.metadata.consolidated_metadata.metadata["g0"].consolidated_metadata.metadata[ "g1" ], "consolidated_metadata", None, ) # test depth=0 nmembers = await group.nmembers(max_depth=0) assert nmembers == 2 # test depth=1 nmembers = await group.nmembers(max_depth=1) assert nmembers == 4 # test depth=None all_children = sorted( [x async for x in group.members(max_depth=None)], key=operator.itemgetter(0) ) assert len(all_children) == 4 nmembers = await group.nmembers(max_depth=None) assert nmembers == 4 # test depth<0 with pytest.raises(ValueError, match="max_depth"): await group.nmembers(max_depth=-1) async def test_require_group(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: root = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) # create foo group _ = await root.create_group("foo", attributes={"foo": 100}) # test that we can get the group using require_group foo_group = await root.require_group("foo") assert foo_group.attrs == {"foo": 100} # test that we can get the group using require_group and overwrite=True if store.supports_deletes: foo_group = await root.require_group("foo", overwrite=True) assert foo_group.attrs == {} _ = await foo_group.create_array( "bar", shape=(10,), dtype="uint8", chunks=(2,), attributes={"foo": 100} ) # test that overwriting a group w/ children fails # TODO: figure out why ensure_no_existing_node is not catching the foo.bar array # # with pytest.raises(ContainsArrayError): # await root.require_group("foo", overwrite=True) # test that requiring a group where an array is fails with pytest.raises(TypeError, match="Incompatible object"): await foo_group.require_group("bar") async def test_require_groups(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: root = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) # create foo group _ = await root.create_group("foo", attributes={"foo": 100}) # create bar group _ = await root.create_group("bar", attributes={"bar": 200}) foo_group, bar_group = await root.require_groups("foo", "bar") assert foo_group.attrs == {"foo": 100} assert bar_group.attrs == {"bar": 200} # get a mix of existing and new groups foo_group, spam_group = await root.require_groups("foo", "spam") assert foo_group.attrs == {"foo": 100} assert spam_group.attrs == {} # no names no_group = await root.require_groups() assert no_group == () async def test_require_array(store: Store, zarr_format: ZarrFormat) -> None: root = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) foo1 = await root.require_array("foo", shape=(10,), dtype="i8", attributes={"foo": 101}) assert foo1.attrs == {"foo": 101} foo2 = await root.require_array("foo", shape=(10,), dtype="i8") assert foo2.attrs == {"foo": 101} # exact = False _ = await root.require_array("foo", shape=10, dtype="f8") # errors w/ exact True with pytest.raises(TypeError, match="Incompatible dtype"): await root.require_array("foo", shape=(10,), dtype="f8", exact=True) with pytest.raises(TypeError, match="Incompatible shape"): await root.require_array("foo", shape=(100, 100), dtype="i8") with pytest.raises(TypeError, match="Incompatible dtype"): await root.require_array("foo", shape=(10,), dtype="f4") _ = await root.create_group("bar") with pytest.raises(TypeError, match="Incompatible object"): await root.require_array("bar", shape=(10,), dtype="int8") @pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.parametrize("api", ["sync", "async"]) @pytest.mark.parametrize("exists", [True, False]) @pytest.mark.parametrize( ("config", "expected"), [ (None, ArrayConfig.from_dict({})), ( {"read_missing_chunks": False}, ArrayConfig.from_dict({"read_missing_chunks": False}), ), ( {"order": "F", "write_empty_chunks": True}, ArrayConfig.from_dict({"order": "F", "write_empty_chunks": True}), ), ( ArrayConfig.from_dict({"read_missing_chunks": False}), ArrayConfig.from_dict({"read_missing_chunks": False}), ), ], ) async def test_require_array_config( zarr_format: ZarrFormat, api: Literal["sync", "async"], exists: bool, config: ArrayConfigLike | None, expected: ArrayConfig, ) -> None: """ require_array applies the config argument whether it creates the array or returns an existing one, with keys missing from a partial config taken from the global defaults. """ group = Group.from_store(MemoryStore(), zarr_format=zarr_format) if exists: group.create_array("a", shape=(4,), dtype="uint8") if api == "sync": observed = group.require_array("a", shape=(4,), dtype="uint8", config=config).config else: observed = ( await group._async_group.require_array("a", shape=(4,), dtype="uint8", config=config) ).config assert observed == expected @pytest.mark.parametrize("exists", [True, False]) def test_require_array_unknown_config_key(exists: bool) -> None: """ Requiring an array with a config containing an unknown key raises TypeError, whether the array exists or is created. """ group = Group.from_store(MemoryStore()) if exists: group.create_array("a", shape=(4,), dtype="uint8") with pytest.raises(TypeError, match=r"Unknown array config keys: \['nope'\]"): group.require_array("a", shape=(4,), dtype="uint8", config={"nope": 1}) @pytest.mark.parametrize("exists", [True, False]) def test_require_array_invalid_config_value(exists: bool) -> None: """ Requiring an array with a config containing an invalid value raises ValueError, whether the array exists or is created. """ group = Group.from_store(MemoryStore()) if exists: group.create_array("a", shape=(4,), dtype="uint8") with pytest.raises(ValueError, match="Expected instance of bool"): group.require_array("a", shape=(4,), dtype="uint8", config={"read_missing_chunks": "yes"}) @pytest.mark.parametrize( ("dtype", "expected"), [ (Int32(), Int32()), (np.dtype("int32"), Int32()), ("int32", Int32()), (None, Float64()), ], ids=["zdtype", "numpy", "str", "none"], ) async def test_require_array_zdtype( store: Store, zarr_format: ZarrFormat, dtype: ZDTypeLike | None, expected: ZDType[Any, Any] ) -> None: """An existing array can be required with a ZDType, as well as a string, a NumPy dtype, or None. See https://github.com/zarr-developers/zarr-python/issues/3377 """ root = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) await root.create_array("foo", shape=(10,), dtype=expected) foo = await root.require_array("foo", shape=(10,), dtype=dtype, exact=True) assert foo._zdtype == expected @pytest.mark.parametrize("consolidate", [True, False]) async def test_members_name(store: Store, consolidate: bool, zarr_format: ZarrFormat): group = Group.from_store(store=store, zarr_format=zarr_format) a = group.create_group(name="a") a.create_array("array", shape=(1,), dtype="uint8") b = a.create_group(name="b") b.create_array("array", shape=(1,), dtype="uint8") if consolidate: if isinstance(store, ZipStore): with pytest.warns(UserWarning, match="Duplicate name"): # noqa: PT031 if zarr_format == 3: with pytest.warns( ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): group = zarr.api.synchronous.consolidate_metadata(store) else: group = zarr.api.synchronous.consolidate_metadata(store) else: if zarr_format == 3: with pytest.warns( ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): group = zarr.api.synchronous.consolidate_metadata(store) else: group = zarr.api.synchronous.consolidate_metadata(store) result = group["a"]["b"] assert result.name == "/a/b" paths = sorted(x.name for _, x in group.members(max_depth=None)) expected = ["/a", "/a/array", "/a/b", "/a/b/array"] assert paths == expected # regression test for https://github.com/zarr-developers/zarr-python/pull/2356 g = zarr.open_group(store, use_consolidated=False) with warnings.catch_warnings(): warnings.simplefilter("error") assert list(g) async def test_open_mutable_mapping(): group = await zarr.api.asynchronous.open_group( store={}, ) assert isinstance(group.store_path.store, MemoryStore) def test_open_mutable_mapping_sync(): group = zarr.open_group( store={}, ) assert isinstance(group.store_path.store, MemoryStore) async def test_open_ambiguous_node(): zarr_json_bytes = default_buffer_prototype().buffer.from_bytes( json.dumps({"zarr_format": 3, "node_type": "group"}).encode("utf-8") ) zgroup_bytes = default_buffer_prototype().buffer.from_bytes( json.dumps({"zarr_format": 2}).encode("utf-8") ) store: dict[str, Buffer] = {"zarr.json": zarr_json_bytes, ".zgroup": zgroup_bytes} with pytest.warns( ZarrUserWarning, match=r"Both zarr\.json \(Zarr format 3\) and \.zgroup \(Zarr format 2\) metadata objects exist at", ): await AsyncGroup.open(store, zarr_format=None) class TestConsolidated: async def test_group_getitem_consolidated(self, store: Store) -> None: root = await AsyncGroup.from_store(store=store) # Set up the test structure with # / # g0/ # group /g0 # g1/ # group /g0/g1 # g2/ # group /g0/g1/g2 # x1/ # group /x0 # x2/ # group /x0/x1 # x3/ # group /x0/x1/x2 g0 = await root.create_group("g0") g1 = await g0.create_group("g1") await g1.create_group("g2") x0 = await root.create_group("x0") x1 = await x0.create_group("x1") await x1.create_group("x2") with pytest.warns( # noqa: PT031 ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): if isinstance(store, ZipStore): with pytest.warns(UserWarning, match="Duplicate name"): await zarr.api.asynchronous.consolidate_metadata(store) else: await zarr.api.asynchronous.consolidate_metadata(store) # On disk, we've consolidated all the metadata in the root zarr.json group = await zarr.api.asynchronous.open(store=store) rg0 = await group.get_group("g0") expected = ConsolidatedMetadata( metadata={ "g1": GroupMetadata( attributes={}, zarr_format=3, consolidated_metadata=ConsolidatedMetadata( metadata={ "g2": GroupMetadata( attributes={}, zarr_format=3, consolidated_metadata=ConsolidatedMetadata(metadata={}), ) } ), ), } ) assert rg0.metadata.consolidated_metadata == expected rg1 = await rg0.get_group("g1") assert rg1.metadata.consolidated_metadata == expected.metadata["g1"].consolidated_metadata rg2 = await rg1.get_group("g2") assert rg2.metadata.consolidated_metadata == ConsolidatedMetadata(metadata={}) async def test_group_delitem_consolidated(self, store: Store, zarr_format: ZarrFormat) -> None: """Deleting a member removes it from the consolidated metadata in memory and in every document that stores it, so the group reopens without it.""" if isinstance(store, ZipStore): raise pytest.skip("Not implemented") root = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) # Set up the test structure with # / # g0/ # group /g0 # g1/ # group /g0/g1 # g2/ # group /g0/g1/g2 # data # array # x1/ # group /x0 # x2/ # group /x0/x1 # x3/ # group /x0/x1/x2 # data # array g0 = await root.create_group("g0") g1 = await g0.create_group("g1") g2 = await g1.create_group("g2") await g2.create_array("data", shape=(1,), dtype="uint8") x0 = await root.create_group("x0") x1 = await x0.create_group("x1") x2 = await x1.create_group("x2") await x2.create_array("data", shape=(1,), dtype="uint8") with warnings.catch_warnings(): warnings.filterwarnings( "ignore", "Consolidated metadata is currently not part", ZarrUserWarning ) await zarr.api.asynchronous.consolidate_metadata(store) group = await zarr.api.asynchronous.open_consolidated(store=store, zarr_format=zarr_format) assert group.metadata.consolidated_metadata is not None assert sorted(group.metadata.consolidated_metadata.metadata) == ["g0", "x0"] await group.delitem("g0") assert sorted(group.metadata.consolidated_metadata.metadata) == ["x0"] reopened = await zarr.api.asynchronous.open_consolidated( store=store, zarr_format=zarr_format ) assert reopened.metadata.consolidated_metadata is not None assert sorted(reopened.metadata.consolidated_metadata.metadata) == ["x0"] def test_group_delitem_consolidated_aliased(self, store: Store) -> None: """A subgroup read from its parent's consolidated metadata shares it, so a member deleted through the subgroup is gone through the parent too.""" if isinstance(store, ZipStore): raise pytest.skip("Not implemented") root = zarr.create_group(store) root.create_group("sub").create_array("b", shape=(4,), chunks=(2,), dtype="i4") with warnings.catch_warnings(): warnings.filterwarnings( "ignore", "Consolidated metadata is currently not part", ZarrUserWarning ) zarr.consolidate_metadata(store) group = zarr.open_group(store, mode="r+", use_consolidated=True) sub = group["sub"] assert isinstance(sub, Group) del sub["b"] assert "b" not in sub assert "b" not in group["sub"] with pytest.raises(KeyError): group["sub/b"] async def test_group_delitem_consolidated_concurrent(self, zarr_format: ZarrFormat) -> None: """Concurrent deletions through one handle each store the deletions made before them, so the stored consolidated metadata lists what the handle lists. Here the deletions finish in the reverse of the order they started in.""" delays = [0.03, 0.02, 0.01] class SlowDeletes(LatencyStore): async def delete_dir(self, prefix: str) -> None: await asyncio.sleep(delays.pop(0)) await super().delete_dir(prefix) store = SlowDeletes(MemoryStore()) root = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) for name in "abcd": await root.create_array(name, shape=(2,), dtype="i4") with warnings.catch_warnings(): warnings.filterwarnings( "ignore", "Consolidated metadata is currently not part", ZarrUserWarning ) await zarr.api.asynchronous.consolidate_metadata(store) group = await zarr.api.asynchronous.open_consolidated(store=store, zarr_format=zarr_format) await asyncio.gather(*(group.delitem(name) for name in "abc")) reopened = await zarr.api.asynchronous.open_consolidated( store=store, zarr_format=zarr_format ) assert group.metadata.consolidated_metadata is not None assert reopened.metadata.consolidated_metadata is not None assert list(group.metadata.consolidated_metadata.metadata) == ["d"] assert list(reopened.metadata.consolidated_metadata.metadata) == ["d"] def test_open_consolidated_raises(self, store: Store) -> None: if isinstance(store, ZipStore): raise pytest.skip("Not implemented") root = Group.from_store(store=store) # fine to be missing by default zarr.open_group(store=store) with pytest.raises(ValueError, match="Consolidated metadata requested."): zarr.open_group(store=store, use_consolidated=True) # Now create consolidated metadata... root.create_group("g0") with pytest.warns( ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): zarr.consolidate_metadata(store) # and explicitly ignore it. group = zarr.open_group(store=store, use_consolidated=False) assert group.metadata.consolidated_metadata is None async def test_open_consolidated_raises_async(self, store: Store) -> None: if isinstance(store, ZipStore): raise pytest.skip("Not implemented") root = await AsyncGroup.from_store(store=store) # fine to be missing by default await zarr.api.asynchronous.open_group(store=store) with pytest.raises(ValueError, match="Consolidated metadata requested."): await zarr.api.asynchronous.open_group(store=store, use_consolidated=True) # Now create consolidated metadata... await root.create_group("g0") with pytest.warns( ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): await zarr.api.asynchronous.consolidate_metadata(store) # and explicitly ignore it. group = await zarr.api.asynchronous.open_group(store=store, use_consolidated=False) assert group.metadata.consolidated_metadata is None class TestGroupMetadata: def test_from_dict_extra_fields(self): data = { "attributes": {"key": "value"}, "_nczarr_superblock": {"version": "2.0.0"}, # a key named like the `extra_fields` parameter is dropped, not passed to __init__ "extra_fields": {"my_extension": {"must_understand": False}}, "zarr_format": 2, } result = GroupMetadata.from_dict(data) expected = GroupMetadata(attributes={"key": "value"}, zarr_format=2) assert result == expected @pytest.mark.parametrize( "extra_fields", [ {}, {"my_extension": {"must_understand": False, "version": 1}}, {"a": {"must_understand": False}, "b": {"must_understand": False, "data": [1, 2]}}, {"extra_fields": {"must_understand": False}}, ], ) def test_v3_extra_fields_round_trip(self, extra_fields: dict[str, Any]) -> None: # https://github.com/zarr-developers/zarr-python/issues/3523 data = {"zarr_format": 3, "node_type": "group", "attributes": {}, **extra_fields} expected = GroupMetadata(extra_fields=extra_fields) result = GroupMetadata.from_dict(data) assert result == expected assert result.extra_fields == extra_fields assert result.to_dict() == data @pytest.mark.parametrize("value", [{"must_understand": True}, {}, "not an object", 1]) def test_from_dict_v3_disallowed_extra_fields(self, value: object) -> None: data = {"zarr_format": 3, "node_type": "group", "my_extension": value} with pytest.raises(MetadataValidationError, match="my_extension"): GroupMetadata.from_dict(data) @pytest.mark.parametrize("value", [{"must_understand": True}, {}, "not an object", 1]) def test_init_disallowed_extra_fields(self, value: object) -> None: with pytest.raises(MetadataValidationError, match="my_extension"): GroupMetadata(extra_fields={"my_extension": value}) # type: ignore[dict-item] @pytest.mark.parametrize("key", sorted(GROUP_METADATA_KEYS)) def test_init_extra_fields_collision(self, key: str) -> None: extra_fields: dict[str, Any] = {key: {"must_understand": False}} with pytest.raises(ValueError, match="collide with keys reserved"): GroupMetadata(extra_fields=extra_fields) def test_init_extra_fields_v2(self) -> None: with pytest.raises(ValueError, match="Zarr format 2 group metadata does not support"): GroupMetadata(zarr_format=2, extra_fields={"my_extension": {"must_understand": False}}) @pytest.mark.filterwarnings("ignore:Consolidated metadata is:zarr.errors.ZarrUserWarning") def test_extra_fields_survive_rewrite(self) -> None: extension = {"must_understand": False} doc = {"zarr_format": 3, "node_type": "group", "attributes": {}, "ext": extension} store = MemoryStore() for key in ("zarr.json", "child/zarr.json"): buffer = default_buffer_prototype().buffer.from_bytes(json.dumps(doc).encode()) sync(store.set(key, buffer)) group = zarr.open_group(store=store, mode="r+") group["child"].attrs["key"] = "value" zarr.consolidate_metadata(store) written = json.loads( sync(store.get("child/zarr.json", prototype=default_buffer_prototype())).to_bytes() ) assert written == {**doc, "attributes": {"key": "value"}} consolidated = zarr.open_consolidated(store) assert consolidated.metadata.extra_fields == {"ext": extension} assert consolidated["child"].metadata.extra_fields == {"ext": extension} assert consolidated["child"].attrs["key"] == "value" class TestInfo: def test_info(self): store = zarr.storage.MemoryStore() A = zarr.group(store=store, path="A") B = A.create_group(name="B") B.create_array(name="x", shape=(1,), dtype="uint8") B.create_array(name="y", shape=(2,), dtype="uint8") result = A.info expected = GroupInfo( _name="A", _read_only=False, _store_type="MemoryStore", _zarr_format=3, ) assert result == expected result = A.info_complete() expected = GroupInfo( _name="A", _read_only=False, _store_type="MemoryStore", _zarr_format=3, _count_members=3, _count_arrays=2, _count_groups=1, ) assert result == expected def test_update_attrs() -> None: # regression test for https://github.com/zarr-developers/zarr-python/issues/2328 root = Group.from_store( MemoryStore(), ) root.attrs["foo"] = "bar" assert root.attrs["foo"] == "bar" @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) def test_delitem_removes_children(store: Store, zarr_format: ZarrFormat) -> None: # https://github.com/zarr-developers/zarr-python/issues/2191 g1 = zarr.group(store=store, zarr_format=zarr_format) g1.create_group("0") g1.create_group("0/0") arr = g1.create_array("0/0/0", shape=(1,), dtype="uint8") arr[:] = 1 del g1["0"] with pytest.raises(KeyError, match="0/0"): g1["0/0"] @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize("impl", ["async", "sync"]) async def test_create_nodes( impl: Literal["async", "sync"], store: Store, zarr_format: ZarrFormat ) -> None: """ Ensure that ``create_nodes`` can create a zarr hierarchy from a model of that hierarchy in dict form. Note that this creates an incomplete Zarr hierarchy. """ node_spec = { "group": GroupMetadata(attributes={"foo": 10}), "group/array_0": meta_from_array(np.arange(3), zarr_format=zarr_format), "group/array_1": meta_from_array(np.arange(4), zarr_format=zarr_format), "group/subgroup/array_0": meta_from_array(np.arange(4), zarr_format=zarr_format), "group/subgroup/array_1": meta_from_array(np.arange(5), zarr_format=zarr_format), } if impl == "sync": observed_nodes = dict(sync_group.create_nodes(store=store, nodes=node_spec)) elif impl == "async": observed_nodes = dict(await _collect_aiterator(create_nodes(store=store, nodes=node_spec))) else: raise ValueError(f"Invalid impl: {impl}") assert node_spec == {k: v.metadata for k, v in observed_nodes.items()} @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_create_nodes_concurrency_limit(store: MemoryStore) -> None: """ Test that the execution time of create_nodes can be constrained by the async concurrency configuration setting. """ set_latency = 0.02 num_groups = 10 groups = {str(idx): GroupMetadata() for idx in range(num_groups)} latency_store = LatencyStore(store, set_latency=set_latency) # check how long it takes to iterate over the groups # if create_nodes is sensitive to IO latency, # this should take (num_groups * get_latency) seconds # otherwise, it should take only marginally more than get_latency seconds with zarr_config.set({"async.concurrency": 1}): start = time.time() _ = tuple(sync_group.create_nodes(store=latency_store, nodes=groups)) elapsed = time.time() - start assert elapsed > num_groups * set_latency @pytest.mark.parametrize( ("a_func", "b_func"), [ (zarr.core.group.AsyncGroup.create_array, zarr.core.group.Group.create_array), (zarr.core.group.AsyncGroup.create_hierarchy, zarr.core.group.Group.create_hierarchy), (zarr.core.group.create_hierarchy, zarr.core.sync_group.create_hierarchy), (zarr.core.group.create_nodes, zarr.core.sync_group.create_nodes), (zarr.core.group.create_rooted_hierarchy, zarr.core.sync_group.create_rooted_hierarchy), (zarr.core.group.get_node, zarr.core.sync_group.get_node), ], # The default ids (from __name__) collide: the method pair and the module-level pair # for create_hierarchy would both be id'd "create_hierarchy-create_hierarchy". ids=lambda func: f"{func.__module__.rsplit('.', maxsplit=1)[-1]}.{func.__qualname__}", ) def test_consistent_signatures( a_func: Callable[[object], object], b_func: Callable[[object], object] ) -> None: """ Ensure that pairs of functions have consistent signatures """ base_sig = inspect.signature(a_func) test_sig = inspect.signature(b_func) wrong: dict[str, list[object]] = { "missing_from_test": [], "missing_from_base": [], "wrong_type": [], } for key, value in base_sig.parameters.items(): if key not in test_sig.parameters: wrong["missing_from_test"].append((key, value)) for key, value in test_sig.parameters.items(): if key not in base_sig.parameters: wrong["missing_from_base"].append((key, value)) if base_sig.parameters[key] != value: wrong["wrong_type"].append({key: {"test": value, "base": base_sig.parameters[key]}}) assert wrong["missing_from_base"] == [] assert wrong["missing_from_test"] == [] assert wrong["wrong_type"] == [] @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize("overwrite", [True, False]) @pytest.mark.parametrize("impl", ["async", "sync"]) async def test_create_hierarchy( impl: Literal["async", "sync"], store: Store, overwrite: bool, zarr_format: ZarrFormat ) -> None: """ Test that ``create_hierarchy`` can create a complete Zarr hierarchy, even if the input describes an incomplete one. """ hierarchy_spec = { "group": GroupMetadata(attributes={"path": "group"}, zarr_format=zarr_format), "group/array_0": meta_from_array( np.arange(3), attributes={"path": "group/array_0"}, zarr_format=zarr_format ), "group/subgroup/array_0": meta_from_array( np.arange(4), attributes={"path": "group/subgroup/array_0"}, zarr_format=zarr_format ), } pre_existing_nodes = { "group/extra": GroupMetadata(zarr_format=zarr_format, attributes={"path": "group/extra"}), "": GroupMetadata(zarr_format=zarr_format, attributes={"name": "root"}), } # we expect create_hierarchy to insert a group that was missing from the hierarchy spec expected_meta = hierarchy_spec | {"group/subgroup": GroupMetadata(zarr_format=zarr_format)} # initialize the group with some nodes _ = dict(sync_group.create_nodes(store=store, nodes=pre_existing_nodes)) if impl == "sync": created = dict( sync_group.create_hierarchy(store=store, nodes=hierarchy_spec, overwrite=overwrite) ) elif impl == "async": created = { k: v async for k, v in create_hierarchy( store=store, nodes=hierarchy_spec, overwrite=overwrite ) } else: raise ValueError(f"Invalid impl: {impl}") if not overwrite: extra_group = sync_group.get_node(store=store, path="group/extra", zarr_format=zarr_format) assert extra_group.metadata.attributes == {"path": "group/extra"} else: with pytest.raises(FileNotFoundError): await get_node(store=store, path="group/extra", zarr_format=zarr_format) assert expected_meta == {k: v.metadata for k, v in created.items()} @pytest.mark.parametrize("zarr_format", [2, 3]) def test_create_hierarchy_unbuildable_node_leaves_store_untouched( monkeypatch: pytest.MonkeyPatch, zarr_format: ZarrFormat ) -> None: """`create_hierarchy` builds every node before it deletes or stores anything, so a node that cannot be built fails with the store untouched, even when overwriting.""" store = MemoryStore() group = zarr.create_group(store, zarr_format=zarr_format) group.create_array("a", shape=(2,), chunks=(1,), dtype="int8")[:] = [1, 2] before = dict(store._store_dict) def unbuildable(**kwargs: object) -> None: raise RuntimeError("cannot build this node") monkeypatch.setattr(zarr.core.group, "_build_node", unbuildable) with pytest.raises(RuntimeError, match="cannot build this node"): dict( zarr.create_hierarchy( store=store, nodes={"a": GroupMetadata(zarr_format=zarr_format)}, overwrite=True ) ) assert store._store_dict == before @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize("extant_node", ["array", "group"]) @pytest.mark.parametrize("impl", ["async", "sync"]) async def test_create_hierarchy_existing_nodes( impl: Literal["async", "sync"], store: Store, extant_node: Literal["array", "group"], zarr_format: ZarrFormat, ) -> None: """ Test that create_hierarchy with overwrite = False will not overwrite an existing array or group, and raises an exception instead. """ extant_node_path = "node" if extant_node == "array": extant_metadata = meta_from_array( np.zeros(4), zarr_format=zarr_format, attributes={"extant": True} ) new_metadata = meta_from_array(np.zeros(4), zarr_format=zarr_format) err_cls = ContainsArrayError else: extant_metadata = GroupMetadata(zarr_format=zarr_format, attributes={"extant": True}) new_metadata = GroupMetadata(zarr_format=zarr_format) err_cls = ContainsGroupError # write the extant metadata tuple(sync_group.create_nodes(store=store, nodes={extant_node_path: extant_metadata})) msg = f"{extant_node} exists in store {store!r} at path {extant_node_path!r}." # ensure that we cannot invoke create_hierarchy with overwrite=False here if impl == "sync": with pytest.raises(err_cls, match=re.escape(msg)): tuple( sync_group.create_hierarchy( store=store, nodes={"node": new_metadata}, overwrite=False ) ) elif impl == "async": with pytest.raises(err_cls, match=re.escape(msg)): tuple( [ x async for x in create_hierarchy( store=store, nodes={"node": new_metadata}, overwrite=False ) ] ) else: raise ValueError(f"Invalid impl: {impl}") # ensure that the extant metadata was not overwritten assert ( await get_node(store=store, path=extant_node_path, zarr_format=zarr_format) ).metadata.attributes == {"extant": True} @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize("overwrite", [True, False]) @pytest.mark.parametrize("group_path", ["", "foo"]) @pytest.mark.parametrize("impl", ["async", "sync"]) async def test_group_create_hierarchy( store: Store, zarr_format: ZarrFormat, overwrite: bool, group_path: str, impl: Literal["async", "sync"], ) -> None: """ Test that the Group.create_hierarchy method creates specified nodes and returns them in a dict. Also test that off-target nodes are not deleted, and that the root group is not deleted """ root_attrs = {"root": True} g = sync_group.create_rooted_hierarchy( store=store, nodes={group_path: GroupMetadata(zarr_format=zarr_format, attributes=root_attrs)}, ) node_spec = { "a": GroupMetadata(zarr_format=zarr_format, attributes={"name": "a"}), "a/b": GroupMetadata(zarr_format=zarr_format, attributes={"name": "a/b"}), "a/b/c": meta_from_array( np.zeros(5), zarr_format=zarr_format, attributes={"name": "a/b/c"} ), } # This node should be kept if overwrite is True extant_spec = {"b": GroupMetadata(zarr_format=zarr_format, attributes={"name": "b"})} if impl == "async": extant_created = dict( await _collect_aiterator(g._async_group.create_hierarchy(extant_spec, overwrite=False)) ) nodes_created = dict( await _collect_aiterator( g._async_group.create_hierarchy(node_spec, overwrite=overwrite) ) ) elif impl == "sync": extant_created = dict(g.create_hierarchy(extant_spec, overwrite=False)) nodes_created = dict(g.create_hierarchy(node_spec, overwrite=overwrite)) all_members = dict(g.members(max_depth=None)) for k, v in node_spec.items(): assert all_members[k].metadata == v == nodes_created[k].metadata # if overwrite is True, the extant nodes should be erased for k, v in extant_spec.items(): if overwrite: assert k in all_members else: assert all_members[k].metadata == v == extant_created[k].metadata # ensure that we left the root group as-is assert ( sync_group.get_node(store=store, path=group_path, zarr_format=zarr_format).attrs.asdict() == root_attrs ) @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize("overwrite", [True, False]) def test_group_create_hierarchy_no_root( store: Store, zarr_format: ZarrFormat, overwrite: bool ) -> None: """ Test that the Group.create_hierarchy method will error if the dict provided contains a root. """ g = Group.from_store(store, zarr_format=zarr_format) tree = { "": GroupMetadata(zarr_format=zarr_format, attributes={"name": "a"}), } with pytest.raises( ValueError, match="It is an error to use this method to create a root node. " ): _ = dict(g.create_hierarchy(tree, overwrite=overwrite)) class TestParseHierarchyDict: """ Tests for the function that parses dicts of str : Metadata pairs, ensuring that the output models a valid Zarr hierarchy """ @staticmethod def test_normed_keys() -> None: """ Test that keys get normalized properly """ nodes = { "a": GroupMetadata(), "/b": GroupMetadata(), "": GroupMetadata(), "/a//c////": GroupMetadata(), } observed = _parse_hierarchy_dict(data=nodes) expected = {normalize_path(k): v for k, v in nodes.items()} assert observed == expected @staticmethod def test_empty() -> None: """ Test that an empty dict passes through """ assert _parse_hierarchy_dict(data={}) == {} @staticmethod def test_implicit_groups() -> None: """ Test that implicit groups were added as needed. """ requested = {"a/b/c": GroupMetadata()} expected = requested | { "": ImplicitGroupMarker(), "a": ImplicitGroupMarker(), "a/b": ImplicitGroupMarker(), } observed = _parse_hierarchy_dict(data=requested) assert observed == expected @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_group_create_hierarchy_invalid_mixed_zarr_format( store: Store, zarr_format: ZarrFormat ) -> None: """ Test that ``Group.create_hierarchy`` will raise an error if the zarr_format of the nodes is different from the parent group. """ other_format = 2 if zarr_format == 3 else 3 g = Group.from_store(store, zarr_format=other_format) tree = { "a": GroupMetadata(zarr_format=zarr_format, attributes={"name": "a"}), "a/b": meta_from_array(np.zeros(5), zarr_format=zarr_format, attributes={"name": "a/c"}), } msg = "The zarr_format of the nodes must be the same as the parent group." with pytest.raises(ValueError, match=msg): _ = tuple(g.create_hierarchy(tree)) @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize("defect", ["array/array", "array/group"]) @pytest.mark.parametrize("impl", ["async", "sync"]) async def test_create_hierarchy_invalid_nested( impl: Literal["async", "sync"], store: Store, defect: tuple[str, str], zarr_format: ZarrFormat ) -> None: """ Test that create_hierarchy will not create a Zarr array that contains a Zarr group or Zarr array. """ if defect == "array/array": hierarchy_spec = { "array_0": meta_from_array(np.arange(3), zarr_format=zarr_format), "array_0/subarray": meta_from_array(np.arange(4), zarr_format=zarr_format), } elif defect == "array/group": hierarchy_spec = { "array_0": meta_from_array(np.arange(3), zarr_format=zarr_format), "array_0/subgroup": GroupMetadata(attributes={"foo": 10}, zarr_format=zarr_format), } msg = "Only Zarr groups can contain other nodes." if impl == "sync": with pytest.raises(ValueError, match=msg): tuple(sync_group.create_hierarchy(store=store, nodes=hierarchy_spec)) elif impl == "async": with pytest.raises(ValueError, match=msg): await _collect_aiterator(create_hierarchy(store=store, nodes=hierarchy_spec)) @pytest.mark.parametrize("store", ["memory"], indirect=True) @pytest.mark.parametrize("impl", ["async", "sync"]) async def test_create_hierarchy_invalid_mixed_format( impl: Literal["async", "sync"], store: Store ) -> None: """ Test that create_hierarchy will not create a Zarr group that contains a both Zarr v2 and Zarr v3 nodes. """ msg = ( "Got data with both Zarr v2 and Zarr v3 nodes, which is invalid. " "The following keys map to Zarr v2 nodes: ['v2']. " "The following keys map to Zarr v3 nodes: ['v3']." "Ensure that all nodes have the same Zarr format." ) nodes = { "v2": GroupMetadata(zarr_format=2), "v3": GroupMetadata(zarr_format=3), } if impl == "sync": with pytest.raises(ValueError, match=re.escape(msg)): tuple( sync_group.create_hierarchy( store=store, nodes=nodes, ) ) elif impl == "async": with pytest.raises(ValueError, match=re.escape(msg)): await _collect_aiterator( create_hierarchy( store=store, nodes=nodes, ) ) else: raise ValueError(f"Invalid impl: {impl}") @pytest.mark.parametrize("store", ["memory", "local"], indirect=True) @pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.parametrize("root_key", ["", "root"]) @pytest.mark.parametrize("impl", ["async", "sync"]) async def test_create_rooted_hierarchy_group( impl: Literal["async", "sync"], store: Store, zarr_format, root_key: str ) -> None: """ Test that the _create_rooted_hierarchy can create a group. """ root_meta = {root_key: GroupMetadata(zarr_format=zarr_format, attributes={"path": root_key})} group_names = ["a", "a/b"] array_names = ["a/b/c", "a/b/d"] # just to ensure that we don't use the same name twice in tests assert set(group_names) & set(array_names) == set() groups_expected_meta = { _join_paths([root_key, node_name]): GroupMetadata( zarr_format=zarr_format, attributes={"path": node_name} ) for node_name in group_names } arrays_expected_meta = { _join_paths([root_key, node_name]): meta_from_array(np.zeros(4), zarr_format=zarr_format) for node_name in array_names } nodes_create = root_meta | groups_expected_meta | arrays_expected_meta if impl == "sync": g = sync_group.create_rooted_hierarchy(store=store, nodes=nodes_create) assert isinstance(g, Group) members = g.members(max_depth=None) elif impl == "async": g = await create_rooted_hierarchy(store=store, nodes=nodes_create) assert isinstance(g, AsyncGroup) members = await _collect_aiterator(g.members(max_depth=None)) else: raise ValueError(f"Unknown implementation: {impl}") assert g.metadata.attributes == {"path": root_key} members_observed_meta = {k: v.metadata for k, v in members} members_expected_meta_relative = { k.removeprefix(root_key).lstrip("/"): v for k, v in (groups_expected_meta | arrays_expected_meta).items() } assert members_observed_meta == members_expected_meta_relative @pytest.mark.parametrize("store", ["memory", "local"], indirect=True) @pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.parametrize("root_key", ["", "root"]) @pytest.mark.parametrize("impl", ["async", "sync"]) async def test_create_rooted_hierarchy_array( impl: Literal["async", "sync"], store: Store, zarr_format, root_key: str ) -> None: """ Test that _create_rooted_hierarchy can create an array. """ root_meta = { root_key: meta_from_array( np.arange(3), zarr_format=zarr_format, attributes={"path": root_key} ) } nodes_create = root_meta if impl == "sync": a = sync_group.create_rooted_hierarchy(store=store, nodes=nodes_create, overwrite=True) assert isinstance(a, Array) elif impl == "async": a = await create_rooted_hierarchy(store=store, nodes=nodes_create, overwrite=True) assert isinstance(a, AsyncArray) else: raise ValueError(f"Invalid impl: {impl}") assert a.metadata.attributes == {"path": root_key} @pytest.mark.parametrize("impl", ["async", "sync"]) async def test_create_rooted_hierarchy_invalid(impl: Literal["async", "sync"]) -> None: """ Ensure _create_rooted_hierarchy will raise a ValueError if the input does not contain a root node. """ zarr_format = 3 nodes = { "a": GroupMetadata(zarr_format=zarr_format), "b": GroupMetadata(zarr_format=zarr_format), } msg = "The input does not specify a root node. " if impl == "sync": with pytest.raises(ValueError, match=msg): sync_group.create_rooted_hierarchy(store=store, nodes=nodes) elif impl == "async": with pytest.raises(ValueError, match=msg): await create_rooted_hierarchy(store=store, nodes=nodes) else: raise ValueError(f"Invalid impl: {impl}") @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_group_members_performance(store: Store) -> None: """ Test that the execution time of Group.members is less than the number of members times the latency for accessing each member. """ get_latency = 0.1 # use the input store to create some groups group_create = zarr.group(store=store) num_groups = 10 # Create some groups for i in range(num_groups): group_create.create_group(f"group{i}") latency_store = LatencyStore(store, get_latency=get_latency) # create a group with some latency on get operations group_read = zarr.group(store=latency_store) # check how long it takes to iterate over the groups # if .members is sensitive to IO latency, # this should take (num_groups * get_latency) seconds # otherwise, it should take only marginally more than get_latency seconds start = time.time() _ = group_read.members() elapsed = time.time() - start assert elapsed < (num_groups * get_latency) @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_group_members_concurrency_limit(store: MemoryStore) -> None: """ Test that the execution time of Group.members can be constrained by the async concurrency configuration setting. """ get_latency = 0.02 # use the input store to create some groups group_create = zarr.group(store=store) num_groups = 10 # Create some groups for i in range(num_groups): group_create.create_group(f"group{i}") latency_store = LatencyStore(store, get_latency=get_latency) # create a group with some latency on get operations group_read = zarr.group(store=latency_store) # check how long it takes to iterate over the groups # if .members is sensitive to IO latency, # this should take (num_groups * get_latency) seconds # otherwise, it should take only marginally more than get_latency seconds with zarr_config.set({"async.concurrency": 1}): start = time.time() _ = group_read.members() elapsed = time.time() - start assert elapsed > num_groups * get_latency @pytest.mark.parametrize("option", ["array", "group", "invalid"]) def test_build_metadata_v3(option: Literal["array", "group", "invalid"]) -> None: """ Test that _build_metadata_v3 returns the correct metadata for a v3 array or group """ match option: case "array": metadata_dict = meta_from_array(np.arange(10), zarr_format=3).to_dict() assert _build_metadata_v3(metadata_dict) == ArrayV3Metadata.from_dict(metadata_dict) case "group": metadata_dict = GroupMetadata(attributes={"foo": 10}, zarr_format=3).to_dict() assert _build_metadata_v3(metadata_dict) == GroupMetadata.from_dict(metadata_dict) case "invalid": metadata_dict = GroupMetadata(zarr_format=3).to_dict() metadata_dict.pop("node_type") # TODO: fix the error message msg = "Required key 'node_type' is missing from the provided metadata document." with pytest.raises(MetadataValidationError, match=msg): _build_metadata_v3(metadata_dict) @pytest.mark.parametrize("roots", [("",), ("a", "b")]) def test_get_roots(roots: tuple[str, ...]): root_nodes = {k: GroupMetadata(attributes={"name": k}) for k in roots} child_nodes = { _join_paths([k, "foo"]): GroupMetadata(attributes={"name": _join_paths([k, "foo"])}) for k in roots } data = root_nodes | child_nodes assert set(_get_roots(data)) == set(roots) def test_open_array_as_group(): z = zarr.create_array(shape=(40, 50), chunks=(10, 10), dtype="f8", store={}) with pytest.raises(ContainsArrayError): zarr.open_group(z.store) zarr-developers-zarr-python-a2f2415/tests/test_indexing.py000066400000000000000000003165701527014137600240610ustar00rootroot00000000000000from __future__ import annotations import itertools import warnings from collections import Counter from dataclasses import asdict from dataclasses import fields as dataclass_fields from typing import TYPE_CHECKING, Any from uuid import uuid4 import numpy as np import numpy.typing as npt import pytest from hypothesis import example, given from hypothesis import strategies as st from numpy.testing import assert_array_equal import zarr from tests.conftest import Expect, ExpectFail from zarr import Array from zarr.core.buffer import default_buffer_prototype from zarr.core.chunk_grids import ChunkGrid from zarr.core.common import ceildiv_int from zarr.core.indexing import ( BasicSelection, CoordinateIndexer, CoordinateSelection, IntArrayDimIndexer, MaskIndexer, OrthogonalSelection, Selection, _ArrayIndexingOrder, _iter_grid, _iter_regions, make_slice_selection, normalize_integer_selection, oindex, oindex_set, replace_ellipsis, ) from zarr.registry import get_ndbuffer_class from zarr.storage import MemoryStore, StorePath if TYPE_CHECKING: from collections.abc import AsyncGenerator from zarr.abc.store import ByteRequest from zarr.core.buffer import BufferPrototype from zarr.core.buffer.core import Buffer from zarr.core.common import ZarrFormat @pytest.fixture async def store() -> AsyncGenerator[StorePath]: return StorePath(await MemoryStore.open()) def zarr_array_from_numpy_array( store: StorePath, a: npt.NDArray[Any], chunk_shape: tuple[int, ...] | None = None, shards: tuple[int, ...] | None = None, ) -> zarr.Array: z = zarr.create_array( store=store / str(uuid4()), shape=a.shape, dtype=a.dtype, chunks=chunk_shape or a.shape, shards=shards, chunk_key_encoding={"name": "v2", "separator": "."}, ) z[()] = a return z class CountingDict(MemoryStore): counter: Counter[tuple[str, str]] @classmethod async def open(cls) -> CountingDict: store = await super().open() store.counter = Counter() return store async def get( self, key: str, prototype: BufferPrototype, byte_range: tuple[int | None, int | None] | None = None, ) -> Buffer | None: key_suffix = "/".join(key.split("/")[1:]) self.counter["__getitem__", key_suffix] += 1 return await super().get(key, prototype, byte_range) async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None = None) -> None: key_suffix = "/".join(key.split("/")[1:]) self.counter["__setitem__", key_suffix] += 1 return await super().set(key, value, byte_range) def get_sync( self, key: str, *, prototype: BufferPrototype | None = None, byte_range: ByteRequest | None = None, ) -> Buffer | None: key_suffix = "/".join(key.split("/")[1:]) self.counter["__getitem__", key_suffix] += 1 return super().get_sync(key, prototype=prototype, byte_range=byte_range) def set_sync(self, key: str, value: Buffer) -> None: key_suffix = "/".join(key.split("/")[1:]) self.counter["__setitem__", key_suffix] += 1 return super().set_sync(key, value) def test_normalize_integer_selection() -> None: """normalize_integer_selection handles positive/negative indices and raises IndexError for out-of-bounds values.""" assert 1 == normalize_integer_selection(1, 100) assert 99 == normalize_integer_selection(-1, 100) with pytest.raises(IndexError): normalize_integer_selection(100, 100) with pytest.raises(IndexError): normalize_integer_selection(1000, 100) with pytest.raises(IndexError): normalize_integer_selection(-1000, 100) def test_replace_ellipsis() -> None: """replace_ellipsis expands Ellipsis to full slice(None) selections for 1D and 2D shapes.""" # 1D, single item assert (0,) == replace_ellipsis(0, (100,)) # 1D assert (slice(None),) == replace_ellipsis(Ellipsis, (100,)) assert (slice(None),) == replace_ellipsis(slice(None), (100,)) assert (slice(None, 100),) == replace_ellipsis(slice(None, 100), (100,)) assert (slice(0, None),) == replace_ellipsis(slice(0, None), (100,)) assert (slice(None),) == replace_ellipsis((slice(None), Ellipsis), (100,)) assert (slice(None),) == replace_ellipsis((Ellipsis, slice(None)), (100,)) # 2D, single item assert (0, 0) == replace_ellipsis((0, 0), (100, 100)) assert (-1, 1) == replace_ellipsis((-1, 1), (100, 100)) # 2D, single col/row assert (0, slice(None)) == replace_ellipsis((0, slice(None)), (100, 100)) assert (0, slice(None)) == replace_ellipsis((0,), (100, 100)) assert (slice(None), 0) == replace_ellipsis((slice(None), 0), (100, 100)) # 2D slice assert (slice(None), slice(None)) == replace_ellipsis(Ellipsis, (100, 100)) assert (slice(None), slice(None)) == replace_ellipsis(slice(None), (100, 100)) assert (slice(None), slice(None)) == replace_ellipsis((slice(None), slice(None)), (100, 100)) assert (slice(None), slice(None)) == replace_ellipsis((Ellipsis, slice(None)), (100, 100)) assert (slice(None), slice(None)) == replace_ellipsis((slice(None), Ellipsis), (100, 100)) assert (slice(None), slice(None)) == replace_ellipsis( (slice(None), Ellipsis, slice(None)), (100, 100) ) assert (slice(None), slice(None)) == replace_ellipsis( (Ellipsis, slice(None), slice(None)), (100, 100) ) assert (slice(None), slice(None)) == replace_ellipsis( (slice(None), slice(None), Ellipsis), (100, 100) ) @pytest.mark.parametrize( ("value", "dtype"), [ (42, "uint8"), pytest.param( (b"aaa", 1, 4.2), [("foo", "S3"), ("bar", "i4"), ("baz", "f8")], marks=pytest.mark.filterwarnings("ignore::zarr.errors.UnstableSpecificationWarning"), ), ], ) @pytest.mark.parametrize("use_out", [True, False]) def test_get_basic_selection_0d(store: StorePath, use_out: bool, value: Any, dtype: Any) -> None: """get_basic_selection on a 0-dimensional array returns the scalar value via Ellipsis and (), including the `out` buffer path.""" # setup arr_np = np.array(value, dtype=dtype) arr_z = zarr_array_from_numpy_array(store, arr_np) assert_array_equal(arr_np, arr_z.get_basic_selection(Ellipsis)) assert_array_equal(arr_np, arr_z[...]) assert arr_np[()] == arr_z.get_basic_selection(()) assert arr_np[()] == arr_z[()] if use_out: # test out param b = default_buffer_prototype().nd_buffer.from_numpy_array(np.zeros_like(arr_np)) arr_z.get_basic_selection(Ellipsis, out=b) assert_array_equal(arr_np, b.as_ndarray_like()) # todo: uncomment the structured array tests when we can make them pass, # or delete them if we formally decide not to support structured dtypes. # test structured array # value = (b"aaa", 1, 4.2) # a = np.array(value, dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")]) # z = zarr_array_from_numpy_array(store, a) # z[()] = value # assert_array_equal(a, z.get_basic_selection(Ellipsis)) # assert_array_equal(a, z[...]) # assert a[()] == z.get_basic_selection(()) # assert a[()] == z[()] # assert b"aaa" == z.get_basic_selection((), fields="foo") # assert b"aaa" == z["foo"] # assert a[["foo", "bar"]] == z.get_basic_selection((), fields=["foo", "bar"]) # assert a[["foo", "bar"]] == z["foo", "bar"] # # test out param # b = NDBuffer.from_numpy_array(np.zeros_like(a)) # z.get_basic_selection(Ellipsis, out=b) # assert_array_equal(a, b) # c = NDBuffer.from_numpy_array(np.zeros_like(a[["foo", "bar"]])) # z.get_basic_selection(Ellipsis, out=c, fields=["foo", "bar"]) # assert_array_equal(a[["foo", "bar"]], c) _BASIC_1D_CASES: list[Expect[BasicSelection, None]] = [ Expect(input=5, output=None, id="single-positive"), Expect(input=-1, output=None, id="single-negative"), Expect(input=slice(3, 18), output=None, id="bounded-slice"), Expect(input=slice(0, 100), output=None, id="over-bounds-slice"), Expect(input=slice(-18, -3), output=None, id="negative-slice"), Expect(input=slice(0, 0), output=None, id="empty-slice"), Expect(input=slice(-1, 0), output=None, id="empty-negative-slice"), Expect(input=slice(None), output=None, id="full-slice"), Expect(input=Ellipsis, output=None, id="ellipsis"), Expect(input=(), output=None, id="empty-tuple"), Expect(input=(Ellipsis, slice(None)), output=None, id="ellipsis-slice"), Expect(input=slice(None, None, 3), output=None, id="stride-3"), Expect(input=slice(3, 27, 5), output=None, id="bounded-stride"), ] _BASIC_1D_BAD_CASES: list[ExpectFail[Any]] = [ ExpectFail( input=slice(None, None, -1), exception=IndexError, id="negative-step", msg="only slices with step >= 1 are supported", ), ExpectFail( input=2.3, exception=IndexError, id="float", msg="unsupported selection item for basic indexing; expected integer or slice, got ", escape=True, ), # get_basic_selection and z[...] word their errors differently for a string # selection, so this case asserts only the exception type (msg=None). ExpectFail( input="foo", exception=IndexError, id="string", msg=None, ), ExpectFail( input=b"xxx", exception=IndexError, id="bytes", msg="unsupported selection item for basic indexing; expected integer or slice, got ", escape=True, ), ExpectFail( input=None, exception=IndexError, id="none", msg="unsupported selection item for basic indexing; expected integer or slice, got ", escape=True, ), ExpectFail( input=(0, 0), exception=IndexError, id="tuple-too-many", msg="too many indices for array; expected 1, got 2", ), ExpectFail( input=(slice(None), slice(None)), exception=IndexError, id="two-slices", msg="too many indices for array; expected 1, got 2", ), ] def _test_get_basic_selection( a: npt.NDArray[Any] | Array, z: Array, selection: BasicSelection ) -> None: expect = a[selection] actual = z.get_basic_selection(selection) assert_array_equal(expect, actual) actual = z[selection] assert_array_equal(expect, actual) # test out param b = default_buffer_prototype().nd_buffer.from_numpy_array( np.empty(shape=expect.shape, dtype=expect.dtype) ) z.get_basic_selection(selection, out=b) assert_array_equal(expect, b.as_numpy_array()) @pytest.mark.parametrize("case", _BASIC_1D_CASES, ids=lambda c: c.id) def test_get_basic_selection_1d(store: StorePath, case: Expect[BasicSelection, None]) -> None: """Basic getitem on a 1D array matches numpy for ints, slices, strides, and full selections.""" a = np.arange(30, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) _test_get_basic_selection(a, z, case.input) @pytest.mark.parametrize("case", _BASIC_1D_BAD_CASES, ids=lambda c: c.id) def test_get_basic_selection_1d_raises(store: StorePath, case: ExpectFail[Any]) -> None: """Basic getitem on a 1D array rejects negative steps and invalid index types with IndexError.""" a = np.arange(30, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) with case.raises(): z.get_basic_selection(case.input) with case.raises(): z[case.input] _BASIC_2D_CASES: list[Expect[BasicSelection, None]] = [ Expect(input=5, output=None, id="single-row"), Expect(input=-1, output=None, id="single-row-neg"), Expect(input=(5, slice(None)), output=None, id="row-and-full-col"), Expect(input=(slice(None), 3), output=None, id="single-col"), Expect(input=(slice(None), -1), output=None, id="single-col-neg"), Expect(input=slice(None), output=None, id="full"), Expect(input=slice(2, 9), output=None, id="row-slice"), Expect(input=slice(0, 0), output=None, id="empty-row-slice"), Expect(input=(slice(2, 9), slice(1, 4)), output=None, id="2d-slice"), Expect(input=(slice(0, 12, 3), slice(0, 5, 2)), output=None, id="strided-2d-slice"), Expect(input=Ellipsis, output=None, id="ellipsis"), Expect(input=(), output=None, id="empty-tuple"), ] _BASIC_2D_BAD_CASES: list[ExpectFail[Any]] = [ ExpectFail( input=2.3, exception=IndexError, id="float", msg="unsupported selection item for basic indexing; expected integer or slice, got ", escape=True, ), ExpectFail( input="foo", exception=IndexError, id="string", msg="unsupported selection item for basic indexing; expected integer or slice, got ", escape=True, ), ExpectFail( input=None, exception=IndexError, id="none", msg="unsupported selection item for basic indexing; expected integer or slice, got ", escape=True, ), ExpectFail( input=(2.3, slice(None)), exception=IndexError, id="float-in-tuple", msg="unsupported selection item for basic indexing; expected integer or slice, got ", escape=True, ), ExpectFail( input=slice(None, None, -1), exception=IndexError, id="negative-step", msg="only slices with step >= 1 are supported", ), ExpectFail( input=(slice(None), slice(None), slice(None)), exception=IndexError, id="too-many-dims", msg="too many indices for array; expected 2, got 3", ), ExpectFail( input=[0, 1], exception=IndexError, id="integer-list", msg="unsupported selection item for basic indexing; expected integer or slice, got ", escape=True, ), ] @pytest.mark.parametrize("case", _BASIC_2D_CASES, ids=lambda c: c.id) def test_get_basic_selection_2d(store: StorePath, case: Expect[BasicSelection, None]) -> None: """Basic getitem on a 2D array matches numpy for rows, cols, slices, and strides.""" a = np.arange(60, dtype=int).reshape(12, 5) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) _test_get_basic_selection(a, z, case.input) @pytest.mark.parametrize("case", _BASIC_2D_BAD_CASES, ids=lambda c: c.id) def test_get_basic_selection_2d_raises(store: StorePath, case: ExpectFail[Any]) -> None: """Basic getitem on a 2D array rejects malformed selections with IndexError.""" a = np.arange(60, dtype=int).reshape(12, 5) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) with case.raises(): z.get_basic_selection(case.input) def test_basic_2d_fancy_fallback(store: StorePath) -> None: """Indexing a 2D array with paired integer lists falls back to fancy (vectorized) indexing.""" a = np.arange(60, dtype=int).reshape(12, 5) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) np.testing.assert_array_equal(z[([0, 1], [0, 1])], a[([0, 1], [0, 1])]) def test_get_basic_selection_1d_rejects_integer_list(store: StorePath) -> None: """get_basic_selection on a 1D array rejects an integer list (basic indexing is int/slice only).""" a = np.arange(30, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) with pytest.raises(IndexError, match="unsupported selection item for basic indexing"): z.get_basic_selection([1, 0]) def test_get_basic_selection_2d_rejects_list_in_tuple(store: StorePath) -> None: """get_basic_selection on a 2D array rejects a list nested in an index tuple.""" a = np.arange(60, dtype=int).reshape(12, 5) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) with pytest.raises(IndexError, match="unsupported selection item for basic indexing"): z.get_basic_selection((slice(None), [0, 1])) def test_fancy_indexing_fallback_on_get_setitem(store: StorePath) -> None: """Paired integer-list indexing falls back to vectorized (fancy) get and set via `__getitem__`/`__setitem__`.""" z = zarr_array_from_numpy_array(store, np.zeros((20, 20))) z[[1, 2, 3], [1, 2, 3]] = 1 np.testing.assert_array_equal( z[:4, :4], [ [0, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1], ], ) np.testing.assert_array_equal(z[[1, 2, 3], [1, 2, 3]], 1) # test broadcasting np.testing.assert_array_equal(z[1, [1, 2, 3]], [1, 0, 0]) # test 1D fancy indexing z2 = zarr_array_from_numpy_array(store, np.zeros(5)) z2[[1, 2, 3]] = 1 np.testing.assert_array_equal(z2[:], [0, 1, 1, 1, 0]) @pytest.mark.parametrize( ("index", "expected_result"), [ # Single iterable of integers ([0, 1], [[0, 1, 2], [3, 4, 5]]), # List first, then slice (([0, 1], slice(None)), [[0, 1, 2], [3, 4, 5]]), # List first, then slice (([0, 1], slice(1, None)), [[1, 2], [4, 5]]), # Slice first, then list ((slice(0, 2), [0, 2]), [[0, 2], [3, 5]]), # Slices only ((slice(0, 2), slice(0, 2)), [[0, 1], [3, 4]]), # List with repeated index (([1, 0, 1], slice(1, None)), [[4, 5], [1, 2], [4, 5]]), # 1D indexing (([1, 0, 1]), [[3, 4, 5], [0, 1, 2], [3, 4, 5]]), ], ) def test_orthogonal_indexing_fallback_on_getitem_2d( store: StorePath, index: Selection, expected_result: npt.ArrayLike ) -> None: """ Tests the orthogonal indexing fallback on __getitem__ for a 2D matrix. In addition to checking expected behavior, all indexing is also checked against numpy. """ # [0, 1, 2], # [3, 4, 5], # [6, 7, 8] a = np.arange(9).reshape(3, 3) z = zarr_array_from_numpy_array(store, a) np.testing.assert_array_equal(z[index], a[index], err_msg="Indexing disagrees with numpy") np.testing.assert_array_equal(z[index], expected_result) def test_setitem_zarr_array_as_value() -> None: """Assigning a zarr array as a value in `__setitem__` does not raise a SyncError (regression for GH3611).""" # Regression test for https://github.com/zarr-developers/zarr-python/issues/3611 # Assigning a zarr Array as the value used to raise # SyncError("Calling sync() from within a running loop") because the codec # pipeline tried to index the zarr array inside an already-running async loop. src = zarr.array(np.arange(10), chunks=(5,)) dst = zarr.zeros(10, chunks=(5,), dtype=src.dtype) # Full assignment dst[:] = src assert_array_equal(dst[:], np.arange(10)) # Slice assignment dst2 = zarr.zeros(10, chunks=(5,), dtype=src.dtype) dst2[2:7] = src[2:7] assert_array_equal(dst2[2:7], np.arange(2, 7)) @pytest.mark.skip(reason="fails on ubuntu, windows; numpy=2.2; in CI") def test_setitem_repeated_index(): """oindex assignment with repeated indices writes the last value for each duplicated index position.""" array = zarr.array(data=np.zeros((4,)), chunks=(1,)) indexer = np.array([-1, -1, 0, 0]) array.oindex[(indexer,)] = [0, 1, 2, 3] np.testing.assert_array_equal(array[:], np.array([3, 0, 0, 1])) indexer = np.array([-1, 0, 0, -1]) array.oindex[(indexer,)] = [0, 1, 2, 3] np.testing.assert_array_equal(array[:], np.array([2, 0, 0, 3])) Index = list[int] | tuple[slice | int | list[int], ...] @pytest.mark.parametrize( ("index", "expected_result"), [ # Single iterable of integers ([0, 1], [[[0, 1, 2], [3, 4, 5], [6, 7, 8]], [[9, 10, 11], [12, 13, 14], [15, 16, 17]]]), # One slice, two integers ((slice(0, 2), 1, 1), [4, 13]), # One integer, two slices ((slice(0, 2), 1, slice(0, 2)), [[3, 4], [12, 13]]), # Two slices and a list ((slice(0, 2), [1, 2], slice(0, 2)), [[[3, 4], [6, 7]], [[12, 13], [15, 16]]]), ], ) def test_orthogonal_indexing_fallback_on_getitem_3d( store: StorePath, index: Selection, expected_result: npt.ArrayLike ) -> None: """ Tests the orthogonal indexing fallback on __getitem__ for a 3D matrix. In addition to checking expected behavior, all indexing is also checked against numpy. """ # [[[ 0, 1, 2], # [ 3, 4, 5], # [ 6, 7, 8]], # [[ 9, 10, 11], # [12, 13, 14], # [15, 16, 17]], # [[18, 19, 20], # [21, 22, 23], # [24, 25, 26]]] a = np.arange(27).reshape(3, 3, 3) z = zarr_array_from_numpy_array(store, a) np.testing.assert_array_equal(z[index], a[index], err_msg="Indexing disagrees with numpy") np.testing.assert_array_equal(z[index], expected_result) @pytest.mark.parametrize( ("index", "expected_result"), [ # Single iterable of integers ([0, 1], [[1, 1, 1], [1, 1, 1], [0, 0, 0]]), # List and slice combined (([0, 1], slice(1, 3)), [[0, 1, 1], [0, 1, 1], [0, 0, 0]]), # Index repetition is ignored on setitem (([0, 1, 1, 1, 1, 1, 1], slice(1, 3)), [[0, 1, 1], [0, 1, 1], [0, 0, 0]]), # Slice with step (([0, 2], slice(None, None, 2)), [[1, 0, 1], [0, 0, 0], [1, 0, 1]]), ], ) def test_orthogonal_indexing_fallback_on_setitem_2d( store: StorePath, index: Selection, expected_result: npt.ArrayLike ) -> None: """ Tests the orthogonal indexing fallback on __setitem__ for a 3D matrix. In addition to checking expected behavior, all indexing is also checked against numpy. """ # Slice + fancy index a = np.zeros((3, 3)) z = zarr_array_from_numpy_array(store, a) z[index] = 1 a[index] = 1 np.testing.assert_array_equal(z[:], expected_result) np.testing.assert_array_equal(z[:], a, err_msg="Indexing disagrees with numpy") def test_fancy_indexing_doesnt_mix_with_implicit_slicing(store: StorePath) -> None: """Fancy indexing that would require implicit slicing over an unspecified axis raises IndexError on a 3D array.""" z2 = zarr_array_from_numpy_array(store, np.zeros((5, 5, 5))) with pytest.raises(IndexError): z2[[1, 2, 3], [1, 2, 3]] = 2 with pytest.raises(IndexError): np.testing.assert_array_equal(z2[[1, 2, 3], [1, 2, 3]], 0) with pytest.raises(IndexError): z2[..., [1, 2, 3]] = 2 # type: ignore[index] with pytest.raises(IndexError): np.testing.assert_array_equal(z2[..., [1, 2, 3]], 0) # type: ignore[index] @pytest.mark.parametrize( ("value", "dtype"), [ (42, "uint8"), pytest.param( (b"aaa", 1, 4.2), [("foo", "S3"), ("bar", "i4"), ("baz", "f8")], marks=pytest.mark.filterwarnings("ignore::zarr.errors.UnstableSpecificationWarning"), ), ], ) def test_set_basic_selection_0d( store: StorePath, value: Any, dtype: str | list[tuple[str, str]] ) -> None: """set_basic_selection and `__setitem__` write scalar values correctly to a 0-dimensional array.""" arr_np = np.array(value, dtype=dtype) arr_np_zeros = np.zeros_like(arr_np, dtype=dtype) arr_z = zarr_array_from_numpy_array(store, arr_np_zeros) assert_array_equal(arr_np_zeros, arr_z) arr_z.set_basic_selection(Ellipsis, value) assert_array_equal(arr_np, arr_z) arr_z[...] = arr_np_zeros[()] assert_array_equal(arr_np_zeros, arr_z) arr_z[...] = value assert_array_equal(arr_np, arr_z) # todo: uncomment the structured array tests when we can make them pass, # or delete them if we formally decide not to support structured dtypes. # arr_z.set_basic_selection(Ellipsis, v["foo"], fields="foo") # assert v["foo"] == arr_z["foo"] # assert arr_np_zeros["bar"] == arr_z["bar"] # assert arr_np_zeros["baz"] == arr_z["baz"] # arr_z["bar"] = v["bar"] # assert v["foo"] == arr_z["foo"] # assert v["bar"] == arr_z["bar"] # assert arr_np_zeros["baz"] == arr_z["baz"] # # multiple field assignment not supported # with pytest.raises(IndexError): # arr_z.set_basic_selection(Ellipsis, v[["foo", "bar"]], fields=["foo", "bar"]) # with pytest.raises(IndexError): # arr_z[..., "foo", "bar"] = v[["foo", "bar"]] def _test_get_orthogonal_selection( a: npt.NDArray[Any], z: Array, selection: OrthogonalSelection ) -> None: expect = oindex(a, selection) actual = z.get_orthogonal_selection(selection) assert_array_equal(expect, actual) actual = z.oindex[selection] assert_array_equal(expect, actual) _ORTHO_1D_BOOL_CASES: list[Expect[OrthogonalSelection, None]] = [ Expect(input=np.zeros(30, dtype=bool), output=None, id="empty-mask"), Expect(input=np.ones(30, dtype=bool), output=None, id="full-mask"), Expect(input=np.arange(30) % 2 == 0, output=None, id="alternating-mask"), Expect(input=np.arange(30) == 7, output=None, id="single-true"), Expect( input=np.isin(np.arange(30), [0, 1, 8, 15, 29]), output=None, id="sparse-cross-chunk", ), ] _ORTHO_1D_BOOL_BAD_CASES: list[ExpectFail[Any]] = [ ExpectFail( input=np.zeros(5, dtype=bool), exception=IndexError, id="mask-too-short", msg="wrong length for dimension; expected 30, got 5", ), ExpectFail( input=np.zeros(50, dtype=bool), exception=IndexError, id="mask-too-long", msg="wrong length for dimension; expected 30, got 50", ), ExpectFail( input=[[True, False], [False, True]], exception=IndexError, id="mask-too-many-dims", msg="must be 1-dimensional only", ), ] @pytest.mark.parametrize("case", _ORTHO_1D_BOOL_CASES, ids=lambda c: c.id) def test_get_orthogonal_selection_1d_bool( store: StorePath, case: Expect[OrthogonalSelection, None] ) -> None: """oindex with a 1D boolean mask matches numpy across chunk boundaries.""" a = np.arange(30, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) _test_get_orthogonal_selection(a, z, case.input) @pytest.mark.parametrize("case", _ORTHO_1D_BOOL_BAD_CASES, ids=lambda c: c.id) def test_get_orthogonal_selection_1d_bool_raises(store: StorePath, case: ExpectFail[Any]) -> None: """oindex rejects masks of the wrong length or dimensionality with IndexError.""" a = np.arange(30, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) with case.raises(): z.oindex[case.input] _ORTHO_1D_INT_CASES: list[Expect[OrthogonalSelection, None]] = [ Expect(input=[0, 8, 15, 29], output=None, id="sorted"), Expect(input=[3, 29, 1, 16], output=None, id="unsorted"), Expect(input=[2, 2, 8, 8], output=None, id="duplicates"), Expect(input=[0, 3, 10, -23, -12, -1], output=None, id="wraparound"), Expect(input=[15], output=None, id="single"), ] _ORTHO_1D_INT_BAD_CASES: list[ExpectFail[Any]] = [ ExpectFail( input=[31], exception=IndexError, id="out-of-bounds-high", msg="index out of bounds for dimension with length 30", ), ExpectFail( input=[-31], exception=IndexError, id="out-of-bounds-low", msg="index out of bounds for dimension with length 30", ), ExpectFail( input=[[2, 4], [6, 8]], exception=IndexError, id="too-many-dims", msg="integer arrays in an orthogonal selection must be 1-dimensional only", ), ] @pytest.mark.parametrize("case", _ORTHO_1D_INT_CASES, ids=lambda c: c.id) def test_get_orthogonal_selection_1d_int( store: StorePath, case: Expect[OrthogonalSelection, None] ) -> None: """oindex with a 1D integer array matches numpy, including wraparound and duplicates.""" a = np.arange(30, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) _test_get_orthogonal_selection(a, z, case.input) @pytest.mark.parametrize("case", _ORTHO_1D_INT_BAD_CASES, ids=lambda c: c.id) def test_get_orthogonal_selection_1d_int_raises(store: StorePath, case: ExpectFail[Any]) -> None: """oindex rejects out-of-bounds or multi-dimensional integer selections with IndexError.""" a = np.arange(30, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) with case.raises(): z.get_orthogonal_selection(case.input) with case.raises(): z.oindex[case.input] _ORTHO_2D_IX0_BOOL = np.isin(np.arange(12), [0, 5, 11]) # rows 0, 5, 11 _ORTHO_2D_IX1_BOOL = np.array([True, False, True, False, True]) # cols 0, 2, 4 _ORTHO_2D_IX0_INT = np.array([0, 5, 11]) _ORTHO_2D_IX1_INT = np.array([0, 2, 4]) _ORTHO_2D_CASES: list[Expect[OrthogonalSelection, None]] = [ Expect(input=(_ORTHO_2D_IX0_BOOL, _ORTHO_2D_IX1_BOOL), output=None, id="both-bool"), Expect(input=(_ORTHO_2D_IX0_BOOL, slice(1, 4)), output=None, id="bool-slice"), Expect(input=(_ORTHO_2D_IX0_BOOL, slice(0, 5, 2)), output=None, id="bool-strided-slice"), Expect(input=(slice(2, 9), _ORTHO_2D_IX1_BOOL), output=None, id="slice-bool"), Expect(input=(slice(0, 12, 4), _ORTHO_2D_IX1_BOOL), output=None, id="strided-slice-bool"), Expect(input=(_ORTHO_2D_IX0_BOOL, 3), output=None, id="bool-int"), Expect(input=(7, _ORTHO_2D_IX1_BOOL), output=None, id="int-bool"), Expect(input=(_ORTHO_2D_IX0_INT, _ORTHO_2D_IX1_INT), output=None, id="both-int"), Expect(input=(_ORTHO_2D_IX0_INT, _ORTHO_2D_IX1_BOOL), output=None, id="int-array-bool-array"), Expect(input=(_ORTHO_2D_IX0_BOOL, _ORTHO_2D_IX1_INT), output=None, id="bool-array-int-array"), Expect(input=7, output=None, id="single-row"), Expect(input=(slice(None), 3), output=None, id="single-col"), Expect(input=(slice(None), slice(None)), output=None, id="full"), Expect(input=slice(2, 9), output=None, id="row-slice"), ] _ORTHO_2D_BAD_CASES: list[ExpectFail[Any]] = [ ExpectFail( input=2.3, exception=IndexError, id="float-index", msg="unsupported selection item for orthogonal indexing", ), # get_orthogonal_selection and oindex raise different messages for a string # selection, so assert only the exception type. ExpectFail( input="foo", exception=IndexError, id="string-index", msg=None, ), ExpectFail( input=None, exception=IndexError, id="none-index", msg="unsupported selection item for orthogonal indexing", ), ExpectFail( input=slice(None, None, -1), exception=IndexError, id="negative-step", msg="only slices with step >= 1 are supported", ), ExpectFail( input=(0, 0, 0), exception=IndexError, id="too-many-dims", msg="too many indices for array", ), ] @pytest.mark.parametrize("case", _ORTHO_2D_CASES, ids=lambda c: c.id) def test_get_orthogonal_selection_2d( store: StorePath, case: Expect[OrthogonalSelection, None] ) -> None: """oindex on a 2D array matches numpy for array/slice/int combinations per axis.""" a = np.arange(60, dtype=int).reshape(12, 5) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) _test_get_orthogonal_selection(a, z, case.input) @pytest.mark.parametrize("case", _ORTHO_2D_BAD_CASES, ids=lambda c: c.id) def test_get_orthogonal_selection_2d_raises(store: StorePath, case: ExpectFail[Any]) -> None: """oindex on a 2D array rejects malformed selections with IndexError.""" a = np.arange(60, dtype=int).reshape(12, 5) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) with case.raises(): z.get_orthogonal_selection(case.input) with case.raises(): z.oindex[case.input] _ORTHO_3D_IX0_BOOL = np.isin(np.arange(7), [0, 3, 6]) # axis 0 _ORTHO_3D_IX1_BOOL = np.isin(np.arange(6), [0, 2, 5]) # axis 1 _ORTHO_3D_IX2_BOOL = np.isin(np.arange(10), [0, 4, 9]) # axis 2 _ORTHO_3D_IX0_INT = np.array([0, 3, 6]) _ORTHO_3D_IX1_INT = np.array([0, 2, 5]) _ORTHO_3D_IX2_INT = np.array([0, 4, 9]) _ORTHO_3D_CASES: list[Expect[OrthogonalSelection, None]] = [ # single value Expect(input=(5, 3, 8), output=None, id="single-value"), Expect(input=(-1, -1, -1), output=None, id="all-negative"), # index all axes with arrays Expect( input=(_ORTHO_3D_IX0_BOOL, _ORTHO_3D_IX1_BOOL, _ORTHO_3D_IX2_BOOL), output=None, id="three-bool-arrays", ), Expect( input=(_ORTHO_3D_IX0_INT, _ORTHO_3D_IX1_INT, _ORTHO_3D_IX2_INT), output=None, id="three-int-arrays", ), # mixed indexing with single array / slices Expect( input=(_ORTHO_3D_IX0_BOOL, slice(1, 5), slice(2, 9)), output=None, id="array-slice-slice" ), Expect( input=(slice(1, 6), _ORTHO_3D_IX1_BOOL, slice(2, 9)), output=None, id="slice-array-slice" ), Expect( input=(slice(1, 6), slice(1, 5), _ORTHO_3D_IX2_BOOL), output=None, id="slice-slice-array" ), Expect( input=(_ORTHO_3D_IX0_BOOL, slice(0, 6, 2), slice(0, 10, 3)), output=None, id="array-strided-strided", ), Expect( input=(slice(0, 7, 2), _ORTHO_3D_IX1_BOOL, slice(0, 10, 3)), output=None, id="strided-array-strided", ), Expect( input=(slice(0, 7, 2), slice(0, 6, 2), _ORTHO_3D_IX2_BOOL), output=None, id="strided-strided-array", ), # mixed indexing with single array / ints Expect(input=(_ORTHO_3D_IX0_BOOL, 3, 8), output=None, id="array-int-int"), Expect(input=(5, _ORTHO_3D_IX1_BOOL, 8), output=None, id="int-array-int"), Expect(input=(5, 3, _ORTHO_3D_IX2_BOOL), output=None, id="int-int-array"), # mixed indexing with single array / slice / int Expect(input=(_ORTHO_3D_IX0_BOOL, slice(1, 5), 8), output=None, id="array-slice-int"), Expect(input=(5, _ORTHO_3D_IX1_BOOL, slice(2, 9)), output=None, id="int-array-slice"), Expect(input=(slice(1, 6), 3, _ORTHO_3D_IX2_BOOL), output=None, id="slice-int-array"), # mixed indexing with two arrays / slice Expect( input=(_ORTHO_3D_IX0_BOOL, _ORTHO_3D_IX1_BOOL, slice(2, 9)), output=None, id="two-arrays-slice", ), Expect( input=(slice(1, 6), _ORTHO_3D_IX1_BOOL, _ORTHO_3D_IX2_BOOL), output=None, id="slice-two-arrays", ), Expect( input=(_ORTHO_3D_IX0_BOOL, slice(1, 5), _ORTHO_3D_IX2_BOOL), output=None, id="array-slice-array", ), # mixed indexing with two arrays / integer Expect(input=(_ORTHO_3D_IX0_BOOL, _ORTHO_3D_IX1_BOOL, 8), output=None, id="two-arrays-int"), Expect(input=(5, _ORTHO_3D_IX1_BOOL, _ORTHO_3D_IX2_BOOL), output=None, id="int-two-arrays"), Expect(input=(_ORTHO_3D_IX0_BOOL, 3, _ORTHO_3D_IX2_BOOL), output=None, id="array-int-array"), ] @pytest.mark.parametrize("case", _ORTHO_3D_CASES, ids=lambda c: c.id) def test_get_orthogonal_selection_3d( store: StorePath, case: Expect[OrthogonalSelection, None] ) -> None: """oindex on a 3D array matches numpy for array/slice/int combinations per axis.""" a = np.arange(420, dtype=int).reshape(7, 6, 10) z = zarr_array_from_numpy_array(store, a, chunk_shape=(3, 2, 4)) _test_get_orthogonal_selection(a, z, case.input) def test_orthogonal_indexing_edge_cases(store: StorePath) -> None: """oindex on a shape-(1, 2, 3) array correctly handles mixing integer, slice, int-list, and bool-list indexers per axis.""" a = np.arange(6).reshape(1, 2, 3) z = zarr_array_from_numpy_array(store, a, chunk_shape=(1, 2, 3)) expect = oindex(a, (0, slice(None), [0, 1, 2])) actual = z.oindex[0, :, [0, 1, 2]] assert_array_equal(expect, actual) expect = oindex(a, (0, slice(None), [True, True, True])) actual = z.oindex[0, :, [True, True, True]] assert_array_equal(expect, actual) @pytest.mark.parametrize("dtype", ["int8", "int64", "uint8", "uint16", "uint32", "uint64"]) def test_unsorted_index_unsigned_dtype(store: StorePath, dtype: str) -> None: a = np.arange(8).reshape(4, 2) z = zarr_array_from_numpy_array(store, a, chunk_shape=(2, 1)) rows = np.array([3, 0], dtype=dtype) assert_array_equal(a[[3, 0], :], z[rows, :]) assert_array_equal(a[[3, 0], [1, 0]], z.vindex[rows, np.array([1, 0], dtype=dtype)]) @pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.parametrize("indexer", ["oindex", "vindex"]) @pytest.mark.parametrize("operation", ["read", "write"]) @pytest.mark.parametrize("index", [4, 2**63, 2**64 - 4, 2**64 - 1]) def test_unsigned_index_out_of_bounds( zarr_format: ZarrFormat, indexer: str, operation: str, index: int ) -> None: """Unsigned indices must be checked before narrowing can turn them negative.""" data = np.arange(16).reshape(4, 4) arr = zarr.create_array({}, data=data, chunks=(2, 2), zarr_format=zarr_format) selection = (np.array([0, 1]), np.array([0, index], dtype="uint64")) accessor = getattr(arr, indexer) if operation == "read": with pytest.raises(IndexError, match="out of bounds"): accessor[selection] else: with pytest.raises(IndexError, match="out of bounds"): accessor[selection] = 99 assert_array_equal(arr[:], data) @pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.parametrize("indexer", ["oindex", "vindex"]) @pytest.mark.parametrize("dtype", ["uint8", "uint16", "uint32", "uint64"]) @pytest.mark.parametrize("indices", [[], [3, 0], [0, 3], [0] * 16 + [3] * 16]) def test_unsigned_index_roundtrip( zarr_format: ZarrFormat, indexer: str, dtype: str, indices: list[int] ) -> None: """Valid unsigned reads and writes retain the behavior fixed by #4286.""" expected = np.arange(4) arr = zarr.create_array({}, data=expected, chunks=(2,), zarr_format=zarr_format) selection = np.array(indices, dtype=dtype) accessor = getattr(arr, indexer) assert_array_equal(accessor[selection], expected[selection]) accessor[selection] = 99 expected[selection] = 99 assert_array_equal(arr[:], expected) @pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.parametrize("indexer", ["oindex", "vindex"]) @pytest.mark.parametrize("readonly", [False, True]) @pytest.mark.parametrize("indices", [[-1, 0], [0, 3]]) def test_index_array_preserved( zarr_format: ZarrFormat, indexer: str, readonly: bool, indices: list[int] ) -> None: """Shared, strided index arrays remain unchanged after reads and writes.""" expected = np.arange(20).reshape(4, 5) arr = zarr.create_array({}, data=expected, chunks=(2, 2), zarr_format=zarr_format) backing = np.array([indices[0], 99, indices[1], 99], dtype=np.intp) original = backing.copy() selection = backing[::2] selection.flags.writeable = not readonly accessor = getattr(arr, indexer) # Reuse the same array on axes of different lengths: -1 must wrap independently. numpy_selection = ( np.ix_(selection, selection) if indexer == "oindex" else (selection, selection) ) assert_array_equal(accessor[selection, selection], expected[numpy_selection]) assert_array_equal(backing, original) accessor[selection, selection] = 99 expected[numpy_selection] = 99 assert_array_equal(arr[:], expected) assert_array_equal(backing, original) @pytest.mark.parametrize("indexer", ["oindex", "vindex"]) @pytest.mark.parametrize("operation", ["read", "write"]) def test_invalid_negative_index_array_preserved(indexer: str, operation: str) -> None: """Even a rejected selection must not modify caller-owned indices or array data.""" expected = np.arange(4) arr = zarr.create_array({}, data=expected, chunks=(2,)) selection = np.array([-5, 0], dtype=np.intp) accessor = getattr(arr, indexer) if operation == "read": with pytest.raises(IndexError, match="out of bounds"): accessor[selection] else: with pytest.raises(IndexError, match="out of bounds"): accessor[selection] = 99 assert_array_equal(selection, [-5, 0]) assert_array_equal(arr[:], expected) def _test_set_orthogonal_selection( v: npt.NDArray[np.int_], a: npt.NDArray[Any], z: Array, selection: OrthogonalSelection ) -> None: for value in 42, oindex(v, selection), oindex(v, selection).tolist(): if isinstance(value, list) and value == []: # skip these cases as cannot preserve all dimensions continue # setup expectation a[:] = 0 oindex_set(a, selection, value) # long-form API z[:] = 0 z.set_orthogonal_selection(selection, value) assert_array_equal(a, z[:]) # short-form API z[:] = 0 z.oindex[selection] = value assert_array_equal(a, z[:]) @pytest.mark.parametrize("case", _ORTHO_1D_BOOL_CASES + _ORTHO_1D_INT_CASES, ids=lambda c: c.id) def test_set_orthogonal_selection_1d( store: StorePath, case: Expect[OrthogonalSelection, None] ) -> None: """set_orthogonal_selection on a 1D array round-trips through numpy for masks and int arrays.""" v = np.arange(30, dtype=int) a = np.empty_like(v) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) _test_set_orthogonal_selection(v, a, z, case.input) def test_set_item_1d_last_two_chunks(store: StorePath): """Regression for GH2849: `__setitem__` correctly writes to the last two chunks of a 1D array and to 0-dimensional scalar arrays.""" # regression test for GH2849 g = zarr.open_group(store=store, zarr_format=3, mode="w") a = g.create_array("bar", shape=(10,), chunks=(3,), dtype=int) data = np.array([7, 8, 9]) a[slice(7, 10)] = data np.testing.assert_array_equal(a[slice(7, 10)], data) z = zarr.open_group(store=store, mode="w") z.create_array("zoo", dtype=float, shape=()) z["zoo"][...] = np.array(1) # why doesn't [:] work? np.testing.assert_equal(z["zoo"][()], np.array(1)) z = zarr.open_group(store=store, mode="w") z.create_array("zoo", dtype=float, shape=()) z["zoo"][...] = 1 # why doesn't [:] work? np.testing.assert_equal(z["zoo"][()], np.array(1)) @pytest.mark.parametrize("case", _ORTHO_2D_CASES, ids=lambda c: c.id) def test_set_orthogonal_selection_2d( store: StorePath, case: Expect[OrthogonalSelection, None] ) -> None: """set_orthogonal_selection on a 2D array round-trips through numpy.""" v = np.arange(60, dtype=int).reshape(12, 5) a = np.empty_like(v) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) _test_set_orthogonal_selection(v, a, z, case.input) @pytest.mark.parametrize("case", _ORTHO_3D_CASES, ids=lambda c: c.id) def test_set_orthogonal_selection_3d( store: StorePath, case: Expect[OrthogonalSelection, None] ) -> None: """set_orthogonal_selection on a 3D array round-trips through numpy.""" v = np.arange(420, dtype=int).reshape(7, 6, 10) a = np.empty_like(v) z = zarr_array_from_numpy_array(store, a, chunk_shape=(3, 2, 4)) _test_set_orthogonal_selection(v, a, z, case.input) def test_orthogonal_indexing_fallback_on_get_setitem(store: StorePath) -> None: """Paired integer-list indexing on a 2D array falls back to orthogonal get and set via `__getitem__`/`__setitem__`.""" z = zarr_array_from_numpy_array(store, np.zeros((20, 20))) z[[1, 2, 3], [1, 2, 3]] = 1 np.testing.assert_array_equal( z[:4, :4], [ [0, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1], ], ) np.testing.assert_array_equal(z[[1, 2, 3], [1, 2, 3]], 1) # test broadcasting np.testing.assert_array_equal(z[1, [1, 2, 3]], [1, 0, 0]) # test 1D fancy indexing z2 = zarr_array_from_numpy_array(store, np.zeros(5)) z2[[1, 2, 3]] = 1 np.testing.assert_array_equal(z2[:], [0, 1, 1, 1, 0]) def _test_get_coordinate_selection( a: npt.NDArray, z: Array, selection: CoordinateSelection ) -> None: expect = a[selection] actual = z.get_coordinate_selection(selection) assert_array_equal(expect, actual) actual = z.vindex[selection] assert_array_equal(expect, actual) _COORD_1D_CASES: list[Expect[CoordinateSelection, None]] = [ Expect(input=5, output=None, id="single"), Expect(input=-1, output=None, id="single-negative"), Expect(input=[0, 3, 10, -23, -12, -1], output=None, id="wraparound"), Expect(input=[3, 25, 8, 17], output=None, id="out-of-order"), Expect(input=[1, 8, 15, 29], output=None, id="sorted"), Expect(input=[29, 15, 8, 1], output=None, id="reversed"), Expect(input=np.array([29, 15, 8, 1], dtype=np.uint32), output=None, id="reversed-uint"), Expect(input=[2, 2, 8, 8], output=None, id="duplicates"), Expect(input=np.array([[2, 4], [6, 8]]), output=None, id="multi-dim"), # sorted-1D fast path (chunk_shape=(7,)): boundaries, contiguous runs, single chunk, full Expect(input=[0, 6, 7, 13, 14, 28, 29], output=None, id="sorted-chunk-boundaries"), Expect(input=[0, 1, 2, 8, 9, 10, 21, 22, 23], output=None, id="sorted-contiguous-runs"), Expect(input=[1, 2, 3, 4, 5, 6], output=None, id="sorted-single-chunk"), Expect(input=list(range(30)), output=None, id="sorted-full"), Expect(input=[0, 0, 7, 7, 7, 29], output=None, id="sorted-duplicates-boundaries"), ] # get_coordinate_selection and vindex word their errors differently for these # invalid-type inputs, so these cases assert only the exception type (msg=None). _COORD_1D_BAD_CASES: list[ExpectFail[Any]] = [ ExpectFail(input=slice(5, 15), exception=IndexError, id="slice", msg=None), ExpectFail(input=slice(None), exception=IndexError, id="full-slice", msg=None), ExpectFail(input=Ellipsis, exception=IndexError, id="ellipsis", msg=None), ExpectFail(input=2.3, exception=IndexError, id="float", msg=None), ExpectFail(input="foo", exception=IndexError, id="string", msg=None), ExpectFail(input=b"xxx", exception=IndexError, id="bytes", msg=None), ExpectFail(input=None, exception=IndexError, id="none", msg=None), ExpectFail(input=(0, 0), exception=IndexError, id="tuple-pair", msg=None), ExpectFail(input=(slice(None), slice(None)), exception=IndexError, id="two-slices", msg=None), ExpectFail( input=[31], exception=IndexError, id="out-of-bounds-high", msg="index out of bounds for dimension with length 30", ), ExpectFail( input=[-31], exception=IndexError, id="out-of-bounds-low", msg="index out of bounds for dimension with length 30", ), ] _COORD_2D_IX0 = np.array([0, 5, 11, 2, 8]) _COORD_2D_IX1 = np.array([1, 3, 4, 0, 2]) _COORD_2D_CASES: list[Expect[CoordinateSelection, None]] = [ Expect(input=(5, 4), output=None, id="single"), Expect(input=(-1, -1), output=None, id="single-negative"), Expect(input=(_COORD_2D_IX0, _COORD_2D_IX1), output=None, id="both-arrays"), # scalar broadcasts in coordinate indexing (numpy and zarr agree) Expect(input=(np.array([0, 5, 11]), 4), output=None, id="array-int"), Expect(input=(7, np.array([0, 2, 4])), output=None, id="int-array"), Expect(input=([3, 3, 4, 2, 5], [1, 3, 4, 0, 2]), output=None, id="not-monotonic-first"), Expect(input=([1, 1, 2, 2, 5], [1, 3, 2, 1, 0]), output=None, id="not-monotonic-second"), Expect( input=(np.array([[1, 1, 2], [2, 2, 5]]), np.array([[1, 3, 2], [1, 0, 0]])), output=None, id="multi-dim", ), ] _COORD_2D_BAD_CASES: list[ExpectFail[Any]] = [ ExpectFail( input=(slice(5, 15), [1, 2, 3]), exception=IndexError, id="slice-with-array", msg=None, ), ExpectFail( input=([1, 2, 3], slice(5, 15)), exception=IndexError, id="array-with-slice", msg=None, ), ExpectFail( input=(Ellipsis, [1, 2, 3]), exception=IndexError, id="ellipsis-with-array", msg=None, ), ExpectFail(input=Ellipsis, exception=IndexError, id="ellipsis", msg=None), ExpectFail( input=(np.array([12]), np.array([0])), exception=IndexError, id="out-of-bounds-axis0", msg="index out of bounds for dimension with length 12", ), ExpectFail( input=(np.array([0]), np.array([5])), exception=IndexError, id="out-of-bounds-axis1", msg="index out of bounds for dimension with length 5", ), ] @pytest.mark.parametrize("case", _COORD_1D_CASES, ids=lambda c: c.id) def test_get_coordinate_selection_1d( store: StorePath, case: Expect[CoordinateSelection, None] ) -> None: """vindex and get_coordinate_selection on a 1D array match numpy for int, list, and multi-dim selections.""" a = np.arange(30, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) _test_get_coordinate_selection(a, z, case.input) @pytest.mark.parametrize( ("chunks", "shards"), [((7,), None), ((7,), (21,))], ids=["chunked", "sharded"], ) def test_get_coordinate_selection_1d_fast_path( store: StorePath, chunks: tuple[int, ...], shards: tuple[int, ...] | None ) -> None: """The sorted-1D-runs fast path in CoordinateIndexer matches numpy on chunked and sharded arrays. Exercises the boundary/run/single-chunk/full-array cases that the fast path optimizes, plus the sharded case where the top-level (shard) grid drives chunk assignment. """ a = np.arange(210, dtype=int) z = zarr.create_array( store=store / str(uuid4()), shape=a.shape, dtype=a.dtype, chunks=chunks, shards=shards, ) z[:] = a rng = np.random.default_rng(0) selections = [ np.sort(rng.choice(210, 60, replace=False)), # scattered sorted np.array([0, 6, 7, 20, 21, 209]), # chunk/shard boundaries np.concatenate([np.arange(s, s + 5) for s in (0, 33, 100, 180)]), # contiguous runs np.array([0, 0, 7, 7, 209]), # sorted with duplicates np.arange(210), # whole array np.array([5]), # single element ] for sel in selections: assert_array_equal(a[sel], z.get_coordinate_selection(sel)) assert_array_equal(a[sel], z.vindex[sel]) def test_coordinate_indexer_1d_last_chunk_boundary_does_not_overflow() -> None: max_intp = np.iinfo(np.intp).max chunk_size = max_intp // 2 + 1 coords = np.arange(max_intp - 4, max_intp, dtype=np.intp) chunk_grid = ChunkGrid.from_sizes((max_intp,), (chunk_size,)) (projection,) = tuple(CoordinateIndexer((coords,), (max_intp,), chunk_grid)) assert projection.chunk_coords == (1,) assert_array_equal(projection.chunk_selection[0], coords - chunk_size) assert projection.out_selection == slice(0, 4) @pytest.mark.parametrize("coord_dtype", [np.int8, np.uint8, np.uint32]) def test_coordinate_selection_1d_narrow_dtype_large_chunk( store: StorePath, coord_dtype: type[np.integer[Any]] ) -> None: source = np.arange(1_000) coords = np.arange(10, dtype=coord_dtype) z = zarr_array_from_numpy_array(store, source, chunk_shape=(1_000,)) assert_array_equal(z.get_coordinate_selection(coords), source[coords]) assert_array_equal(z.vindex[coords], source[coords]) assert_array_equal(z[coords], source[coords]) expected = source.copy() expected[coords] = -1 z.set_coordinate_selection(coords, -1) assert_array_equal(z[:], expected) z[:] = source z.vindex[coords] = -1 assert_array_equal(z[:], expected) def test_coordinate_indexer_1d_sparse_selection_uses_general_path( monkeypatch: pytest.MonkeyPatch, ) -> None: coords = np.array([0, 99]) chunk_grid = ChunkGrid.from_sizes((100,), (1,)) def unexpected_searchsorted(*args: Any, **kwargs: Any) -> None: pytest.fail("sparse coordinate selection should not call searchsorted") monkeypatch.setattr(np, "searchsorted", unexpected_searchsorted) projections = tuple(CoordinateIndexer((coords,), (100,), chunk_grid)) assert tuple(projection.chunk_coords for projection in projections) == ((0,), (99,)) @pytest.mark.parametrize("kind", ["coordinate", "coordinate-fast", "integer", "mask"]) def test_lazy_dense_indexer_attributes(kind: str) -> None: """Dense compatibility arrays are lazy, cached, mutable, and warning-free.""" grid = ChunkGrid.from_sizes((20,), (3,)) coords = np.array([1, 4, 4, 19]) if kind == "coordinate-fast": coords = np.repeat(coords, 100) elif kind == "mask": coords = np.unique(coords) indexer = ( IntArrayDimIndexer(coords, 20, grid._dimensions[0]) if kind == "integer" else CoordinateIndexer((coords,), (20,), grid) ) if kind == "mask": mask = np.zeros(20, dtype=bool) mask[coords] = True indexer = MaskIndexer((mask,), (20,), grid) expected_counts = np.bincount(coords // 3, minlength=7) expected = {"chunk_nitems_cumsum": np.cumsum(expected_counts)} if kind == "integer": expected["chunk_nitems"] = expected_counts with warnings.catch_warnings(): warnings.simplefilter("error") tuple(indexer) repr(indexer) assert all(name not in vars(indexer) for name in expected) assert set(expected) <= {f.name for f in dataclass_fields(indexer)} for name, values in expected.items(): dense = getattr(indexer, name) assert_array_equal(dense, values) assert getattr(indexer, name) is dense assert_array_equal(asdict(indexer)[name], values) dense[0] = 123 assert getattr(indexer, name)[0] == 123 # Iteration respects edits to the materialized cumulative offsets. cumulative = indexer.chunk_nitems_cumsum cumulative[:] = expected["chunk_nitems_cumsum"] cumulative[-1] = cumulative[-2] final = tuple(indexer)[-1] output = ( final.dim_out_sel if isinstance(indexer, IntArrayDimIndexer) else final.out_selection ) assert output == slice(cumulative[-2], cumulative[-2]) @pytest.mark.parametrize("kind", ["coordinate", "integer"]) def test_indexer_unknown_attribute(kind: str) -> None: grid = ChunkGrid.from_sizes((20,), (3,)) coords = np.array([1, 4, 19]) indexer = ( IntArrayDimIndexer(coords, 20, grid._dimensions[0]) if kind == "integer" else CoordinateIndexer((coords,), (20,), grid) ) with pytest.raises(AttributeError, match="missing_attribute"): _ = indexer.missing_attribute def test_sparse_selections_on_arrays_with_many_chunks(store: StorePath) -> None: """Coordinate and orthogonal selections must scale with the number of selected points, not the total number of chunks in the array. This array has 2**44 chunks, so any dense per-chunk allocation fails before the assertions are reached. See gh-4174.""" z = zarr.create_array( store=store / str(uuid4()), shape=(2**22, 2**22), chunks=(1, 1), dtype="int32", fill_value=-1, ) # unsorted coords with duplicates force the argsort branch of the general path rows = np.array([7, 0, 3, 0]) cols = np.array([1, 5, 2**22 - 1, 5]) z.set_coordinate_selection((rows, cols), np.array([10, 20, 30, 20], dtype="int32")) assert_array_equal( z.get_coordinate_selection((rows, cols)), np.array([10, 20, 30, 20], dtype="int32") ) # points in untouched chunks come back as fill_value assert_array_equal( z.get_coordinate_selection((np.array([0, 42]), np.array([0, 42]))), np.array([-1, -1], dtype="int32"), ) # orthogonal integer-array selections: increasing, decreasing, and unsorted order for coords in ([3, 5, 6], [6, 5, 3], [5, 6, 3]): vals = np.arange(1, len(coords) + 1, dtype="int32") z.set_orthogonal_selection((np.array(coords), 0), vals) assert_array_equal(z.get_orthogonal_selection((np.array(coords), 0)), vals) def test_coordinate_indexer_many_chunks() -> None: """Both CoordinateIndexer paths (sorted-1D fast path and general path) produce correct projections on a grid whose chunk count (2**62) is far too large for any dense per-chunk array. See gh-4174.""" dim_len = 2**62 chunk_grid = ChunkGrid.from_sizes((dim_len,), (1,)) # sorted coords, sparse relative to their span: the general path without a sort coords = np.array([3, 4, 4, dim_len - 1]) projections = tuple(CoordinateIndexer((coords,), (dim_len,), chunk_grid)) assert tuple(p.chunk_coords for p in projections) == ((3,), (4,), (dim_len - 1,)) assert [p.out_selection for p in projections] == [slice(0, 1), slice(1, 3), slice(3, 4)] for p in projections: assert_array_equal(p.chunk_selection[0], np.zeros(len(p.chunk_selection[0]), dtype=int)) # sorted coords, dense relative to their span: the searchsorted fast path coords = np.concatenate([np.full(100, 3), np.full(100, 7)]) projections = tuple(CoordinateIndexer((coords,), (dim_len,), chunk_grid)) assert tuple(p.chunk_coords for p in projections) == ((3,), (7,)) assert [p.out_selection for p in projections] == [slice(0, 100), slice(100, 200)] # unsorted coords: the general (argsort) path coords = np.array([dim_len - 1, 5]) projections = tuple(CoordinateIndexer((coords,), (dim_len,), chunk_grid)) assert tuple(p.chunk_coords for p in projections) == ((5,), (dim_len - 1,)) assert [list(p.out_selection) for p in projections] == [[1], [0]] @pytest.mark.parametrize( ("coords", "expected_out_sels"), [ pytest.param([3, 5, 5], [slice(0, 1), slice(1, 3)], id="increasing"), pytest.param([5, 3, 1], [[2], [1], [0]], id="decreasing"), pytest.param([5, 1, 3], [[1], [2], [0]], id="unordered"), ], ) def test_int_array_dim_indexer_many_chunks(coords: list[int], expected_out_sels: list[Any]) -> None: """IntArrayDimIndexer produces correct projections for every ordering of the selection on a dimension with 2**62 chunks, where any dense per-chunk array would fail. See gh-4174.""" dim_len = 2**62 (dim_grid,) = ChunkGrid.from_sizes((dim_len,), (1,))._dimensions indexer = IntArrayDimIndexer(np.array(coords), dim_len, dim_grid) projections = tuple(indexer) assert [p.dim_chunk_ix for p in projections] == sorted(set(coords)) for p, expected_out_sel in zip(projections, expected_out_sels, strict=True): # chunk size is 1, so every selected point maps to offset 0 within its chunk assert_array_equal(p.dim_chunk_sel, np.zeros(len(p.dim_chunk_sel), dtype=int)) if isinstance(expected_out_sel, slice): assert p.dim_out_sel == expected_out_sel else: assert list(p.dim_out_sel) == expected_out_sel def test_get_coordinate_selection_1d_irregular_grid(store: StorePath) -> None: """Coordinate selections on an irregular (rectilinear) chunk grid bypass the sorted-1D fast path (which requires a regular grid) and still match numpy via the general path.""" a = np.arange(30, dtype=int) with zarr.config.set({"array.rectilinear_chunks": True}): z = zarr.create_array( store=store / str(uuid4()), shape=a.shape, dtype=a.dtype, chunks=((3, 3, 4, 5, 5, 5, 5),), ) z[:] = a for sel in (np.array([1, 8, 15, 29]), np.array([0, 3, 3, 29]), np.arange(30)): assert_array_equal(a[sel], z.get_coordinate_selection(sel)) @pytest.mark.parametrize("case", _COORD_1D_BAD_CASES, ids=lambda c: c.id) def test_get_coordinate_selection_1d_raises(store: StorePath, case: ExpectFail[Any]) -> None: """get_coordinate_selection and vindex both raise IndexError for invalid 1D selections.""" a = np.arange(30, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) with case.raises(): z.get_coordinate_selection(case.input) # type: ignore[arg-type] with case.raises(): z.vindex[case.input] # type: ignore[index] @pytest.mark.parametrize("case", _COORD_2D_CASES, ids=lambda c: c.id) def test_get_coordinate_selection_2d( store: StorePath, case: Expect[CoordinateSelection, None] ) -> None: """vindex and get_coordinate_selection on a 2D array match numpy for coordinate selections.""" a = np.arange(60, dtype=int).reshape(12, 5) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) _test_get_coordinate_selection(a, z, case.input) @pytest.mark.parametrize("case", _COORD_2D_BAD_CASES, ids=lambda c: c.id) def test_get_coordinate_selection_2d_raises(store: StorePath, case: ExpectFail[Any]) -> None: """get_coordinate_selection raises IndexError when slices or Ellipsis appear in a 2D coordinate selection.""" a = np.arange(60, dtype=int).reshape(12, 5) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) with case.raises(): z.get_coordinate_selection(case.input) # type: ignore[arg-type] def _test_set_coordinate_selection( v: npt.NDArray, a: npt.NDArray, z: Array, selection: CoordinateSelection ) -> None: for value in 42, v[selection], v[selection].tolist(): # setup expectation a[:] = 0 a[selection] = value # test long-form API z[:] = 0 z.set_coordinate_selection(selection, value) assert_array_equal(a, z[:]) # test short-form API z[:] = 0 z.vindex[selection] = value assert_array_equal(a, z[:]) @pytest.mark.parametrize("case", _COORD_1D_CASES, ids=lambda c: c.id) def test_set_coordinate_selection_1d( store: StorePath, case: Expect[CoordinateSelection, None] ) -> None: """set_coordinate_selection and vindex assignment on a 1D array round-trip through numpy.""" v = np.arange(30, dtype=int) a = np.empty_like(v) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) _test_set_coordinate_selection(v, a, z, case.input) @pytest.mark.parametrize("case", _COORD_2D_CASES, ids=lambda c: c.id) def test_set_coordinate_selection_2d( store: StorePath, case: Expect[CoordinateSelection, None] ) -> None: """set_coordinate_selection and vindex assignment on a 2D array round-trip through numpy.""" v = np.arange(60, dtype=int).reshape(12, 5) a = np.empty_like(v) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) _test_set_coordinate_selection(v, a, z, case.input) def _test_get_block_selection( a: npt.NDArray[Any], z: Array, selection: BasicSelection, expected_idx: slice | tuple[slice, ...], ) -> None: expect = a[expected_idx] actual = z.get_block_selection(selection) assert_array_equal(expect, actual) actual = z.blocks[selection] assert_array_equal(expect, actual) _BLOCK_1D_CASES: list[Expect[BasicSelection, slice]] = [ Expect(input=0, output=slice(0, 7), id="block-0"), Expect(input=2, output=slice(14, 21), id="block-mid"), Expect(input=4, output=slice(28, 30), id="block-last"), Expect(input=-1, output=slice(28, 30), id="block-neg-1"), Expect(input=-2, output=slice(21, 28), id="block-neg-2"), Expect(input=slice(3), output=slice(0, 21), id="slice-to-3"), Expect(input=slice(None, 2), output=slice(0, 14), id="slice-none-2"), Expect(input=slice(1, 2), output=slice(7, 14), id="slice-1-2"), Expect(input=slice(-2, -1), output=slice(21, 28), id="slice-neg"), Expect(input=slice(None), output=slice(0, 30), id="full"), ] _BLOCK_1D_BAD_CASES: list[ExpectFail[Any]] = [ ExpectFail(input=slice(3, 8, 2), exception=IndexError, id="strided-slice"), ExpectFail(input=2.3, exception=IndexError, id="float"), ExpectFail(input=b"xxx", exception=IndexError, id="bytes"), ExpectFail(input=None, exception=IndexError, id="none"), ExpectFail(input=(0, 0), exception=IndexError, id="tuple-pair"), ExpectFail(input=(slice(None), slice(None)), exception=IndexError, id="two-slices"), ExpectFail(input=[0, 5, 3], exception=IndexError, id="int-list"), ExpectFail(input=5, exception=IndexError, id="out-of-bounds-high"), ExpectFail(input=-6, exception=IndexError, id="out-of-bounds-low"), ] _BLOCK_2D_CASES: list[Expect[BasicSelection, tuple[slice, slice]]] = [ Expect(input=(0, 0), output=(slice(0, 5), slice(0, 2)), id="single-00"), Expect(input=(1, 1), output=(slice(5, 10), slice(2, 4)), id="single-mid"), Expect(input=(-1, -1), output=(slice(10, 12), slice(4, 5)), id="neg"), Expect(input=(slice(0, 2), 0), output=(slice(0, 10), slice(0, 2)), id="slice-rows"), Expect(input=(2, slice(1, 3)), output=(slice(10, 12), slice(2, 5)), id="slice-cols"), Expect(input=(slice(0, 2), slice(0, 2)), output=(slice(0, 10), slice(0, 4)), id="both-slices"), Expect(input=(slice(None), slice(None)), output=(slice(0, 12), slice(0, 5)), id="full"), ] _BLOCK_2D_BAD_CASES: list[ExpectFail[Any]] = [ ExpectFail(input=(slice(5, 15), [1, 2, 3]), exception=IndexError, id="slice-with-array"), ExpectFail(input=(Ellipsis, [1, 2, 3]), exception=IndexError, id="ellipsis-with-array"), ExpectFail(input=(slice(15, 20), slice(None)), exception=IndexError, id="out-of-bounds"), ] @pytest.mark.parametrize("case", _BLOCK_1D_CASES, ids=lambda c: c.id) def test_get_block_selection_1d(store: StorePath, case: Expect[BasicSelection, slice]) -> None: """get_block_selection / .blocks on a 1D array selects whole chunks matching the array slice.""" a = np.arange(30, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) _test_get_block_selection(a, z, case.input, case.output) @pytest.mark.parametrize("case", _BLOCK_1D_BAD_CASES, ids=lambda c: c.id) def test_get_block_selection_1d_raises(store: StorePath, case: ExpectFail[Any]) -> None: """get_block_selection / .blocks on a 1D array rejects invalid block selections with IndexError.""" a = np.arange(30, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) with case.raises(): z.get_block_selection(case.input) with case.raises(): z.blocks[case.input] @pytest.mark.parametrize("case", _BLOCK_2D_CASES, ids=lambda c: c.id) def test_get_block_selection_2d( store: StorePath, case: Expect[BasicSelection, tuple[slice, slice]] ) -> None: """get_block_selection / .blocks on a 2D array selects whole chunk regions matching the array slices.""" a = np.arange(60, dtype=int).reshape(12, 5) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) _test_get_block_selection(a, z, case.input, case.output) @pytest.mark.parametrize("case", _BLOCK_2D_BAD_CASES, ids=lambda c: c.id) def test_get_block_selection_2d_raises(store: StorePath, case: ExpectFail[Any]) -> None: """get_block_selection on a 2D array rejects invalid or out-of-bounds block selections with IndexError.""" a = np.arange(60, dtype=int).reshape(12, 5) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) with case.raises(): z.get_block_selection(case.input) def _test_set_block_selection( v: npt.NDArray[Any], a: npt.NDArray[Any], z: zarr.Array, selection: BasicSelection, expected_idx: slice | tuple[slice, ...], ) -> None: for value in 42, v[expected_idx], v[expected_idx].tolist(): # setup expectation a[:] = 0 a[expected_idx] = value # test long-form API z[:] = 0 z.set_block_selection(selection, value) assert_array_equal(a, z[:]) # test short-form API z[:] = 0 z.blocks[selection] = value assert_array_equal(a, z[:]) @pytest.mark.parametrize("case", _BLOCK_1D_CASES, ids=lambda c: c.id) def test_set_block_selection_1d(store: StorePath, case: Expect[BasicSelection, slice]) -> None: """set_block_selection / .blocks assignment on a 1D array round-trips through numpy for each block selection.""" v = np.arange(30, dtype=int) a = np.empty_like(v) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) _test_set_block_selection(v, a, z, case.input, case.output) @pytest.mark.parametrize("case", _BLOCK_1D_BAD_CASES, ids=lambda c: c.id) def test_set_block_selection_1d_raises(store: StorePath, case: ExpectFail[Any]) -> None: """set_block_selection / .blocks assignment on a 1D array rejects invalid block selections with IndexError.""" a = np.arange(30, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) with case.raises(): z.set_block_selection(case.input, 42) with case.raises(): z.blocks[case.input] = 42 @pytest.mark.parametrize("case", _BLOCK_2D_CASES, ids=lambda c: c.id) def test_set_block_selection_2d( store: StorePath, case: Expect[BasicSelection, tuple[slice, slice]] ) -> None: """set_block_selection / .blocks assignment on a 2D array round-trips through numpy for each block selection.""" v = np.arange(60, dtype=int).reshape(12, 5) a = np.empty_like(v) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) _test_set_block_selection(v, a, z, case.input, case.output) @pytest.mark.parametrize("case", _BLOCK_2D_BAD_CASES, ids=lambda c: c.id) def test_set_block_selection_2d_raises(store: StorePath, case: ExpectFail[Any]) -> None: """set_block_selection on a 2D array rejects invalid or out-of-bounds block selections with IndexError.""" a = np.arange(60, dtype=int).reshape(12, 5) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) with case.raises(): z.set_block_selection(case.input, 42) def _test_get_mask_selection(a: npt.NDArray[Any], z: Array, selection: npt.NDArray) -> None: expect = a[selection] actual = z.get_mask_selection(selection) assert_array_equal(expect, actual) actual = z.vindex[selection] assert_array_equal(expect, actual) actual = z[selection] assert_array_equal(expect, actual) _MASK_1D_CASES: list[Expect[Any, None]] = [ Expect(input=np.zeros(30, dtype=bool), output=None, id="all-false"), Expect(input=np.ones(30, dtype=bool), output=None, id="all-true"), Expect(input=np.arange(30) % 2 == 0, output=None, id="alternating"), Expect( input=np.isin(np.arange(30), [0, 7, 14, 29]), output=None, id="sparse-cross-chunk", ), ] # msg=None for all 1d bad cases: get_mask_selection and vindex raise different # messages for the same input, so no single substring satisfies both assertions. _MASK_1D_BAD_CASES: list[ExpectFail[Any]] = [ ExpectFail(input=slice(5, 15), exception=IndexError, id="slice"), ExpectFail(input=slice(None), exception=IndexError, id="full-slice"), ExpectFail(input=Ellipsis, exception=IndexError, id="ellipsis"), ExpectFail(input=2.3, exception=IndexError, id="float"), ExpectFail(input="foo", exception=IndexError, id="string"), ExpectFail(input=b"xxx", exception=IndexError, id="bytes"), ExpectFail(input=None, exception=IndexError, id="none"), ExpectFail(input=(0, 0), exception=IndexError, id="tuple-pair"), ExpectFail(input=(slice(None), slice(None)), exception=IndexError, id="two-slices"), ExpectFail(input=np.zeros(5, dtype=bool), exception=IndexError, id="mask-too-short"), ExpectFail(input=np.zeros(50, dtype=bool), exception=IndexError, id="mask-too-long"), ExpectFail(input=[[True, False], [False, True]], exception=IndexError, id="too-many-dims"), ] def _make_sparse_2d_mask() -> npt.NDArray[np.bool_]: """Build a deterministic sparse (12, 5) boolean mask with Trues at (0,0), (5,2), (11,4), (2,3).""" mask = np.zeros((12, 5), dtype=bool) for r, c in [(0, 0), (5, 2), (11, 4), (2, 3)]: mask[r, c] = True return mask _MASK_2D_CASES: list[Expect[Any, None]] = [ Expect(input=np.zeros((12, 5), dtype=bool), output=None, id="all-false"), Expect(input=np.ones((12, 5), dtype=bool), output=None, id="all-true"), Expect( input=(np.add.outer(np.arange(12), np.arange(5)) % 2).astype(bool), output=None, id="checkerboard", ), Expect( input=_make_sparse_2d_mask(), output=None, id="sparse", ), ] _MASK_2D_BAD_CASES: list[ExpectFail[Any]] = [ ExpectFail(input=np.zeros((12, 3), dtype=bool), exception=IndexError, id="too-few-cols"), ExpectFail(input=np.zeros((20, 5), dtype=bool), exception=IndexError, id="too-many-rows"), ExpectFail(input=[True, False], exception=IndexError, id="wrong-ndim"), ] @pytest.mark.parametrize("case", _MASK_1D_CASES, ids=lambda c: c.id) def test_get_mask_selection_1d(store: StorePath, case: Expect[Any, None]) -> None: """get_mask_selection / vindex / getitem on a 1D array match numpy for boolean masks.""" a = np.arange(30, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) _test_get_mask_selection(a, z, case.input) @pytest.mark.parametrize("case", _MASK_1D_BAD_CASES, ids=lambda c: c.id) def test_get_mask_selection_1d_raises(store: StorePath, case: ExpectFail[Any]) -> None: """get_mask_selection / vindex on a 1D array reject non-boolean-mask and mis-shaped selections.""" a = np.arange(30, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) with case.raises(): z.get_mask_selection(case.input) # type: ignore[arg-type] with case.raises(): z.vindex[case.input] # type: ignore[index] @pytest.mark.parametrize("case", _MASK_2D_CASES, ids=lambda c: c.id) def test_get_mask_selection_2d(store: StorePath, case: Expect[Any, None]) -> None: """get_mask_selection / vindex / getitem on a 2D array match numpy for boolean masks.""" a = np.arange(60, dtype=int).reshape(12, 5) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) _test_get_mask_selection(a, z, case.input) @pytest.mark.parametrize("case", _MASK_2D_BAD_CASES, ids=lambda c: c.id) def test_get_mask_selection_2d_raises(store: StorePath, case: ExpectFail[Any]) -> None: """vindex on a 2D array rejects masks of the wrong shape or dimensionality.""" a = np.arange(60, dtype=int).reshape(12, 5) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) with case.raises(): z.vindex[case.input] # type: ignore[index] def _test_set_mask_selection( v: npt.NDArray, a: npt.NDArray, z: Array, selection: npt.NDArray ) -> None: a[:] = 0 z[:] = 0 a[selection] = v[selection] z.set_mask_selection(selection, v[selection]) assert_array_equal(a, z[:]) z[:] = 0 z.vindex[selection] = v[selection] assert_array_equal(a, z[:]) z[:] = 0 z[selection] = v[selection] assert_array_equal(a, z[:]) @pytest.mark.parametrize("case", _MASK_1D_CASES, ids=lambda c: c.id) def test_set_mask_selection_1d(store: StorePath, case: Expect[Any, None]) -> None: """set_mask_selection / vindex / setitem on a 1D array match numpy for boolean masks.""" v = np.arange(30, dtype=int) a = np.empty_like(v) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) _test_set_mask_selection(v, a, z, case.input) @pytest.mark.parametrize("case", _MASK_1D_BAD_CASES, ids=lambda c: c.id) def test_set_mask_selection_1d_raises(store: StorePath, case: ExpectFail[Any]) -> None: """set_mask_selection / vindex on a 1D array reject non-boolean-mask and mis-shaped selections.""" a = np.arange(30, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) with case.raises(): z.set_mask_selection(case.input, 42) # type: ignore[arg-type] with case.raises(): z.vindex[case.input] = 42 # type: ignore[index] @pytest.mark.parametrize("case", _MASK_2D_CASES, ids=lambda c: c.id) def test_set_mask_selection_2d(store: StorePath, case: Expect[Any, None]) -> None: """set_mask_selection / vindex / setitem on a 2D array match numpy for boolean masks.""" v = np.arange(60, dtype=int).reshape(12, 5) a = np.empty_like(v) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) _test_set_mask_selection(v, a, z, case.input) def test_get_selection_out(store: StorePath) -> None: """get_*_selection writes results into a provided out buffer, matching numpy.""" # basic selections a = np.arange(30) z = zarr_array_from_numpy_array(store, a, chunk_shape=(7,)) selections = [ slice(5, 15), slice(0, 30), slice(1, 2), ] for selection in selections: expect = a[selection] out = get_ndbuffer_class().from_numpy_array(np.empty(expect.shape)) z.get_basic_selection(selection, out=out) assert_array_equal(expect, out.as_numpy_array()[:]) with pytest.raises(TypeError): z.get_basic_selection(Ellipsis, out=[]) # type: ignore[arg-type] # orthogonal selections a = np.arange(60, dtype=int).reshape(12, 5) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) selections = [ # index both axes with bool array (_ORTHO_2D_IX0_BOOL, _ORTHO_2D_IX1_BOOL), # mixed indexing with bool array / slice (_ORTHO_2D_IX0_BOOL, slice(1, 4)), (slice(2, 9), _ORTHO_2D_IX1_BOOL), # mixed indexing with bool array / int (_ORTHO_2D_IX0_BOOL, 3), (7, _ORTHO_2D_IX1_BOOL), # mixed int array / bool array (_ORTHO_2D_IX0_BOOL, _ORTHO_2D_IX1_INT), (_ORTHO_2D_IX0_INT, _ORTHO_2D_IX1_BOOL), ] for selection in selections: expect = oindex(a, selection) out = get_ndbuffer_class().from_numpy_array(np.zeros(expect.shape, dtype=expect.dtype)) z.get_orthogonal_selection(selection, out=out) assert_array_equal(expect, out.as_numpy_array()[:]) # coordinate selections a = np.arange(60, dtype=int).reshape(12, 5) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) selections = [ # index both axes with array (np.array([0, 5, 11]), np.array([0, 2, 4])), # mixed indexing with array / int (np.array([0, 5, 11]), 3), (7, np.array([0, 2, 4])), ] for selection in selections: expect = a[selection] out = get_ndbuffer_class().from_numpy_array(np.zeros(expect.shape, dtype=expect.dtype)) z.get_coordinate_selection(selection, out=out) assert_array_equal(expect, out.as_numpy_array()[:]) @pytest.mark.xfail(reason="fields are not supported in v3") def test_get_selections_with_fields(store: StorePath) -> None: """Would verify that basic, orthogonal, coordinate, and mask selections with structured-array `fields` arguments return the correct sub-fields (xfail: fields unsupported in v3).""" a = np.array( [("aaa", 1, 4.2), ("bbb", 2, 8.4), ("ccc", 3, 12.6)], dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")], ) z = zarr_array_from_numpy_array(store, a, chunk_shape=(2,)) fields_fixture: list[str | list[str]] = [ "foo", ["foo"], ["foo", "bar"], ["foo", "baz"], ["bar", "baz"], ["foo", "bar", "baz"], ["bar", "foo"], ["baz", "bar", "foo"], ] for fields in fields_fixture: # total selection expect = a[fields] actual = z.get_basic_selection(Ellipsis, fields=fields) assert_array_equal(expect, actual) # alternative API if isinstance(fields, str): actual = z[fields] assert_array_equal(expect, actual) elif len(fields) == 2: actual = z[fields[0], fields[1]] assert_array_equal(expect, actual) if isinstance(fields, str): actual = z[..., fields] assert_array_equal(expect, actual) elif len(fields) == 2: actual = z[..., fields[0], fields[1]] assert_array_equal(expect, actual) # basic selection with slice expect = a[fields][0:2] actual = z.get_basic_selection(slice(0, 2), fields=fields) assert_array_equal(expect, actual) # alternative API if isinstance(fields, str): actual = z[0:2, fields] assert_array_equal(expect, actual) elif len(fields) == 2: actual = z[0:2, fields[0], fields[1]] assert_array_equal(expect, actual) # basic selection with single item expect = a[fields][1] actual = z.get_basic_selection(1, fields=fields) assert_array_equal(expect, actual) # alternative API if isinstance(fields, str): actual = z[1, fields] assert_array_equal(expect, actual) elif len(fields) == 2: actual = z[1, fields[0], fields[1]] assert_array_equal(expect, actual) # orthogonal selection ix = [0, 2] expect = a[fields][ix] actual = z.get_orthogonal_selection(ix, fields=fields) assert_array_equal(expect, actual) # alternative API if isinstance(fields, str): actual = z.oindex[ix, fields] assert_array_equal(expect, actual) elif len(fields) == 2: actual = z.oindex[ix, fields[0], fields[1]] assert_array_equal(expect, actual) # coordinate selection ix = [0, 2] expect = a[fields][ix] actual = z.get_coordinate_selection(ix, fields=fields) assert_array_equal(expect, actual) # alternative API if isinstance(fields, str): actual = z.vindex[ix, fields] assert_array_equal(expect, actual) elif len(fields) == 2: actual = z.vindex[ix, fields[0], fields[1]] assert_array_equal(expect, actual) # mask selection ix = [True, False, True] expect = a[fields][ix] actual = z.get_mask_selection(ix, fields=fields) assert_array_equal(expect, actual) # alternative API if isinstance(fields, str): actual = z.vindex[ix, fields] assert_array_equal(expect, actual) elif len(fields) == 2: actual = z.vindex[ix, fields[0], fields[1]] assert_array_equal(expect, actual) # missing/bad fields with pytest.raises(IndexError): z.get_basic_selection(Ellipsis, fields=["notafield"]) with pytest.raises(IndexError): z.get_basic_selection(Ellipsis, fields=slice(None)) # type: ignore[arg-type] @pytest.mark.xfail(reason="fields are not supported in v3") def test_set_selections_with_fields(store: StorePath) -> None: """Would verify that basic, orthogonal, coordinate, and mask set-selections with structured-array `fields` correctly write individual fields and reject multi-field assignment (xfail: fields unsupported in v3).""" v = np.array( [("aaa", 1, 4.2), ("bbb", 2, 8.4), ("ccc", 3, 12.6)], dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")], ) a = np.empty_like(v) z = zarr_array_from_numpy_array(store, v, chunk_shape=(2,)) fields_fixture: list[str | list[str]] = [ "foo", [], ["foo"], ["foo", "bar"], ["foo", "baz"], ["bar", "baz"], ["foo", "bar", "baz"], ["bar", "foo"], ["baz", "bar", "foo"], ] for fields in fields_fixture: # currently multi-field assignment is not supported in numpy, so we won't support # it either if isinstance(fields, list) and len(fields) > 1: with pytest.raises(IndexError): z.set_basic_selection(Ellipsis, v, fields=fields) with pytest.raises(IndexError): z.set_orthogonal_selection([0, 2], v, fields=fields) # type: ignore[arg-type] with pytest.raises(IndexError): z.set_coordinate_selection([0, 2], v, fields=fields) with pytest.raises(IndexError): z.set_mask_selection([True, False, True], v, fields=fields) # type: ignore[arg-type] else: if isinstance(fields, list) and len(fields) == 1: # work around numpy does not support multi-field assignment even if there # is only one field key = fields[0] elif isinstance(fields, list) and len(fields) == 0: # work around numpy ambiguity about what is a field selection key = Ellipsis else: key = fields # setup expectation a[:] = ("", 0, 0) z[:] = ("", 0, 0) assert_array_equal(a, z[:]) a[key] = v[key] # total selection z.set_basic_selection(Ellipsis, v[key], fields=fields) assert_array_equal(a, z[:]) # basic selection with slice a[:] = ("", 0, 0) z[:] = ("", 0, 0) a[key][0:2] = v[key][0:2] z.set_basic_selection(slice(0, 2), v[key][0:2], fields=fields) assert_array_equal(a, z[:]) # orthogonal selection a[:] = ("", 0, 0) z[:] = ("", 0, 0) ix = [0, 2] a[key][ix] = v[key][ix] z.set_orthogonal_selection(ix, v[key][ix], fields=fields) assert_array_equal(a, z[:]) # coordinate selection a[:] = ("", 0, 0) z[:] = ("", 0, 0) ix = [0, 2] a[key][ix] = v[key][ix] z.set_coordinate_selection(ix, v[key][ix], fields=fields) assert_array_equal(a, z[:]) # mask selection a[:] = ("", 0, 0) z[:] = ("", 0, 0) ix = [True, False, True] a[key][ix] = v[key][ix] z.set_mask_selection(ix, v[key][ix], fields=fields) assert_array_equal(a, z[:]) def test_slice_selection_uints() -> None: """make_slice_selection accepts unsigned integer indices without error and produces correct shape.""" arr = np.arange(24).reshape((4, 6)) idx = np.uint64(3) slice_sel = make_slice_selection((idx,)) assert arr[tuple(slice_sel)].shape == (1, 6) def test_numpy_int_indexing(store: StorePath) -> None: """Indexing with a plain Python int and with `np.int64` both return the correct scalar element.""" a = np.arange(1050) z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) assert a[42] == z[42] assert a[np.int64(42)] == z[np.int64(42)] @pytest.mark.parametrize( ("shape", "chunks", "ops"), [ # 1D test cases ((1070,), (50,), [("__getitem__", (slice(200, 400),))]), ((1070,), (50,), [("__getitem__", (slice(200, 400, 100),))]), ( (1070,), (50,), [ ("__getitem__", (slice(200, 400),)), ("__setitem__", (slice(200, 400, 100),)), ], ), # 2D test cases ( (40, 50), (5, 8), [ ("__getitem__", (slice(6, 37, 13), (slice(4, 10)))), ("__setitem__", (slice(None), (slice(None)))), ], ), ], ) async def test_accessed_chunks( shape: tuple[int, ...], chunks: tuple[int, ...], ops: list[tuple[str, tuple[slice, ...]]] ) -> None: """Only the chunks intersected by a slice selection are read or written, verified via a `CountingDict` store.""" # Test that only the required chunks are accessed during basic selection operations # shape: array shape # chunks: chunk size # ops: list of tuples with (optype, tuple of slices) # optype = "__getitem__" or "__setitem__", tuple length must match number of dims # Use a counting dict as the backing store so we can track the items access store = await CountingDict.open() z = zarr_array_from_numpy_array(StorePath(store), np.zeros(shape), chunk_shape=chunks) for ii, (optype, slices) in enumerate(ops): # Resolve the slices into the accessed chunks for each dimension chunks_per_dim = [] for N, C, sl in zip(shape, chunks, slices, strict=True): chunk_ind = np.arange(N, dtype=int)[sl] // C chunks_per_dim.append(np.unique(chunk_ind)) # Combine and generate the cartesian product to determine the chunks keys that # will be accessed chunks_accessed = [".".join(map(str, comb)) for comb in itertools.product(*chunks_per_dim)] counts_before = store.counter.copy() # Perform the operation if optype == "__getitem__": z[slices] else: z[slices] = ii # Get the change in counts delta_counts = store.counter - counts_before # Check that the access counts for the operation have increased by one for all # the chunks we expect to be included for ci in chunks_accessed: assert delta_counts.pop((optype, ci)) == 1 # If the chunk was partially written to it will also have been read once. We # don't determine if the chunk was actually partial here, just that the # counts are consistent that this might have happened if optype == "__setitem__": assert ("__getitem__", ci) not in delta_counts or delta_counts.pop( ("__getitem__", ci) ) == 1 # Check that no other chunks were accessed assert len(delta_counts) == 0 @pytest.mark.parametrize( "selection", [ # basic selection [...], [1, ...], [slice(None)], [1, 3], [[1, 2, 3], 4], [np.arange(12)], [slice(2, 9)], [slice(1, 3), 3], [[1, 3]], # mask selection [np.tile([True, False, True, False, True], (12, 1))], [np.full((12, 5), False)], # coordinate selection [[1, 2, 3, 4], [0, 1, 2, 3]], [[10, 11, 5], [4, 0, 2]], ], ) def test_indexing_equals_numpy(store: StorePath, selection: Selection) -> None: """Indexing a zarr array with assorted basic/mask/coordinate selections matches numpy.""" a = np.arange(60, dtype=int).reshape(12, 5) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) # note: in python 3.10 a[*selection] is not valid unpacking syntax expected = a[*selection,] actual = z[*selection,] assert_array_equal(expected, actual, err_msg=f"selection: {selection}") @pytest.mark.parametrize( "selection", [ [np.tile([True, False], 6), np.tile([True, False, True, False, True], 1)], [np.full(12, False), np.array([True, False, True, False, True])], [np.full(12, True), np.full(5, True)], [np.full(12, True), [True, False, True, False, True]], ], ) def test_orthogonal_bool_indexing_like_numpy_ix( store: StorePath, selection: list[npt.ArrayLike] ) -> None: """Orthogonal boolean indexing on each axis matches numpy's np.ix_ semantics.""" a = np.arange(60, dtype=int).reshape(12, 5) z = zarr_array_from_numpy_array(store, a, chunk_shape=(5, 2)) expected = a[np.ix_(*selection)] # note: in python 3.10 z[*selection] is not valid unpacking syntax actual = z[*selection,] assert_array_equal(expected, actual, err_msg=f"{selection=}") @pytest.mark.parametrize("ndim", [1, 2, 3]) @pytest.mark.parametrize("origin_0d", [None, (0,), (1,)]) @pytest.mark.parametrize("selection_shape_0d", [None, (2,), (3,)]) def test_iter_grid( ndim: int, origin_0d: tuple[int] | None, selection_shape_0d: tuple[int] | None ) -> None: """ Test that iter_grid works as expected for 1, 2, and 3 dimensions. """ grid_shape = (10, 5, 7)[:ndim] if origin_0d is not None: origin_kwarg = origin_0d * ndim origin = origin_kwarg else: origin_kwarg = None origin = (0,) * ndim if selection_shape_0d is not None: selection_shape_kwarg = selection_shape_0d * ndim selection_shape = selection_shape_kwarg else: selection_shape_kwarg = None selection_shape = tuple(gs - o for gs, o in zip(grid_shape, origin, strict=False)) observed = tuple( _iter_grid(grid_shape, origin=origin_kwarg, selection_shape=selection_shape_kwarg) ) # generate a numpy array of indices, and index it coord_array = np.array(list(itertools.product(*[range(s) for s in grid_shape]))).reshape( (*grid_shape, ndim) ) coord_array_indexed = coord_array[ tuple(slice(o, o + s, 1) for o, s in zip(origin, selection_shape, strict=False)) + (range(ndim),) ] expected = tuple(map(tuple, coord_array_indexed.reshape(-1, ndim).tolist())) assert observed == expected def test_iter_grid_invalid() -> None: """ Ensure that a selection_shape that exceeds the grid_shape + origin produces an indexing error. """ with pytest.raises(IndexError): list(_iter_grid((5,), origin=(0,), selection_shape=(10,))) def test_indexing_with_zarr_array(store: StorePath) -> None: """Regression for GH2133: indexing a zarr array with another zarr array (boolean or integer) as the indexer produces the same result as indexing with the equivalent numpy array.""" # regression test for https://github.com/zarr-developers/zarr-python/issues/2133 a = np.arange(10) za = zarr.array(a, chunks=2, store=store, path="a") ix = [False, True, False, True, False, True, False, True, False, True] ii = [0, 2, 4, 5] zix = zarr.array(ix, chunks=2, store=store, dtype="bool", path="ix") zii = zarr.array(ii, chunks=2, store=store, dtype="i4", path="ii") assert_array_equal(a[ix], za[zix]) assert_array_equal(a[ix], za.oindex[zix]) assert_array_equal(a[ix], za.vindex[zix]) assert_array_equal(a[ii], za[zii]) assert_array_equal(a[ii], za.oindex[zii]) @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) @pytest.mark.parametrize("shape", [(0, 2, 3), (0,), (3, 0)]) def test_zero_sized_chunks(store: StorePath, shape: list[int]) -> None: """Arrays with zero-extent dimensions can be created and indexed without error; reading back returns the fill value.""" # Chunk sizes must be >= 1 per spec; use 1 for zero-extent dimensions. chunks = tuple(max(1, s) for s in shape) z = zarr.create_array(store=store, shape=shape, chunks=chunks, zarr_format=3, dtype="f8") z[...] = 42 assert_array_equal(z[...], np.zeros(shape, dtype="f8")) @pytest.mark.parametrize( "pipeline_path", [ "zarr.core.codec_pipeline.BatchedCodecPipeline", "zarr.core.codec_pipeline.FusedCodecPipeline", ], ids=["batched", "fused"], ) @pytest.mark.parametrize("shards", [None, (4, 4)], ids=["chunked", "sharded"]) @pytest.mark.parametrize( ("kind", "selection", "value_shape"), [ # GH2469: array-level check, the value has twice the selected elements. pytest.param("vindex", (np.array([1, 2]), np.array([1, 2])), (2, 2), id="coord-2d-value"), # Right element count, wrong rank: a mask takes a flat value. pytest.param("vindex", np.eye(4, dtype=bool), (2, 2), id="mask-2d-value"), # Right element count, an axis the selection does not have. pytest.param( "oindex", (np.array([3, 1, 2]), np.array([0, 2])), (3, 2, 1), id="oindex-extra-axis" ), ], ) def test_set_selection_rejects_value_with_wrong_rank( store: StorePath, kind: str, selection: Any, value_shape: tuple[int, ...], shards: tuple[int, ...] | None, pipeline_path: str, ) -> None: """These wrong-rank values are rejected on chunked and sharded arrays alike. The sharding codec re-derives an indexer from the selection it is handed and ravels the value when it is the selection's broadcast shape minus integer-indexed axes. The cases here pin that a matching element count alone does not make the codec accept a value the chunked path rejects. That is not a general law: storage layout can change which writes are rejected. ``oindex[np.array([3, 1]), np.array([0, 2])]`` with a ``(2, 2, 1)`` value is accepted on a chunked ``(4, 4)`` array with ``(2, 2)`` chunks, because each chunk receives a ``(1, 1, 1)`` piece numpy can broadcast, while the sharded array raises ``ValueError`` and numpy rejects it outright. Only the rejection is asserted: a write that fails inside the chunk merge may already have touched other chunks. """ a = np.zeros((4, 4), dtype=np.int32) value = np.arange(np.prod(value_shape), dtype=np.int32).reshape(value_shape) with zarr.config.set({"codec_pipeline.path": pipeline_path}): z = zarr_array_from_numpy_array(store, a, chunk_shape=(2, 2), shards=shards) with pytest.raises(ValueError, match="Attempting to set|shape mismatch"): getattr(z, kind)[selection] = value def test_iter_chunk_regions(): """_iter_chunk_regions yields slices that exactly cover each chunk, and reading/writing each region round-trips correctly.""" chunks = (2, 3) a = zarr.create((10, 10), chunks=chunks) a[:] = 1 for region in a._iter_chunk_regions(): assert_array_equal(a[region], np.ones_like(a[region])) a[region] = 0 assert_array_equal(a[region], np.zeros_like(a[region])) @pytest.mark.parametrize( ("domain_shape", "region_shape", "origin", "selection_shape"), [ ((9,), (1,), None, (9,)), ((9,), (1,), (0,), (9,)), ((3,), (2,), (0,), (1,)), ((9,), (2,), (2,), (2,)), ((9, 9), (2, 1), None, None), ((9, 9), (4, 1), None, None), ], ) @pytest.mark.parametrize("order", ["lexicographic"]) @pytest.mark.parametrize("trim_excess", [True, False]) def test_iter_regions( domain_shape: tuple[int, ...], region_shape: tuple[int, ...], origin: tuple[int, ...] | None, selection_shape: tuple[int, ...] | None, order: _ArrayIndexingOrder, trim_excess: bool, ) -> None: """ Test that iter_regions properly iterates over contiguous regions of a gridded domain. """ expected_slices_by_dim: list[list[slice]] = [] origin_parsed: tuple[int, ...] selection_shape_parsed: tuple[int, ...] if origin is None: origin_parsed = (0,) * len(domain_shape) else: origin_parsed = origin if selection_shape is None: selection_shape_parsed = tuple( ceildiv_int(ds, rs) - o for ds, o, rs in zip(domain_shape, origin_parsed, region_shape, strict=True) ) else: selection_shape_parsed = selection_shape for d_s, r_s, o, ss in zip( domain_shape, region_shape, origin_parsed, selection_shape_parsed, strict=True ): _expected_slices: list[slice] = [] start = o * r_s for incr in range(start, start + ss * r_s, r_s): if trim_excess: term = min(incr + r_s, d_s) else: term = incr + r_s _expected_slices.append(slice(incr, term, 1)) expected_slices_by_dim.append(_expected_slices) expected = tuple(itertools.product(*expected_slices_by_dim)) observed = tuple( _iter_regions( domain_shape, region_shape, origin=origin, selection_shape=selection_shape, order=order, trim_excess=trim_excess, ) ) assert observed == expected class TestAsync: @pytest.mark.parametrize( ("indexer", "expected"), [ # int ((0,), np.array([1, 2])), ((1,), np.array([3, 4])), ((0, 1), np.array(2)), # slice ((slice(None),), np.array([[1, 2], [3, 4]])), ((slice(0, 1),), np.array([[1, 2]])), ((slice(1, 2),), np.array([[3, 4]])), ((slice(0, 2),), np.array([[1, 2], [3, 4]])), ((slice(0, 0),), np.empty(shape=(0, 2), dtype="i8")), # ellipsis ((...,), np.array([[1, 2], [3, 4]])), ((0, ...), np.array([1, 2])), ((..., 0), np.array([1, 3])), ((0, 1, ...), np.array(2)), # combined ((0, slice(None)), np.array([1, 2])), ((slice(None), 0), np.array([1, 3])), ((slice(None), slice(None)), np.array([[1, 2], [3, 4]])), # array of ints (([0]), np.array([[1, 2]])), (([1]), np.array([[3, 4]])), (([0], [1]), np.array(2)), (([0, 1], [0]), np.array([[1], [3]])), (([0, 1], [0, 1]), np.array([[1, 2], [3, 4]])), # boolean array (np.array([True, True]), np.array([[1, 2], [3, 4]])), (np.array([True, False]), np.array([[1, 2]])), (np.array([False, True]), np.array([[3, 4]])), (np.array([False, False]), np.empty(shape=(0, 2), dtype="i8")), ], ) async def test_async_oindex(self, store, indexer, expected): """The async `oindex.getitem` interface returns the correct orthogonally-indexed result for int, slice, ellipsis, array, and boolean indexers.""" z = zarr.create_array(store=store, shape=(2, 2), chunks=(1, 1), zarr_format=3, dtype="i8") z[...] = np.array([[1, 2], [3, 4]]) async_zarr = z._async_array result = await async_zarr.oindex.getitem(indexer) assert_array_equal(result, expected) async def test_async_oindex_with_zarr_array(self, store): """The async `oindex.getitem` interface accepts a zarr boolean array as the indexer and returns the correct rows.""" group = zarr.create_group(store=store, zarr_format=3) z1 = group.create_array(name="z1", shape=(2, 2), chunks=(1, 1), dtype="i8") z1[...] = np.array([[1, 2], [3, 4]]) async_zarr = z1._async_array # create boolean zarr array to index with z2 = group.create_array(name="z2", shape=(2,), chunks=(1,), dtype="?") z2[...] = np.array([True, False]) result = await async_zarr.oindex.getitem(z2) expected = np.array([[1, 2]]) assert_array_equal(result, expected) @pytest.mark.parametrize( ("indexer", "expected"), [ (([0], [0]), np.array(1)), (([0, 1], [0, 1]), np.array([1, 4])), (np.array([[False, True], [False, True]]), np.array([2, 4])), ], ) async def test_async_vindex(self, store, indexer, expected): """The async `vindex.getitem` interface returns the correct vectorized-indexed result for coordinate and boolean indexers.""" z = zarr.create_array(store=store, shape=(2, 2), chunks=(1, 1), zarr_format=3, dtype="i8") z[...] = np.array([[1, 2], [3, 4]]) async_zarr = z._async_array result = await async_zarr.vindex.getitem(indexer) assert_array_equal(result, expected) async def test_async_vindex_with_zarr_array(self, store): """The async `vindex.getitem` interface accepts a zarr 2D boolean array as the indexer and returns the correct elements.""" group = zarr.create_group(store=store, zarr_format=3) z1 = group.create_array(name="z1", shape=(2, 2), chunks=(1, 1), dtype="i8") z1[...] = np.array([[1, 2], [3, 4]]) async_zarr = z1._async_array # create boolean zarr array to index with z2 = group.create_array(name="z2", shape=(2, 2), chunks=(1, 1), dtype="?") z2[...] = np.array([[False, True], [False, True]]) result = await async_zarr.vindex.getitem(z2) expected = np.array([2, 4]) assert_array_equal(result, expected) async def test_async_invalid_indexer(self, store): """The async `vindex.getitem` and `oindex.getitem` interfaces raise IndexError when given an unsupported indexer type.""" z = zarr.create_array(store=store, shape=(2, 2), chunks=(1, 1), zarr_format=3, dtype="i8") z[...] = np.array([[1, 2], [3, 4]]) async_zarr = z._async_array with pytest.raises(IndexError): await async_zarr.vindex.getitem("invalid_indexer") with pytest.raises(IndexError): await async_zarr.oindex.getitem("invalid_indexer") @given( coordinates=st.lists(st.integers(0, 2**62 - 1), max_size=40), chunk_size=st.integers(1, 100), ) @example(coordinates=[2**62 - 1], chunk_size=3) def test_sparse_indexer_projection_reconstructs_coordinates( coordinates: list[int], chunk_size: int ) -> None: """Projection placement reconstructs arbitrary sparse requests without dense grids.""" extent = 2**62 coords = np.array(coordinates, dtype=np.intp) grid = ChunkGrid.from_sizes((extent,), (chunk_size,)) (dimension,) = grid._dimensions expected_chunks = sorted({c // chunk_size for c in coordinates}) for indexer in ( CoordinateIndexer((coords,), (extent,), grid), IntArrayDimIndexer(coords, extent, dimension), ): reconstructed = np.empty_like(coords) covered = np.zeros(coords.shape, dtype=np.intp) visited = [] for projection in indexer: if isinstance(indexer, CoordinateIndexer): chunk = projection.chunk_coords[0] selection = projection.chunk_selection[0] output = projection.out_selection else: chunk = projection.dim_chunk_ix selection = projection.dim_chunk_sel output = projection.dim_out_sel visited.append(chunk) reconstructed[output] = selection + chunk * chunk_size covered[output] += 1 assert visited == expected_chunks assert_array_equal(reconstructed, coords) assert_array_equal(covered, np.ones_like(coords)) assert len(indexer.chunk_run_ends) == len(expected_chunks) zarr-developers-zarr-python-a2f2415/tests/test_info.py000066400000000000000000000105111527014137600231710ustar00rootroot00000000000000import textwrap import pytest from zarr.codecs.bytes import BytesCodec from zarr.core._info import ArrayInfo, GroupInfo, human_readable_size from zarr.core.common import ZarrFormat from zarr.core.dtype.npy.int import Int32 ZARR_FORMATS = [2, 3] @pytest.mark.parametrize("zarr_format", ZARR_FORMATS) def test_group_info_repr(zarr_format: ZarrFormat) -> None: info = GroupInfo( _name="a", _store_type="MemoryStore", _read_only=False, _zarr_format=zarr_format ) result = repr(info) expected = textwrap.dedent(f"""\ Name : a Type : Group Zarr format : {zarr_format} Read-only : False Store type : MemoryStore""") assert result == expected @pytest.mark.parametrize("zarr_format", ZARR_FORMATS) def test_group_info_complete(zarr_format: ZarrFormat) -> None: info = GroupInfo( _name="a", _store_type="MemoryStore", _zarr_format=zarr_format, _read_only=False, _count_arrays=10, _count_groups=4, _count_members=14, ) result = repr(info) expected = textwrap.dedent(f"""\ Name : a Type : Group Zarr format : {zarr_format} Read-only : False Store type : MemoryStore No. members : 14 No. arrays : 10 No. groups : 4""") assert result == expected @pytest.mark.parametrize("zarr_format", ZARR_FORMATS) def test_array_info(zarr_format: ZarrFormat) -> None: info = ArrayInfo( _zarr_format=zarr_format, _data_type=Int32(endianness="little"), _fill_value=0, _shape=(100, 100), _chunk_shape=(10, 100), _order="C", _read_only=True, _store_type="MemoryStore", _serializer=BytesCodec(endian="little"), ) result = repr(info) assert result == textwrap.dedent(f"""\ Type : Array Zarr format : {zarr_format} Data type : Int32(endianness='little') Fill value : 0 Shape : (100, 100) Chunk shape : (10, 100) Order : C Read-only : True Store type : MemoryStore Filters : () Serializer : BytesCodec(endian='little') Compressors : ()""") @pytest.mark.parametrize("zarr_format", ZARR_FORMATS) @pytest.mark.parametrize("bytes_things", [(1_000_000, "976.6K", 500_000, "488.3K", "2.0", 5)]) def test_array_info_complete( zarr_format: ZarrFormat, bytes_things: tuple[int, str, int, str, str, int] ) -> None: ( count_bytes, count_bytes_formatted, count_bytes_stored, count_bytes_stored_formatted, storage_ratio_formatted, count_chunks_initialized, ) = bytes_things info = ArrayInfo( _zarr_format=zarr_format, _data_type=Int32(endianness="little"), _fill_value=0, _shape=(100, 100), _chunk_shape=(10, 100), _order="C", _read_only=True, _store_type="MemoryStore", _serializer=BytesCodec(endian="little"), _count_bytes=count_bytes, _count_bytes_stored=count_bytes_stored, _count_chunks_initialized=count_chunks_initialized, ) result = repr(info) assert result == textwrap.dedent(f"""\ Type : Array Zarr format : {zarr_format} Data type : Int32(endianness='little') Fill value : 0 Shape : (100, 100) Chunk shape : (10, 100) Order : C Read-only : True Store type : MemoryStore Filters : () Serializer : BytesCodec(endian='little') Compressors : () No. bytes : {count_bytes} ({count_bytes_formatted}) No. bytes stored : {count_bytes_stored} ({count_bytes_stored_formatted}) Storage ratio : {storage_ratio_formatted} Chunks Initialized : 5""") @pytest.mark.parametrize( ("size", "expected"), [ (1, "1"), (2**10, "1.0K"), (2**20, "1.0M"), (2**30, "1.0G"), (2**40, "1.0T"), (2**50, "1.0P"), ], ) def test_human_readable_size(size: int, expected: str) -> None: result = human_readable_size(size) assert result == expected zarr-developers-zarr-python-a2f2415/tests/test_json.py000066400000000000000000000102441527014137600232120ustar00rootroot00000000000000from __future__ import annotations import math from typing import TYPE_CHECKING import pytest from zarr.core._json import ( buffer_to_json, buffer_to_json_object, get_json, json_to_buffer, set_json, ) from zarr.core.buffer import cpu, default_buffer_prototype from zarr.storage import MemoryStore from zarr.storage._common import StorePath if TYPE_CHECKING: from zarr.core.common import JSON def test_json_to_buffer_round_trips() -> None: """`buffer_to_json` inverts `json_to_buffer` for an arbitrary JSON value.""" obj: JSON = {"zarr_format": 3, "node_type": "group", "attributes": {"a": [1, 2, 3]}} buffer = json_to_buffer(obj) assert buffer_to_json(buffer) == obj def test_json_to_buffer_uses_given_prototype() -> None: """`json_to_buffer` constructs the buffer from the supplied prototype.""" prototype = default_buffer_prototype() buffer = json_to_buffer({"x": 1}, prototype=prototype) assert isinstance(buffer, prototype.buffer) def test_json_to_buffer_allows_nan_by_default() -> None: """`json_to_buffer` permits NaN by default (writes it as `NaN`).""" buffer = json_to_buffer({"fill_value": math.nan}) decoded = buffer_to_json(buffer) assert isinstance(decoded, dict) assert math.isnan(decoded["fill_value"]) def test_json_to_buffer_allow_nan_false_rejects_nan() -> None: """`json_to_buffer(allow_nan=False)` raises on a non-finite value.""" with pytest.raises(ValueError, match="Out of range float"): json_to_buffer({"fill_value": math.nan}, allow_nan=False) def test_json_to_buffer_indent_controls_formatting() -> None: """`json_to_buffer(indent=...)` controls whitespace in the serialized bytes.""" obj: JSON = {"a": 1, "b": 2} compact = json_to_buffer(obj).to_bytes() indented = json_to_buffer(obj, indent=2).to_bytes() assert b"\n" not in compact assert b"\n" in indented # both still round-trip to the same value assert buffer_to_json(json_to_buffer(obj, indent=2)) == obj async def test_get_json_reads_existing_key() -> None: """`get_json` returns the parsed document stored at an existing key.""" store = MemoryStore() obj: JSON = {"zarr_format": 3, "node_type": "array"} await set_json(store, "zarr.json", obj) assert await get_json(store, "zarr.json") == obj async def test_get_json_returns_none_for_missing_key() -> None: """`get_json` returns None (rather than raising) when the key is absent.""" store = MemoryStore() assert await get_json(store, "does-not-exist") is None async def test_set_json_then_get_json_round_trips() -> None: """`set_json` followed by `get_json` returns the original value.""" store = MemoryStore() obj: JSON = {"a": 1, "b": [2, 3], "c": {"d": None}} await set_json(store, "doc.json", obj) assert await get_json(store, "doc.json") == obj async def test_storepath_get_json_reads_existing_key() -> None: """`StorePath.get_json` reads and parses the document at its own path.""" store = MemoryStore() obj: JSON = {"zarr_format": 2} await set_json(store, "group/.zgroup", obj) sp = StorePath(store, "group/.zgroup") assert await sp.get_json() == obj async def test_storepath_get_json_returns_none_for_missing() -> None: """`StorePath.get_json` returns None when its path is absent.""" store = MemoryStore() sp = StorePath(store, "missing") assert await sp.get_json() is None def test_buffer_to_json_on_cpu_buffer() -> None: """`buffer_to_json` works on a plain CPU buffer built from raw bytes.""" buffer = cpu.Buffer.from_bytes(b'{"hello": "world"}') assert buffer_to_json(buffer) == {"hello": "world"} def test_buffer_to_json_object_returns_dict() -> None: """`buffer_to_json_object` returns the parsed object as a dict.""" buffer = cpu.Buffer.from_bytes(b'{"node_type": "group"}') assert buffer_to_json_object(buffer) == {"node_type": "group"} def test_buffer_to_json_object_rejects_non_object() -> None: """`buffer_to_json_object` raises TypeError when the document is not an object.""" buffer = cpu.Buffer.from_bytes(b"[1, 2, 3]") with pytest.raises(TypeError, match="Expected a JSON object"): buffer_to_json_object(buffer) zarr-developers-zarr-python-a2f2415/tests/test_json_parse.py000066400000000000000000000062251527014137600244100ustar00rootroot00000000000000"""Tests for :mod:`zarr.core.json_parse`. ``convert`` delegates JSON type coercion to :func:`msgspec.convert` (translating ``msgspec.ValidationError`` into ``ValueError``). The final group covers the ``parse_storage_transformers`` generator-exhaustion regression. """ from __future__ import annotations from typing import Literal import pytest from zarr.core.json_parse import convert, parse_field from zarr.core.metadata.v3 import parse_storage_transformers class TestConvert: def test_literal(self) -> None: assert convert(3, Literal[3]) == 3 assert convert("array", Literal["array", "group"]) == "array" def test_literal_rejects_non_member(self) -> None: with pytest.raises(ValueError, match="Expected instance of"): convert(4, Literal[3]) with pytest.raises(ValueError, match="Expected instance of"): convert("Q", Literal["C", "F"]) def test_sequence_coerced_to_tuple(self) -> None: assert convert([1, 2, 3], tuple[int, ...]) == (1, 2, 3) assert convert([1, 2], tuple[int, int]) == (1, 2) def test_int(self) -> None: assert convert(5, int) == 5 def test_bool_int_strictness(self) -> None: # bool is an int subclass, but the two must not be interchangeable. with pytest.raises(ValueError): convert(True, int) with pytest.raises(ValueError): convert(1, bool) # ... and True must not satisfy Literal[1]. with pytest.raises(ValueError): convert(True, Literal[1]) class TestParseField: def test_valid_passthrough(self) -> None: assert parse_field(3, Literal[3], "zarr_format") == 3 def test_wraps_with_field_context(self) -> None: with pytest.raises(ValueError, match="Failed to parse input for 'zarr_format'"): parse_field(4, Literal[3], "zarr_format") def test_custom_error_type_and_chaining(self) -> None: class MyError(ValueError): pass with pytest.raises(MyError, match="Failed to parse input for 'node_type'") as exc_info: parse_field(5, Literal["array"], "node_type", error=MyError) # the generic type error is chained as the cause assert isinstance(exc_info.value.__cause__, ValueError) class TestStorageTransformersRegression: """`parse_storage_transformers` used to call `len(tuple(data))` and then return `data` itself, exhausting a one-shot iterable and returning a value typed as a tuple but not actually a tuple.""" def test_none(self) -> None: assert parse_storage_transformers(None) == () def test_empty(self) -> None: assert parse_storage_transformers([]) == () def test_list_returns_tuple(self) -> None: result = parse_storage_transformers([{"a": 1}]) assert result == ({"a": 1},) assert isinstance(result, tuple) def test_generator_not_exhausted(self) -> None: result = parse_storage_transformers(iter([{"a": 1}, {"b": 2}])) assert result == ({"a": 1}, {"b": 2}) def test_non_iterable_rejected(self) -> None: with pytest.raises(TypeError, match="Expected an iterable"): parse_storage_transformers(5) zarr-developers-zarr-python-a2f2415/tests/test_metadata/000077500000000000000000000000001527014137600234465ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/test_metadata/__init__.py000066400000000000000000000000001527014137600255450ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/test_metadata/conftest.py000066400000000000000000000025471527014137600256550ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING, Any from zarr.codecs.bytes import BytesCodec if TYPE_CHECKING: from zarr.core.metadata.v3 import ArrayMetadataJSON_V3 def minimal_metadata_dict_v3( extra_fields: dict[str, Any] | None = None, **overrides: Any ) -> ArrayMetadataJSON_V3: """Build a minimal valid V3 array metadata JSON dict. The output matches the shape of ``ArrayV3Metadata.to_dict()`` — all fields that ``to_dict`` always emits are included. Parameters ---------- extra_fields : dict, optional Extra keys to inject into the dict (e.g. extension fields). **overrides Override any of the standard metadata fields. """ d: ArrayMetadataJSON_V3 = { "zarr_format": 3, "node_type": "array", "shape": (4, 4), "data_type": "uint8", "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (4, 4)}}, "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, "fill_value": 0, "codecs": (BytesCodec(endian="little").to_dict(),), # type: ignore[typeddict-item] "attributes": {}, "storage_transformers": (), } d.update(overrides) # type: ignore[typeddict-item] if extra_fields is not None: d.update(extra_fields) # type: ignore[typeddict-item] return d zarr-developers-zarr-python-a2f2415/tests/test_metadata/test_consolidated.py000066400000000000000000001121431527014137600275310ustar00rootroot00000000000000from __future__ import annotations import json from typing import TYPE_CHECKING, Any import numpy as np import pytest from numcodecs import Blosc import zarr.api.asynchronous import zarr.api.synchronous import zarr.storage from zarr import AsyncGroup from zarr.api.asynchronous import ( consolidate_metadata, group, open, open_consolidated, ) from zarr.core.buffer import cpu, default_buffer_prototype from zarr.core.dtype import parse_dtype from zarr.core.group import ConsolidatedMetadata, GroupMetadata from zarr.core.metadata import ArrayV3Metadata from zarr.core.metadata.v2 import ArrayV2Metadata from zarr.errors import ZarrUserWarning from zarr.storage import StorePath if TYPE_CHECKING: from zarr.abc.store import Store from zarr.core.common import JSON, ZarrFormat @pytest.fixture async def memory_store_with_hierarchy(memory_store: Store) -> Store: g = await group(store=memory_store, attributes={"foo": "bar"}) dtype = "uint8" await g.create_array(name="air", shape=(1, 2, 3), dtype=dtype) await g.create_array(name="lat", shape=(1,), dtype=dtype) await g.create_array(name="lon", shape=(2,), dtype=dtype) await g.create_array(name="time", shape=(3,), dtype=dtype) child = await g.create_group("child", attributes={"key": "child"}) await child.create_array("array", shape=(4, 4), attributes={"key": "child"}, dtype=dtype) grandchild = await child.create_group("grandchild", attributes={"key": "grandchild"}) await grandchild.create_array( "array", shape=(4, 4), attributes={"key": "grandchild"}, dtype=dtype ) await grandchild.create_group("empty_group", attributes={"key": "empty"}) return memory_store class TestConsolidated: @pytest.mark.filterwarnings("ignore:Consolidated metadata") async def test_getitem_consolidated_empty_leaf_group( self, memory_store: zarr.storage.MemoryStore, zarr_format: ZarrFormat ) -> None: # This test writes the bytes directly, rather than using the zarr API, to mimic # how older versions of zarr-python wrote the consolidated metadata. # Notably, zarr-python 2.x does not include a # # "consolidated_metadata": {"metadata": {}} # # field on the leaf group nodes. if zarr_format == 2: zmetadata: dict[str, JSON] = { "metadata": { ".zattrs": {}, ".zgroup": {"zarr_format": 2}, "raw/.zattrs": {}, "raw/.zgroup": {"zarr_format": 2}, "raw/varm/.zattrs": {}, "raw/varm/.zgroup": {"zarr_format": 2}, }, "zarr_consolidated_format": 1, } await memory_store.set( ".zgroup", cpu.Buffer.from_bytes(json.dumps({"zarr_format": 2}).encode()) ) await memory_store.set(".zattrs", cpu.Buffer.from_bytes(json.dumps({}).encode())) await memory_store.set( ".zmetadata", cpu.Buffer.from_bytes(json.dumps(zmetadata).encode()) ) else: zmetadata = { "attributes": {}, "zarr_format": 3, "consolidated_metadata": { "kind": "inline", "must_understand": False, "metadata": { "raw": { "attributes": {}, "zarr_format": 3, "node_type": "group", }, "raw/varm": { "attributes": {}, "zarr_format": 3, "node_type": "group", }, }, }, "node_type": "group", } await memory_store.set( "zarr.json", cpu.Buffer.from_bytes(json.dumps(zmetadata).encode()) ) group = await zarr.api.asynchronous.open_consolidated( store=memory_store, zarr_format=zarr_format ) raw = await group.get_group("raw") assert raw.metadata.consolidated_metadata is not None varm = await raw.get_group("varm") assert varm.metadata.consolidated_metadata == ConsolidatedMetadata(metadata={}) async def test_open_consolidated_false_raises(self) -> None: store = zarr.storage.MemoryStore() with pytest.raises(TypeError, match="use_consolidated"): await zarr.api.asynchronous.open_consolidated(store, use_consolidated=False) # type: ignore[arg-type] def test_open_consolidated_false_raises_sync(self) -> None: store = zarr.storage.MemoryStore() with pytest.raises(TypeError, match="use_consolidated"): zarr.open_consolidated(store, use_consolidated=False) # type: ignore[arg-type] async def test_consolidated(self, memory_store_with_hierarchy: Store) -> None: # TODO: Figure out desired keys in # TODO: variety in the hierarchies # More nesting # arrays under arrays # single array # etc. with pytest.warns( ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): await consolidate_metadata(memory_store_with_hierarchy) group2 = await AsyncGroup.open(memory_store_with_hierarchy) array_metadata: dict[str, JSON] = { "attributes": {}, "chunk_key_encoding": { "configuration": {"separator": "/"}, "name": "default", }, "codecs": ( {"configuration": {"endian": "little"}, "name": "bytes"}, {"configuration": {"level": 0, "checksum": False}, "name": "zstd"}, ), "data_type": "uint8", "fill_value": 0, "node_type": "array", # "shape": (1, 2, 3), "zarr_format": 3, } expected = GroupMetadata( attributes={"foo": "bar"}, consolidated_metadata=ConsolidatedMetadata( kind="inline", must_understand=False, metadata={ "air": ArrayV3Metadata.from_dict( { "shape": (1, 2, 3), "chunk_grid": { "configuration": {"chunk_shape": (1, 2, 3)}, "name": "regular", }, **array_metadata, } ), "lat": ArrayV3Metadata.from_dict( { "shape": (1,), "chunk_grid": { "configuration": {"chunk_shape": (1,)}, "name": "regular", }, **array_metadata, } ), "lon": ArrayV3Metadata.from_dict( { "shape": (2,), "chunk_grid": { "configuration": {"chunk_shape": (2,)}, "name": "regular", }, **array_metadata, } ), "time": ArrayV3Metadata.from_dict( { "shape": (3,), "chunk_grid": { "configuration": {"chunk_shape": (3,)}, "name": "regular", }, **array_metadata, } ), "child": GroupMetadata( attributes={"key": "child"}, consolidated_metadata=ConsolidatedMetadata( metadata={ "array": ArrayV3Metadata.from_dict( { **array_metadata, "attributes": {"key": "child"}, "shape": (4, 4), "chunk_grid": { "configuration": {"chunk_shape": (4, 4)}, "name": "regular", }, } ), "grandchild": GroupMetadata( attributes={"key": "grandchild"}, consolidated_metadata=ConsolidatedMetadata( metadata={ # known to be empty child group "empty_group": GroupMetadata( consolidated_metadata=ConsolidatedMetadata( metadata={} ), attributes={"key": "empty"}, ), "array": ArrayV3Metadata.from_dict( { **array_metadata, "attributes": {"key": "grandchild"}, "shape": (4, 4), "chunk_grid": { "configuration": {"chunk_shape": (4, 4)}, "name": "regular", }, } ), } ), ), }, ), ), }, ), ) assert group2.metadata == expected group3 = await open(store=memory_store_with_hierarchy) assert group3.metadata == expected group4 = await open_consolidated(store=memory_store_with_hierarchy) assert group4.metadata == expected buf = await memory_store_with_hierarchy.get( "zarr.json", prototype=default_buffer_prototype() ) assert buf is not None result_raw = json.loads(buf.to_bytes())["consolidated_metadata"] assert result_raw["kind"] == "inline" assert sorted(result_raw["metadata"]) == [ "air", "child", "child/array", "child/grandchild", "child/grandchild/array", "child/grandchild/empty_group", "lat", "lon", "time", ] def test_consolidated_sync(self, memory_store: Store) -> None: g = zarr.api.synchronous.group(store=memory_store, attributes={"foo": "bar"}) dtype = "uint8" g.create_array(name="air", shape=(1, 2, 3), dtype=dtype) g.create_array(name="lat", shape=(1,), dtype=dtype) g.create_array(name="lon", shape=(2,), dtype=dtype) g.create_array(name="time", shape=(3,), dtype=dtype) with pytest.warns( ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): zarr.api.synchronous.consolidate_metadata(memory_store) group2 = zarr.Group.open(memory_store) array_metadata: dict[str, JSON] = { "attributes": {}, "chunk_key_encoding": { "configuration": {"separator": "/"}, "name": "default", }, "codecs": ( {"configuration": {"endian": "little"}, "name": "bytes"}, {"configuration": {"level": 0, "checksum": False}, "name": "zstd"}, ), "data_type": dtype, "fill_value": 0, "node_type": "array", # "shape": (1, 2, 3), "zarr_format": 3, } expected = GroupMetadata( attributes={"foo": "bar"}, consolidated_metadata=ConsolidatedMetadata( kind="inline", must_understand=False, metadata={ "air": ArrayV3Metadata.from_dict( { "shape": (1, 2, 3), "chunk_grid": { "configuration": {"chunk_shape": (1, 2, 3)}, "name": "regular", }, **array_metadata, } ), "lat": ArrayV3Metadata.from_dict( { "shape": (1,), "chunk_grid": { "configuration": {"chunk_shape": (1,)}, "name": "regular", }, **array_metadata, } ), "lon": ArrayV3Metadata.from_dict( { "shape": (2,), "chunk_grid": { "configuration": {"chunk_shape": (2,)}, "name": "regular", }, **array_metadata, } ), "time": ArrayV3Metadata.from_dict( { "shape": (3,), "chunk_grid": { "configuration": {"chunk_shape": (3,)}, "name": "regular", }, **array_metadata, } ), }, ), ) assert group2.metadata == expected group3 = zarr.api.synchronous.open(store=memory_store) assert group3.metadata == expected group4 = zarr.api.synchronous.open_consolidated(store=memory_store) assert group4.metadata == expected async def test_not_writable_raises(self, memory_store: zarr.storage.MemoryStore) -> None: await group(store=memory_store, attributes={"foo": "bar"}) read_store = zarr.storage.MemoryStore(store_dict=memory_store._store_dict, read_only=True) with pytest.raises(ValueError, match="does not support writing"): await consolidate_metadata(read_store) async def test_non_root_node(self, memory_store_with_hierarchy: Store) -> None: with pytest.warns( ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): await consolidate_metadata(memory_store_with_hierarchy, path="child") root = await AsyncGroup.open(memory_store_with_hierarchy) child = await AsyncGroup.open(StorePath(memory_store_with_hierarchy) / "child") assert root.metadata.consolidated_metadata is None assert child.metadata.consolidated_metadata is not None assert "air" not in child.metadata.consolidated_metadata.metadata assert "grandchild" in child.metadata.consolidated_metadata.metadata def test_consolidated_metadata_from_dict(self) -> None: data: dict[str, JSON] = {"must_understand": False} # missing kind with pytest.raises(ValueError, match="kind='None'"): ConsolidatedMetadata.from_dict(data) # invalid kind data["kind"] = "invalid" with pytest.raises(ValueError, match="kind='invalid'"): ConsolidatedMetadata.from_dict(data) # missing metadata data["kind"] = "inline" with pytest.raises(TypeError, match="Unexpected type for 'metadata'"): ConsolidatedMetadata.from_dict(data) data["kind"] = "inline" # empty is fine data["metadata"] = {} ConsolidatedMetadata.from_dict(data) def test_flatten(self) -> None: array_metadata: dict[str, Any] = { "attributes": {}, "chunk_key_encoding": { "configuration": {"separator": "/"}, "name": "default", }, "codecs": ({"configuration": {"endian": "little"}, "name": "bytes"},), "data_type": "float64", "fill_value": np.float64(0.0), "node_type": "array", # "shape": (1, 2, 3), "zarr_format": 3, } metadata = ConsolidatedMetadata( kind="inline", must_understand=False, metadata={ "air": ArrayV3Metadata.from_dict( { "shape": (1, 2, 3), "chunk_grid": { "configuration": {"chunk_shape": (1, 2, 3)}, "name": "regular", }, **array_metadata, } ), "lat": ArrayV3Metadata.from_dict( { "shape": (1,), "chunk_grid": { "configuration": {"chunk_shape": (1,)}, "name": "regular", }, **array_metadata, } ), "child": GroupMetadata( attributes={"key": "child"}, consolidated_metadata=ConsolidatedMetadata( metadata={ "array": ArrayV3Metadata.from_dict( { **array_metadata, "attributes": {"key": "child"}, "shape": (4, 4), "chunk_grid": { "configuration": {"chunk_shape": (4, 4)}, "name": "regular", }, } ), "grandchild": GroupMetadata( attributes={"key": "grandchild"}, consolidated_metadata=ConsolidatedMetadata( metadata={ "array": ArrayV3Metadata.from_dict( { **array_metadata, "attributes": {"key": "grandchild"}, "shape": (4, 4), "chunk_grid": { "configuration": {"chunk_shape": (4, 4)}, "name": "regular", }, } ) } ), ), }, ), ), }, ) result = metadata.flattened_metadata expected = { "air": metadata.metadata["air"], "lat": metadata.metadata["lat"], "child": GroupMetadata( attributes={"key": "child"}, consolidated_metadata=ConsolidatedMetadata(metadata={}) ), "child/array": metadata.metadata["child"].consolidated_metadata.metadata["array"], # type: ignore[union-attr] "child/grandchild": GroupMetadata( attributes={"key": "grandchild"}, consolidated_metadata=ConsolidatedMetadata(metadata={}), ), "child/grandchild/array": ( metadata.metadata["child"] .consolidated_metadata.metadata["grandchild"] # type: ignore[union-attr] .consolidated_metadata.metadata["array"] ), } assert result == expected def test_invalid_metadata_raises(self) -> None: payload: dict[str, JSON] = { "kind": "inline", "must_understand": False, "metadata": { "foo": [1, 2, 3] # invalid }, } with pytest.raises(TypeError, match="key='foo', type='list'"): ConsolidatedMetadata.from_dict(payload) def test_to_dict_empty(self) -> None: meta = ConsolidatedMetadata( metadata={ "empty": GroupMetadata( attributes={"key": "empty"}, consolidated_metadata=ConsolidatedMetadata(metadata={}), ) } ) result = meta.to_dict() expected = { "kind": "inline", "must_understand": False, "metadata": { "empty": { "attributes": {"key": "empty"}, "consolidated_metadata": { "kind": "inline", "must_understand": False, "metadata": {}, }, "node_type": "group", "zarr_format": 3, } }, } assert result == expected @pytest.mark.parametrize("zarr_format", [2, 3]) async def test_to_dict_order( self, memory_store: zarr.storage.MemoryStore, zarr_format: ZarrFormat ) -> None: with zarr.config.set(default_zarr_format=zarr_format): g = await group(store=memory_store) # Create groups in non-lexicographix order dtype = "float32" await g.create_array(name="b", shape=(1,), dtype=dtype) child = await g.create_group("c", attributes={"key": "child"}) await g.create_array(name="a", shape=(1,), dtype=dtype) await child.create_array("e", shape=(1,), dtype=dtype) await child.create_array("d", shape=(1,), dtype=dtype) # Consolidate metadata and re-open store if zarr_format == 3: with pytest.warns( ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): await zarr.api.asynchronous.consolidate_metadata(memory_store) else: await zarr.api.asynchronous.consolidate_metadata(memory_store) g2 = await zarr.api.asynchronous.open_group(store=memory_store) assert g2.metadata.consolidated_metadata is not None assert list(g2.metadata.consolidated_metadata.metadata) == ["a", "b", "c"] assert list(g2.metadata.consolidated_metadata.flattened_metadata) == [ "a", "b", "c", "c/d", "c/e", ] @pytest.mark.parametrize("zarr_format", [2, 3]) async def test_open_consolidated_raises_async(self, zarr_format: ZarrFormat) -> None: store = zarr.storage.MemoryStore() await AsyncGroup.from_store(store, zarr_format=zarr_format) with pytest.raises(ValueError): await zarr.api.asynchronous.open_consolidated(store, zarr_format=zarr_format) with pytest.raises(ValueError): await zarr.api.asynchronous.open_consolidated(store, zarr_format=None) @pytest.fixture async def v2_consolidated_metadata_empty_dataset( self, memory_store: zarr.storage.MemoryStore ) -> AsyncGroup: zgroup_bytes = cpu.Buffer.from_bytes(json.dumps({"zarr_format": 2}).encode()) zmetadata_bytes = cpu.Buffer.from_bytes( b'{"metadata":{".zgroup":{"zarr_format":2}},"zarr_consolidated_format":1}' ) return AsyncGroup._from_bytes_v2( StorePath(memory_store, path=""), zgroup_bytes, zattrs_bytes=None, consolidated_metadata_bytes=zmetadata_bytes, ) async def test_consolidated_metadata_backwards_compatibility( self, v2_consolidated_metadata_empty_dataset: AsyncGroup ) -> None: """ Test that consolidated metadata handles a missing .zattrs key. This is necessary for backwards compatibility with zarr-python 2.x. See https://github.com/zarr-developers/zarr-python/issues/2694 """ store = zarr.storage.MemoryStore() await zarr.api.asynchronous.open(store=store, zarr_format=2) await zarr.api.asynchronous.consolidate_metadata(store) result = await zarr.api.asynchronous.open_consolidated(store, zarr_format=2) assert result.metadata == v2_consolidated_metadata_empty_dataset.metadata async def test_consolidated_metadata_v2(self) -> None: store = zarr.storage.MemoryStore() g = await AsyncGroup.from_store(store, attributes={"key": "root"}, zarr_format=2) dtype = parse_dtype("uint8", zarr_format=2) await g.create_array(name="a", shape=(1,), attributes={"key": "a"}, dtype=dtype) g1 = await g.create_group(name="g1", attributes={"key": "g1"}) await g1.create_group(name="g2", attributes={"key": "g2"}) await zarr.api.asynchronous.consolidate_metadata(store) result = await zarr.api.asynchronous.open_consolidated(store, zarr_format=2) expected = GroupMetadata( attributes={"key": "root"}, zarr_format=2, consolidated_metadata=ConsolidatedMetadata( metadata={ "a": ArrayV2Metadata( shape=(1,), dtype=dtype, attributes={"key": "a"}, chunks=(1,), fill_value=0, compressor=Blosc(), order="C", ), "g1": GroupMetadata( attributes={"key": "g1"}, zarr_format=2, consolidated_metadata=ConsolidatedMetadata( metadata={ "g2": GroupMetadata( attributes={"key": "g2"}, zarr_format=2, consolidated_metadata=ConsolidatedMetadata(metadata={}), ) } ), ), } ), ) assert result.metadata == expected @pytest.mark.parametrize("zarr_format", [2, 3]) async def test_use_consolidated_false( self, memory_store: zarr.storage.MemoryStore, zarr_format: ZarrFormat ) -> None: with zarr.config.set(default_zarr_format=zarr_format): g = await group(store=memory_store, attributes={"foo": "bar"}) await g.create_group(name="a") # test a stale read if zarr_format == 3: with pytest.warns( ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): await zarr.api.asynchronous.consolidate_metadata(memory_store) else: await zarr.api.asynchronous.consolidate_metadata(memory_store) await g.create_group(name="b") stale = await zarr.api.asynchronous.open_group(store=memory_store) assert len([x async for x in stale.members()]) == 1 assert stale.metadata.consolidated_metadata assert list(stale.metadata.consolidated_metadata.metadata) == ["a"] # bypass stale data good = await zarr.api.asynchronous.open_group( store=memory_store, use_consolidated=False ) assert len([x async for x in good.members()]) == 2 # reconsolidate if zarr_format == 3: with pytest.warns( ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): await zarr.api.asynchronous.consolidate_metadata(memory_store) else: await zarr.api.asynchronous.consolidate_metadata(memory_store) good = await zarr.api.asynchronous.open_group(store=memory_store) assert len([x async for x in good.members()]) == 2 assert good.metadata.consolidated_metadata assert sorted(good.metadata.consolidated_metadata.metadata) == ["a", "b"] async def test_stale_child_metadata_ignored( self, memory_store: zarr.storage.MemoryStore ) -> None: # https://github.com/zarr-developers/zarr-python/issues/2921 # When consolidating metadata, we should ignore any (possibly stale) metadata # from previous consolidations, *including at child nodes*. root = await zarr.api.asynchronous.group(store=memory_store, zarr_format=3) await root.create_group("foo") await zarr.api.asynchronous.consolidate_metadata(memory_store, path="foo") await root.create_group("foo/bar/spam") with pytest.warns( ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): await zarr.api.asynchronous.consolidate_metadata(memory_store) reopened = await zarr.api.asynchronous.open_consolidated(store=memory_store, zarr_format=3) result = [x[0] async for x in reopened.members(max_depth=None)] expected = ["foo", "foo/bar", "foo/bar/spam"] assert result == expected async def test_use_consolidated_for_children_members( self, memory_store: zarr.storage.MemoryStore ) -> None: # A test that has *unconsolidated* metadata at the root group, but discovers # a child group with consolidated metadata. root = await zarr.api.asynchronous.create_group(store=memory_store) await root.create_group("a/b") # Consolidate metadata at "a/b" await zarr.api.asynchronous.consolidate_metadata(memory_store, path="a/b") # Add a new group a/b/c, that's not present in the CM at "a/b" await root.create_group("a/b/c") # Now according to the consolidated metadata, "a" has children ["b"] # but according to the unconsolidated metadata, "a" has children ["b", "c"] group = await zarr.api.asynchronous.open_group(store=memory_store, path="a") with pytest.warns(ZarrUserWarning, match="Object at 'c' not found"): result = sorted([x[0] async for x in group.members(max_depth=None)]) expected = ["b"] assert result == expected result = sorted( [x[0] async for x in group.members(max_depth=None, use_consolidated_for_children=False)] ) expected = ["b", "b/c"] assert result == expected async def test_absolute_path_for_subgroup(self, memory_store: zarr.storage.MemoryStore) -> None: root = await zarr.api.asynchronous.create_group(store=memory_store) await root.create_group("a/b") with pytest.warns( ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): await zarr.api.asynchronous.consolidate_metadata(memory_store) group = await zarr.api.asynchronous.open_group(store=memory_store) subgroup = await group.get_group("/a") members = [x async for x in subgroup.keys()] # noqa: SIM118 assert members == ["b"] @pytest.mark.parametrize("fill_value", [np.nan, np.inf, -np.inf]) async def test_consolidated_metadata_encodes_special_chars( memory_store: Store, zarr_format: ZarrFormat, fill_value: float ) -> None: root = await group(store=memory_store, zarr_format=zarr_format) _time = await root.create_array("time", shape=(12,), dtype=np.float64, fill_value=fill_value) if zarr_format == 3: with pytest.warns( ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): await zarr.api.asynchronous.consolidate_metadata(memory_store) else: await zarr.api.asynchronous.consolidate_metadata(memory_store) root = await group(store=memory_store, zarr_format=zarr_format) root_buffer = root.metadata.to_buffer_dict(default_buffer_prototype()) if zarr_format == 2: root_metadata = json.loads(root_buffer[".zmetadata"].to_bytes().decode("utf-8"))["metadata"] elif zarr_format == 3: root_metadata = json.loads(root_buffer["zarr.json"].to_bytes().decode("utf-8"))[ "consolidated_metadata" ]["metadata"] expected_fill_value = _time._zdtype.to_json_scalar(fill_value, zarr_format=2) if zarr_format == 2: assert root_metadata["time/.zarray"]["fill_value"] == expected_fill_value elif zarr_format == 3: assert root_metadata["time"]["fill_value"] == expected_fill_value class NonConsolidatedStore(zarr.storage.MemoryStore): """A store that doesn't support consolidated metadata""" @property def supports_consolidated_metadata(self) -> bool: return False async def test_consolidate_metadata_raises_for_self_consolidating_stores() -> None: """Verify calling consolidate_metadata on a non supporting stores raises an error.""" memory_store = NonConsolidatedStore() root = await zarr.api.asynchronous.create_group(store=memory_store) await root.create_group("a/b") with pytest.raises(TypeError, match="doesn't support consolidated metadata"): await zarr.api.asynchronous.consolidate_metadata(memory_store) async def test_open_group_in_non_consolidating_stores() -> None: memory_store = NonConsolidatedStore() root = await zarr.api.asynchronous.create_group(store=memory_store) await root.create_group("a/b") # Opening a group without consolidatedion works as expected await AsyncGroup.open(memory_store, use_consolidated=False) # let the Store opt out of consolidation await AsyncGroup.open(memory_store, use_consolidated=None) # Opening a group with use_consolidated=True should fail with pytest.raises(ValueError, match="doesn't support consolidated metadata"): await AsyncGroup.open(memory_store, use_consolidated=True) @pytest.mark.parametrize( "order", [ # keys grouped by parent, the order zarr-python used to write before it # started sorting the persisted keys ["a", "b", "a/x", "a/y", "b/x", "b/y"], # sibling subtrees interleaved, which is what the (depth, casefold) sort # produces for names differing only by case ["a", "b", "a/x", "b/x", "a/y", "b/y"], # reversed, to cover a parent appearing after its children in the mapping ["b/y", "b/x", "a/y", "a/x", "b", "a"], ], ) def test_flat_to_nested_is_order_independent(order: list[str]) -> None: """The persisted key order is arbitrary, so nesting must not depend on it.""" group_metadata: dict[str, JSON] = {"zarr_format": 3, "node_type": "group", "attributes": {}} consolidated = ConsolidatedMetadata.from_dict( { "kind": "inline", "must_understand": False, "metadata": dict.fromkeys(order, group_metadata), } ) assert sorted(consolidated.metadata) == ["a", "b"] for name in ("a", "b"): child = consolidated.metadata[name] assert isinstance(child, GroupMetadata) assert child.consolidated_metadata is not None assert sorted(child.consolidated_metadata.metadata) == ["x", "y"] async def test_consolidated_metadata_case_differing_siblings(memory_store: Store) -> None: """Sibling nodes whose names differ only by case each keep their own children. Regression test for https://github.com/zarr-developers/zarr-python/issues/4226 """ root = await zarr.api.asynchronous.create_group(store=memory_store) for name in ("Study", "study"): child = await root.create_group(f"obs/{name}") await child.create_array(name="categories", shape=(2,), dtype="uint8") await child.create_array(name="codes", shape=(2,), dtype="uint8") with pytest.warns( ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): await consolidate_metadata(memory_store) consolidated = await open_consolidated(store=memory_store) result = sorted([key async for key, _ in consolidated.members(max_depth=None)]) assert result == [ "obs", "obs/Study", "obs/Study/categories", "obs/Study/codes", "obs/study", "obs/study/categories", "obs/study/codes", ] zarr-developers-zarr-python-a2f2415/tests/test_metadata/test_io.py000066400000000000000000000174001527014137600254700ustar00rootroot00000000000000"""Tests for comparing stored metadata documents with the documents metadata would store, and for storing only the documents that differ.""" from __future__ import annotations import json from typing import TYPE_CHECKING, Any, Literal import pytest import zarr from zarr.core.buffer import cpu from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.core.metadata.io import ( ABSENT, ARRAY_DOCUMENTS, DocumentChange, diff_documents, read_documents, upsert_metadata, ) from zarr.core.sync import sync from zarr.storage import MemoryStore, StorePath if TYPE_CHECKING: from zarr.abc.store import Store from zarr.core.buffer import Buffer from zarr.core.common import JSON from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata V3_DOC: dict[str, JSON] = { "zarr_format": 3, "node_type": "array", "shape": [3], "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [3]}}, "fill_value": 0, } def _with(doc: dict[str, JSON], **changes: JSON) -> dict[str, JSON]: return {**doc, **changes} @pytest.mark.parametrize( ("stored", "new", "expected"), [ ({"zarr.json": V3_DOC}, {"zarr.json": V3_DOC}, ()), ( {"zarr.json": V3_DOC}, {"zarr.json": _with(V3_DOC, fill_value=1)}, ((("zarr.json", "fill_value"), 0, 1),), ), ( { "zarr.json": _with( V3_DOC, chunk_grid={"name": "regular", "configuration": {"chunk_shape": [0]}} ) }, {"zarr.json": V3_DOC}, ((("zarr.json", "chunk_grid", "configuration", "chunk_shape", 0), 0, 3),), ), ( {"zarr.json": V3_DOC}, {"zarr.json": _with(V3_DOC, shape=[3, 4])}, ((("zarr.json", "shape", 1), ABSENT, 4),), ), ( {"zarr.json": _with(V3_DOC, attributes={"a": 1})}, {"zarr.json": _with(V3_DOC, dimension_names=["x"])}, ( (("zarr.json", "attributes"), {"a": 1}, ABSENT), (("zarr.json", "dimension_names"), ABSENT, ["x"]), ), ), ( {"zarr.json": _with(V3_DOC, fill_value=True)}, {"zarr.json": _with(V3_DOC, fill_value=1)}, ((("zarr.json", "fill_value"), True, 1),), ), ( {"zarr.json": _with(V3_DOC, fill_value=1.0)}, {"zarr.json": _with(V3_DOC, fill_value=1)}, ((("zarr.json", "fill_value"), 1.0, 1),), ), ( {"zarr.json": _with(V3_DOC, fill_value=float("nan"))}, {"zarr.json": _with(V3_DOC, fill_value=float("nan"))}, (), ), ( {"zarr.json": _with(V3_DOC, shape={"0": 3})}, {"zarr.json": V3_DOC}, ((("zarr.json", "shape"), {"0": 3}, [3]),), ), ( {".zarray": {"shape": [3], "chunks": [0]}}, {".zarray": {"shape": [3], "chunks": [3]}, ".zattrs": {}}, (((".zarray", "chunks", 0), 0, 3), ((".zattrs",), ABSENT, {})), ), ], ids=[ "identical", "changed-scalar", "changed-nested-list", "longer-list", "removed-and-added-key", "bool-is-not-int", "float-is-not-int", "nan-is-nan", "object-is-not-array", "v2-two-documents", ], ) def test_diff_documents( stored: dict[str, JSON], new: dict[str, JSON], expected: tuple[Any, ...] ) -> None: """Documents are compared value by value, each change named by its JSON path under the document's key; values other than objects and arrays are identical only if their JSON encodings are. `diff_documents` is total over JSON: it raises no error.""" assert diff_documents(stored, new) == tuple(DocumentChange(*change) for change in expected) class _CountingStore(MemoryStore): """A memory store that counts the values set in it.""" sets = 0 async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None = None) -> None: self.sets += 1 await super().set(key, value, byte_range) def _documents(store: Store) -> dict[str, Any]: assert isinstance(store, MemoryStore) return {key: json.loads(value.to_bytes()) for key, value in store._store_dict.items()} def _legacy(zarr_format: Literal[2, 3]) -> tuple[StorePath, ArrayV2Metadata | ArrayV3Metadata]: """An array stored with chunk shape `[0]`, and the metadata its repair reads.""" store = _CountingStore() array = zarr.create_array( store, shape=(3,), chunks=(3,), dtype="int16", zarr_format=zarr_format ) key = ".zarray" if zarr_format == 2 else "zarr.json" doc = _documents(store)[key] if zarr_format == 2: doc["chunks"] = [0] else: doc["chunk_grid"]["configuration"]["chunk_shape"] = [0] sync(store.set(key, cpu.Buffer.from_bytes(json.dumps(doc).encode()))) return StorePath(store), array.metadata def _upsert( store_path: StorePath, metadata: ArrayV2Metadata | ArrayV3Metadata ) -> tuple[DocumentChange, ...]: """Upsert `metadata` against the documents stored at `store_path`.""" stored = sync(read_documents(store_path, ARRAY_DOCUMENTS[metadata.zarr_format])) return sync(upsert_metadata(store_path, metadata, stored)) @pytest.mark.parametrize("zarr_format", [2, 3]) def test_upsert_metadata_stores_documents_that_differ(zarr_format: Literal[2, 3]) -> None: """The documents that differ from the stored ones are stored, and the changes are returned.""" store_path, metadata = _legacy(zarr_format) assert isinstance(store_path.store, _CountingStore) store_path.store.sets = 0 key = ".zarray" if zarr_format == 2 else "zarr.json" chunk_path = ( ("chunks", 0) if zarr_format == 2 else ("chunk_grid", "configuration", "chunk_shape", 0) ) changes = _upsert(store_path, metadata) assert changes == (DocumentChange((key, *chunk_path), 0, 3),) assert store_path.store.sets == 1 stored = _documents(store_path.store) assert stored[key] == json.loads(metadata.to_buffer_dict(cpu.buffer_prototype)[key].to_bytes()) @pytest.mark.parametrize("zarr_format", [2, 3]) def test_upsert_metadata_identical_stores_nothing(zarr_format: Literal[2, 3]) -> None: """Metadata identical to what is stored stores nothing.""" store = _CountingStore() array = zarr.create_array( store, shape=(3,), chunks=(3,), dtype="int16", zarr_format=zarr_format ) store.sets = 0 assert _upsert(StorePath(store), array.metadata) == () assert store.sets == 0 def test_upsert_metadata_unstorable_leaves_store_untouched() -> None: """Metadata that may not be stored (a rectilinear chunk grid without the rectilinear chunks flag) fails before the store is read or written, naming the array.""" store_path, _ = _legacy(3) before = _documents(store_path.store) with zarr.config.set({"array.rectilinear_chunks": True}): metadata = zarr.create_array( MemoryStore(), shape=(3,), chunks=[[1, 2]], dtype="int16" ).metadata with ( zarr.config.set({"array.rectilinear_chunks": False}), pytest.raises(ValueError, match="experimental and disabled") as info, ): _upsert(store_path, metadata) assert info.value.__notes__ == [f"Array {str(store_path)!r}: nothing was stored."] assert _documents(store_path.store) == before def test_upsert_metadata_stored_document_not_an_object() -> None: """A stored document that is not a JSON object is not overwritten.""" store_path, metadata = _legacy(3) sync(store_path.store.set("zarr.json", cpu.Buffer.from_bytes(b"[]"))) with pytest.raises(TypeError, match="Expected a JSON object, got list"): _upsert(store_path, metadata) assert _documents(store_path.store) == {"zarr.json": []} zarr-developers-zarr-python-a2f2415/tests/test_metadata/test_repair.py000066400000000000000000002073611527014137600263520ustar00rootroot00000000000000"""Tests for the repairs that read invalid stored array metadata documents.""" from __future__ import annotations import asyncio import dataclasses import json import re import warnings import zipfile from typing import TYPE_CHECKING, Any, Literal, cast import numpy as np import pytest import zarr from zarr.codecs import ShardingCodec from zarr.codecs.numcodecs import Quantize from zarr.core.array import AsyncArray from zarr.core.group import ConsolidatedMetadata from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.core.metadata.repair import ( RECREATE_HINT, RESAVE_HINT, repair_array_document, ) from zarr.core.metadata.v3 import RectilinearChunkGridMetadata, RegularChunkGridMetadata from zarr.core.sync import sync from zarr.dtype import Int16 from zarr.errors import ZarrUserWarning from zarr.storage import LocalStore, MemoryStore, StorePath from zarr.storage._common import make_store_path if TYPE_CHECKING: from collections.abc import Callable from pathlib import Path from zarr.core.common import JSON, ZarrFormat from zarr.types import AnyArray def _v2_doc(shape: list[int], chunks: list[Any]) -> dict[str, JSON]: return { "zarr_format": 2, "shape": shape, "chunks": chunks, "dtype": " dict[str, JSON]: bytes_codec: dict[str, JSON] = {"name": "bytes", "configuration": {"endian": "little"}} codecs: list[JSON] = [bytes_codec] if inner is not None: codecs = [ { "name": "sharding_indexed", "configuration": { "chunk_shape": inner, "codecs": [bytes_codec], "index_codecs": [bytes_codec, {"name": "crc32c"}], "index_location": "end", }, } ] return { "zarr_format": 3, "node_type": "array", "shape": shape, "data_type": "int16", "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": chunk_shape}}, "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, "fill_value": 0, "codecs": codecs, } def _stored_chunks(doc: dict[str, Any]) -> Any: return ( doc["chunks"] if doc["zarr_format"] == 2 else doc["chunk_grid"]["configuration"]["chunk_shape"] ) def _chunk_shapes(metadata: ArrayV2Metadata | ArrayV3Metadata) -> tuple[Any, ...]: """The chunk shape of `metadata`, then the inner chunk shape of each sharding codec, from the outermost in.""" if isinstance(metadata, ArrayV2Metadata): return (metadata.chunks,) assert isinstance(metadata.chunk_grid, RegularChunkGridMetadata) shapes: list[Any] = [metadata.chunk_grid.chunk_shape] codecs: tuple[Any, ...] = metadata.codecs while sharding := next((c for c in codecs if isinstance(c, ShardingCodec)), None): shapes.append(sharding.chunk_shape) codecs = sharding.codecs return tuple(shapes) def _nested_sharded_doc(inner: list[Any], nested: list[Any]) -> dict[str, JSON]: """A Zarr format 3 document of shape `[8]` in one shard of chunk shape `inner`, whose codecs shard each chunk again, in chunks of shape `nested`.""" doc = _v3_doc([8], [8], inner=inner) outer = cast("dict[str, Any]", cast("list[JSON]", doc["codecs"])[0]) configuration = outer["configuration"] configuration["codecs"] = [{**outer, "configuration": {**configuration, "chunk_shape": nested}}] return doc @pytest.mark.parametrize( ("doc", "expected", "repaired", "warning"), [ (_v2_doc([10, 10], [4, 5]), ((4, 5),), "no", None), (_v3_doc([0, 0], [1, 1]), ((1, 1),), "no", None), (_v3_doc([10], [4], inner=[2]), ((4,), (2,)), "no", None), (_v2_doc([0, 4], [0, 4]), ((1, 4),), "moved", None), (_v3_doc([0], [False]), ((1,),), "moved", None), (_v2_doc([5], [True]), ((1,),), "respelled", None), (_v3_doc([5, 4], [True, 4]), ((1, 4),), "respelled", None), ( _v2_doc([3], [0]), ((1,),), "moved", ( r"^The stored chunk shape \[0\] is invalid: .* read as \[1\], reading 0 in " r"dimension 0 as 1, as no chunk can be stored under a chunk size of 0\.$" ), ), ( _v3_doc([4, 3], [4, 0]), ((4, 1),), "moved", r"reading 0 in dimension 1 as .* under a chunk size of 0\.$", ), ( _v2_doc([0, 3], [0, 0]), ((1, 1),), "moved", r"read as \[1, 1\], reading 0 in dimension 1 as .* under a chunk size of 0\.$", ), (_v3_doc([0], [0], inner=[4]), ((4,), (4,)), "moved", None), ( _v3_doc([10], [0], inner=[4]), ((4,), (4,)), "moved", r"as 4, the inner chunk size, as no chunk can be stored under a chunk size of 0", ), (_v3_doc([0, 3], [0, 3], inner=[2, 3]), ((2, 3), (2, 3)), "moved", None), (_v3_doc([5], [True], inner=[True]), ((1,), (1,)), "respelled", None), (_nested_sharded_doc([4], [2]), ((8,), (4,), (2,)), "no", None), (_nested_sharded_doc([4], [True]), ((8,), (4,), (1,)), "respelled", None), ], ids=[ "v2-valid", "v3-valid-empty-axes", "v3-valid-sharded", "v2-zero-empty-axis", "v3-false-empty-axis", "v2-true", "v3-true", "v2-zero-grown-axis", "v3-zero-grown-axis", "v2-zero-empty-and-grown-axes", "v3-sharded-zero-empty-axis", "v3-sharded-zero-grown-axis", "v3-sharded-zero-2d", "v3-sharded-true-inner-and-outer", "v3-nested-sharded-valid", "v3-nested-sharded-true", ], ) def test_repair_array_document( doc: dict[str, JSON], expected: tuple[Any, ...], repaired: Literal["no", "respelled", "moved"], warning: str | None, ) -> None: """Valid documents pass unchanged. A stored chunk size of 0 or `false` is read as 1 (the inner chunk size when sharded), however long the axis, and `true` as 1, in the chunk shape and in the inner chunk shape of every sharding codec, nested or not. `from_dict` marks the metadata of a document whose repair moves chunks (a chunk size read as another size), so the array stores the repair before it writes chunks, but not one that only respells a value (`true` as 1). It warns once, naming the array, only where a chunk size of 0 was stored for a non-empty axis (which then holds only its fill value), saying how that part was read and that recreating the array loses nothing (see `RECREATE_HINT`). The other readings give what zarr read before, so they are silent.""" repaired_doc, readings = repair_array_document(doc, cast("ZarrFormat", doc["zarr_format"])) assert { k: v for k, v in repaired_doc.items() if k not in ("chunks", "chunk_grid", "codecs") } == {k: v for k, v in doc.items() if k not in ("chunks", "chunk_grid", "codecs")} assert bool(readings) is (repaired != "no") if repaired == "no": assert repaired_doc is doc metadata_cls = ArrayV2Metadata if doc["zarr_format"] == 2 else ArrayV3Metadata with warnings.catch_warnings(record=True) as record: warnings.simplefilter("always") metadata = metadata_cls.from_dict(dict(doc), path="group/array") assert _chunk_shapes(metadata) == expected assert metadata._stored_document == (doc if repaired == "moved" else None) messages = [str(w.message) for w in record] if warning is None: assert messages == [] else: [message] = messages assert message.startswith("Array 'group/array': ") assert message.endswith(RECREATE_HINT) assert re.search( warning, message.removeprefix("Array 'group/array': ").removesuffix(f" {RECREATE_HINT}") ) @pytest.mark.parametrize( ("doc", "error"), [ (_v2_doc([4], [0]) | {"order": "Z"}, "Failed to parse input for 'order'"), (_v3_doc([4], [0], inner=[2, 2]), "need to have the same number of dimensions"), ], ids=["v2", "v3-sharded"], ) def test_invalid_repaired_document_raises_without_warning(doc: dict[str, JSON], error: str) -> None: """A document the repairs read that the metadata constructor then rejects raises that error, without first warning how it was read.""" metadata_cls = ArrayV2Metadata if doc["zarr_format"] == 2 else ArrayV3Metadata assert repair_array_document(doc, cast("ZarrFormat", doc["zarr_format"]))[1] with warnings.catch_warnings(): warnings.simplefilter("error", ZarrUserWarning) with pytest.raises(ValueError, match=error): metadata_cls.from_dict(doc) def _read_strictly(doc: dict[str, JSON]) -> ArrayV2Metadata | ArrayV3Metadata: """Read `doc`, failing on any warning that it was repaired.""" metadata_cls = ArrayV2Metadata if doc["zarr_format"] == 2 else ArrayV3Metadata with warnings.catch_warnings(): warnings.simplefilter("error", ZarrUserWarning) return metadata_cls.from_dict(doc) def _open_strictly(path: Path, mode: Literal["r", "a", "r+"] = "r") -> AnyArray: """Open the array at `path`, failing on any warning that its document was repaired.""" with warnings.catch_warnings(): warnings.simplefilter("error", ZarrUserWarning) array = zarr.open_array(store=path, mode=mode) assert isinstance(array, zarr.Array) return array def test_stored_negative_chunk_size_rejected() -> None: """No known writer stored a negative chunk size: it is rejected, not repaired.""" with pytest.raises(ValueError, match="^Expected all values to be non-negative"): _read_strictly(_v2_doc([4], [-1])) def test_stored_chunk_shape_ndim_mismatch_rejected() -> None: """A chunk shape with the wrong number of dimensions is not repaired, so its 0 is rejected.""" with pytest.raises(ValueError, match="^Dimension 0: chunk edge length must be >= 1, got 0$"): _read_strictly(_v3_doc([4, 4], [0])) @pytest.mark.parametrize("inner", [[0], [False]]) def test_stored_zero_chunk_size_of_shard_with_invalid_inner_chunk_shape_rejected( inner: list[Any], ) -> None: """A stored chunk size of 0 of a sharded array is read in multiples of the inner chunk size; if that is not an integer of at least 1, the 0 is not repaired, so it is rejected.""" with pytest.raises(ValueError, match="^Dimension 0: chunk edge length must be >= 1, got 0$"): _read_strictly(_v3_doc([4], [0], inner=inner)) def _rectilinear_doc(shape: list[int], chunk_shapes: list[Any]) -> dict[str, JSON]: return _v3_doc(shape, [1] * len(shape)) | { "chunk_grid": { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": chunk_shapes}, } } @pytest.mark.parametrize( ("chunk_shapes", "expected"), [ ([[[4, 2]], [[5, 2]]], ((4, 4), (5, 5))), ([[[4.0, 2]], [[5, 2]]], ((4, 4), (5, 5))), ([[3.0, 5.0], [[5, 2]]], ((3, 5), (5, 5))), ([[[4.0, 2], 2.0], [[5, 2]]], ((4, 4, 2), (5, 5))), ([[[4.0, 2]], [10]], ((4, 4), (10,))), ([[[4.0, 2]], [5.0, 5]], ((4, 4), (5, 5))), ([[True, 4], [[5, 2]]], ((1, 4), (5, 5))), ([[[True, 2], 3], [[5, 2]]], ((1, 1, 3), (5, 5))), ], ids=[ "valid", "float-rle-size", "float-edges", "float-rle-size-and-edge", "float-sharded-outer", "float-2d", "true-edge", "true-rle-size", ], ) def test_read_invalid_edges_in_rectilinear_grid( chunk_shapes: list[Any], expected: tuple[tuple[int, ...], ...] ) -> None: """A stored rectilinear chunk grid whose explicit edges or run-length encoded sizes are integral floats or JSON `true`, as zarr-python wrote them when given float or `True` edges, is read with those edges as the `int`s they equal, silently: zarr read these edges so before. The reading moves no chunks, so the metadata is not marked for re-saving.""" shape = [sum(edges) for edges in expected] doc = _rectilinear_doc(shape, chunk_shapes) with zarr.config.set({"array.rectilinear_chunks": True}): metadata = _read_strictly(doc) assert metadata.chunk_grid == RectilinearChunkGridMetadata(chunk_shapes=expected) assert metadata._stored_document is None @pytest.mark.parametrize( ("doc", "error"), [ (_v3_doc([20], [10.0]), "Dimension 0: chunk edge length must be an int, got 10.0"), (_v3_doc([4], [4.5]), "Dimension 0: chunk edge length must be an int, got 4.5"), ( _v3_doc([8], [4], inner=[2.0]), "Expected an iterable of integers. Got [2.0] instead.", ), (_v2_doc([20], [10.0]), "Expected an iterable of integers. Got [10.0] instead."), ( _rectilinear_doc([8], [[[4, 2.0]]]), "Dimension 0: RLE repeat count must be an int, got 2.0", ), ( _rectilinear_doc([8], [4.0]), "Dimension 0: chunk edge length must be an int, got 4.0", ), ( _rectilinear_doc([8], [[0.0, 8]]), "Dimension 0: chunk edge length must be an int, got 0.0", ), ], ids=[ "regular", "regular-fractional", "sharding-inner", "v2", "rle-count", "rectilinear-bare", "rectilinear-zero", ], ) def test_stored_float_chunk_size_rejected(doc: dict[str, JSON], error: str) -> None: """A float chunk size is read only where zarr-python stored one, as an edge of at least 1 of a rectilinear chunk grid. Anywhere else it is rejected, as zarr 3.4.0 rejected a stored regular chunk size of `10.0`.""" with zarr.config.set({"array.rectilinear_chunks": True}), pytest.raises(TypeError) as info: _read_strictly(doc) assert info.match(re.escape(error)) @pytest.mark.parametrize( ("chunks", "stored"), [ ([[4, 4], [5, 5]], [[[4.0, 2]], [[5, 2]]]), ([[1, 3, 4], [5, 5]], [[True, 3, 4], [[5, 2]]]), ], ids=["float", "true"], ) def test_invalid_edges_round_trip( tmp_path: Path, chunks: list[list[int]], stored: list[Any] ) -> None: """A store whose rectilinear chunk grid holds the float or `true` edges zarr-python wrote opens silently and reads its data. Those edges are read as the values they equal, so a write stores its chunks and leaves the document as it was stored.""" path = tmp_path / "rectilinear.zarr" data = np.arange(80, dtype="int16").reshape(8, 10) with zarr.config.set({"array.rectilinear_chunks": True}): zarr.create_array(path, shape=data.shape, chunks=chunks, dtype="int16")[...] = data _rewrite_doc( path, 3, lambda doc: doc["chunk_grid"]["configuration"].update(chunk_shapes=stored) ) arr = _open_strictly(path, mode="a") np.testing.assert_array_equal(arr[...], data) arr[0, 0] = -1 written = json.loads((path / "zarr.json").read_text())["chunk_grid"]["configuration"] assert json.dumps(written["chunk_shapes"]) == json.dumps(stored) data[0, 0] = -1 np.testing.assert_array_equal(_open_strictly(path)[...], data) def test_v2_constructor_rejects_chunks_of_wrong_length() -> None: with pytest.raises(ValueError, match="`chunks` has length 1, but `shape` has length 2"): ArrayV2Metadata(shape=(4, 4), chunks=(2,), dtype=Int16(), fill_value=0, order="C") def _v2_metadata(chunks: Any) -> ArrayV2Metadata: return ArrayV2Metadata(shape=(4,), chunks=chunks, dtype=Int16(), fill_value=0, order="C") def _rectilinear(chunk_shapes: tuple[Any, ...]) -> RectilinearChunkGridMetadata: with zarr.config.set({"array.rectilinear_chunks": True}): return RectilinearChunkGridMetadata(chunk_shapes=chunk_shapes) def _rectilinear_from_dict(chunk_shapes: list[Any]) -> RectilinearChunkGridMetadata: with zarr.config.set({"array.rectilinear_chunks": True}): return RectilinearChunkGridMetadata.from_dict( { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": chunk_shapes}, } ) def _second_edge(grid: RectilinearChunkGridMetadata) -> int: edges = grid.chunk_shapes[0] assert isinstance(edges, tuple) return edges[1] CHUNK_EDGE_SITES: dict[str, Callable[[Any], object]] = { "regular": lambda size: RegularChunkGridMetadata(chunk_shape=(size,)).chunk_shape[0], "rectilinear-bare": lambda size: _rectilinear((size,)).chunk_shapes[0], "rectilinear-edge": lambda size: _second_edge(_rectilinear(((4, size),))), "rectilinear-bare-json": lambda size: _rectilinear_from_dict([size]).chunk_shapes[0], "rectilinear-edge-json": lambda size: _second_edge(_rectilinear_from_dict([[4, size]])), "rectilinear-rle-json": lambda size: _second_edge(_rectilinear_from_dict([[[size, 2]]])), } """Each place chunk grid metadata reads a chunk edge length, returning the edge it read.""" @pytest.mark.parametrize("site", CHUNK_EDGE_SITES) @pytest.mark.parametrize( ("size", "expected"), [(4, 4), (True, 1)], ids=["int", "bool"], ) def test_metadata_reads_integer_chunk_edge(site: str, size: object, expected: int) -> None: """Chunk grid metadata reads an `int` or a `bool` as the `int` chunk edge length it equals.""" edge = CHUNK_EDGE_SITES[site](size) assert type(edge) is int assert edge == expected @pytest.mark.parametrize("site", CHUNK_EDGE_SITES) @pytest.mark.parametrize( "size", [4.0, np.float64(4.0), 4.5, float("inf"), "4", None, np.int64(4)], ids=["float", "numpy-float", "fractional", "inf", "str", "none", "numpy-int"], ) def test_metadata_rejects_non_integer_chunk_edge(site: str, size: object) -> None: """A chunk edge length in metadata built in code is an `int`: a float is rejected, even an integral one (stored documents with integral floats are read by the repairs), and so is a NumPy integer, as zarr 3.4.0 rejected one.""" with pytest.raises( TypeError, match=re.escape(f"Dimension 0: chunk edge length must be an int, got {size!r}"), ): CHUNK_EDGE_SITES[site](size) @pytest.mark.parametrize("site", CHUNK_EDGE_SITES) @pytest.mark.parametrize("size", [0, False, -1]) def test_metadata_rejects_chunk_edge_below_one(site: str, size: int) -> None: """Chunk grid metadata built in code is strict: a chunk edge length below 1 is rejected, without a warning.""" with warnings.catch_warnings(): warnings.simplefilter("error", ZarrUserWarning) with pytest.raises( ValueError, match=f"Dimension 0: chunk edge length must be >= 1, got {size}" ): CHUNK_EDGE_SITES[site](size) @pytest.mark.parametrize("chunk_shape", [4, np.int64(4), None, "44", {"4": 4}]) def test_regular_chunk_grid_rejects_chunk_shape_not_a_sequence(chunk_shape: Any) -> None: """A regular chunk shape is an iterable of chunk edge lengths, but not a string or a mapping, which is rejected as a whole, not entry by entry.""" with pytest.raises( TypeError, match=re.escape( f"A chunk shape must be an iterable of chunk edge lengths, got {chunk_shape!r}" ), ): RegularChunkGridMetadata(chunk_shape=chunk_shape) def _sharding_chunk_shape(chunks: Any) -> tuple[tuple[int, ...], object]: codec = ShardingCodec(chunk_shape=chunks) configuration = cast("dict[str, JSON]", codec.to_dict()["configuration"]) return codec.chunk_shape, configuration["chunk_shape"] CHUNK_SHAPE_SITES: dict[str, Callable[[Any], tuple[tuple[int, ...], object]]] = { "v2": lambda chunks: ((md := _v2_metadata(chunks)).chunks, md.to_dict()["chunks"]), "sharding-inner": _sharding_chunk_shape, } """`ArrayV2Metadata` and `ShardingCodec` read a chunk shape as an array shape, returning the chunk shape and the value `to_dict` writes for it.""" @pytest.mark.parametrize("site", CHUNK_SHAPE_SITES) @pytest.mark.parametrize( ("chunks", "expected"), [ ((4,), (4,)), ([4], (4,)), (4, (4,)), (np.int64(4), (4,)), ((np.int64(4),), (4,)), (np.array([4]), (4,)), ((True,), (1,)), (range(4, 5), (4,)), ], ) def test_chunk_shape_read_as_array_shape( site: str, chunks: object, expected: tuple[int, ...] ) -> None: """`ArrayV2Metadata` and `ShardingCodec` read their chunk shape as `parse_shapelike` reads an array shape: an integer or an iterable of integers, including NumPy integers and bools.""" parsed, written = CHUNK_SHAPE_SITES[site](chunks) assert parsed == expected assert all(type(size) is int for size in parsed) assert written == expected @pytest.mark.parametrize("chunks", [(0,), (False,)]) def test_v2_chunk_size_zero_written_as_given(chunks: tuple[Any, ...]) -> None: """`ArrayV2Metadata` accepts a chunk size of 0 and writes it back as given; reading a stored 0 is `zarr.core.metadata.repair`' business. `ShardingCodec` rejects one.""" metadata = _v2_metadata(chunks) assert metadata.chunks == (0,) assert metadata.to_dict()["chunks"] == (0,) @pytest.mark.parametrize("site", CHUNK_SHAPE_SITES) def test_chunk_shape_read_as_array_shape_rejects_negative(site: str) -> None: with pytest.raises(ValueError, match="Expected all values to be non-negative"): CHUNK_SHAPE_SITES[site]((-1,)) @pytest.mark.parametrize("site", CHUNK_SHAPE_SITES) @pytest.mark.parametrize("chunks", [(4.0,), "4", None]) def test_chunk_shape_read_as_array_shape_rejects_non_integer(site: str, chunks: object) -> None: with pytest.raises(TypeError, match="Expected an"): CHUNK_SHAPE_SITES[site](chunks) def _rewrite_doc(path: Path, zarr_format: Literal[2, 3], edit: Any) -> None: doc_path = path / (".zarray" if zarr_format == 2 else "zarr.json") doc = json.loads(doc_path.read_text()) edit(doc) doc_path.write_text(json.dumps(doc)) @pytest.mark.parametrize( ("zarr_format", "shape", "stored", "inner", "expected", "warns"), [ (2, (0, 4), [0, 4], None, (1, 4), False), (3, (5,), [True], None, (1,), False), (3, (10,), [0], (4,), (4,), True), ], ids=["v2-empty-2d", "v3-true", "v3-sharded-grown"], ) def test_legacy_chunk_size_round_trip( tmp_path: Path, zarr_format: Literal[2, 3], shape: tuple[int, ...], stored: list[Any], inner: tuple[int, ...] | None, expected: tuple[int, ...], warns: bool, ) -> None: """A store whose metadata holds a chunk size written by older software opens (with a warning where a non-empty axis was stored with chunk size 0), reads and appends under the repaired grid, and re-saves valid metadata.""" path = tmp_path / "legacy.zarr" arr = zarr.create_array( store=path, shape=shape, chunks=inner or expected, shards=expected if inner else None, dtype="int16", fill_value=0, zarr_format=zarr_format, ) data = np.arange(np.prod(shape), dtype="int16").reshape(shape) arr[...] = data def store_legacy(doc: dict[str, Any]) -> None: if zarr_format == 2: doc["chunks"] = stored else: doc["chunk_grid"]["configuration"]["chunk_shape"] = stored _rewrite_doc(path, zarr_format, store_legacy) with warnings.catch_warnings(record=True) as record: warnings.simplefilter("always", ZarrUserWarning) arr = zarr.open_array(store=path, mode="a") assert [ re.match(r"^Array '.*legacy\.zarr': .* is read as", str(w.message)) is not None for w in record ] == [True] * warns assert (arr.shards or arr.chunks) == expected np.testing.assert_array_equal(arr[...], data) block = np.full((2, *shape[1:]), 7, dtype="int16") arr.append(block) arr.update_attributes({}) with warnings.catch_warnings(): warnings.simplefilter("error", ZarrUserWarning) reopened = zarr.open_array(store=path) np.testing.assert_array_equal(reopened[...], np.concatenate([data, block])) @pytest.mark.parametrize( ("zarr_format", "shape", "inner", "expected"), [(2, (3,), None, (1,)), (3, (3,), None, (1,)), (3, (10,), (4,), (4,))], ids=["v2", "v3", "v3-sharded"], ) @pytest.mark.parametrize("api", ["sync", "async", "async-concurrent"]) def test_write_stores_repaired_metadata_first( tmp_path: Path, zarr_format: Literal[2, 3], shape: tuple[int, ...], inner: tuple[int, ...] | None, expected: tuple[int, ...], api: str, ) -> None: """Writing chunks to an array read from a repaired document first stores the repaired metadata, so readers that do not repair (or read it differently) see the chunks the write stored.""" path = tmp_path / "legacy.zarr" zarr.create_array( store=path, shape=shape, chunks=inner or expected, shards=expected if inner else None, dtype="int16", fill_value=0, zarr_format=zarr_format, ) _rewrite_doc( path, zarr_format, lambda doc: ( doc.update(chunks=[0]) if zarr_format == 2 else doc["chunk_grid"]["configuration"].update(chunk_shape=[0]) ), ) data = np.arange(1, shape[0] + 1, dtype="int16") with pytest.warns(ZarrUserWarning, match="is read as"): arr = zarr.open_array(store=path, mode="r+") if api == "sync": arr[:] = data elif api == "async": sync(arr.async_array.setitem(slice(None), data)) else: async def write_twice() -> None: # Both writes find the metadata not yet stored, and both store it. await asyncio.gather(*(arr.async_array.setitem(slice(None), data) for _ in "ab")) sync(write_twice()) assert arr.metadata._stored_document is None with warnings.catch_warnings(): warnings.simplefilter("error", ZarrUserWarning) reopened = zarr.open_array(store=path, mode="r") assert (reopened.shards or reopened.chunks) == expected np.testing.assert_array_equal(reopened[...], data) def _legacy_array(path: Path, zarr_format: Literal[2, 3]) -> None: """Store an array of shape (3,) whose stored chunk shape is `[0]`.""" zarr.create_array(store=path, shape=(3,), chunks=(3,), dtype="int16", zarr_format=zarr_format) _rewrite_doc( path, zarr_format, lambda doc: ( doc.update(chunks=[0]) if zarr_format == 2 else doc["chunk_grid"]["configuration"].update(chunk_shape=[0]) ), ) @pytest.mark.parametrize("zarr_format", [2, 3]) def test_stale_handle_write_keeps_newer_metadata( tmp_path: Path, zarr_format: Literal[2, 3] ) -> None: """A handle read from a repaired document stores the repair of what the store holds when it first writes chunks; if another handle stored valid metadata since, it stores no metadata and writes only its chunks.""" path = tmp_path / "legacy.zarr" _legacy_array(path, zarr_format) with pytest.warns(ZarrUserWarning, match="is read as"): stale = zarr.open_array(store=path, mode="r+") with pytest.warns(ZarrUserWarning, match="is read as"): other = zarr.open_array(store=path, mode="r+") other.append(np.arange(1, 7, dtype="int16")) other.attrs["x"] = 1 stale[0] = 9 assert stale.metadata._stored_document is None reopened = _open_strictly(path) assert reopened.shape == (9,) assert reopened.attrs.asdict() == {"x": 1} np.testing.assert_array_equal(reopened[...], [9, 0, 0, 1, 2, 3, 4, 5, 6]) @pytest.mark.parametrize("zarr_format", [2, 3]) def test_stale_handle_write_keeps_valid_document_as_written( tmp_path: Path, zarr_format: Literal[2, 3] ) -> None: """If the document the store holds when a handle read from a repaired document first writes chunks needs no repair, it is left as written, even where zarr would encode the same metadata differently (as another implementation may have written it).""" path = tmp_path / "legacy.zarr" _legacy_array(path, zarr_format) with pytest.warns(ZarrUserWarning, match="is read as"): stale = zarr.open_array(store=path, mode="r+") # Valid metadata for the same array, as another writer might store it: chunk size # 1, without the optional members zarr writes, as compact JSON. doc_path = path / (".zarray" if zarr_format == 2 else "zarr.json") doc = json.loads(doc_path.read_text()) for optional in ("dimension_separator", "attributes", "storage_transformers"): doc.pop(optional, None) _stored_chunks(doc)[0] = 1 doc_path.write_text(json.dumps(doc, separators=(",", ":"))) written = doc_path.read_bytes() stale[0] = 9 assert doc_path.read_bytes() == written np.testing.assert_array_equal(_open_strictly(path)[...], [9, 0, 0]) @pytest.mark.parametrize("zarr_format", [2, 3]) def test_recreate_hint_recipe(tmp_path: Path, zarr_format: Literal[2, 3]) -> None: """The recipe the warning gives for an array read from a stored chunk size of 0 recreates it with the chunk shape asked for, keeping its data type, fill value, attributes and codecs, and the array then reopens without a warning.""" path = tmp_path / "legacy.zarr" zarr.create_array( store=path, shape=(3,), chunks=(3,), dtype="int16", fill_value=7, attributes={"units": "m"}, zarr_format=zarr_format, ) _rewrite_doc( path, zarr_format, lambda doc: ( doc.update(chunks=[0]) if zarr_format == 2 else doc["chunk_grid"]["configuration"].update(chunk_shape=[0]) ), ) recipe = ( "`zarr.from_array(array.store, name=array.path, data=array, chunks=, " "overwrite=True, write_data=False)`" ) with pytest.warns(ZarrUserWarning, match=re.escape(recipe)): array = zarr.open_array(store=path, mode="r+") zarr.from_array( array.store, name=array.path, data=array, chunks=(2,), overwrite=True, write_data=False ) reopened = _open_strictly(path) assert reopened.chunks == (2,) assert reopened.dtype == array.dtype assert reopened.fill_value == 7 assert reopened.attrs.asdict() == {"units": "m"} assert reopened.compressors == array.compressors assert reopened.filters == array.filters np.testing.assert_array_equal(reopened[...], [7, 7, 7]) def _store_zero(doc: dict[str, Any]) -> None: _stored_chunks(doc)[0] = 0 def _resize_to_10(doc: dict[str, Any]) -> None: doc["shape"] = [10] def _store_chunk_size_3(doc: dict[str, Any]) -> None: _stored_chunks(doc)[0] = 3 def _halve_inner_chunk_shape(doc: dict[str, Any]) -> None: doc["codecs"][0]["configuration"]["chunk_shape"] = [2] @pytest.mark.parametrize( ("zarr_format", "sharded", "change"), [ (2, False, _store_chunk_size_3), (3, False, _store_chunk_size_3), (3, True, _halve_inner_chunk_shape), ], ids=["v2-chunk-size", "v3-chunk-size", "v3-sharded-inner-chunk-shape"], ) def test_stale_handle_write_after_chunk_grid_change_raises( tmp_path: Path, zarr_format: Literal[2, 3], sharded: bool, change: Callable[[dict[str, Any]], None], ) -> None: """If the document the store holds when a handle read from a repaired document first writes chunks lays out chunks differently from the handle's metadata (another writer stored a different chunk size, or changed the inner chunk shape), the handle's chunks would not be found under it: the write raises and stores nothing.""" path = tmp_path / "legacy.zarr" if sharded: zarr.create_array(store=path, shape=(3,), chunks=(4,), shards=(4,), dtype="int16") _rewrite_doc(path, 3, _store_zero) else: _legacy_array(path, zarr_format) with pytest.warns(ZarrUserWarning, match="is read as"): stale = zarr.open_array(store=path, mode="r+") _rewrite_doc(path, zarr_format, change) documents = {p.name: p.read_bytes() for p in path.iterdir()} with pytest.raises(ValueError, match="has changed since this array was opened; reopen"): stale[0:3] = [7, 8, 9] assert stale.metadata._stored_document is not None assert {p.name: p.read_bytes() for p in path.iterdir()} == documents @pytest.mark.parametrize("zarr_format", [2, 3]) def test_stale_handle_write_after_resize_keeping_zero( tmp_path: Path, zarr_format: Literal[2, 3] ) -> None: """A stored chunk size of 0 is read as 1 however long the axis is, so a resize by software that kept it lays out chunks as the handle does: the handle's first write stores the repair of the resized document, then its chunks.""" path = tmp_path / "legacy.zarr" _legacy_array(path, zarr_format) with pytest.warns(ZarrUserWarning, match="is read as"): stale = zarr.open_array(store=path, mode="r+") _rewrite_doc(path, zarr_format, _resize_to_10) stale[0:3] = [7, 8, 9] reopened = _open_strictly(path) assert (reopened.shape, reopened.chunks) == ((10,), (1,)) np.testing.assert_array_equal(reopened[...], [7, 8, 9, 0, 0, 0, 0, 0, 0, 0]) def _set_attribute(array: AnyArray) -> None: array.attrs["x"] = 1 def _grow(array: AnyArray) -> None: array.resize((9,)) @pytest.mark.parametrize("operation", [_set_attribute, _grow], ids=["attrs", "resize"]) @pytest.mark.parametrize("zarr_format", [2, 3]) def test_failed_metadata_save_keeps_stored_document( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, zarr_format: Literal[2, 3], operation: Callable[[AnyArray], None], ) -> None: """If storing an array's metadata fails, the array still stands for the document the store holds, which still needs its repair.""" path = tmp_path / "legacy.zarr" _legacy_array(path, zarr_format) doc_name = ".zarray" if zarr_format == 2 else "zarr.json" with pytest.warns(ZarrUserWarning, match="is read as"): arr = zarr.open_array(store=path, mode="r+") stored = (path / doc_name).read_bytes() original_set = LocalStore.set async def failing_set(self: LocalStore, key: str, *args: Any, **kwargs: Any) -> None: if key == doc_name: raise OSError(f"cannot store {key}") await original_set(self, key, *args, **kwargs) monkeypatch.setattr(LocalStore, "set", failing_set) with pytest.raises(OSError, match=f"cannot store {re.escape(doc_name)}"): operation(arr) assert arr.metadata._stored_document is not None assert (path / doc_name).read_bytes() == stored @pytest.mark.parametrize("zarr_format", [2, 3]) def test_write_without_stored_document(zarr_format: Literal[2, 3]) -> None: """An array read from a repaired document that no store holds (as `AsyncArray.from_dict` builds one) writes its chunks as any array does: there is no stored document to repair.""" store = MemoryStore() doc = _v2_doc([3], [0]) if zarr_format == 2 else _v3_doc([3], [0]) with pytest.warns(ZarrUserWarning, match="is read as"): array = zarr.Array(AsyncArray.from_dict(StorePath(store), doc)) repaired = array.metadata._stored_document is not None array[:] = [1, 2, 3] assert (repaired, array.metadata._stored_document) == (True, None) np.testing.assert_array_equal(array[:], [1, 2, 3]) assert not [key for key in store._store_dict if key.endswith((".zarray", "zarr.json"))] @pytest.mark.parametrize("zarr_format", [2, 3]) def test_respelled_document_write_stores_only_chunks( tmp_path: Path, zarr_format: Literal[2, 3] ) -> None: """A document whose repair only respells a value (a stored chunk size `true`, read as 1 as zarr read it before) moves no chunks, so a write stores its chunks and no metadata: in a `ZipStore`, which cannot replace an entry, the write adds no second metadata entry and gives no warning.""" path = tmp_path / "legacy.zip" doc = _v2_doc([4, 4], [True, 4]) if zarr_format == 2 else _v3_doc([4, 4], [True, 4]) name = ".zarray" if zarr_format == 2 else "zarr.json" with zipfile.ZipFile(path, "w") as archive: archive.writestr(name, json.dumps(doc)) store = zarr.storage.ZipStore(path, mode="a") with warnings.catch_warnings(): warnings.simplefilter("error") array = zarr.open_array(store, mode="a") array[0] = 5 store.close() entries = [info.filename for info in zipfile.ZipFile(path).infolist()] assert entries.count(name) == 1 assert len(entries) == 2 np.testing.assert_array_equal( zarr.open_array(zarr.storage.ZipStore(path, mode="r"))[0], [5, 5, 5, 5] ) @pytest.mark.parametrize( "codec", [ ShardingCodec(chunk_shape=(1,)), { "name": "sharding_indexed", "configuration": { "chunk_shape": [np.int64(2)], "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], "index_codecs": [ {"name": "bytes", "configuration": {"endian": "little"}}, {"name": "crc32c"}, ], "index_location": "end", }, }, ], ids=["codec-instance", "numpy-integer"], ) def test_from_dict_keeps_values_that_are_not_json(codec: Any) -> None: """`from_dict` takes metadata built in code as well as stored documents, so the repairs read values that are not JSON (codec instances, NumPy integers) as they are, without encoding them, and the constructors read them as before.""" doc: dict[str, Any] = _v3_doc([4], [4]) doc["codecs"] = [codec] metadata = ArrayV3Metadata.from_dict(doc) assert metadata._stored_document is None assert isinstance(metadata.codecs[0], ShardingCodec) @pytest.mark.parametrize(("shape", "expected"), [((0,), (1,)), ((3,), (1,))]) def test_array_from_metadata_with_chunk_size_zero(shape: tuple[int], expected: tuple[int]) -> None: """`ArrayV2Metadata` accepts a chunk size of 0, as a stored document may hold it. An array built from such metadata reads it as the repairs read that document, silently (no data was read or written under it): `create_hierarchy` stores the metadata as given and yields such an array, which stores the repair before its first write.""" metadata = ArrayV2Metadata(shape=shape, chunks=(0,), dtype=Int16(), fill_value=0, order="C") store = MemoryStore() with warnings.catch_warnings(): warnings.simplefilter("error", ZarrUserWarning) nodes = dict(zarr.create_hierarchy(store=store, nodes={"a": metadata})) array = nodes["a"] assert isinstance(array, zarr.Array) assert array.chunks == expected assert json.loads(store._store_dict["a/.zarray"].to_bytes())["chunks"] == [0] array[...] = 1 # Writing an empty selection stores no chunks, so it stores no metadata either. resaved = list(expected) if array.size else [0] assert json.loads(store._store_dict["a/.zarray"].to_bytes())["chunks"] == resaved np.testing.assert_array_equal(zarr.open_array(store, path="a")[...], np.ones(shape)) def test_array_from_metadata_with_numpy_scalar_codec_configuration() -> None: """An array is built from metadata whose codec configuration holds NumPy scalars (which are not JSON values), as zarr always built one.""" array = zarr.create_array( MemoryStore(), shape=(4,), chunks=(2,), dtype="f8", filters=[Quantize(digits=3, dtype="f8")] ) assert isinstance(array.metadata, ArrayV3Metadata) # The codec keeps its configuration as given, though it is typed as JSON. digits = cast("JSON", np.int64(3)) codecs = (Quantize(digits=digits, dtype="f8"), *array.metadata.codecs[1:]) metadata = dataclasses.replace(array.metadata, codecs=codecs) assert AsyncArray(metadata, StorePath(MemoryStore())).metadata is metadata def _rewrite_consolidated( path: Path, zarr_format: Literal[2, 3], name: str, edit: Callable[[dict[str, Any]], None] ) -> None: """Edit the document of the member `name` in the consolidated metadata at `path`.""" if zarr_format == 2: document = json.loads((path / ".zmetadata").read_text()) edit(document["metadata"][f"{name}/.zarray"]) (path / ".zmetadata").write_text(json.dumps(document)) else: _rewrite_doc(path, 3, lambda doc: edit(doc["consolidated_metadata"]["metadata"][name])) def _consolidated_member(path: Path, zarr_format: Literal[2, 3], name: str) -> Any: if zarr_format == 2: return json.loads((path / ".zmetadata").read_text())["metadata"][f"{name}/.zarray"] return json.loads((path / "zarr.json").read_text())["consolidated_metadata"]["metadata"][name] def _flagged_consolidated_group(path: Path, zarr_format: Literal[2, 3], member: str) -> None: """A group whose consolidated copy of the array `member` holds the stored chunk size 0, and whose array `b` is valid.""" group = zarr.open_group(path, mode="w", zarr_format=zarr_format) parent, _, name = member.rpartition("/") (group.require_group(parent) if parent else group).create_array( name, shape=(3,), chunks=(3,), dtype="int16" ) group.create_array("b", shape=(1,), chunks=(1,), dtype="int16") zarr.consolidate_metadata(path) _rewrite_consolidated(path, zarr_format, member, _store_zero) def _record_store_access(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, str]]: """Record every `get` and `set` of a `LocalStore`, as `(method, key)`.""" accesses: list[tuple[str, str]] = [] for method in ("get", "set"): original = getattr(LocalStore, method) async def recording( self: LocalStore, key: str, *args: Any, _method: str = method, _original: Any = original, **kwargs: Any, ) -> Any: accesses.append((_method, key)) return await _original(self, key, *args, **kwargs) monkeypatch.setattr(LocalStore, method, recording) return accesses @pytest.mark.filterwarnings("ignore:Consolidated metadata is currently not part:UserWarning") @pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.parametrize("member", ["a", "g/a"]) @pytest.mark.parametrize("operation", ["attrs", "update_attributes_async", "delete-member"]) def test_group_write_stores_repaired_consolidated_member_as_stored( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, zarr_format: Literal[2, 3], member: str, operation: str, ) -> None: """A group write stores only the group's own documents, reading none: it reads and writes no member document, and stores the consolidated copy of a member read from a document that had to be repaired exactly as it was stored, so every reader of the consolidated metadata reads that member as repaired again.""" path = tmp_path / "group.zarr" _flagged_consolidated_group(path, zarr_format, member) stored = json.dumps(_consolidated_member(path, zarr_format, member)) with pytest.warns(ZarrUserWarning, match="is read as"): group = zarr.open_group(path, mode="r+", use_consolidated=True) accesses = _record_store_access(monkeypatch) if operation == "attrs": group.attrs["x"] = 1 elif operation == "update_attributes_async": sync(group.update_attributes_async({"x": 1})) else: del group["b"] own = {"zarr.json"} if zarr_format == 3 else {".zgroup", ".zattrs", ".zmetadata"} assert {method for method, _ in accesses} == {"set"} assert {key for _, key in accesses} == own assert json.dumps(_consolidated_member(path, zarr_format, member)) == stored with pytest.warns(ZarrUserWarning, match="is read as"): zarr.open_group(path, mode="r", use_consolidated=True) @pytest.mark.filterwarnings("ignore:Consolidated metadata is currently not part:UserWarning") @pytest.mark.parametrize("zarr_format", [2, 3]) def test_concurrent_deletions_leave_no_repaired_member( tmp_path: Path, zarr_format: Literal[2, 3] ) -> None: """Members deleted concurrently through one consolidated group handle stay deleted, also those read from documents that had to be repaired: no group write stores a member document.""" path = tmp_path / "group.zarr" zarr.open_group(path, mode="w", zarr_format=zarr_format) for name in ("a", "b"): _legacy_array(path / name, zarr_format) with pytest.warns(ZarrUserWarning, match="is read as"): zarr.consolidate_metadata(path) with pytest.warns(ZarrUserWarning, match="is read as"): group = zarr.open_group(path, mode="r+", use_consolidated=True) async def delete_both() -> None: await asyncio.gather(*(group._async_group.delitem(name) for name in ("a", "b"))) sync(delete_both()) assert not (path / "a").exists() assert not (path / "b").exists() def _consolidated_legacy_member(path: Path, zarr_format: Literal[2, 3]) -> Path: """A group whose array `a` is stored with chunk shape `[0]`, consolidated; the path of the array's own document.""" zarr.open_group(path, mode="w", zarr_format=zarr_format) _legacy_array(path / "a", zarr_format) with pytest.warns(ZarrUserWarning, match="is read as"): zarr.consolidate_metadata(path) return path / "a" / (".zarray" if zarr_format == 2 else "zarr.json") @pytest.mark.filterwarnings("ignore:Consolidated metadata is currently not part:UserWarning") @pytest.mark.parametrize("zarr_format", [2, 3]) def test_consolidated_repaired_member_stored_by_its_first_write( tmp_path: Path, zarr_format: Literal[2, 3] ) -> None: """Consolidating a group copies the document of a member that had to be repaired as it is stored, and leaves that document as it is. The first chunk write through the consolidated metadata stores the member's repair before its chunks, the one write that stores it; consolidating again then copies the repair.""" path = tmp_path / "group.zarr" document = _consolidated_legacy_member(path, zarr_format) legacy = document.read_bytes() assert json.dumps(_consolidated_member(path, zarr_format, "a")).encode() == legacy with pytest.warns(ZarrUserWarning, match="is read as"): array = zarr.open_group(path, mode="r+", use_consolidated=True)["a"] assert isinstance(array, zarr.Array) array[:] = [7, 8, 9] np.testing.assert_array_equal(_open_strictly(path / "a")[...], [7, 8, 9]) zarr.consolidate_metadata(path) assert _stored_chunks(_consolidated_member(path, zarr_format, "a")) == [1] @pytest.mark.filterwarnings("ignore:Consolidated metadata is currently not part:UserWarning") @pytest.mark.parametrize("zarr_format", [2, 3]) def test_consolidated_repaired_member_write_after_chunk_grid_change_raises( tmp_path: Path, zarr_format: Literal[2, 3] ) -> None: """A member read from its consolidated copy, whose own document has since been stored with a different chunk size, would write chunks no reader finds: its first write raises and stores nothing.""" path = tmp_path / "group.zarr" _consolidated_legacy_member(path, zarr_format) with pytest.warns(ZarrUserWarning, match="is read as"): array = zarr.open_group(path, mode="r+", use_consolidated=True)["a"] assert isinstance(array, zarr.Array) _rewrite_doc(path / "a", zarr_format, _store_chunk_size_3) documents = {p: p.read_bytes() for p in path.rglob("*") if p.is_file()} with pytest.raises(ValueError, match="has changed since this array was opened; reopen"): array[0:3] = [7, 8, 9] assert {p: p.read_bytes() for p in path.rglob("*") if p.is_file()} == documents @pytest.mark.filterwarnings("ignore:Consolidated metadata is currently not part:UserWarning") @pytest.mark.parametrize("zarr_format", [2, 3]) def test_consolidated_repaired_member_attributes_kept_by_group_write( tmp_path: Path, zarr_format: Literal[2, 3] ) -> None: """Setting attributes of a member read from consolidated metadata stores the member's repaired document with them. A later write of the group, whose consolidated metadata shares the member's metadata, then stores that repair: the new attributes, not the legacy document as it was stored before.""" path = tmp_path / "group.zarr" _consolidated_legacy_member(path, zarr_format) with pytest.warns(ZarrUserWarning, match="is read as"): group = zarr.open_group(path, mode="r+", use_consolidated=True) group["a"].attrs["x"] = 1 group.attrs["y"] = 2 with warnings.catch_warnings(record=True) as record: warnings.simplefilter("always") attributes = dict(zarr.open_group(path, mode="r", use_consolidated=True)["a"].attrs) assert attributes == {"x": 1} assert _stored_chunks(_consolidated_member(path, zarr_format, "a")) == [1] assert not [w for w in record if "is read as" in str(w.message)] @pytest.mark.parametrize("zarr_format", [2, 3]) def test_repaired_metadata_keeps_the_document_it_read(zarr_format: Literal[2, 3]) -> None: """Metadata read from a document that had to be repaired keeps that document as it was read, whatever becomes of the caller's dict (or the objects in it, which the metadata's attributes may hold): consolidated metadata stores it as read.""" doc = _v2_doc([0], [0]) if zarr_format == 2 else _v3_doc([0], [0]) doc["attributes"] = {"k": [1]} read = json.loads(json.dumps(doc)) metadata_cls = ArrayV2Metadata if zarr_format == 2 else ArrayV3Metadata metadata = metadata_cls.from_dict(doc) cast("list[int]", metadata.attributes["k"]).append(2) doc["shape"] = [4] assert ConsolidatedMetadata(metadata={"a": metadata}).to_dict()["metadata"] == {"a": read} @pytest.mark.parametrize("zarr_format", [2, 3]) def test_empty_write_stores_no_metadata(tmp_path: Path, zarr_format: Literal[2, 3]) -> None: """A write of an empty selection stores no chunks, so it stores no metadata either.""" path = tmp_path / "legacy.zarr" _legacy_array(path, zarr_format) documents = {p.name: p.read_bytes() for p in path.iterdir()} with pytest.warns(ZarrUserWarning, match="is read as"): array = zarr.open_array(store=path, mode="r+") array[0:0] = np.empty(0, dtype="int16") assert array.metadata._stored_document is not None assert {p.name: p.read_bytes() for p in path.iterdir()} == documents @pytest.mark.parametrize("zarr_format", [2, 3]) def test_async_array_from_dict_names_array(tmp_path: Path, zarr_format: Literal[2, 3]) -> None: """`AsyncArray.from_dict` names the array at its store path in the repair warning.""" store_path = sync(make_store_path(tmp_path / "legacy.zarr")) doc = _v2_doc([4], [0]) if zarr_format == 2 else _v3_doc([4], [0]) with pytest.warns(ZarrUserWarning, match=f"^Array {re.escape(repr(str(store_path)))}: "): AsyncArray.from_dict(store_path, doc) @pytest.mark.filterwarnings("ignore:Consolidated metadata is currently not part:UserWarning") @pytest.mark.parametrize("zarr_format", [2, 3]) def test_legacy_chunk_size_consolidated(tmp_path: Path, zarr_format: Literal[2, 3]) -> None: """Consolidated metadata goes through the same repair as the arrays' own documents, with one warning naming each array by its path; re-saving the arrays and consolidating again leaves a group that opens without a warning.""" path = tmp_path / "group.zarr" group = zarr.open_group(path, mode="w", zarr_format=zarr_format) names = ("a", "b") for name in names: group.create_array(name, shape=(2,), chunks=(2,), dtype="int32") zarr.consolidate_metadata(path) # What zarr-python wrote for `chunks=(0,)`, in both copies. for name in names: if zarr_format == 2: _rewrite_doc(path / name, 2, lambda doc: doc.update(chunks=[0])) zmetadata = json.loads((path / ".zmetadata").read_text()) zmetadata["metadata"][f"{name}/.zarray"]["chunks"] = [0] (path / ".zmetadata").write_text(json.dumps(zmetadata)) else: _rewrite_doc( path / name, 3, lambda doc: doc["chunk_grid"]["configuration"].update(chunk_shape=[0]), ) _rewrite_doc( path, 3, lambda doc, name=name: doc["consolidated_metadata"]["metadata"][name]["chunk_grid"][ "configuration" ].update(chunk_shape=[0]), ) for use_consolidated in (True, False): with warnings.catch_warnings(record=True) as record: warnings.simplefilter("always", ZarrUserWarning) group = zarr.open_group(path, mode="r+", use_consolidated=use_consolidated) arrays = [group[name] for name in names] assert all("zarr.consolidate_metadata" in str(w.message) for w in record) assert sorted(str(w.message).split(": ")[0] for w in record) == [ f"Array '{group.store_path / name}'" for name in names ] for array in arrays: assert isinstance(array, zarr.Array) assert array.chunks == (1,) array.update_attributes({}) zarr.consolidate_metadata(path) with warnings.catch_warnings(): warnings.simplefilter("error", ZarrUserWarning) warnings.filterwarnings("ignore", "Consolidated metadata is currently not part") for use_consolidated in (True, False): reopened = zarr.open_group(path, mode="r", use_consolidated=use_consolidated) for name in names: array = reopened[name] assert isinstance(array, zarr.Array) assert array.chunks == (1,) # A document copied verbatim from a store that zarr 3.2.1 wrote for # `create_array(shape=(6, 20), chunks=(2, (5, 10, 5)), dtype="float32")`. MIXED_REGULAR_GRID_DOC = """{ "shape": [6, 20], "data_type": "float32", "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [2, [5, 10, 5]]}}, "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, "fill_value": 0.0, "codecs": [ {"name": "bytes", "configuration": {"endian": "little"}}, {"name": "zstd", "configuration": {"level": 0, "checksum": false}} ], "attributes": {}, "zarr_format": 3, "node_type": "array", "storage_transformers": [] }""" def _mixed_doc(shape: list[int], chunk_shape: list[Any]) -> dict[str, JSON]: doc: dict[str, JSON] = json.loads(MIXED_REGULAR_GRID_DOC) doc["shape"] = shape doc["chunk_grid"] = {"name": "regular", "configuration": {"chunk_shape": chunk_shape}} return doc @pytest.mark.parametrize( ("doc", "expected", "warning"), [ (json.loads(MIXED_REGULAR_GRID_DOC), (2, (5, 10, 5)), r"^The stored chunk grid .* \[1\]"), ( _mixed_doc([6, 20, 4], [2, [5, 10, 5], [1, 3]]), (2, (5, 10, 5), (1, 3)), r"^The stored chunk grid .* in dimensions \[1, 2\]", ), (_mixed_doc([6, 20], [2, [20]]), (2, (20,)), r"^The stored chunk grid .* \[1\]"), (_mixed_doc([6, 12], [2, [5, 10, 5]]), (2, (5, 10, 5)), r"^The stored chunk grid"), (_mixed_doc([0, 20], [2, [5, 10, 5]]), (2, (5, 10, 5)), r"^The stored chunk grid"), (_mixed_doc([6, 0], [2, [5, 10, 5]]), (2, (5, 10, 5)), r"^The stored chunk grid"), # JSON true is read as 1 silently; only the grid reading warns. (_mixed_doc([6, 20], [True, [5, 10, 5]]), (1, (5, 10, 5)), r"^The stored chunk grid"), (_mixed_doc([6, 20], [2, [5.0, 10.0, 5.0]]), (2, (5, 10, 5)), r"^The stored chunk grid"), ( _mixed_doc([4, 10_000], [0, [10] * 1000]), (1, (10,) * 1000), r"^The stored chunk shape \[0, \[10, 10, .*\.\.\. is invalid: .* read as \[1, \[10, .*\.\.\.,", ), ], ids=[ "written", "3d", "one-edge", "shrunk", "empty-int-axis", "empty-edge-axis", "true", "float-edges", "long", ], ) def test_read_edge_lists_in_regular_grid( doc: dict[str, JSON], expected: tuple[int | tuple[int, ...], ...], warning: str ) -> None: """A `regular` chunk grid whose chunk shape mixes chunk sizes with lists of chunk edge lengths is read as the rectilinear chunk grid it describes, without the rectilinear chunks flag. `from_dict` warns once, naming the array and the axes, quoting a bounded part of the chunk shape, and saying that re-saving requires the flag.""" with ( zarr.config.set({"array.rectilinear_chunks": False}), warnings.catch_warnings(record=True) as record, ): warnings.simplefilter("always") metadata = ArrayV3Metadata.from_dict(doc, path="group/array") assert metadata.chunk_grid == RectilinearChunkGridMetadata(chunk_shapes=expected) [message] = [str(w.message) for w in record] assert re.search(warning, message.removeprefix("Array 'group/array': ")) assert message.startswith("Array 'group/array': ") assert ( "Re-saving the metadata stores that rectilinear chunk grid, so each step that " "follows requires `zarr.config.set({'array.rectilinear_chunks': True})`. " ) in message assert message.endswith(RESAVE_HINT) # The 1000-edge list is abbreviated: the message is two sentences and two hints, not # a dump of the edges. assert len(message) < 1600 def _rejected_without_warning(doc: dict[str, JSON]) -> pytest.ExceptionInfo[Exception]: with warnings.catch_warnings(): warnings.simplefilter("error", ZarrUserWarning) with pytest.raises((TypeError, ValueError)) as info: ArrayV3Metadata.from_dict(doc) return info def test_regular_grid_of_only_edge_lists_rejected() -> None: """A regular chunk shape made only of edge lists was never stored (a rectilinear chunk grid was), so it is not read as rectilinear.""" info = _rejected_without_warning(_mixed_doc([6, 20], [[1, 5], [5, 10, 5]])) assert info.match(re.escape("Dimension 0: chunk edge length must be an int, got [1, 5]")) def test_run_length_encoded_edges_in_regular_grid_rejected() -> None: """Run-length encoded edges were never stored in a regular chunk shape.""" info = _rejected_without_warning(_mixed_doc([6, 20], [2, [[5, 2], 10]])) assert info.match(re.escape("Dimension 1: chunk edge length must be an int, got [[5, 2], 10]")) @pytest.mark.parametrize("edge", [5.5, "5"], ids=["fractional", "string"]) def test_non_int_edge_in_regular_grid_rejected(edge: object) -> None: """An edge that is not an integer is reported as such, not blamed on its list.""" info = _rejected_without_warning(_mixed_doc([6, 20], [2, [edge, 15]])) assert info.match(re.escape(f"Dimension 1: chunk edge length must be an int, got {edge!r}")) def test_edge_below_one_in_regular_grid_rejected() -> None: info = _rejected_without_warning(_mixed_doc([6, 20], [2, [0, 20]])) assert info.match("Dimension 1: chunk edge length must be >= 1, got 0") def test_short_edges_in_regular_grid_rejected() -> None: info = _rejected_without_warning(_mixed_doc([6, 20], [2, [5, 10]])) assert info.match("sum to 15 but array shape extent is 20") def _store_mixed_array(path: Path) -> np.ndarray[Any, np.dtype[np.float32]]: """Write the chunks of the verbatim document and the document itself at `path`.""" data = np.arange(120, dtype="float32").reshape(6, 20) with zarr.config.set({"array.rectilinear_chunks": True}): arr = zarr.create_array(path, shape=data.shape, chunks=(2, (5, 10, 5)), dtype="float32") arr[...] = data (path / "zarr.json").write_text(MIXED_REGULAR_GRID_DOC) return data def _update_attributes(arr: zarr.Array[Any], data: np.ndarray[Any, Any]) -> np.ndarray[Any, Any]: arr.update_attributes({}) return data def _write(arr: zarr.Array[Any], data: np.ndarray[Any, Any]) -> np.ndarray[Any, Any]: arr[:2] = -data[:2] return np.concatenate([-data[:2], data[2:]]) @pytest.mark.parametrize("store_metadata", [_update_attributes, _write], ids=["re-save", "write"]) def test_edge_lists_in_regular_grid_round_trip( tmp_path: Path, store_metadata: Callable[[zarr.Array[Any], np.ndarray[Any, Any]], np.ndarray[Any, Any]], ) -> None: """A store holding the verbatim document opens without the rectilinear chunks flag and reads its data. With the flag, re-saving the metadata, or writing chunks (which stores the metadata first), stores the rectilinear chunk grid, which then opens cleanly.""" path = tmp_path / "mixed.zarr" data = _store_mixed_array(path) with ( zarr.config.set({"array.rectilinear_chunks": False}), pytest.warns(ZarrUserWarning, match="read as that rectilinear chunk grid"), ): arr = zarr.open_array(path, mode="a") np.testing.assert_array_equal(arr[...], data) with zarr.config.set({"array.rectilinear_chunks": True}): expected = store_metadata(arr, data) assert json.loads((path / "zarr.json").read_text())["chunk_grid"] == { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [2, [5, 10, 5]]}, } with warnings.catch_warnings(): warnings.simplefilter("error", ZarrUserWarning) reopened = zarr.open_array(path) np.testing.assert_array_equal(reopened[...], expected) def _store_mixed_group(path: Path) -> None: """A group at `path` holding the verbatim document at `mixed` and a regular array `n`, with consolidated metadata that quotes the verbatim document.""" group = zarr.open_group(path, mode="w") _store_mixed_array(path / "mixed") group.create_array("n", data=np.arange(4), chunks=(2,)) with zarr.config.set({"array.rectilinear_chunks": True}): zarr.consolidate_metadata(path) group_doc = json.loads((path / "zarr.json").read_text()) group_doc["consolidated_metadata"]["metadata"]["mixed"] = json.loads(MIXED_REGULAR_GRID_DOC) (path / "zarr.json").write_text(json.dumps(group_doc)) def _resize(path: Path) -> None: zarr.open_array(path / "mixed", mode="a").resize((6, 5)) def _write_chunks(path: Path) -> None: zarr.open_array(path / "mixed", mode="a")[...] = 1 def _delete_member(path: Path) -> None: del zarr.open_group(path, mode="a")["n"] def _overwrite_hierarchy(path: Path) -> None: mixed = zarr.open_array(path / "mixed") list(zarr.create_hierarchy(store=LocalStore(path), nodes={"n": mixed.metadata}, overwrite=True)) def _overwrite_with_create(path: Path) -> None: zarr.create(shape=(4,), chunks=[[2, 2]], dtype="int64", store=path / "n", overwrite=True) def _overwrite_with_create_array(path: Path) -> None: zarr.create_array(path / "n", shape=(4,), chunks=[[2, 2]], dtype="int64", overwrite=True) def _set_group_attribute(path: Path) -> None: zarr.open_group(path, mode="a").attrs["x"] = 1 @pytest.mark.filterwarnings( "ignore:.*read as that rectilinear chunk grid:zarr.errors.ZarrUserWarning" ) @pytest.mark.filterwarnings("ignore:Consolidated metadata is currently not part:UserWarning") @pytest.mark.parametrize( "action", [ _resize, _write_chunks, _overwrite_hierarchy, _overwrite_with_create, _overwrite_with_create_array, ], ids=["resize", "write", "overwrite-hierarchy", "overwrite-create", "overwrite-create-array"], ) def test_store_untouched_without_flag(tmp_path: Path, action: Callable[[Path], None]) -> None: """An operation that would store the rectilinear chunk grid read from the verbatim document fails without the flag before it deletes or writes anything.""" path = tmp_path / "group.zarr" _store_mixed_group(path) stored = {p: p.read_bytes() for p in path.rglob("*") if p.is_file()} with ( zarr.config.set({"array.rectilinear_chunks": False}), pytest.raises(ValueError, match="experimental and disabled by default"), ): action(path) assert {p: p.read_bytes() for p in path.rglob("*") if p.is_file()} == stored RECTILINEAR_GRID: dict[str, Any] = { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [2, [5, 10, 5]]}, } @pytest.mark.filterwarnings("ignore:Consolidated metadata is currently not part:UserWarning") @pytest.mark.parametrize("member", ["mixed", "sub/mixed"]) def test_consolidate_edge_lists_in_regular_grid(tmp_path: Path, member: str) -> None: """Consolidating a group holding the verbatim document copies it as stored, with or without the flag, and leaves the member's document as it is, so the group opens and reads the member without the flag. Once the member's metadata is re-saved (which needs the flag), consolidating copies its rectilinear chunk grid.""" path = tmp_path / "group.zarr" zarr.open_group(path, mode="w").create_group("sub") data = _store_mixed_array(path / member) for flag in (False, True): with ( zarr.config.set({"array.rectilinear_chunks": flag}), pytest.warns(ZarrUserWarning, match="read as that rectilinear chunk grid"), ): zarr.consolidate_metadata(path) assert (path / member / "zarr.json").read_text() == MIXED_REGULAR_GRID_DOC assert _consolidated_member(path, 3, member) == json.loads(MIXED_REGULAR_GRID_DOC) with ( zarr.config.set({"array.rectilinear_chunks": False}), pytest.warns(ZarrUserWarning, match="read as that rectilinear chunk grid"), ): mixed = zarr.open_group(path, mode="r")[member] assert isinstance(mixed, zarr.Array) np.testing.assert_array_equal(mixed[...], data) with zarr.config.set({"array.rectilinear_chunks": True}): with pytest.warns(ZarrUserWarning, match="read as that rectilinear chunk grid"): zarr.open_array(path / member, mode="a").update_attributes({}) zarr.consolidate_metadata(path) assert _consolidated_member(path, 3, member)["chunk_grid"] == RECTILINEAR_GRID with warnings.catch_warnings(): warnings.simplefilter("error", ZarrUserWarning) reopened = zarr.open_group(path, mode="r")[member] assert isinstance(reopened, zarr.Array) np.testing.assert_array_equal(reopened[...], data) @pytest.mark.filterwarnings( "ignore:.*read as that rectilinear chunk grid:zarr.errors.ZarrUserWarning" ) @pytest.mark.filterwarnings("ignore:Consolidated metadata is currently not part:UserWarning") @pytest.mark.parametrize("action", [_delete_member, _set_group_attribute], ids=["delete", "attrs"]) @pytest.mark.parametrize("flag", [False, True]) def test_group_write_stores_mixed_member_as_stored( tmp_path: Path, action: Callable[[Path], None], flag: bool ) -> None: """A group write, with or without the flag, stores the consolidated copy of the verbatim document as stored and leaves the member's own document as it is: the group opens and reads the member without the flag.""" path = tmp_path / "group.zarr" _store_mixed_group(path) with zarr.config.set({"array.rectilinear_chunks": flag}): action(path) assert _consolidated_member(path, 3, "mixed") == json.loads(MIXED_REGULAR_GRID_DOC) assert (path / "mixed" / "zarr.json").read_text() == MIXED_REGULAR_GRID_DOC with zarr.config.set({"array.rectilinear_chunks": False}): mixed = zarr.open_group(path, mode="r", use_consolidated=True)["mixed"] assert isinstance(mixed, zarr.Array) assert mixed.read_chunk_sizes == ((2, 2, 2), (5, 10, 5)) @pytest.mark.filterwarnings("ignore:Consolidated metadata is currently not part:UserWarning") def test_delete_member_without_flag_keeps_group(tmp_path: Path) -> None: """A deletion that would store a rectilinear chunk grid in the consolidated metadata fails without the flag before it deletes anything, and leaves the group listing the member.""" path = tmp_path / "group.zarr" with zarr.config.set({"array.rectilinear_chunks": True}): group = zarr.open_group(path, mode="w") group.create_array("r", shape=(6, 20), chunks=(2, (5, 10, 5)), dtype="float32") group.create_array("n", shape=(2,), chunks=(1,), dtype="int8") zarr.consolidate_metadata(path) group = zarr.open_group(path, mode="a", use_consolidated=True) stored = {p: p.read_bytes() for p in path.rglob("*") if p.is_file()} with zarr.config.set({"array.rectilinear_chunks": False}): with pytest.raises(ValueError, match="experimental and disabled"): del group["n"] assert group.metadata.consolidated_metadata is not None assert sorted(group.metadata.consolidated_metadata.metadata) == ["n", "r"] assert {p: p.read_bytes() for p in path.rglob("*") if p.is_file()} == stored zarr-developers-zarr-python-a2f2415/tests/test_metadata/test_v2.py000066400000000000000000000373711527014137600254210ustar00rootroot00000000000000from __future__ import annotations import json from typing import TYPE_CHECKING, Literal import numpy as np import pytest import zarr.api.asynchronous import zarr.storage from zarr.core.buffer import cpu from zarr.core.buffer.core import default_buffer_prototype from zarr.core.dtype.npy.float import Float32, Float64 from zarr.core.dtype.npy.int import Int16 from zarr.core.group import ConsolidatedMetadata, GroupMetadata from zarr.core.metadata import ArrayV2Metadata from zarr.core.metadata.v2 import parse_zarr_format from zarr.errors import ZarrUserWarning if TYPE_CHECKING: from pathlib import Path from typing import Any from zarr.abc.codec import Codec from zarr.core.common import JSON from zarr.core.dtype.common import DTypeName_V2 def test_parse_zarr_format_valid() -> None: assert parse_zarr_format(2) == 2 # The explicit id for "3" avoids colliding with the auto-generated id for the int 3. @pytest.mark.parametrize("data", [None, 1, 3, 4, 5, pytest.param("3", id="3-str")]) def test_parse_zarr_format_invalid(data: Any) -> None: with pytest.raises(ValueError, match="Failed to parse input for 'zarr_format'"): parse_zarr_format(data) @pytest.mark.parametrize("attributes", [None, {"foo": "bar"}]) @pytest.mark.parametrize("filters", [None, [{"id": "gzip", "level": 1}]]) @pytest.mark.parametrize("compressor", [None, {"id": "gzip", "level": 1}]) @pytest.mark.parametrize("fill_value", [None, 0, 1]) @pytest.mark.parametrize("order", ["C", "F"]) @pytest.mark.parametrize("dimension_separator", [".", "/", None]) def test_metadata_to_dict( compressor: Codec | None, filters: tuple[Codec] | None, fill_value: Any, order: Literal["C", "F"], dimension_separator: Literal[".", "/"] | None, attributes: dict[str, Any] | None, ) -> None: shape = (1, 2, 3) chunks = (1,) * len(shape) data_type = "|u1" metadata_dict = { "zarr_format": 2, "shape": shape, "chunks": chunks, "dtype": data_type, "order": order, "compressor": compressor, "filters": filters, "fill_value": fill_value, } if attributes is not None: metadata_dict["attributes"] = attributes if dimension_separator is not None: metadata_dict["dimension_separator"] = dimension_separator metadata = ArrayV2Metadata.from_dict(metadata_dict) observed = metadata.to_dict() expected = metadata_dict.copy() if attributes is None: assert observed["attributes"] == {} observed.pop("attributes") if dimension_separator is None: expected_dimension_sep = "." assert observed["dimension_separator"] == expected_dimension_sep observed.pop("dimension_separator") assert observed == expected def test_filters_empty_tuple_warns() -> None: metadata_dict = { "zarr_format": 2, "shape": (1,), "chunks": (1,), "dtype": "|u1", "order": "C", "compressor": None, "filters": (), "fill_value": 0, } with pytest.warns( ZarrUserWarning, match="Found an empty list of filters in the array metadata document." ): meta = ArrayV2Metadata.from_dict(metadata_dict) assert meta.filters is None class TestConsolidated: @pytest.fixture async def v2_consolidated_metadata( self, memory_store: zarr.storage.MemoryStore ) -> zarr.storage.MemoryStore: zmetadata: dict[str, JSON] = { "metadata": { ".zattrs": { "Conventions": "COARDS", }, ".zgroup": {"zarr_format": 2}, "air/.zarray": { "chunks": [730], "compressor": None, "dtype": " None: # .zgroup, .zattrs, .metadata store = v2_consolidated_metadata group = zarr.open_consolidated(store=store, zarr_format=2) assert group.metadata.consolidated_metadata is not None expected = ConsolidatedMetadata( metadata={ "air": ArrayV2Metadata( shape=(730,), fill_value=0, chunks=(730,), attributes={"_ARRAY_DIMENSIONS": ["time"], "dataset": "NMC Reanalysis"}, dtype=Int16(endianness="little"), order="C", filters=None, dimension_separator=".", compressor=None, ), "time": ArrayV2Metadata( shape=(730,), fill_value=0.0, chunks=(730,), attributes={ "_ARRAY_DIMENSIONS": ["time"], "calendar": "standard", "long_name": "Time", "standard_name": "time", "units": "hours since 1800-01-01", }, dtype=Float32(endianness="little"), order="C", filters=None, dimension_separator=".", compressor=None, ), "nested": GroupMetadata( attributes={"key": "value"}, zarr_format=2, consolidated_metadata=ConsolidatedMetadata( metadata={ "array": ArrayV2Metadata( shape=(730,), fill_value=0.0, chunks=(730,), attributes={ "calendar": "standard", }, dtype=Float32(endianness="little"), order="C", filters=None, dimension_separator=".", compressor=None, ) } ), ), }, kind="inline", must_understand=False, ) result = group.metadata.consolidated_metadata assert result == expected async def test_getitem_consolidated( self, v2_consolidated_metadata: zarr.storage.MemoryStore ) -> None: store = v2_consolidated_metadata group = await zarr.api.asynchronous.open_consolidated(store=store, zarr_format=2) air = await group.getitem("air") assert isinstance(air, zarr.AsyncArray) assert air.metadata.shape == (730,) def test_from_dict_extra_fields() -> None: data = { "_nczarr_array": {"dimrefs": ["/dim1", "/dim2"], "storage": "chunked"}, "attributes": {"key": "value"}, "chunks": [8], "compressor": None, "dtype": " None: """Two metadata objects with an identical NaN fill_value compare equal. NaN is not equal to itself under IEEE 754, so the default dataclass __eq__ reports two otherwise-identical metadata objects as unequal. Metadata equality must treat matching NaN fill values as equal (see issue #2929). """ a = ArrayV2Metadata( shape=(8,), dtype=Float64(), chunks=(8,), fill_value=np.float64("nan"), order="C" ) b = ArrayV2Metadata( shape=(8,), dtype=Float64(), chunks=(8,), fill_value=np.float64("nan"), order="C" ) assert a == b def test_eq_distinct_fill_value() -> None: """Metadata objects that differ only in fill_value do not compare equal.""" a = ArrayV2Metadata(shape=(8,), dtype=Float64(), chunks=(8,), fill_value=0.0, order="C") b = ArrayV2Metadata(shape=(8,), dtype=Float64(), chunks=(8,), fill_value=1.0, order="C") assert a != b @pytest.mark.parametrize("fill_value", [np.float64("inf"), np.float64("-inf")]) def test_eq_inf_fill_value(fill_value: np.float64) -> None: """Two metadata objects with an identical infinite fill_value compare equal.""" a = ArrayV2Metadata(shape=(8,), dtype=Float64(), chunks=(8,), fill_value=fill_value, order="C") b = ArrayV2Metadata(shape=(8,), dtype=Float64(), chunks=(8,), fill_value=fill_value, order="C") assert a == b def test_hash_consistent_with_eq_nan_fill_value() -> None: """Equal metadata objects with a NaN fill_value hash equal. NaN hashes by identity, so a field-based hash would break the ``a == b implies hash(a) == hash(b)`` invariant for objects that compare equal under the to_dict-based __eq__. """ a = ArrayV2Metadata( shape=(8,), dtype=Float64(), chunks=(8,), fill_value=np.float64("nan"), order="C" ) b = ArrayV2Metadata( shape=(8,), dtype=Float64(), chunks=(8,), fill_value=np.float64("nan"), order="C" ) assert a == b assert hash(a) == hash(b) def test_eq_non_metadata() -> None: """Comparison against a non-metadata object returns False rather than erroring.""" a = ArrayV2Metadata(shape=(8,), dtype=Float64(), chunks=(8,), fill_value=0.0, order="C") assert a != object() def test_zstd_checksum() -> None: compressor_config: dict[str, JSON] = {"id": "zstd", "level": 5, "checksum": False} arr = zarr.create_array( {}, shape=(10,), chunks=(10,), dtype="int32", compressors=compressor_config, zarr_format=2, ) metadata = json.loads( arr.metadata.to_buffer_dict(default_buffer_prototype())[".zarray"].to_bytes() ) assert "checksum" not in metadata["compressor"] @pytest.mark.parametrize("fill_value", [np.void((0, 0), np.dtype([("foo", "i4"), ("bar", "i4")]))]) def test_structured_dtype_fill_value_serialization( tmp_path: Path, fill_value: np.void | np.dtype[Any] ) -> None: zarr_format: Literal[2] = 2 group_path = tmp_path / "test.zarr" root_group = zarr.open_group(group_path, mode="w", zarr_format=zarr_format) dtype = np.dtype([("foo", "i4"), ("bar", "i4")]) root_group.create_array( name="structured_dtype", shape=(100, 100), chunks=(100, 100), dtype=dtype, fill_value=fill_value, ) zarr.consolidate_metadata(root_group.store, zarr_format=zarr_format) root_group = zarr.open_group(group_path, mode="r") observed = root_group.metadata.consolidated_metadata.metadata["structured_dtype"].fill_value # type: ignore[union-attr] assert observed == fill_value @pytest.mark.parametrize( ("stored", "canonical", "values"), [ *((f"{b}b1", "|b1", np.array([True, False, True])) for b in "<>|"), *((f"{b}i1", "|i1", np.array([-128, 0, 127], dtype=np.int8)) for b in "<>|"), *((f"{b}u1", "|u1", np.array([0, 128, 255], dtype=np.uint8)) for b in "<>|"), ( [["a", "u1"]], [["a", "|i1"], ["b", "|u1"]], np.array([(-1, 1), (0, 128), (1, 255)], dtype=[("a", "i1"), ("b", "u1")]), ), ], ids=str, ) def test_open_v2_byte_order_irrelevant_dtype( tmp_path: Path, stored: DTypeName_V2, canonical: DTypeName_V2, values: np.ndarray[Any, Any] ) -> None: """ An array whose data type is written with any byte order character, where the byte order is not relevant, reads the stored data and writes the data type back with the canonical "|". """ metadata = { "zarr_format": 2, "shape": [3], "chunks": [3], "dtype": stored, "compressor": None, "fill_value": None, "order": "C", "filters": None, } metadata_path = tmp_path / ".zarray" metadata_path.write_text(json.dumps(metadata)) (tmp_path / "0").write_bytes(values.tobytes()) array = zarr.open_array(tmp_path, mode="r+") np.testing.assert_array_equal(array[:], values) array.update_attributes({"foo": "bar"}) assert json.loads(metadata_path.read_text())["dtype"] == canonical zarr-developers-zarr-python-a2f2415/tests/test_metadata/test_v3.py000066400000000000000000000450771527014137600254240ustar00rootroot00000000000000"""Tests for zarr v3 metadata classes and parsing helpers.""" from __future__ import annotations import json import re from typing import TYPE_CHECKING import pytest from tests.conftest import Expect, ExpectFail from tests.test_metadata.conftest import minimal_metadata_dict_v3 from zarr.core.buffer import default_buffer_prototype from zarr.core.chunk_grids import ChunkGrid, is_regular_1d, is_regular_nd from zarr.core.config import config from zarr.core.dtype import Float64, UInt8 from zarr.core.group import GroupMetadata, parse_node_type from zarr.core.metadata.v2 import ArrayV2Metadata from zarr.core.metadata.v3 import ( ARRAY_METADATA_KEYS, ArrayMetadataJSON_V3, ArrayV3Metadata, RegularChunkGridMetadata, create_chunk_grid_metadata, parse_codecs, parse_dimension_names, parse_node_type_array, parse_zarr_format, ) from zarr.errors import ( MetadataValidationError, NodeTypeValidationError, UnknownCodecError, ) if TYPE_CHECKING: from typing import Any # --------------------------------------------------------------------------- # Parsing helpers # --------------------------------------------------------------------------- def test_parse_zarr_format_valid() -> None: """The integer 3 is the only valid zarr_format for v3.""" assert parse_zarr_format(3) == 3 @pytest.mark.parametrize("data", [None, 1, 2, 4, 5, "3"]) def test_parse_zarr_format_invalid(data: Any) -> None: """Non-3 values are rejected.""" with pytest.raises(MetadataValidationError): parse_zarr_format(data) def test_parse_node_type_valid() -> None: """'array' and 'group' are the only valid node types.""" assert parse_node_type("array") == "array" assert parse_node_type("group") == "group" @pytest.mark.parametrize("data", [None, 2, "other"]) def test_parse_node_type_invalid(data: Any) -> None: """Non-string and unrecognized values are rejected.""" with pytest.raises(MetadataValidationError): parse_node_type(data) def test_parse_node_type_array_valid() -> None: """parse_node_type_array accepts only 'array'.""" assert parse_node_type_array("array") == "array" @pytest.mark.parametrize("data", [None, "group"]) def test_parse_node_type_array_invalid(data: Any) -> None: """parse_node_type_array rejects 'group' and non-string values.""" with pytest.raises(NodeTypeValidationError): parse_node_type_array(data) @pytest.mark.parametrize("data", [None, ("a", "b", "c"), ["a", "a", "a"], ()]) def test_parse_dimension_names_valid(data: Any) -> None: """None, tuples of strings, lists of strings, and empty tuples are accepted.""" result = parse_dimension_names(data) if data is None: assert result is None else: assert result == tuple(data) @pytest.mark.parametrize("data", [[1, 2, "a"], [None, 3]]) def test_parse_dimension_names_invalid(data: Any) -> None: """Iterables containing non-string elements are rejected.""" with pytest.raises(TypeError, match="Expected either None or"): parse_dimension_names(data) def test_parse_codecs_unknown_raises(monkeypatch: pytest.MonkeyPatch) -> None: """An unregistered codec name raises UnknownCodecError.""" from collections import defaultdict import zarr.registry from zarr.registry import Registry monkeypatch.setattr(zarr.registry, "_codec_registries", defaultdict(Registry)) with pytest.raises(UnknownCodecError): parse_codecs([{"name": "unknown"}]) # --------------------------------------------------------------------------- # Chunk-grid regularity helpers # --------------------------------------------------------------------------- _REGULAR_1D_CASES: list[Expect[list[int], bool]] = [ Expect(input=[], output=True, id="empty"), Expect(input=[10], output=True, id="single-chunk"), Expect(input=[10, 10, 10], output=True, id="all-equal"), Expect(input=[10, 10, 10, 7], output=True, id="smaller-boundary"), Expect(input=[10, 10, 10, 10], output=True, id="exact-multiple"), Expect(input=[10, 5, 10], output=False, id="middle-mismatch"), Expect(input=[10, 10, 10, 12], output=False, id="last-larger"), # The first chunk anchors the size; later mismatches in the middle fail # before the boundary check. Expect(input=[5, 10, 5], output=False, id="middle-larger"), ] @pytest.mark.parametrize("case", _REGULAR_1D_CASES, ids=lambda c: c.id) def test_is_regular_1d_sequence(case: Expect[list[int], bool]) -> None: """`is_regular_1d` accepts plain Python sequences.""" assert is_regular_1d(case.input) is case.output assert is_regular_1d(tuple(case.input)) is case.output @pytest.mark.parametrize( "case", [ Expect(input=[[10, 10, 10], [5, 5]], output=True, id="all-regular"), Expect(input=[[10, 10, 7], [5, 5, 5, 3]], output=True, id="all-regular-with-boundary"), Expect(input=[[10, 10, 10], [5, 8, 5]], output=False, id="second-dim-irregular"), Expect(input=[[10, 5, 10], [5, 5]], output=False, id="first-dim-irregular"), Expect(input=[], output=True, id="zero-dims"), ], ids=lambda c: c.id, ) def test_is_regular_nd_sequence(case: Expect[list[list[int]], bool]) -> None: """`is_regular_nd` returns True iff every per-dim spec is regular.""" assert is_regular_nd(case.input) is case.output def test_create_chunk_grid_metadata_unknown_dimension_type() -> None: """`create_chunk_grid_metadata` rejects dimension grids it does not recognize.""" grid = ChunkGrid(dimensions=(object(),)) # type: ignore[arg-type] with pytest.raises(TypeError, match="Unknown dimension grid type"): create_chunk_grid_metadata(grid) def test_regular_chunk_grid_rejects_edge_lists() -> None: """A regular chunk grid only accepts integer chunk edge lengths.""" with pytest.raises( TypeError, match=re.escape("Dimension 1: chunk edge length must be an int, got (5, 10, 5)"), ): RegularChunkGridMetadata(chunk_shape=(2, (5, 10, 5))) # type: ignore[arg-type] # --------------------------------------------------------------------------- # Types # --------------------------------------------------------------------------- def test_array_metadata_keys_matches_typeddict() -> None: """ Test that the variable modelling the set of keys for array v3 metadata matches the keys of the typeddict model for the metadata. """ assert ARRAY_METADATA_KEYS == set(ArrayMetadataJSON_V3.__annotations__.keys()) # --------------------------------------------------------------------------- # ArrayV3Metadata: round-trip # --------------------------------------------------------------------------- # Codecs after evolution for single-byte (uint8) and multi-byte (float64) types. _UINT8_CODECS = ({"name": "bytes"},) _FLOAT64_CODECS = ({"name": "bytes", "configuration": {"endian": "little"}},) @pytest.mark.parametrize( "case", [ Expect( input={}, output=minimal_metadata_dict_v3(codecs=_UINT8_CODECS), id="minimal", ), Expect( input={"attributes": {"key": "value"}}, output=minimal_metadata_dict_v3(attributes={"key": "value"}, codecs=_UINT8_CODECS), id="with_attributes", ), Expect( input={"dimension_names": ("x", "y")}, output=minimal_metadata_dict_v3(dimension_names=("x", "y"), codecs=_UINT8_CODECS), id="with_dimension_names", ), Expect( input={"storage_transformers": ()}, output=minimal_metadata_dict_v3(storage_transformers=(), codecs=_UINT8_CODECS), id="with_storage_transformers", ), Expect( input={"data_type": "float64", "fill_value": 0.0}, output=minimal_metadata_dict_v3( data_type="float64", fill_value=0.0, codecs=_FLOAT64_CODECS ), id="float64", ), Expect( input={"chunk_key_encoding": {"name": "v2", "configuration": {"separator": "."}}}, output=minimal_metadata_dict_v3( chunk_key_encoding={"name": "v2", "configuration": {"separator": "."}}, codecs=_UINT8_CODECS, ), id="v2_chunk_key_encoding", ), Expect( input={"data_type": "float64", "fill_value": "NaN"}, output=minimal_metadata_dict_v3( data_type="float64", fill_value="NaN", codecs=_FLOAT64_CODECS ), id="nan_fill_value", ), Expect( input={"data_type": "float64", "fill_value": "Infinity"}, output=minimal_metadata_dict_v3( data_type="float64", fill_value="Infinity", codecs=_FLOAT64_CODECS ), id="inf_fill_value", ), Expect( input={"data_type": "float64", "fill_value": "-Infinity"}, output=minimal_metadata_dict_v3( data_type="float64", fill_value="-Infinity", codecs=_FLOAT64_CODECS ), id="neg_inf_fill_value", ), Expect( input={ "attributes": {}, "storage_transformers": (), "extra_fields": {"my_ext": {"must_understand": False, "data": [1, 2, 3]}}, }, output=minimal_metadata_dict_v3( attributes={}, storage_transformers=(), codecs=_UINT8_CODECS, extra_fields={"my_ext": {"must_understand": False, "data": [1, 2, 3]}}, ), id="extra_fields", ), ], ids=lambda case: case.id, ) def test_array_metadata_roundtrip(case: Expect[dict[str, Any], dict[str, Any]]) -> None: """from_dict(d).to_dict() produces the expected output, including codec evolution.""" d = minimal_metadata_dict_v3(**case.input) m = ArrayV3Metadata.from_dict(d) # type: ignore[arg-type] assert m.to_dict() == case.output # --------------------------------------------------------------------------- # ArrayV3Metadata: failure modes # --------------------------------------------------------------------------- @pytest.mark.parametrize( "case", [ ExpectFail( input={"dimension_names": ("x", "y", "z")}, exception=ValueError, msg="dimension_names.*shape", id="dimension_names_length_mismatch", ), ExpectFail( input={"data_type": "uint8", "fill_value": {}}, exception=TypeError, id="invalid_fill_value_type", ), ], ids=lambda case: case.id, ) def test_array_metadata_from_dict_fails(case: ExpectFail[dict[str, Any]]) -> None: """from_dict rejects invalid metadata documents.""" d = minimal_metadata_dict_v3(**case.input) with case.raises(): ArrayV3Metadata.from_dict(d) # type: ignore[arg-type] @pytest.mark.parametrize( "case", [ ExpectFail( input=minimal_metadata_dict_v3(extra_fields={"my_ext": {"must_understand": True}}), exception=MetadataValidationError, msg="disallowed extra fields", id="must_understand_true", ), ExpectFail( input=minimal_metadata_dict_v3(extra_fields={"my_ext": 42}), exception=MetadataValidationError, msg="disallowed extra fields", id="non_dict_extra_field", ), ], ids=lambda case: case.id, ) def test_array_metadata_extra_fields_rejected(case: ExpectFail[dict[str, Any]]) -> None: """from_dict rejects extra fields that don't conform to the spec.""" with case.raises(): ArrayV3Metadata.from_dict(case.input) @pytest.mark.parametrize("value", [{"must_understand": True}, {}, 42]) def test_init_extra_fields_disallowed(value: object) -> None: """Extra field values that are not objects with `must_understand: false` are rejected.""" with pytest.raises(MetadataValidationError, match="disallowed extra fields"): ArrayV3Metadata( shape=(10,), data_type=UInt8(), chunk_grid={"name": "regular", "configuration": {"chunk_shape": (10,)}}, chunk_key_encoding={"name": "default", "configuration": {"separator": "/"}}, fill_value=0, codecs=({"name": "bytes", "configuration": {"endian": "little"}},), attributes={}, dimension_names=None, extra_fields={"my_ext": value}, # type: ignore[dict-item] ) def test_init_extra_fields_collision() -> None: """Extra field keys that collide with reserved metadata field names are rejected.""" extra_fields: dict[str, object] = {"shape": (10,), "data_type": "uint8"} with pytest.raises(ValueError, match="collide with keys reserved"): ArrayV3Metadata( shape=(10,), data_type=UInt8(), chunk_grid={"name": "regular", "configuration": {"chunk_shape": (10,)}}, chunk_key_encoding={"name": "default", "configuration": {"separator": "/"}}, fill_value=0, codecs=({"name": "bytes", "configuration": {"endian": "little"}},), attributes={}, dimension_names=None, extra_fields=extra_fields, # type: ignore[arg-type] ) # --------------------------------------------------------------------------- # Equality # --------------------------------------------------------------------------- def test_eq_nan_fill_value() -> None: """Two metadata objects with an identical NaN fill_value compare equal. NaN is not equal to itself under IEEE 754, so the default dataclass __eq__ reports two otherwise-identical metadata objects as unequal. Metadata equality must treat matching NaN fill values as equal (see issue #2929). """ a = ArrayV3Metadata.from_dict(minimal_metadata_dict_v3(data_type="float64", fill_value="NaN")) # type: ignore[arg-type] b = ArrayV3Metadata.from_dict(minimal_metadata_dict_v3(data_type="float64", fill_value="NaN")) # type: ignore[arg-type] assert a == b def test_eq_distinct_fill_value() -> None: """Metadata objects that differ only in fill_value do not compare equal.""" a = ArrayV3Metadata.from_dict(minimal_metadata_dict_v3(data_type="float64", fill_value=0.0)) # type: ignore[arg-type] b = ArrayV3Metadata.from_dict(minimal_metadata_dict_v3(data_type="float64", fill_value=1.0)) # type: ignore[arg-type] assert a != b @pytest.mark.parametrize("fill_value", ["Infinity", "-Infinity"]) def test_eq_inf_fill_value(fill_value: str) -> None: """Two metadata objects with an identical infinite fill_value compare equal.""" a = ArrayV3Metadata.from_dict( minimal_metadata_dict_v3(data_type="float64", fill_value=fill_value) # type: ignore[arg-type] ) b = ArrayV3Metadata.from_dict( minimal_metadata_dict_v3(data_type="float64", fill_value=fill_value) # type: ignore[arg-type] ) assert a == b def test_hash_consistent_with_eq_nan_fill_value() -> None: """Equal metadata objects with a NaN fill_value hash equal. NaN hashes by identity, so a field-based hash would break the ``a == b implies hash(a) == hash(b)`` invariant for objects that compare equal under the to_dict-based __eq__. """ a = ArrayV3Metadata.from_dict(minimal_metadata_dict_v3(data_type="float64", fill_value="NaN")) # type: ignore[arg-type] b = ArrayV3Metadata.from_dict(minimal_metadata_dict_v3(data_type="float64", fill_value="NaN")) # type: ignore[arg-type] assert a == b assert hash(a) == hash(b) def test_eq_non_metadata() -> None: """Comparison against a non-metadata object returns False rather than erroring.""" a = ArrayV3Metadata.from_dict(minimal_metadata_dict_v3(data_type="float64", fill_value=0.0)) # type: ignore[arg-type] assert a != object() def test_eq_across_zarr_formats() -> None: """A v2 and v3 metadata describing the same array do not compare equal. Each __eq__ guards on its own concrete type and returns NotImplemented otherwise, so the two versions are never equal even when they describe the same array. """ v3 = ArrayV3Metadata.from_dict(minimal_metadata_dict_v3(data_type="float64", fill_value=0.0)) # type: ignore[arg-type] v2 = ArrayV2Metadata(shape=(4, 4), dtype=Float64(), chunks=(4, 4), fill_value=0.0, order="C") assert v2 != v3 assert v3 != v2 # --------------------------------------------------------------------------- # JSON indent # --------------------------------------------------------------------------- @pytest.mark.parametrize("indent", [2, 4, None]) def test_json_indent(indent: int | None) -> None: """The json_indent config setting controls indentation in to_buffer_dict output.""" with config.set({"json_indent": indent}): m = GroupMetadata() d = m.to_buffer_dict(default_buffer_prototype())["zarr.json"].to_bytes() assert d == json.dumps(json.loads(d), indent=indent).encode() # --------------------------------------------------------------------------- # GroupMetadata.to_dict # --------------------------------------------------------------------------- @pytest.mark.parametrize("attributes", [None, {"foo": "bar"}]) def test_group_metadata_to_dict(attributes: dict[str, Any] | None) -> None: """GroupMetadata.to_dict produces the expected v3 JSON structure.""" meta = GroupMetadata(attributes=attributes) assert meta.to_dict() == { "zarr_format": 3, "node_type": "group", "attributes": attributes or {}, } @pytest.mark.parametrize("attributes", [None, {"foo": "bar"}]) def test_group_metadata_to_dict_consolidated(attributes: dict[str, Any] | None) -> None: """GroupMetadata.to_dict includes consolidated_metadata when present.""" from zarr import consolidate_metadata, create_group from zarr.errors import ZarrUserWarning store: dict[str, object] = {} group = create_group(store, attributes=attributes, zarr_format=3) group.create_group("foo") with pytest.warns( ZarrUserWarning, match="Consolidated metadata is currently not part in the Zarr format 3 specification.", ): group = consolidate_metadata(store) assert group.metadata.to_dict() == { "zarr_format": 3, "node_type": "group", "attributes": attributes or {}, "consolidated_metadata": { "kind": "inline", "must_understand": False, "metadata": { "foo": { "attributes": {}, "zarr_format": 3, "node_type": "group", "consolidated_metadata": { "kind": "inline", "metadata": {}, "must_understand": False, }, } }, }, } zarr-developers-zarr-python-a2f2415/tests/test_pipeline_parity.py000066400000000000000000000512731527014137600254450ustar00rootroot00000000000000"""Pipeline parity test — exhaustive matrix of read/write scenarios. For every cell of the matrix (codec config x layout x operation sequence x runtime config), assert that ``FusedCodecPipeline`` and ``BatchedCodecPipeline`` produce semantically identical results: * Same returned array contents on read. * Same set of store keys after writes (catches divergent empty-shard handling: one pipeline deletes, the other writes an empty blob). * Reading each pipeline's store contents through the *other* pipeline yields the same array (catches "wrote a layout that only one pipeline can read" bugs). Pipeline-divergence bugs (e.g. one pipeline writes a dense shard layout while the other writes a compact layout) fail this test loudly with a clear diff, instead of waiting for a downstream test to trip over the symptom. Byte-for-byte equality of store contents is intentionally NOT checked: codecs like gzip embed the wall-clock timestamp in their output, so two compressions of the same data done at different seconds produce different bytes despite being semantically identical. The matrix axes are: * codec chain — bytes-only, gzip, with/without sharding * layout — chunk_shape, shard_shape (None for no sharding) * write sequence — full overwrite, partial in middle, scalar to one cell, multiple overlapping writes, sequence ending in fill values * runtime config — write_empty_chunks True/False """ from __future__ import annotations import warnings from contextlib import contextmanager from typing import TYPE_CHECKING, Any import numpy as np import pytest import zarr from zarr.codecs.bytes import BytesCodec from zarr.codecs.crc32c_ import Crc32cCodec from zarr.codecs.gzip import GzipCodec from zarr.codecs.sharding import ( SUBCHUNK_WRITE_ORDER, IndexLocation, ShardingCodec, SubchunkWriteOrder, ) from zarr.codecs.transpose import TransposeCodec from zarr.core.config import config as zarr_config from zarr.errors import ZarrUserWarning from zarr.storage import MemoryStore if TYPE_CHECKING: from collections.abc import Callable, Iterator # --------------------------------------------------------------------------- # Reference helpers # --------------------------------------------------------------------------- def _store_snapshot(store: MemoryStore) -> dict[str, bytes]: """Return {key: bytes} for every entry in the store.""" return {k: bytes(v.to_bytes()) for k, v in store._store_dict.items()} # --------------------------------------------------------------------------- # Matrix definitions # --------------------------------------------------------------------------- # Each codec config is (filters, serializer, compressors). We only vary the # pieces that actually affect the pipeline. compressors=None means a # fixed-size chain (the byte-range fast path is eligible when sharded). CodecConfig = dict[str, Any] CODEC_CONFIGS: list[tuple[str, CodecConfig]] = [ ("bytes-only", {"compressors": None}), ("gzip", {"compressors": GzipCodec(level=1)}), # Explicit serializer byte order: the on-disk byte order is carried by the # BytesCodec, not the dtype. Guards the bulk whole-shard decode against ignoring # endian (it would otherwise reinterpret stored bytes as native — silent # corruption). dtype is int32 so endianness is observable. Both orders are # listed so that one of them is non-native on any host. ( "bytes-big-endian", {"compressors": None, "serializer": BytesCodec(endian="big"), "dtype": "int32"}, ), ( "bytes-little-endian", {"compressors": None, "serializer": BytesCodec(endian="little"), "dtype": "int32"}, ), # crc32c as a bytes->bytes codec after the serializer: the bulk fast path # must NOT silently drop checksum verification (it falls through to the # per-chunk path). Parity still requires identical bytes + contents across # pipelines. (crc32c is a BytesBytesCodec, so it goes in `compressors`.) ( "bytes-crc32c", {"compressors": [Crc32cCodec()], "serializer": BytesCodec(), "dtype": "int32"}, ), ] # (id, kwargs) — chunks/shards layout. kwargs are passed to create_array. LayoutConfig = dict[str, Any] LAYOUT_CONFIGS: list[tuple[str, LayoutConfig]] = [ ("1d-unsharded", {"shape": (100,), "chunks": (10,), "shards": None}), ("1d-1chunk-per-shard", {"shape": (100,), "chunks": (10,), "shards": (10,)}), ("1d-multi-chunk-per-shard", {"shape": (100,), "chunks": (10,), "shards": (50,)}), ("2d-unsharded", {"shape": (20, 20), "chunks": (5, 5), "shards": None}), ("2d-sharded", {"shape": (20, 20), "chunks": (5, 5), "shards": (10, 10)}), # Nested sharding: outer chunk (10,10) sharded into inner chunks (5,5). # Restricted to the codec configs that don't set their own `serializer` # (bytes-only, gzip): this layout supplies an explicit nested-ShardingCodec # `serializer`, and a codec config that also sets `serializer` (e.g. # bytes-big-endian) would silently clobber it via dict merge, dropping # sharding from the test entirely rather than exercising it. The gzip # config applies as an outer bytes-bytes codec around the outer # ShardingCodec -- this is the regression coverage for the fused pipeline # applying outer AA/BB codecs around sharding (see # `pipeline_supports_partial_decode`/`pipeline_supports_partial_encode`). ( "2d-nested-sharded", { "shape": (20, 20), "chunks": (10, 10), "shards": None, "serializer": ShardingCodec( chunk_shape=(10, 10), codecs=[ShardingCodec(chunk_shape=(5, 5))], ), "_codec_ids": {"bytes-only", "gzip"}, }, ), ] WriteOp = tuple[Any, Any] # (selection, value) WriteSequence = tuple[str, list[WriteOp]] def _full_overwrite(shape: tuple[int, ...]) -> list[WriteOp]: return [((slice(None),) * len(shape), np.arange(int(np.prod(shape))).reshape(shape) + 1)] def _partial_middle(shape: tuple[int, ...]) -> list[WriteOp]: if len(shape) == 1: n = shape[0] return [((slice(n // 4, 3 * n // 4),), 7)] # 2D: write a centered block rs = slice(shape[0] // 4, 3 * shape[0] // 4) cs = slice(shape[1] // 4, 3 * shape[1] // 4) return [((rs, cs), 7)] def _scalar_one_cell(shape: tuple[int, ...]) -> list[WriteOp]: if len(shape) == 1: return [((shape[0] // 2,), 99)] return [((shape[0] // 2, shape[1] // 2), 99)] def _overlapping(shape: tuple[int, ...]) -> list[WriteOp]: if len(shape) == 1: n = shape[0] return [ ((slice(0, n // 2),), 1), ((slice(n // 4, 3 * n // 4),), 2), ((slice(n // 2, n),), 3), ] rs1, cs1 = slice(0, shape[0] // 2), slice(0, shape[1] // 2) rs2, cs2 = slice(shape[0] // 4, 3 * shape[0] // 4), slice(shape[1] // 4, 3 * shape[1] // 4) return [((rs1, cs1), 1), ((rs2, cs2), 2)] def _ends_in_fill(shape: tuple[int, ...]) -> list[WriteOp]: """Write something then overwrite it with fill — exercises empty-chunk handling.""" full = (slice(None),) * len(shape) return [(full, 5), (full, 0)] def _ends_in_partial_fill(shape: tuple[int, ...]) -> list[WriteOp]: """Write data, then overwrite half with fill — some chunks become empty.""" full: tuple[slice, ...] half: tuple[slice, ...] if len(shape) == 1: full = (slice(None),) half = (slice(0, shape[0] // 2),) else: full = (slice(None), slice(None)) half = (slice(0, shape[0] // 2), slice(None)) return [(full, 5), (half, 0)] SEQUENCES: list[tuple[str, Callable[[tuple[int, ...]], list[WriteOp]]]] = [ ("full-overwrite", _full_overwrite), ("partial-middle", _partial_middle), ("scalar-one-cell", _scalar_one_cell), ("overlapping", _overlapping), ("ends-in-fill", _ends_in_fill), ("ends-in-partial-fill", _ends_in_partial_fill), ] WRITE_EMPTY_CHUNKS = [False, True] # --------------------------------------------------------------------------- # Matrix iteration (pruned) # --------------------------------------------------------------------------- def _matrix() -> Iterator[Any]: for codec_id, codec_kwargs in CODEC_CONFIGS: for layout_id, layout in LAYOUT_CONFIGS: allowed = layout.get("_codec_ids") if allowed is not None and codec_id not in allowed: continue for seq_id, seq_fn in SEQUENCES: for wec in WRITE_EMPTY_CHUNKS: yield pytest.param( codec_kwargs, layout, seq_fn, wec, id=f"{layout_id}-{codec_id}-{seq_id}-wec{wec}", ) # --------------------------------------------------------------------------- # The parity test # --------------------------------------------------------------------------- @contextmanager def _ignore_sharding_combo_warning() -> Iterator[None]: """Suppress the "combining sharding_indexed disables partial reads" warning. Only the nested-sharded-plus-outer-codec matrix cell emits this; scoping the ignore filter to just its message/category (rather than blanket-disabling warnings) keeps every other warning in the run promoted to an error as usual. """ with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=r"Combining a `sharding_indexed` codec.*", category=ZarrUserWarning, ) yield def _write_under_pipeline( pipeline_path: str, codec_kwargs: CodecConfig, layout: LayoutConfig, sequence: list[WriteOp], write_empty_chunks: bool, ) -> tuple[MemoryStore, Any]: """Apply a sequence of writes via the chosen pipeline. Returns (store with the written data, final array contents read back). """ # Strip private metadata keys (e.g. "_codec_ids") before passing to create_array. array_layout = {k: v for k, v in layout.items() if not k.startswith("_")} # dtype defaults to float64 but a codec config may override it (e.g. an # endian-sensitive int dtype). Merge so the override wins without a dup kwarg. create_kwargs = {"dtype": "float64", **array_layout, **codec_kwargs} store = MemoryStore() with zarr_config.set({"codec_pipeline.path": pipeline_path}): with _ignore_sharding_combo_warning(): arr = zarr.create_array( store=store, fill_value=0, config={"write_empty_chunks": write_empty_chunks}, **create_kwargs, ) for sel, val in sequence: arr[sel] = val contents = arr[...] return store, contents def _read_under_pipeline(pipeline_path: str, store: MemoryStore) -> Any: """Re-open an existing store under the chosen pipeline and read it whole.""" with zarr_config.set({"codec_pipeline.path": pipeline_path}): with _ignore_sharding_combo_warning(): arr = zarr.open_array(store=store, mode="r") return arr[...] _BATCHED = "zarr.core.codec_pipeline.BatchedCodecPipeline" _FUSED = "zarr.core.codec_pipeline.FusedCodecPipeline" @pytest.mark.parametrize( ("codec_kwargs", "layout", "sequence_fn", "write_empty_chunks"), list(_matrix()), ) def test_pipeline_parity( codec_kwargs: CodecConfig, layout: LayoutConfig, sequence_fn: Callable[[tuple[int, ...]], list[WriteOp]], write_empty_chunks: bool, ) -> None: """FusedCodecPipeline must be semantically identical to BatchedCodecPipeline. Three checks, in order of decreasing diagnostic value: 1. Both pipelines return the same array contents after the same write sequence (catches semantic correctness bugs). 2. Both pipelines produce the same set of store keys (catches empty-shard divergence: one deletes, the other doesn't). 3. Each pipeline can correctly read the *other* pipeline's output (catches layout-divergence bugs that would prevent interop, e.g. dense vs compact shard layouts). Byte-for-byte store equality is intentionally not checked: codecs like gzip embed wall-clock timestamps that vary between runs. """ sequence = sequence_fn(layout["shape"]) batched_store, batched_arr = _write_under_pipeline( _BATCHED, codec_kwargs, layout, sequence, write_empty_chunks ) sync_store, sync_arr = _write_under_pipeline( _FUSED, codec_kwargs, layout, sequence, write_empty_chunks ) # 1. Array contents must agree. np.testing.assert_array_equal( sync_arr, batched_arr, err_msg="FusedCodecPipeline returned different array contents than BatchedCodecPipeline", ) # 2. Store key sets must agree. batched_keys = set(batched_store._store_dict) - {"zarr.json"} sync_keys = set(sync_store._store_dict) - {"zarr.json"} assert sync_keys == batched_keys, ( f"Pipelines disagree on which store keys exist.\n" f" only in batched: {sorted(batched_keys - sync_keys)}\n" f" only in sync: {sorted(sync_keys - batched_keys)}" ) # 3. Cross-read: each pipeline must correctly read the other's output. sync_reads_batched = _read_under_pipeline(_FUSED, batched_store) batched_reads_sync = _read_under_pipeline(_BATCHED, sync_store) np.testing.assert_array_equal( sync_reads_batched, batched_arr, err_msg="FusedCodecPipeline could not correctly read BatchedCodecPipeline's output", ) np.testing.assert_array_equal( batched_reads_sync, sync_arr, err_msg="BatchedCodecPipeline could not correctly read FusedCodecPipeline's output", ) # --------------------------------------------------------------------------- # Partial-read parity across subchunk write orders # --------------------------------------------------------------------------- # # Note: general partial-read coverage (scalar single-element and strided reads # from sharded arrays, which hit the sharding codec's partial-decode path) lives # in tests/test_codec_pipeline_suite.py as Scenarios. Those run each pipeline # against a numpy reference -- strictly stronger than checking the two pipelines # only against each other, and they cover both the sync (_decode_partial_sync) # and async (_decode_partial_single) partial-decode variants. What remains here # is the cross-pipeline byte-identical-layout check, which the per-pipeline # suite structurally cannot express. @pytest.mark.parametrize("subchunk_write_order", SUBCHUNK_WRITE_ORDER) @pytest.mark.parametrize("index_location", ["start", "end"]) def test_pipeline_parity_subchunk_write_order( subchunk_write_order: SubchunkWriteOrder, index_location: IndexLocation ) -> None: """Both pipelines must agree across every subchunk_write_order, including a PARTIAL write into an already-dense fixed-size shard. This is the regression net for the byte-range write fast path, which derives each chunk's physical slot from its rank in subchunk_write_order. A wrong (e.g. hardcoded morton) assumption corrupts non-default orders silently, so we assert both identical contents AND identical stored bytes across pipelines. write_empty_chunks=True keeps every slot present, making the shard dense and the byte-range write path eligible. """ # 2D, fixed-size (no compression). The shard (array `chunks`) must hold # MULTIPLE inner chunks, and be non-square, so morton / lexicographic / # colexicographic produce physically DIFFERENT layouts — with one inner # chunk per shard all orders coincide and a wrong-order bug is invisible. # inner chunk = (2, 2); shard = (6, 4) -> a 3x2 grid of inner chunks. shape, shard_shape, inner_chunk = (12, 8), (6, 4), (2, 2) serializer = ShardingCodec( chunk_shape=inner_chunk, codecs=[BytesCodec()], index_location=index_location, subchunk_write_order=subchunk_write_order, ) ref = np.arange(int(np.prod(shape)), dtype="int32").reshape(shape) def run(pipeline_path: str) -> tuple[dict[str, bytes], Any]: store = MemoryStore() with zarr_config.set({"codec_pipeline.path": pipeline_path}): arr = zarr.create_array( store=store, shape=shape, chunks=shard_shape, # array "chunks" == shard size for a ShardingCodec serializer dtype="int32", fill_value=-1, serializer=serializer, compressors=None, config={"write_empty_chunks": True}, ) arr[:] = ref # dense full write arr[3:9, 1:6] = 777 # partial write INTO the dense shard contents = arr[...] return _store_snapshot(store), contents batched_bytes, batched_contents = run(_BATCHED) sync_bytes, sync_contents = run(_FUSED) # Contents must always match across pipelines and equal the reference — # this catches a wrong-order byte-range write (it corrupts the data). expected = ref.copy() expected[3:9, 1:6] = 777 np.testing.assert_array_equal(batched_contents, expected) np.testing.assert_array_equal( sync_contents, batched_contents, err_msg=f"pipeline contents diverged for subchunk_write_order={subchunk_write_order!r}", ) # The two pipelines must also produce byte-identical shards — a stronger # check that they agree on physical layout. This holds for EVERY order # (without special-casing any by name): both pipelines lay chunks out via # the same `_subchunk_order_iter`, so for a given codec instance they must # land on the same bytes whatever that order resolves to. We make no # assumption here about what any particular order "means" — only that the # two implementations agree. assert sync_bytes == batched_bytes, ( f"pipelines wrote different bytes for subchunk_write_order={subchunk_write_order!r} " f"(index_location={index_location!r}) — byte-range write fast path likely assumed " f"the wrong physical chunk order" ) # --------------------------------------------------------------------------- # Outer array-array / bytes-bytes codecs around a sharding serializer # --------------------------------------------------------------------------- # # Regression coverage for FusedCodecPipeline.supports_partial_decode/encode: # it used to allow AA/BB codecs outside the sharding codec, so its partial # branches called ShardingCodec._decode_partial_sync/_encode_partial_sync # directly on the raw stored value, skipping any outer filter/compressor. # That corrupted on-disk bytes for an outer bytes-bytes codec (unreadable by # the other pipeline) and silently produced wrong data for an outer # array-array codec. Both configs below force the partial branches: a # region write and a region read are included alongside the full ones. _OUTER_AA_BB_CONFIGS: list[tuple[str, CodecConfig]] = [ ( "outer-gzip-around-sharding", { "serializer": ShardingCodec(chunk_shape=(2, 2)), "compressors": [GzipCodec(level=1)], }, ), ( "outer-transpose-around-sharding", { "filters": [TransposeCodec(order=(1, 0))], "serializer": ShardingCodec(chunk_shape=(2, 2)), "compressors": None, }, ), ] @pytest.mark.parametrize(("config_id", "codec_kwargs"), _OUTER_AA_BB_CONFIGS) @pytest.mark.parametrize( ("writer", "reader"), [(_BATCHED, _FUSED), (_FUSED, _BATCHED)], ids=["batched-write-fused-read", "fused-write-batched-read"], ) def test_pipeline_parity_outer_aa_bb_codecs( config_id: str, codec_kwargs: CodecConfig, writer: str, reader: str, ) -> None: """Data written under one pipeline with outer AA/BB codecs must read back correctly under the other, including through a partial write and a partial read. """ shape = (8, 8) data = (np.arange(int(np.prod(shape))).reshape(shape) + 1).astype("uint16") store = MemoryStore() with zarr_config.set({"codec_pipeline.path": writer}): with _ignore_sharding_combo_warning(): arr = zarr.create_array( store=store, shape=shape, chunks=(4, 4), dtype=data.dtype, fill_value=0, **codec_kwargs, ) arr[...] = data arr[2:5, 1:3] = 99 # region write -- exercises the partial-encode branch expected = data.copy() expected[2:5, 1:3] = 99 with zarr_config.set({"codec_pipeline.path": reader}): with _ignore_sharding_combo_warning(): arr2 = zarr.open_array(store=store, mode="r") full = arr2[...] partial = arr2[1:3, 2:7] # region read -- exercises the partial-decode branch np.testing.assert_array_equal(full, expected) np.testing.assert_array_equal(partial, expected[1:3, 2:7]) zarr-developers-zarr-python-a2f2415/tests/test_properties.py000066400000000000000000000731001527014137600244350ustar00rootroot00000000000000import itertools import json import numbers import warnings from collections.abc import Generator from typing import Any import numpy as np import pytest from numpy.lib.recfunctions import repack_fields from numpy.testing import assert_array_equal import zarr from zarr.core.buffer import default_buffer_prototype pytest.importorskip("hypothesis") import hypothesis.extra.numpy as npst import hypothesis.strategies as st from hypothesis import assume, event, given, settings from tests.conftest import declared_chunk_data_sizes from zarr.abc.store import Store from zarr.core.common import ZARR_JSON, ZARRAY_JSON, ZATTRS_JSON from zarr.core.dtype import get_data_type_from_json, get_data_type_from_native_dtype from zarr.core.dtype.common import HasItemSize from zarr.core.dtype.npy.structured import Struct from zarr.core.dtype.wrapper import ZDType from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.core.metadata.v3 import RectilinearChunkGridMetadata, RectilinearChunkGridMetadataJSON from zarr.core.sync import sync from zarr.errors import ZarrUserWarning from zarr.storage import MemoryStore from zarr.testing.strategies import ( _rectilinear_chunks, array_metadata, arrays, basic_indices, block_indices, block_test_arrays, complex_rectilinear_arrays, numpy_arrays, orthogonal_indices, rectilinear_arrays, rectilinear_chunk_shape_declarations, rectilinear_chunks, sharded_arrays, simple_arrays, stores, structured_dtypes, zarr_formats, zdtypes, ) @pytest.fixture(autouse=True) def _enable_rectilinear_chunks() -> Generator[None, None, None]: """Enable rectilinear chunks for all property tests since strategies may generate them.""" with zarr.config.set({"array.rectilinear_chunks": True}): yield def deep_equal(a: Any, b: Any) -> bool: """Deep equality check with handling of special cases for array metadata classes""" if isinstance(a, (complex, np.complexfloating)) and isinstance( b, (complex, np.complexfloating) ): a_real, a_imag = float(a.real), float(a.imag) b_real, b_imag = float(b.real), float(b.imag) if np.isnan(a_real) and np.isnan(b_real): real_eq = True else: real_eq = a_real == b_real if np.isnan(a_imag) and np.isnan(b_imag): imag_eq = True else: imag_eq = a_imag == b_imag return real_eq and imag_eq if isinstance(a, (float, np.floating)) and isinstance(b, (float, np.floating)): if np.isnan(a) and np.isnan(b): return True return a == b if isinstance(a, np.datetime64) and isinstance(b, np.datetime64): if np.isnat(a) and np.isnat(b): return True return a == b if isinstance(a, np.ndarray) and isinstance(b, np.ndarray): if a.shape != b.shape: return False return all(itertools.starmap(deep_equal, zip(a.flat, b.flat, strict=False))) if isinstance(a, dict) and isinstance(b, dict): if set(a.keys()) != set(b.keys()): return False return all(deep_equal(a[k], b[k]) for k in a) if isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)): if len(a) != len(b): return False return all(itertools.starmap(deep_equal, zip(a, b, strict=False))) return a == b @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @given(data=st.data()) def test_array_roundtrip(data: st.DataObject) -> None: nparray = data.draw(numpy_arrays()) zarray = data.draw(arrays(arrays=st.just(nparray))) assert_array_equal(nparray, zarray[:]) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @given(array=arrays()) def test_array_creates_implicit_groups(array): path = array.path ancestry = path.split("/")[:-1] for i in range(len(ancestry)): parent = "/".join(ancestry[: i + 1]) if array.metadata.zarr_format == 2: assert ( sync(array.store.get(f"{parent}/.zgroup", prototype=default_buffer_prototype())) is not None ) elif array.metadata.zarr_format == 3: assert ( sync(array.store.get(f"{parent}/zarr.json", prototype=default_buffer_prototype())) is not None ) # this decorator removes timeout; not ideal but it should avoid intermittent CI failures @settings(deadline=None) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @given(data=st.data()) async def test_basic_indexing(data: st.DataObject) -> None: zarray = data.draw(st.one_of(simple_arrays(), rectilinear_arrays())) nparray = zarray[:] indexer = data.draw(basic_indices(shape=nparray.shape)) # sync get actual = zarray[indexer] assert_array_equal(nparray[indexer], actual) # async get async_zarray = zarray._async_array actual = await async_zarray.getitem(indexer) assert_array_equal(nparray[indexer], actual) # sync set new_data = data.draw(numpy_arrays(shapes=st.just(actual.shape), dtype=nparray.dtype)) zarray[indexer] = new_data nparray[indexer] = new_data assert_array_equal(nparray, zarray[:]) # TODO test async setitem? @settings(deadline=None) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @given(data=st.data()) async def test_basic_indexing_complex_rectilinear(data: st.DataObject) -> None: nparray, zarray = data.draw(complex_rectilinear_arrays()) indexer = data.draw(basic_indices(shape=nparray.shape)) assert_array_equal(nparray[indexer], zarray[indexer]) @given(data=st.data()) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") async def test_oindex(data: st.DataObject) -> None: # integer_array_indices can't handle 0-size dimensions. # A sharded array is drawn as its own arm: simple_arrays shards only a few # percent of its draws, and the sharding codec's write path for a selection # with two or more array-indexed axes (GH4284) needs real weight here. That # path only exists for a value with two or more axes, hence min_dims=2. zarray = data.draw( st.one_of( simple_arrays(shapes=npst.array_shapes(max_dims=4, min_side=1)), rectilinear_arrays(shapes=npst.array_shapes(max_dims=4, min_side=1, max_side=20)), sharded_arrays( shapes=npst.array_shapes(min_dims=2, max_dims=4, min_side=1, max_side=8) ), ) ) nparray = zarray[:] zindexer, npindexer = data.draw(orthogonal_indices(shape=nparray.shape)) # sync get actual = zarray.oindex[zindexer] assert_array_equal(nparray[npindexer], actual) # async get async_zarray = zarray._async_array actual = await async_zarray.oindex.getitem(zindexer) assert_array_equal(nparray[npindexer], actual) # sync set for idxr, size in zip(zindexer, nparray.shape, strict=True): if isinstance(idxr, np.ndarray) and idxr.size != np.unique(idxr % size).size: # behaviour of setitem with repeated indices is not guaranteed in practice # Negative and positive spellings of the same index are duplicates too. assume(False) # The sharding codec sees a coordinate selection (the GH4284 path) when the # chunk selection has more than one array axis or drops an integer axis. n_array_axes = sum(isinstance(idxr, np.ndarray) for idxr in zindexer) coordinate_path = n_array_axes > 1 or any(isinstance(idxr, int) for idxr in zindexer) event( f"oindex write: {'sharded' if zarray.shards is not None else 'unsharded'}, " f"{'coordinate' if coordinate_path else 'orthogonal'} chunk selection" ) new_data = data.draw(numpy_arrays(shapes=st.just(actual.shape), dtype=nparray.dtype)) nparray[npindexer] = new_data zarray.oindex[zindexer] = new_data assert_array_equal(nparray, zarray[:]) # note: async oindex setitem not yet implemented @given(data=st.data()) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") async def test_vindex(data: st.DataObject) -> None: # integer_array_indices can't handle 0-size dimensions. zarray = data.draw( st.one_of( simple_arrays(shapes=npst.array_shapes(max_dims=4, min_side=1)), rectilinear_arrays(shapes=npst.array_shapes(max_dims=3, min_side=1, max_side=20)), sharded_arrays(), ) ) nparray = zarray[:] indexer = data.draw( npst.integer_array_indices( shape=nparray.shape, result_shape=npst.array_shapes(min_side=1, max_dims=None) ) ) # sync get actual = zarray.vindex[indexer] assert_array_equal(nparray[indexer], actual) # async get async_zarray = zarray._async_array actual = await async_zarray.vindex.getitem(indexer) assert_array_equal(nparray[indexer], actual) # sync set # Reads preserve the supplied indices; normalize negative indices explicitly when # detecting repeated points rather than relying on a read to mutate the indexer. points = np.stack( [ (idxr % size).ravel() for idxr, size in zip(np.broadcast_arrays(*indexer), nparray.shape, strict=True) ], axis=-1, ) if len(np.unique(points, axis=0)) != len(points): # behaviour of setitem with repeated coordinates is not guaranteed in practice assume(False) new_data = data.draw(numpy_arrays(shapes=st.just(actual.shape), dtype=nparray.dtype)) nparray[indexer] = new_data zarray.vindex[indexer] = new_data assert_array_equal(nparray, zarray[:]) # note: async vindex setitem not yet implemented @settings(deadline=None) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @given(data=st.data()) def test_mask_indexing(data: st.DataObject) -> None: zarray = data.draw(st.one_of(simple_arrays(), rectilinear_arrays())) nparray = zarray[:] mask = data.draw(npst.arrays(dtype=np.bool_, shape=st.just(nparray.shape))) expected = nparray[mask] # sync get, via both the dedicated method and the vindex interface assert_array_equal(expected, zarray.get_mask_selection(mask)) assert_array_equal(expected, zarray.vindex[mask]) # sync set, via both interfaces new_data = data.draw(numpy_arrays(shapes=st.just(expected.shape), dtype=nparray.dtype)) nparray[mask] = new_data zarray.set_mask_selection(mask, new_data) assert_array_equal(nparray, zarray[:]) zarray.vindex[mask] = new_data assert_array_equal(nparray, zarray[:]) @settings(deadline=None) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @given(data=st.data()) def test_block_indexing(data: st.DataObject) -> None: # Block indexing addresses whole inner chunks. block_indices() builds its # array-space oracle from cumulative chunk offsets, so it works for regular # (uniform), rectilinear, and sharded grids alike; block_test_arrays draws # across that matrix (rectilinear + sharded is unsupported and not drawn). zarray, nparray = data.draw(block_test_arrays()) # The block grid is worked out from the stored declaration, not by zarr's grid code. assert isinstance(zarray.metadata, ArrayV3Metadata) grid = zarray.metadata.chunk_grid declared = ( grid.chunk_shapes if isinstance(grid, RectilinearChunkGridMetadata) else grid.chunk_shape ) chunk_sizes = tuple( declared_chunk_data_sizes(d, n) for d, n in zip(declared, zarray.shape, strict=True) ) assert zarray.write_chunk_sizes == chunk_sizes block_indexer, array_indexer = data.draw(block_indices(chunk_sizes=chunk_sizes)) expected = nparray[array_indexer] # sync get, via both the .blocks interface and the dedicated method assert_array_equal(expected, zarray.blocks[block_indexer]) assert_array_equal(expected, zarray.get_block_selection(block_indexer)) # sync set, via both interfaces new_data = data.draw(numpy_arrays(shapes=st.just(expected.shape), dtype=nparray.dtype)) nparray[array_indexer] = new_data zarray.blocks[block_indexer] = new_data assert_array_equal(nparray, zarray[:]) zarray.set_block_selection(block_indexer, new_data) assert_array_equal(nparray, zarray[:]) @given(store=stores, meta=array_metadata()) # type: ignore[misc] @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") async def test_roundtrip_array_metadata_from_store( store: Store, meta: ArrayV2Metadata | ArrayV3Metadata ) -> None: """ Verify that the I/O for metadata in a store are lossless. This test serializes an ArrayV2Metadata or ArrayV3Metadata object to a dict of buffers via `to_buffer_dict`, writes each buffer to a store under keys prefixed with "0/", and then reads them back. The test asserts that each retrieved buffer exactly matches the original buffer. """ asdict = meta.to_buffer_dict(prototype=default_buffer_prototype()) for key, expected in asdict.items(): await store.set(f"0/{key}", expected) actual = await store.get(f"0/{key}", prototype=default_buffer_prototype()) assert actual == expected @given(data=st.data(), zarr_format=zarr_formats) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_roundtrip_array_metadata_from_json(data: st.DataObject, zarr_format: int) -> None: """ Verify that JSON serialization and deserialization of metadata is lossless. For Zarr v2: - The metadata is split into two JSON documents (one for array data and one for attributes). The test merges the attributes back before deserialization. For Zarr v3: - All metadata is stored in a single JSON document. No manual merger is necessary. The test then converts both the original and round-tripped metadata objects into dictionaries using `dataclasses.asdict` and uses a deep equality check to verify that the roundtrip has preserved all fields (including special cases like NaN, Infinity, complex numbers, and datetime values). """ metadata = data.draw(array_metadata(zarr_formats=st.just(zarr_format))) buffer_dict = metadata.to_buffer_dict(prototype=default_buffer_prototype()) if zarr_format == 2: zarray_dict = json.loads(buffer_dict[ZARRAY_JSON].to_bytes().decode()) zattrs_dict = json.loads(buffer_dict[ZATTRS_JSON].to_bytes().decode()) # zattrs and zarray are separate in v2, we have to add attributes back prior to `from_dict` zarray_dict["attributes"] = zattrs_dict metadata_roundtripped = ArrayV2Metadata.from_dict(zarray_dict) else: zarray_dict = json.loads(buffer_dict[ZARR_JSON].to_bytes().decode()) metadata_roundtripped = ArrayV3Metadata.from_dict(zarray_dict) orig = metadata.to_dict() rt = metadata_roundtripped.to_dict() assert deep_equal(orig, rt), f"Roundtrip mismatch:\nOriginal: {orig}\nRoundtripped: {rt}" def _struct_depth(zdtype: ZDType[Any, Any]) -> int: if not isinstance(zdtype, Struct): return 0 return 1 + max(_struct_depth(field_dtype) for _, field_dtype in zdtype.fields) @given(zdtype=zdtypes(), zarr_format=zarr_formats) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_zdtype_json_roundtrip(zdtype: ZDType[Any, Any], zarr_format: int) -> None: """ Generated built-in data types, including nested structs, round-trip through their JSON forms for both Zarr formats within the strategy's documented parameter ranges. Zarr format 3 data type names do not carry endianness (the bytes codec does), so for that format the JSON form is compared instead of the data type instance. """ event(f"dtype={type(zdtype).__name__}") event(f"struct_depth={_struct_depth(zdtype)}") as_json = zdtype.to_json(zarr_format=zarr_format) # type: ignore[arg-type] roundtripped = get_data_type_from_json(as_json, zarr_format=zarr_format) assert roundtripped.to_json(zarr_format=zarr_format) == as_json # type: ignore[arg-type] if zarr_format == 2: assert roundtripped == zdtype @given(zdtype=zdtypes()) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_zdtype_native_roundtrip(zdtype: ZDType[Any, Any]) -> None: """ Generated built-in data types round-trip through their native NumPy dtypes within the strategy's parameter ranges, and reported item sizes match native dtype itemsizes. The NumPy object dtype is shared by several Zarr data types, so resolving it is ambiguous by design and must raise instead. """ event(f"dtype={type(zdtype).__name__}") native = zdtype.to_native_dtype() if native.kind == "O": event("native=object") with pytest.raises(ValueError, match="ambiguous"): get_data_type_from_native_dtype(native) return roundtripped = get_data_type_from_native_dtype(native) assert roundtripped == zdtype assert roundtripped.to_native_dtype() == native if isinstance(zdtype, HasItemSize): assert zdtype.item_size == native.itemsize def _extended_descr_features(descr: list[Any]) -> set[str]: """Classify NumPy's serialized field records independently of Zarr's dtype conversion.""" features = set() for field in descr: if isinstance(field[0], tuple): features.add("title") if len(field) == 3: features.add("subarray") if isinstance(field[1], list): features.update(_extended_descr_features(field[1])) return features @given(dtype=structured_dtypes(allow_extended=True)) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_structured_dtype_never_silently_changes(dtype: np.dtype[np.void]) -> None: """ Generated structured dtypes with titles or subarrays are rejected by the current conversion. Other generated dtypes preserve their fields, warning on layout changes. """ unsupported_features = _extended_descr_features(dtype.descr) if unsupported_features: with pytest.raises(ValueError, match="|".join(sorted(unsupported_features))): get_data_type_from_native_dtype(dtype) event("outcome=rejected") return with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always", ZarrUserWarning) zdtype = get_data_type_from_native_dtype(dtype) event("outcome=accepted") native = zdtype.to_native_dtype() assert native == repack_fields(dtype, recurse=True) layout_warnings = [ w for w in caught if issubclass(w.category, ZarrUserWarning) and "packed" in str(w.message) ] assert bool(layout_warnings) == (native != dtype) # @st.composite # def advanced_indices(draw, *, shape): # basic_idxr = draw( # basic_indices( # shape=shape, min_dims=len(shape), max_dims=len(shape), allow_ellipsis=False # ).filter(lambda x: isinstance(x, tuple)) # ) # int_idxr = draw( # npst.integer_array_indices(shape=shape, result_shape=npst.array_shapes(max_dims=1)) # ) # args = tuple( # st.sampled_from((l, r)) for l, r in zip_longest(basic_idxr, int_idxr, fillvalue=slice(None)) # ) # return draw(st.tuples(*args)) # @given(st.data()) # def test_roundtrip_object_array(data): # nparray = data.draw(np_arrays) # zarray = data.draw(arrays(arrays=st.just(nparray))) # assert_array_equal(nparray, zarray[:]) def serialized_complex_float_is_valid( serialized: tuple[numbers.Real | str, numbers.Real | str], ) -> bool: """ Validate that the serialized representation of a complex float conforms to the spec. The specification requires that a serialized complex float must be either: - A JSON number, or - One of the strings "NaN", "Infinity", or "-Infinity". Args: serialized: The value produced by JSON serialization for a complex floating point number. Returns: bool: True if the serialized value is valid according to the spec, False otherwise. """ return ( isinstance(serialized, tuple) and len(serialized) == 2 and all(serialized_float_is_valid(x) for x in serialized) ) def serialized_float_is_valid(serialized: numbers.Real | str) -> bool: """ Validate that the serialized representation of a float conforms to the spec. The specification requires that a serialized float must be either: - A JSON number, or - One of the strings "NaN", "Infinity", or "-Infinity". Args: serialized: The value produced by JSON serialization for a floating point number. Returns: bool: True if the serialized value is valid according to the spec, False otherwise. """ if isinstance(serialized, numbers.Real): return True return serialized in ("NaN", "Infinity", "-Infinity") @given(meta=array_metadata()) # type: ignore[misc] @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_array_metadata_meets_spec(meta: ArrayV2Metadata | ArrayV3Metadata) -> None: """ Validate that the array metadata produced by the library conforms to the relevant spec (V2 vs V3). For ArrayV2Metadata: - Ensures that 'zarr_format' is 2. - Verifies that 'filters' is either None or a tuple (and not an empty tuple). For ArrayV3Metadata: - Ensures that 'zarr_format' is 3. For both versions: - If the dtype is a floating point of some kind, verifies of fill values: * NaN is serialized as the string "NaN" * Positive Infinity is serialized as the string "Infinity" * Negative Infinity is serialized as the string "-Infinity" * Other fill values are preserved as-is. - If the dtype is a complex number of some kind, verifies that each component of the fill value (real and imaginary) satisfies the serialization rules for floating point numbers. - If the dtype is a datetime of some kind, verifies that `NaT` values are serialized as "NaT". Note: This test validates spec-compliance for array metadata serialization. It is a work-in-progress and should be expanded as further edge cases are identified. """ asdict_dict = meta.to_dict() # version-specific validations if isinstance(meta, ArrayV2Metadata): assert asdict_dict["filters"] != () assert asdict_dict["filters"] is None or isinstance(asdict_dict["filters"], tuple) assert asdict_dict["zarr_format"] == 2 else: assert asdict_dict["zarr_format"] == 3 # version-agnostic validations dtype_native = meta.dtype.to_native_dtype() if dtype_native.kind == "f": assert serialized_float_is_valid(asdict_dict["fill_value"]) elif dtype_native.kind == "c": # fill_value should be a two-element array [real, imag]. assert serialized_complex_float_is_valid(asdict_dict["fill_value"]) elif dtype_native.kind in ("M", "m") and np.isnat(meta.fill_value): assert asdict_dict["fill_value"] == -9223372036854775808 @given(data=st.data()) def test_rectilinear_chunks_declares_edges_per_dimension(data: st.DataObject) -> None: """`rectilinear_chunks` draws an explicit edge list for every dimension, summing to the extent (any positive edges, for a zero extent); a 0-d shape has none.""" shape = data.draw( npst.array_shapes(min_dims=0, max_dims=3, min_side=0, max_side=20), label="shape" ) chunks = data.draw(rectilinear_chunks(shape=shape), label="chunks") assert len(chunks) == len(shape) for edges, extent in zip(chunks, shape, strict=True): assert isinstance(edges, list) assert edges assert all(type(edge) is int and edge >= 1 for edge in edges) assert extent == 0 or sum(edges) == extent def test_chunks_param_from_rectilinear_bare_int_roundtrip() -> None: """Bare-int dims in rectilinear metadata (the spec's step-size shorthand, produced by a scalar dimension of a mixed chunk spec) must pass through the `chunks=` conversion unchanged. Wrapping one in a single-element list turns "repeat to cover the axis" into "exactly one chunk" and re-creation fails the sum-to-span check. Edge tuples become lists, as zarr 3.4.0 returned them.""" from zarr.core.metadata.v3 import RectilinearChunkGridMetadata from zarr.storage import MemoryStore from zarr.testing.strategies import chunks_param_from_rectilinear with zarr.config.set({"array.rectilinear_chunks": True}): src = zarr.create_array(MemoryStore(), shape=(3, 3), chunks=([1, 2], 1), dtype="uint8") grid = src.metadata.chunk_grid # type: ignore[union-attr] assert isinstance(grid, RectilinearChunkGridMetadata) assert grid.chunk_shapes == ((1, 2), 1) chunks = chunks_param_from_rectilinear(grid) # a list of lists, not tuples (`[1, 2] != (1, 2)`) assert chunks == [[1, 2], 1] dst = zarr.create_array( MemoryStore(), shape=src.shape, chunks=chunks, dtype="uint8", ) assert dst.metadata.chunk_grid == grid # type: ignore[union-attr] @given(data=st.data()) def test_rectilinear_chunk_grid_declarations(data: st.DataObject) -> None: """Every `chunk_shapes` declaration the rectilinear spec allows — bare-int steps, edge lists written in full or run-length encoded in any grouping, edges overhanging the extent — parses to its expanded edges, and the re-serialized form parses back to the same grid.""" shape = data.draw(npst.array_shapes(max_dims=3, min_side=0, max_side=20), label="shape") declaration, chunk_shapes = data.draw( rectilinear_chunk_shape_declarations(shape=shape), label="declaration" ) stored: RectilinearChunkGridMetadataJSON = { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": declaration}, } meta = RectilinearChunkGridMetadata.from_dict(stored) assert meta.chunk_shapes == chunk_shapes serialized = json.loads(json.dumps(meta.to_dict())) assert serialized["name"] == "rectilinear" assert RectilinearChunkGridMetadata.from_dict(serialized) == meta # The declaration is read correctly inside a whole stored metadata document. document = { "zarr_format": 3, "node_type": "array", "shape": list(shape), "data_type": "uint8", "chunk_grid": stored, "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, "fill_value": 0, "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], "attributes": {}, } assert ArrayV3Metadata.from_dict(document).chunk_grid == meta # type: ignore[arg-type] def _rle_expand(dim: list[Any]) -> list[int]: """Expand one stored run-length encoded dimension: bare edges and `[size, count]` pairs.""" edges: list[int] = [] for item in dim: if type(item) is int: edges.append(item) else: size, count = item edges.extend([size] * count) return edges @given(data=st.data()) def test_create_array_stores_declared_rectilinear_chunks(data: st.DataObject) -> None: """A `chunks=` specification mixing bare ints and edge lists in any arrangement is stored as a rectilinear grid whose `chunk_shapes` are exactly the specification: bare ints stay bare ints, and edge lists keep their edges (gh-4374, gh-4272). Checked on the stored JSON.""" shape = data.draw(npst.array_shapes(max_dims=3, min_side=0, max_side=20), label="shape") chunks = data.draw(_rectilinear_chunks(shape=shape), label="chunks") arr = zarr.create_array(MemoryStore(), shape=shape, chunks=chunks, dtype="uint8") zarr_json = sync(arr.store.get(ZARR_JSON, prototype=default_buffer_prototype())) assert zarr_json is not None stored = json.loads(zarr_json.to_bytes())["chunk_grid"] assert stored["name"] == "rectilinear" assert stored["configuration"]["kind"] == "inline" stored_dims = stored["configuration"]["chunk_shapes"] assert [dim if type(dim) is int else _rle_expand(dim) for dim in stored_dims] == chunks @given(data=st.data()) def test_rectilinear_zero_length_axis_round_trip(data: st.DataObject) -> None: """An array declared with rectilinear `chunks=` over a zero-length axis holds the data written after that axis grows, by `append` or by `resize`, also when it grows past the declared edges.""" shape = list(data.draw(npst.array_shapes(max_dims=3, min_side=0, max_side=6), label="shape")) axis = data.draw(st.integers(0, len(shape) - 1), label="zero-length axis") shape[axis] = 0 chunks = data.draw(_rectilinear_chunks(shape=tuple(shape)), label="chunks") store = MemoryStore() arr = zarr.create_array(store, shape=tuple(shape), chunks=chunks, dtype="int16", fill_value=-1) assert_array_equal(arr[...], np.full(shape, -1, dtype="int16")) declared = chunks[axis] declared_span = sum(declared) if isinstance(declared, list) else declared rows = data.draw(st.integers(1, 2 * declared_span + 2), label="rows") if isinstance(declared, list): event("grown axis declared as an edge list") if rows > declared_span: event("grown past the declared edges") grown = [*shape] grown[axis] = rows values = data.draw(npst.arrays(np.dtype("int16"), tuple(grown)), label="values") if data.draw(st.booleans(), label="append"): arr.append(values, axis=axis) else: arr.resize(tuple(grown)) arr[...] = values assert arr.shape == tuple(grown) assert_array_equal(arr[...], values) assert_array_equal(zarr.open_array(store, mode="r")[...], values) zarr-developers-zarr-python-a2f2415/tests/test_registry.py000066400000000000000000000442741527014137600241230ustar00rootroot00000000000000from __future__ import annotations import json import numcodecs.registry import pytest import zarr import zarr.registry from zarr.core.buffer import default_buffer_prototype from zarr.core.config import BadConfigError, config from zarr.core.metadata.v3 import parse_codecs from zarr.errors import UnknownCodecError from zarr.registry import ( _CODEC_PACKAGE_PREFIXES, _CODEC_PACKAGES, _NUMCODEC_PACKAGE_PREFIXES, _NUMCODECS_CODEC_DOCS_URL, _ZARR_CODEC_DOCS_URL, _missing_codec_message, _packages_for_codec, get_codec_class, get_numcodec, ) from zarr.storage import MemoryStore @pytest.fixture def unregistered_v3_codec(monkeypatch: pytest.MonkeyPatch) -> str: """Guarantee a Zarr format 3 codec name resolves to nothing. The registry is entry-point driven, so a name the hint table advertises resolves for real in any environment where the advertised package happens to be installed. """ name = "n5_default" monkeypatch.setitem(zarr.registry._codec_registries, name, zarr.registry.Registry()) return name @pytest.fixture def unregistered_v2_codec(monkeypatch: pytest.MonkeyPatch) -> str: """As above, for a numcodecs codec id.""" codec_id = "wavpack" monkeypatch.delitem(numcodecs.registry.codec_registry, codec_id, raising=False) monkeypatch.delitem(numcodecs.registry.entries, codec_id, raising=False) return codec_id @pytest.mark.parametrize( ("name", "expected"), [ ("n5_default", ("zarr-n5",)), ("gribberish", ("gribberish",)), ("imagecodecs_jpeg2k", ("imagecodecs-zarr", "virtual-tiff")), ("omfiles.pfor", ("omfiles",)), ("any-numcodecs.array-array", ("zarr-any-numcodecs",)), ("totally-made-up", ()), ], ) def test_packages_for_codec_v3(name: str, expected: tuple[str, ...]) -> None: """Exact names and prefixes both resolve; unknown names resolve to nothing.""" assert _packages_for_codec(name, zarr_format=3) == expected @pytest.mark.parametrize( ("name", "expected"), [ ("wavpack", ("wavpack-numcodecs",)), ("grib", ("kerchunk",)), ("zfpy", ("numcodecs[zfpy]",)), ("crc32c", ("numcodecs[crc32c]",)), ("gribscan.rawgrib", ("gribscan",)), ("imagecodecs_jpeg2k", ("imagecodecs-numcodecs",)), ("totally-made-up", ()), ], ) def test_packages_for_numcodec_v2(name: str, expected: tuple[str, ...]) -> None: """Zarr format 2 codec ids resolve against the numcodecs table.""" assert _packages_for_codec(name, zarr_format=2) == expected def test_packages_for_codec_is_format_specific() -> None: """The same name can mean different packages in each format's registry.""" assert _packages_for_codec("imagecodecs_jpeg2k", zarr_format=3) == ( "imagecodecs-zarr", "virtual-tiff", ) assert _packages_for_codec("imagecodecs_jpeg2k", zarr_format=2) == ("imagecodecs-numcodecs",) # `crc32c` is a codec zarr implements in format 3, so only format 2 gets a hint for it. assert _packages_for_codec("crc32c", zarr_format=3) == () def test_missing_codec_message_with_known_packages() -> None: """The message names the codec, the docs page, and every known package.""" msg = _missing_codec_message("n5_default", zarr_format=3) assert msg == ( "An implementation for codec 'n5_default' is not available. Register one explicitly " f"using the codec registry (see {_ZARR_CODEC_DOCS_URL}), or install a Python package " "that registers a codec implementation with zarr. Known packages supporting this " "codec: zarr-n5." ) def test_missing_codec_message_without_known_packages() -> None: """With no known package we still explain how to register one by hand.""" msg = _missing_codec_message("totally-made-up", zarr_format=3) assert msg == ( "An implementation for codec 'totally-made-up' is not available. Register one " f"explicitly using the codec registry (see {_ZARR_CODEC_DOCS_URL}), or install a " "Python package that registers a codec implementation with zarr." ) assert "Known packages" not in msg def test_missing_codec_message_for_zarr_format_2() -> None: """Format 2 codecs live in the numcodecs registry, so the message must say so.""" msg = _missing_codec_message("wavpack", zarr_format=2) assert msg == ( "An implementation for codec 'wavpack' is not available. Register one explicitly " f"using the codec registry (see {_NUMCODECS_CODEC_DOCS_URL}), or install a Python " "package that registers a codec implementation with numcodecs. Known packages " "supporting this codec: wavpack-numcodecs." ) def test_no_prefix_shadows_another_prefix() -> None: """First-match prefix lookup is only deterministic while no prefix contains another.""" for table in (_CODEC_PACKAGE_PREFIXES, _NUMCODEC_PACKAGE_PREFIXES): for a in table: for b in table: assert a == b or not a.startswith(b) def test_mapping_does_not_shadow_builtin_codecs() -> None: """A codec zarr implements itself must never appear in the format 3 hint table.""" import zarr.codecs # noqa: F401 (importing registers the built-in codecs) from zarr.registry import _codec_registries # Select on the implementing class's module rather than on "is this registry non-empty": # a third-party entry-point codec lazy-loaded by an earlier test would otherwise show up # here and fail the assertion even though it shadows nothing. implemented = { name for name, reg in _codec_registries.items() if any(cls.__module__.startswith("zarr.") for cls in reg.values()) } assert implemented, "expected importing zarr.codecs to populate the registry" assert not (implemented & set(_CODEC_PACKAGES)) for prefix in _CODEC_PACKAGE_PREFIXES: assert not any(name.startswith(prefix) for name in implemented) def test_get_codec_class_unknown_raises_with_package_hint( monkeypatch: pytest.MonkeyPatch, ) -> None: """An unregistered codec with a known package names that package in the error.""" from collections import defaultdict monkeypatch.setattr(zarr.registry, "_codec_registries", defaultdict(zarr.registry.Registry)) with pytest.raises(UnknownCodecError, match="Known packages supporting this codec: zarr-n5"): get_codec_class("n5_default") def test_get_codec_class_unknown_raises_without_package_hint( monkeypatch: pytest.MonkeyPatch, ) -> None: """An unregistered codec we know nothing about still explains manual registration.""" from collections import defaultdict monkeypatch.setattr(zarr.registry, "_codec_registries", defaultdict(zarr.registry.Registry)) with pytest.raises(UnknownCodecError, match="An implementation for codec 'nope' is not"): get_codec_class("nope") def test_unknown_codec_error_is_exported() -> None: """UnknownCodecError is public API now that we tell users to catch it.""" import zarr.errors assert "UnknownCodecError" in zarr.errors.__all__ def test_get_numcodec_unknown_raises_with_package_hint(unregistered_v2_codec: str) -> None: """A Zarr format 2 codec id we don't have names the package that provides it.""" with pytest.raises( UnknownCodecError, match="Known packages supporting this codec: wavpack-numcodecs" ): get_numcodec({"id": unregistered_v2_codec}) def test_get_numcodec_unknown_points_at_numcodecs_registry() -> None: """The Zarr format 2 message links the numcodecs registry, not zarr's extending guide.""" with pytest.raises(UnknownCodecError) as excinfo: get_numcodec({"id": "definitely-not-a-real-codec"}) assert _NUMCODECS_CODEC_DOCS_URL in str(excinfo.value) assert _ZARR_CODEC_DOCS_URL not in str(excinfo.value) def test_get_numcodec_known_codec_still_works() -> None: """The happy path is untouched.""" from numcodecs import GZip assert get_numcodec({"id": "gzip", "level": 2}) == GZip(level=2) # type: ignore[typeddict-unknown-key] def test_get_numcodec_without_an_id_keeps_the_numcodecs_error() -> None: """With no codec id there is nothing to look up, so numcodecs' own error stands.""" # Not asserting on numcodecs' exception class: it only gained a dedicated # `numcodecs.errors.UnknownCodecError` in 0.15.1, and zarr supports numcodecs >= 0.14. with pytest.raises(ValueError) as excinfo: get_numcodec({"level": 2}) # type: ignore[typeddict-item,typeddict-unknown-key] assert not isinstance(excinfo.value, UnknownCodecError) def test_get_numcodec_does_not_relabel_a_bad_configuration() -> None: """A registered codec rejecting its config is not a missing codec. Telling the user to install a package they already have would be the same misleading error this module exists to remove, pointing the other way. """ with pytest.raises(ValueError) as excinfo: get_numcodec({"id": "bitround", "keepbits": -1}) # type: ignore[typeddict-unknown-key] assert not isinstance(excinfo.value, UnknownCodecError) assert str(excinfo.value) == "keepbits must be zero or positive" @pytest.mark.parametrize("via", ["zarr", "numcodecs"]) def test_get_numcodec_does_not_relabel_a_missing_inner_codec(via: str) -> None: """A wrapper codec failing on an inner codec must not be blamed on the outer id. The outer id is registered, so relabelling it would advertise a package the user has already installed. Covers both routes a real wrapper takes to resolve its inner codec: zarr's ``get_numcodec`` and numcodecs' own ``get_codec``. """ import numcodecs.registry from numcodecs.abc import Codec resolve = get_numcodec if via == "zarr" else numcodecs.registry.get_codec class WrapperCodec(Codec): # type: ignore[misc] codec_id = "test_wrapper" @classmethod def from_config(cls, config: dict[str, object]) -> WrapperCodec: resolve({"id": "some-missing-inner-codec"}) return cls() def encode(self, buf: object) -> object: return buf def decode(self, buf: object, out: object = None) -> object: return buf numcodecs.registry.register_codec(WrapperCodec, codec_id="test_wrapper") try: with pytest.raises(ValueError) as excinfo: get_numcodec({"id": "test_wrapper"}) assert "some-missing-inner-codec" in str(excinfo.value) assert "test_wrapper" not in str(excinfo.value) finally: numcodecs.registry.codec_registry.pop("test_wrapper", None) @pytest.mark.parametrize("data", ["abc", ["ab"]]) def test_get_numcodec_non_mapping_input_still_raises_value_error(data: object) -> None: """Input that is not a mapping must reach numcodecs rather than raising AttributeError. Reading ``id`` off the input unconditionally turned numcodecs' ValueError into an AttributeError, which is not a ValueError and so escaped handlers that caught it. Both params end in a ValueError from numcodecs, by different routes: ``"abc"`` fails ``dict(config)`` coercion, while ``["ab"]`` coerces to ``{"a": "b"}`` and then has no id. """ with pytest.raises(ValueError): get_numcodec(data) # type: ignore[arg-type] @pytest.mark.parametrize( ("name", "expected"), [ ("imagecodecs_jpeg2k", ("imagecodecs-zarr", "virtual-tiff")), ("imagecodecs_jpeg8", ("virtual-tiff",)), ("imagecodecs_jetraw", ("virtual-tiff",)), ("imagecodecs_jpegls", ("imagecodecs-zarr",)), ("imagecodecs_avif", ("imagecodecs-zarr",)), ("imagecodecs_blosc", ("imagecodecs-zarr",)), ("imagecodecs_lzma", ("imagecodecs-zarr",)), ("imagecodecs_wavpack", ("imagecodecs-zarr",)), ("imagecodecs_zstd1", ("imagecodecs-zarr",)), ("imagecodecs_not_a_codec", ()), ], ) def test_missing_imagecodec_reports_packages( name: str, expected: tuple[str, ...], monkeypatch: pytest.MonkeyPatch ) -> None: """Missing imagecodecs name their providers without guessing from the prefix.""" monkeypatch.setitem(zarr.registry._codec_registries, name, zarr.registry.Registry()) message = ( f"An implementation for codec {name!r} is not available. Register one explicitly " f"using the codec registry (see {_ZARR_CODEC_DOCS_URL}), or install a Python package " "that registers a codec implementation with zarr." ) if expected: message += f" Known packages supporting this codec: {', '.join(expected)}." with pytest.raises(UnknownCodecError) as excinfo: get_codec_class(name) assert str(excinfo.value) == message def test_resolve_codec_reports_missing_codec() -> None: """The other public entry point into `get_codec_class` gets the message too.""" import zarr with pytest.raises(UnknownCodecError, match="Known packages supporting this codec: zarr-n5"): zarr.create_array({}, shape=(4,), dtype="uint8", compressors=[{"name": "n5_default"}]) async def test_open_array_with_missing_v3_codec_reports_package( unregistered_v3_codec: str, ) -> None: """Opening a Zarr format 3 array naming a codec we lack points at the package.""" store = MemoryStore() metadata = { "zarr_format": 3, "node_type": "array", "shape": [4], "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, "chunk_key_encoding": {"name": "default"}, "data_type": "float64", "fill_value": 0.0, "codecs": [{"name": "bytes"}, {"name": unregistered_v3_codec}], "attributes": {}, } await store.set( "zarr.json", default_buffer_prototype().buffer.from_bytes(json.dumps(metadata).encode()), ) with pytest.raises(UnknownCodecError, match="Known packages supporting this codec: zarr-n5"): zarr.open_array(store=store, mode="r") async def test_open_array_with_missing_v2_codec_reports_package( unregistered_v2_codec: str, ) -> None: """Same, for a Zarr format 2 array whose compressor id we lack.""" store = MemoryStore() metadata = { "zarr_format": 2, "shape": [4], "chunks": [4], "dtype": " None: """A config pinning an implementation that isn't registered is a config error.""" with config.set({"codecs.bytes": "some.package.RemovedBytesCodec"}): with pytest.raises(BadConfigError, match="RemovedBytesCodec"): get_codec_class("bytes") def test_parse_codecs_with_unregistered_config_pin_raises_bad_config_error() -> None: """The same, through the array-open path, and never as a bare KeyError.""" with config.set({"codecs.bytes": "some.package.RemovedBytesCodec"}): with pytest.raises(BadConfigError, match="RemovedBytesCodec"): parse_codecs([{"name": "bytes"}]) @pytest.fixture def keyerror_codec(monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest) -> str: """Register a codec whose ``from_dict`` raises a KeyError and return its name. Parametrize indirectly with the KeyError's args tuple: ``("required_option",)`` for a codec that indexes its configuration directly, ``()`` for a bare ``raise KeyError``. """ from zarr.codecs import BytesCodec from zarr.registry import register_codec keyerror_args: tuple[object, ...] = request.param class KeyErrorCodec(BytesCodec): @classmethod def from_dict(cls, data: object) -> KeyErrorCodec: raise KeyError(*keyerror_args) name = "test_keyerror" monkeypatch.setitem(zarr.registry._codec_registries, name, zarr.registry.Registry()) register_codec(name, KeyErrorCodec) return name @pytest.mark.parametrize( ("keyerror_codec", "expected_message"), [ ( ("required_option",), "KeyError 'required_option' while parsing the configuration for codec 'test_keyerror'.", ), ((), "KeyError while parsing the configuration for codec 'test_keyerror'."), ], indirect=["keyerror_codec"], ids=["with_key", "argless"], ) def test_parse_codecs_converts_keyerror_from_from_dict( keyerror_codec: str, expected_message: str ) -> None: """A KeyError out of a codec's from_dict becomes a MetadataValidationError naming the codec. A bare KeyError out of metadata parsing is caught by the array-then-group fallback in `zarr.api.asynchronous.open`, which then reports an unrelated group error. The KeyError may carry no arguments, in which case the message simply omits the key. """ from zarr.errors import MetadataValidationError with pytest.raises(MetadataValidationError) as excinfo: parse_codecs([{"name": keyerror_codec, "configuration": {}}]) assert str(excinfo.value) == expected_message assert isinstance(excinfo.value.__cause__, KeyError) @pytest.mark.parametrize("keyerror_codec", [()], indirect=True, ids=["argless"]) async def test_open_with_argless_keyerror_codec_raises_value_error(keyerror_codec: str) -> None: """`zarr.open` on an array whose codec raises a bare KeyError reports the codec, not IndexError. An argless KeyError must propagate as a MetadataValidationError naming the codec. Opening still fails because the codec configuration could not be parsed. """ from zarr.errors import MetadataValidationError store = MemoryStore() metadata = { "zarr_format": 3, "node_type": "array", "shape": [4], "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, "chunk_key_encoding": {"name": "default"}, "data_type": "float64", "fill_value": 0.0, "codecs": [{"name": keyerror_codec}], "attributes": {}, } await store.set( "zarr.json", default_buffer_prototype().buffer.from_bytes(json.dumps(metadata).encode()), ) with pytest.raises(MetadataValidationError, match=keyerror_codec): zarr.open(store=store, mode="r") zarr-developers-zarr-python-a2f2415/tests/test_regression/000077500000000000000000000000001527014137600240465ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/test_regression/__init__.py000066400000000000000000000000001527014137600261450ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/test_regression/scripts/000077500000000000000000000000001527014137600255355ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/test_regression/scripts/__init__.py000066400000000000000000000000001527014137600276340ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/test_regression/scripts/v2.18.py000066400000000000000000000047361527014137600266770ustar00rootroot00000000000000# /// script # requires-python = ">=3.12" # dependencies = [ # "zarr==2.18", # "numcodecs==0.15" # ] # /// import argparse import zarr from zarr._storage.store import BaseStore def copy_group( *, node: zarr.hierarchy.Group, store: zarr.storage.BaseStore, path: str, overwrite: bool ) -> zarr.hierarchy.Group: result = zarr.group(store=store, path=path, overwrite=overwrite) result.attrs.put(node.attrs.asdict()) for key, child in node.items(): child_path = f"{path}/{key}" if isinstance(child, zarr.hierarchy.Group): copy_group(node=child, store=store, path=child_path, overwrite=overwrite) elif isinstance(child, zarr.core.Array): copy_array(node=child, store=store, overwrite=overwrite, path=child_path) return result def copy_array( *, node: zarr.core.Array, store: BaseStore, path: str, overwrite: bool ) -> zarr.core.Array: result = zarr.create( shape=node.shape, dtype=node.dtype, fill_value=node.fill_value, chunks=node.chunks, compressor=node.compressor, filters=node.filters, order=node.order, dimension_separator=node._dimension_separator, store=store, path=path, overwrite=overwrite, ) result.attrs.put(node.attrs.asdict()) result[:] = node[:] return result def copy_node( node: zarr.hierarchy.Group | zarr.core.Array, store: BaseStore, path: str, overwrite: bool ) -> zarr.hierarchy.Group | zarr.core.Array: if isinstance(node, zarr.hierarchy.Group): return copy_group(node=node, store=store, path=path, overwrite=overwrite) elif isinstance(node, zarr.core.Array): return copy_array(node=node, store=store, path=path, overwrite=overwrite) else: raise TypeError(f"Unexpected node type: {type(node)}") # pragma: no cover def cli() -> None: parser = argparse.ArgumentParser( description="Copy a zarr hierarchy from one location to another" ) parser.add_argument("source", type=str, help="Path to the source zarr hierarchy") parser.add_argument("destination", type=str, help="Path to the destination zarr hierarchy") args = parser.parse_args() src, dst = args.source, args.destination root_src = zarr.open(src, mode="r") result = copy_node(node=root_src, store=zarr.NestedDirectoryStore(dst), path="", overwrite=True) print(f"successfully created {result} at {dst}") def main() -> None: cli() if __name__ == "__main__": main() zarr-developers-zarr-python-a2f2415/tests/test_regression/scripts/v3.0.8.py000066400000000000000000000036361527014137600267530ustar00rootroot00000000000000# /// script # requires-python = "==3.12" # dependencies = [ # "zarr==3.0.8", # "numcodecs==0.16.3" # ] # /// import argparse import zarr from zarr.abc.store import Store def copy_group( *, node: zarr.Group, store: Store, path: str, overwrite: bool ) -> zarr.Group: result = zarr.create_group( store=store, path=path, overwrite=overwrite, attributes=node.attrs.asdict(), zarr_format=node.metadata.zarr_format) for key, child in node.members(): child_path = f"{path}/{key}" if isinstance(child, zarr.Group): copy_group(node=child, store=store, path=child_path, overwrite=overwrite) else: copy_array(node=child, store=store, overwrite=overwrite, path=child_path) return result def copy_array( *, node: zarr.Array, store: Store, path: str, overwrite: bool ) -> zarr.Array: result = zarr.from_array(store, name=path, data=node, write_data=True) return result def copy_node( node: zarr.Group | zarr.Array, store: Store, path: str, overwrite: bool ) -> zarr.Group | zarr.Array: if isinstance(node, zarr.Group): return copy_group(node=node, store=store, path=path, overwrite=overwrite) else: return copy_array(node=node, store=store, path=path, overwrite=overwrite) def cli() -> None: parser = argparse.ArgumentParser( description="Copy a zarr hierarchy from one location to another" ) parser.add_argument("source", type=str, help="Path to the source zarr hierarchy") parser.add_argument("destination", type=str, help="Path to the destination zarr hierarchy") args = parser.parse_args() src, dst = args.source, args.destination root_src = zarr.open(src, mode="r") result = copy_node(node=root_src, store=dst, path="", overwrite=True) print(f"successfully created {result} at {dst}") def main() -> None: cli() if __name__ == "__main__": main() zarr-developers-zarr-python-a2f2415/tests/test_regression/test_v2_dtype_regression.py000066400000000000000000000204511527014137600314550ustar00rootroot00000000000000import subprocess import sys from dataclasses import dataclass from itertools import product from pathlib import Path from typing import TYPE_CHECKING, Literal import numpy as np import pytest from numcodecs import LZ4, LZMA, Blosc, GZip, VLenBytes, VLenUTF8, Zstd import zarr import zarr.abc import zarr.abc.codec import zarr.codecs as zarrcodecs from zarr.abc.numcodec import Numcodec from zarr.core.chunk_key_encodings import V2ChunkKeyEncoding from zarr.core.dtype.npy.bytes import VariableLengthBytes from zarr.core.dtype.npy.string import VariableLengthUTF8 from zarr.storage import LocalStore from zarr.types import ArrayV2, ArrayV3 if TYPE_CHECKING: from zarr.core.dtype import ZDTypeLike ZarrPythonVersion = Literal["2.18", "3.0.8"] def runner_installed() -> bool: """ Check if a PEP-723 compliant python script runner is installed. """ try: subprocess.check_output(["uv", "--version"]) return True # noqa: TRY300 except FileNotFoundError: return False @dataclass(kw_only=True) class ArrayParams: values: np.ndarray[tuple[int], np.dtype[np.generic]] fill_value: np.generic | str | int | bytes filters: tuple[Numcodec, ...] = () serializer: str | None = None compressor: Numcodec basic_codecs: tuple[Numcodec, ...] = GZip(), Blosc(), LZ4(), LZMA(), Zstd() basic_dtypes = "|b", ">i2", ">i4", ">f4", ">f8", "c8", "c16", "M8[10us]", "m8[4ps]" string_dtypes = "U4" bytes_dtypes = ">S1", "V10", " ArrayV2: """ Writes a zarr array to a temporary directory based on the provided ArrayParams. The array is returned. """ dest = tmp_path / "in" store = LocalStore(dest) array_params: ArrayParams = request.param compressor = array_params.compressor chunk_key_encoding = V2ChunkKeyEncoding(separator="/") dtype: ZDTypeLike if array_params.values.dtype == np.dtype("|O") and array_params.serializer == "vlen-utf8": dtype = VariableLengthUTF8() # type: ignore[assignment] filters = array_params.filters + (VLenUTF8(),) elif array_params.values.dtype == np.dtype("|O") and array_params.serializer == "vlen-bytes": dtype = VariableLengthBytes() filters = array_params.filters + (VLenBytes(),) else: dtype = array_params.values.dtype filters = array_params.filters z = zarr.create_array( store, shape=array_params.values.shape, dtype=dtype, chunks=array_params.values.shape, compressors=compressor, filters=filters, fill_value=array_params.fill_value, order="C", chunk_key_encoding=chunk_key_encoding, write_data=True, zarr_format=2, ) z[:] = array_params.values return z @pytest.fixture def source_array_v3(tmp_path: Path, request: pytest.FixtureRequest) -> ArrayV3: """ Writes a zarr array to a temporary directory based on the provided ArrayParams. The array is returned. """ dest = tmp_path / "in" store = LocalStore(dest) array_params: ArrayParams = request.param chunk_key_encoding = V2ChunkKeyEncoding(separator="/") dtype: ZDTypeLike serializer: Literal["auto"] | zarr.abc.codec.Codec if array_params.values.dtype == np.dtype("|O") and array_params.serializer == "vlen-utf8": dtype = VariableLengthUTF8() # type: ignore[assignment] serializer = zarrcodecs.VLenUTF8Codec() elif array_params.values.dtype == np.dtype("|O") and array_params.serializer == "vlen-bytes": dtype = VariableLengthBytes() serializer = zarrcodecs.VLenBytesCodec() else: dtype = array_params.values.dtype serializer = "auto" if array_params.compressor == GZip(): compressor = zarrcodecs.GzipCodec() else: msg = ( "This test is only compatible with gzip compression at the moment, because the author" "did not want to implement a complete abstraction layer for v2 and v3 codecs in this test." ) raise ValueError(msg) z = zarr.create_array( store, shape=array_params.values.shape, dtype=dtype, chunks=array_params.values.shape, compressors=compressor, filters=array_params.filters, serializer=serializer, fill_value=array_params.fill_value, chunk_key_encoding=chunk_key_encoding, write_data=True, zarr_format=3, ) z[:] = array_params.values return z # TODO: make this dynamic based on the installed scripts script_paths = [Path(__file__).resolve().parent / "scripts" / "v2.18.py"] @pytest.mark.skipif( sys.platform == "darwin" and sys.version_info >= (3, 14), reason="Numcodecs pinned to 0.15 does not build on newer macos installations with newer python versions: see discussion https://github.com/zarr-developers/zarr-python/pull/3564#issuecomment-4081145034", ) @pytest.mark.skipif(not runner_installed(), reason="no python script runner installed") @pytest.mark.parametrize( "source_array_v2", array_cases_v2_18, indirect=True, ids=tuple(map(str, array_cases_v2_18)) ) @pytest.mark.parametrize("script_path", script_paths) def test_roundtrip_v2(source_array_v2: ArrayV2, tmp_path: Path, script_path: Path) -> None: out_path = tmp_path / "out" copy_op = subprocess.run( [ "uv", "run", str(script_path), str(source_array_v2.store).removeprefix("file://"), str(out_path), ], capture_output=True, text=True, check=False, ) assert copy_op.returncode == 0, f"stdout {copy_op.stdout}\n stderr{copy_op.stderr}" out_array = zarr.open_array(store=out_path, mode="r", zarr_format=2) assert source_array_v2.metadata.to_dict() == out_array.metadata.to_dict() assert np.array_equal(source_array_v2[:], out_array[:]) @pytest.mark.skipif(not runner_installed(), reason="no python script runner installed") @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") @pytest.mark.parametrize( "source_array_v3", array_cases_v3_08, indirect=True, ids=tuple(map(str, array_cases_v3_08)) ) def test_roundtrip_v3(source_array_v3: ArrayV3, tmp_path: Path) -> None: script_path = Path(__file__).resolve().parent / "scripts" / "v3.0.8.py" out_path = tmp_path / "out" copy_op = subprocess.run( [ "uv", "run", str(script_path), str(source_array_v3.store).removeprefix("file://"), str(out_path), ], capture_output=True, text=True, check=False, ) assert copy_op.returncode == 0 out_array = zarr.open_array(store=out_path, mode="r", zarr_format=3) assert source_array_v3.metadata.to_dict() == out_array.metadata.to_dict() assert np.array_equal(source_array_v3[:], out_array[:]) zarr-developers-zarr-python-a2f2415/tests/test_resize.py000066400000000000000000000162621527014137600235500ustar00rootroot00000000000000import asyncio from collections.abc import Generator from itertools import islice, product from unittest import mock import numpy as np import pytest from hypothesis import given from hypothesis import strategies as st from zarr.core.array import ShardsLike, _iter_chunk_coords_to_delete, create_array from zarr.core.buffer import default_buffer_prototype from zarr.core.chunk_grids import ChunkGrid from zarr.core.common import ChunksLike, ZarrFormat from zarr.core.config import config from zarr.storage import MemoryStore @pytest.fixture(autouse=True) def _enable_rectilinear_chunks() -> Generator[None, None, None]: with config.set({"array.rectilinear_chunks": True}): yield @given(st.lists(st.tuples(st.integers(0, 8), st.integers(0, 8)), max_size=4)) def test_resize_chunk_difference_matches_sets(dimensions: list[tuple[int, int]]) -> None: old_shape = tuple(old for old, _ in dimensions) new_shape = tuple(new for _, new in dimensions) expected = set(product(*(range(n) for n in old_shape))) - set( product(*(range(n) for n in new_shape)) ) actual = list(_iter_chunk_coords_to_delete(old_shape, new_shape)) assert set(actual) == expected assert len(actual) == len(expected) def test_resize_chunk_difference_is_lazy_for_large_axes() -> None: # Both the per-axis ranges and their product are too large to materialize. coords = _iter_chunk_coords_to_delete((2**40, 2**40), (0, 1)) assert list(islice(coords, 3)) == [(0, 0), (0, 1), (0, 2)] async def test_resize_shrinking_skips_full_grid(zarr_format: ZarrFormat) -> None: store = MemoryStore() array = await create_array( store, shape=(2**40, 2), chunks=(1, 1), dtype="uint8", zarr_format=zarr_format ) expected = [array.metadata.encode_chunk_key((2**40 - 1, i)) for i in range(2)] # Fail safely on the old implementation instead of exhausting the test machine. with ( mock.patch.object(ChunkGrid, "all_chunk_coords", side_effect=AssertionError("full grid")), mock.patch.object(store, "delete", wraps=store.delete) as delete, ): await array.resize((2**40 - 1, 2)) assert array.shape == (2**40 - 1, 2) assert sorted(call.args[0] for call in delete.call_args_list) == expected @pytest.mark.parametrize("limit", [1, 4, None]) async def test_resize_bounds_pending_deletions(limit: int | None) -> None: store = MemoryStore() array = await create_array(store, shape=(3000,), chunks=(1,), dtype="uint8") initial_tasks = len(asyncio.all_tasks()) workers = limit if limit is not None else 1000 peak_tasks = 0 in_flight = 0 in_flight_at_start: list[int] = [] async def delete(key: str) -> None: nonlocal peak_tasks, in_flight peak_tasks = max(peak_tasks, len(asyncio.all_tasks()) - initial_tasks) in_flight_at_start.append(in_flight) in_flight += 1 await asyncio.sleep(0) in_flight -= 1 with config.set({"async.concurrency": limit}), mock.patch.object(store, "delete", delete): await array.resize((0,)) assert len(in_flight_at_start) == 3000 assert 0 < peak_tasks <= workers # The pool fills up: no deletion ever starts with `workers` others in flight. assert max(in_flight_at_start) == workers - 1 if workers > 1: # Once full, the pool never drains: a finished deletion is replaced # while the others are still in flight. assert min(in_flight_at_start[workers:]) >= 1 @pytest.mark.parametrize("new_shape", [(4, 5), (10, 5), (4, 12), (7, 8), (0, 9)]) @pytest.mark.parametrize( ("zarr_format", "chunks", "shards"), [ (2, (3, 4), None), (3, (3, 4), None), (3, ((2, 3, 3), (3, 2, 4)), None), (3, (1, 1), (3, 4)), (3, (1, 1), ((2, 3, 3), (3, 2, 4))), ], ids=["v2", "v3", "rectilinear", "sharded", "rectilinear-sharded"], ) @pytest.mark.parametrize("delete_outside_chunks", [True, False]) async def test_resize_keeps_retained_chunk_bytes( new_shape: tuple[int, ...], zarr_format: ZarrFormat, chunks: ChunksLike, shards: ShardsLike, delete_outside_chunks: bool, ) -> None: store = MemoryStore() array = await create_array( store, shape=(8, 9), chunks=chunks, shards=shards, dtype="int32", zarr_format=zarr_format ) data = np.arange(1, 73, dtype="int32").reshape(8, 9) await array.setitem(..., data) before = {key: await store.get(key, default_buffer_prototype()) async for key in store.list()} old_coords = set(array._chunk_grid.all_chunk_coords()) new_grid = ChunkGrid.from_metadata(array.metadata.update_shape(new_shape)) removed = old_coords - set(new_grid.all_chunk_coords()) if delete_outside_chunks else set() removed_keys = {array.metadata.encode_chunk_key(coords) for coords in removed} with mock.patch.object(store, "delete", wraps=store.delete) as delete: await array.resize(new_shape, delete_outside_chunks=delete_outside_chunks) assert array.shape == new_shape assert {call.args[0] for call in delete.call_args_list} == removed_keys assert delete.call_count == len(removed_keys) remaining_keys = {key async for key in store.list()} assert remaining_keys == before.keys() - removed_keys for coords in old_coords - removed: key = array.metadata.encode_chunk_key(coords) original = before[key] current = await store.get(key, default_buffer_prototype()) assert original is not None assert current is not None assert current.to_bytes() == original.to_bytes() overlap = tuple(slice(0, min(old, new)) for old, new in zip((8, 9), new_shape, strict=True)) np.testing.assert_array_equal(await array.getitem(overlap), data[overlap]) async def test_resize_delete_failure_preserves_metadata() -> None: store = MemoryStore() array = await create_array(store, shape=(4,), chunks=(1,), dtype="uint8") metadata_before = await store.get("zarr.json", default_buffer_prototype()) grid_before = array._chunk_grid with ( mock.patch.object(store, "delete", side_effect=OSError("delete failed")), pytest.raises(OSError, match="delete failed"), ): await array.resize((0,)) metadata_after = await store.get("zarr.json", default_buffer_prototype()) assert metadata_before is not None assert metadata_after is not None assert metadata_before.to_bytes() == metadata_after.to_bytes() assert array.shape == (4,) assert array._chunk_grid is grid_before async def test_resize_delete_failure_stops_remaining_deletions() -> None: store = MemoryStore() array = await create_array(store, shape=(3000,), chunks=(1,), dtype="uint8") calls = 0 async def delete(key: str) -> None: nonlocal calls calls += 1 call = calls await asyncio.sleep(0) if call == 10: raise OSError("delete failed") with ( config.set({"async.concurrency": 4}), mock.patch.object(store, "delete", delete), ): with pytest.raises(OSError, match="delete failed"): await array.resize((0,)) calls_at_raise = calls # Give any surviving workers a chance to run. for _ in range(10): await asyncio.sleep(0) assert calls == calls_at_raise < 3000 zarr-developers-zarr-python-a2f2415/tests/test_store/000077500000000000000000000000001527014137600230225ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/test_store/__init__.py000066400000000000000000000000001527014137600251210ustar00rootroot00000000000000zarr-developers-zarr-python-a2f2415/tests/test_store/test_core.py000066400000000000000000000411301527014137600253620ustar00rootroot00000000000000import pickle import tempfile from collections.abc import Awaitable, Callable, Generator from pathlib import Path from typing import Any, Literal import pytest from packaging.version import parse as parse_version import zarr from zarr import Group from zarr.abc.store import Store from zarr.core.buffer import cpu from zarr.core.common import ZARR_JSON, AccessModeLiteral, ZarrFormat from zarr.storage import FsspecStore, LocalStore, MemoryStore, StoreLike, StorePath, ZipStore from zarr.storage._common import ( _contains_node_v3, contains_array, contains_group, make_store, make_store_path, ) from zarr.storage._utils import ( _join_paths, _normalize_path_keys, _normalize_paths, _relativize_path, normalize_path, ) # contains_array and contains_group share this signature. _ContainsFunc = Callable[[StorePath, ZarrFormat], Awaitable[bool]] @pytest.fixture( params=["none", "temp_dir_str", "temp_dir_path", "store_path", "memory_store", "dict"] ) def store_like( request: pytest.FixtureRequest, ) -> Generator[str | Path | StorePath | MemoryStore | dict[Any, Any] | None, None, None]: if request.param == "none": yield None elif request.param == "temp_dir_str": with tempfile.TemporaryDirectory() as temp_dir: yield temp_dir elif request.param == "temp_dir_path": with tempfile.TemporaryDirectory() as temp_dir: yield Path(temp_dir) elif request.param == "store_path": yield StorePath(store=MemoryStore(store_dict={}), path="/") elif request.param == "memory_store": yield MemoryStore(store_dict={}) elif request.param == "dict": yield {} @pytest.mark.parametrize("path", ["foo", "foo/bar"]) @pytest.mark.parametrize("write_group", [True, False]) @pytest.mark.parametrize("zarr_format", [2, 3]) async def test_contains_group( local_store: LocalStore, path: str, write_group: bool, zarr_format: ZarrFormat ) -> None: """ Test that the contains_group method correctly reports the existence of a group. """ root = Group.from_store(store=local_store, zarr_format=zarr_format) if write_group: root.create_group(path) store_path = StorePath(local_store, path=path) assert await contains_group(store_path, zarr_format=zarr_format) == write_group @pytest.mark.parametrize("path", ["foo", "foo/bar"]) @pytest.mark.parametrize("write_array", [True, False]) @pytest.mark.parametrize("zarr_format", [2, 3]) async def test_contains_array( local_store: LocalStore, path: str, write_array: bool, zarr_format: ZarrFormat ) -> None: """ Test that the contains array method correctly reports the existence of an array. """ root = Group.from_store(store=local_store, zarr_format=zarr_format) if write_array: root.create_array(path, shape=(100,), chunks=(10,), dtype="i4") store_path = StorePath(local_store, path=path) assert await contains_array(store_path, zarr_format=zarr_format) == write_array @pytest.mark.parametrize("func", [contains_array, contains_group]) async def test_contains_invalid_format_raises(local_store: LocalStore, func: _ContainsFunc) -> None: """ Test contains_group and contains_array raise errors for invalid zarr_formats """ store_path = StorePath(local_store) with pytest.raises(ValueError, match="Invalid zarr_format provided. Got 3.0, expected 2 or 3"): assert await func(store_path, "3.0") # type: ignore[arg-type] async def _write_zarr_json(store_path: StorePath, data: bytes) -> None: """Write raw bytes to the v3 metadata key under `store_path`.""" await (store_path / ZARR_JSON).set(cpu.Buffer.from_bytes(data)) @pytest.mark.parametrize("func", [contains_array, contains_group]) async def test_contains_malformed_json_returns_false( local_store: LocalStore, func: _ContainsFunc ) -> None: """A v3 metadata document that is not valid JSON reads as 'not present'.""" store_path = StorePath(local_store, path="foo") await _write_zarr_json(store_path, b"{not valid json") assert await func(store_path, 3) is False @pytest.mark.parametrize("func", [contains_array, contains_group]) async def test_contains_non_object_json_returns_false( local_store: LocalStore, func: _ContainsFunc ) -> None: """A v3 metadata document that is valid JSON but not an object reads as 'not present'.""" store_path = StorePath(local_store, path="foo") await _write_zarr_json(store_path, b"[1, 2, 3]") assert await func(store_path, 3) is False @pytest.mark.parametrize("func", [contains_array, contains_group]) async def test_contains_missing_node_type_returns_false( local_store: LocalStore, func: _ContainsFunc ) -> None: """A v3 metadata document with no 'node_type' key reads as 'not present'.""" store_path = StorePath(local_store, path="foo") await _write_zarr_json(store_path, b'{"zarr_format": 3}') assert await func(store_path, 3) is False @pytest.mark.parametrize("func", [contains_array, contains_group]) async def test_contains_non_utf8_bytes_returns_false( local_store: LocalStore, func: _ContainsFunc ) -> None: """A v3 metadata document that is not valid UTF-8 reads as 'not present' (not an error).""" store_path = StorePath(local_store, path="foo") await _write_zarr_json(store_path, b"\x80\x81\x82\x83") assert await func(store_path, 3) is False async def test_contains_node_v3_malformed_json_returns_nothing(local_store: LocalStore) -> None: """`_contains_node_v3` returns 'nothing' when the document is not valid JSON.""" store_path = StorePath(local_store, path="foo") await _write_zarr_json(store_path, b"{not valid json") assert await _contains_node_v3(store_path) == "nothing" async def test_contains_node_v3_non_object_json_returns_nothing(local_store: LocalStore) -> None: """`_contains_node_v3` returns 'nothing' when the document is not a JSON object.""" store_path = StorePath(local_store, path="foo") await _write_zarr_json(store_path, b"[1, 2, 3]") assert await _contains_node_v3(store_path) == "nothing" async def test_contains_node_v3_missing_node_type_returns_nothing(local_store: LocalStore) -> None: """`_contains_node_v3` returns 'nothing' when the document lacks a 'node_type' key.""" store_path = StorePath(local_store, path="foo") await _write_zarr_json(store_path, b'{"zarr_format": 3}') assert await _contains_node_v3(store_path) == "nothing" async def test_contains_node_v3_non_utf8_bytes_returns_nothing(local_store: LocalStore) -> None: """`_contains_node_v3` returns 'nothing' when the document is not valid UTF-8.""" store_path = StorePath(local_store, path="foo") await _write_zarr_json(store_path, b"\x80\x81\x82\x83") assert await _contains_node_v3(store_path) == "nothing" @pytest.mark.parametrize("path", [None, "", "bar"]) async def test_make_store_path_none(path: str) -> None: """ Test that creating a store_path with None creates a memorystore """ store_path = await make_store_path(None, path=path) assert isinstance(store_path.store, MemoryStore) assert store_path.path == normalize_path(path) @pytest.mark.parametrize("path", [None, "", "bar"]) @pytest.mark.parametrize("store_type", [str, Path]) @pytest.mark.parametrize("mode", ["r", "w"]) async def test_make_store_path_local( tmp_path: Path, store_type: type[str] | type[Path] | type[LocalStore], path: str, mode: AccessModeLiteral, ) -> None: """ Test the various ways of invoking make_store_path that create a LocalStore """ store_like = store_type(str(tmp_path)) store_path = await make_store_path(store_like, path=path, mode=mode) assert isinstance(store_path.store, LocalStore) assert Path(store_path.store.root) == Path(tmp_path) assert store_path.path == normalize_path(path) assert store_path.read_only == (mode == "r") @pytest.mark.parametrize("path", [None, "", "bar"]) @pytest.mark.parametrize("mode", ["r", "w"]) async def test_make_store_path_store_path( tmp_path: Path, path: str, mode: AccessModeLiteral ) -> None: """ Test invoking make_store_path when the input is another store_path. In particular we want to ensure that a new path is handled correctly. """ ro = mode == "r" store_like = await StorePath.open( LocalStore(str(tmp_path), read_only=ro), path="root", mode=mode ) store_path = await make_store_path(store_like, path=path, mode=mode) assert isinstance(store_path.store, LocalStore) assert Path(store_path.store.root) == tmp_path path_normalized = normalize_path(path) assert store_path.path == (store_like / path_normalized).path assert store_path.read_only == ro @pytest.mark.parametrize("modes", [(True, "w"), (False, "x")]) async def test_store_path_invalid_mode_raises( tmp_path: Path, modes: tuple[bool, Literal["w", "x"]] ) -> None: """ Test that ValueErrors are raise for invalid mode. """ with pytest.raises(ValueError): await StorePath.open(LocalStore(str(tmp_path), read_only=modes[0]), path="", mode=modes[1]) # type: ignore[arg-type] async def test_store_path_open_w_keeps_keys(tmp_path: Path) -> None: """ Opening a store path with mode 'w' leaves the keys under it in place: the node created at the path replaces them once it is valid, not the act of opening the path. """ store = LocalStore(str(tmp_path)) await (StorePath(store, "root") / "key").set(cpu.Buffer.from_bytes(b"data")) store_path = await StorePath.open(store, path="root", mode="w") assert not await store_path.is_empty() async def test_make_store_invalid_mode_raises() -> None: """ Test that make_store raises ValueError for a mode outside the access-mode literals. """ with pytest.raises(ValueError, match="Invalid mode"): await make_store({}, mode="x") # type: ignore[arg-type] async def test_make_store_path_invalid() -> None: """ Test that invalid types raise TypeError """ with pytest.raises(TypeError, match="Unsupported type for store_like: 'int'"): await make_store_path(1) async def test_make_store_path_fsspec() -> None: pytest.importorskip("fsspec") pytest.importorskip("requests") pytest.importorskip("aiohttp") store_path = await make_store_path("http://foo.com/bar") assert isinstance(store_path.store, FsspecStore) async def test_make_store_path_storage_options_raises(store_like: StoreLike) -> None: with pytest.raises(TypeError, match="storage_options"): await make_store_path(store_like, storage_options={"foo": "bar"}) # universal-pathlib 0.2.x emits this from its own subclass registry when a local UPath is built. @pytest.mark.filterwarnings( "ignore:Detected a customized `__new__` method in subclass:DeprecationWarning" ) @pytest.mark.parametrize( ("url", "expected"), [ ("memory://bucket/foo.zarr", FsspecStore), ("s3://bucket/foo.zarr", FsspecStore), ("file://{tmp}/foo.zarr", LocalStore), ("{tmp}/foo.zarr", LocalStore), ], ) async def test_make_store_upath(url: str, expected: type[Store], tmp_path: Path) -> None: """ A remote UPath becomes an FsspecStore, and a local one becomes a LocalStore, so that UPath("/data") and Path("/data") agree. See https://github.com/zarr-developers/zarr-python/issues/4244. """ upath = pytest.importorskip("upath") fsspec = pytest.importorskip("fsspec") if url.startswith("s3://"): pytest.importorskip("s3fs") if url.startswith("memory://") and parse_version(fsspec.__version__) < parse_version( "2024.12.0" ): # MemoryFileSystem is synchronous, so it can only be used once fsspec is new enough to # supply AsyncFileSystemWrapper. pytest.skip("No AsyncFileSystemWrapper") store = await make_store(upath.UPath(url.format(tmp=tmp_path))) assert isinstance(store, expected) if isinstance(store, LocalStore): # The local branch rebuilds the root from the UPath, so a mangled path would still # produce a LocalStore. Pin the root down too, since "file://{tmp}" has no leading # slash on Windows. assert store.root == tmp_path / "foo.zarr" async def test_make_store_upath_storage_options_raises() -> None: """A UPath carries its own storage options, so a separate mapping is ambiguous.""" upath = pytest.importorskip("upath") with pytest.raises(TypeError, match="storage_options"): await make_store(upath.UPath("memory://bucket/foo.zarr"), storage_options={"foo": "bar"}) async def test_unsupported() -> None: with pytest.raises(TypeError, match="Unsupported type for store_like: 'int'"): await make_store_path(1) @pytest.mark.parametrize( "path", [ "/foo/bar", "//foo/bar", "foo///bar", "foo/bar///", Path("foo/bar"), b"foo/bar", ], ) def test_normalize_path_valid(path: str | bytes | Path) -> None: assert normalize_path(path) == "foo/bar" def test_normalize_path_upath() -> None: upath = pytest.importorskip("upath") assert normalize_path(upath.UPath("foo/bar", protocol="memory")) == "memory:/foo/bar" def test_normalize_path_none() -> None: assert normalize_path(None) == "" @pytest.mark.parametrize("path", [".", ".."]) def test_normalize_path_invalid(path: str) -> None: with pytest.raises(ValueError, match="is invalid because its string representation contains"): normalize_path(path) @pytest.mark.parametrize("paths", [("", "foo"), ("foo", "bar")]) def test_join_paths(paths: tuple[str, str]) -> None: """ Test that _join_paths joins paths in a way that is robust to an empty string """ observed = _join_paths(paths) if paths[0] == "": assert observed == paths[1] else: assert observed == "/".join(paths) class TestNormalizePaths: @staticmethod def test_valid() -> None: """ Test that path normalization works as expected """ paths = ["a", "b", "c", "d", "", "//a///b//"] assert _normalize_paths(paths) == tuple(normalize_path(p) for p in paths) @staticmethod @pytest.mark.parametrize("paths", [("", "/"), ("///a", "a")]) def test_invalid(paths: tuple[str, str]) -> None: """ Test that name collisions after normalization raise a ``ValueError`` """ msg = ( f"After normalization, the value '{paths[1]}' collides with '{paths[0]}'. " f"Both '{paths[1]}' and '{paths[0]}' normalize to the same value: '{normalize_path(paths[0])}'. " f"You should use either '{paths[1]}' or '{paths[0]}', but not both." ) with pytest.raises(ValueError, match=msg): _normalize_paths(paths) def test_normalize_path_keys() -> None: """ Test that ``_normalize_path_keys`` just applies the normalize_path function to each key of its input """ data = {"a": 10, "//b": 10} assert _normalize_path_keys(data) == {normalize_path(k): v for k, v in data.items()} @pytest.mark.parametrize( ("path", "prefix", "expected"), [ ("a", "", "a"), ("a/b/c", "a/b", "c"), ("a/b/c", "a", "b/c"), ], ) def test_relativize_path_valid(path: str, prefix: str, expected: str) -> None: """ Test the normal behavior of the _relativize_path function. Prefixes should be removed from the path argument. """ assert _relativize_path(path=path, prefix=prefix) == expected def test_relativize_path_invalid() -> None: path = "a/b/c" prefix = "b" msg = f"The first component of {path} does not start with {prefix}." with pytest.raises(ValueError, match=msg): _relativize_path(path="a/b/c", prefix="b") def test_different_open_mode(tmp_path: Path) -> None: # Test with a store that implements .with_read_only() store = MemoryStore() zarr.create((100,), store=store, zarr_format=2, path="a") arr = zarr.open_array(store=store, path="a", zarr_format=2, mode="r") assert arr.store.read_only # Test with a store that doesn't implement .with_read_only() zarr_path = tmp_path / "foo.zarr" zip_store = ZipStore(zarr_path, mode="w") zarr.create((100,), store=zip_store, zarr_format=2, path="a") with pytest.raises( ValueError, match="Store is not read-only but mode is 'r'. Unable to create a read-only copy of the store. Please use a read-only store or a storage class that implements .with_read_only().", ): zarr.open_array(store=zip_store, path="a", zarr_format=2, mode="r") def test_storage_module_pickles_by_name() -> None: """ Pickling `zarr.storage` re-imports the module, so objects that refer to it can be serialized, for example by Dask. See issue #4029. """ assert pickle.loads(pickle.dumps(zarr.storage)) is zarr.storage zarr-developers-zarr-python-a2f2415/tests/test_store/test_fsspec.py000066400000000000000000000735351527014137600257330ustar00rootroot00000000000000from __future__ import annotations import functools import http.server import json import re import threading import warnings from typing import TYPE_CHECKING, Any import numpy as np import pytest from packaging.version import parse as parse_version import zarr.api.asynchronous from zarr import Array from zarr.abc.store import OffsetByteRequest from zarr.core.buffer import Buffer, cpu, default_buffer_prototype from zarr.core.sync import _collect_aiterator, sync from zarr.errors import ZarrUserWarning from zarr.storage import FsspecStore from zarr.storage._common import make_store from zarr.storage._fsspec import _make_async from zarr.testing.store import StoreTests if TYPE_CHECKING: import pathlib from collections.abc import Generator from pathlib import Path import botocore.client import s3fs from zarr.core.common import JSON # Warning filter due to https://github.com/boto/boto3/issues/3889 pytestmark = [ pytest.mark.filterwarnings( re.escape("ignore:datetime.datetime.utcnow() is deprecated:DeprecationWarning") ), # FsspecStore.from_url() and from_mapper() now close the aiohttp session on store.close(). # This filter covers stores that are GC'd without an explicit close() call, and any # residual sessions from aiobotocore's ClientCreatorContext (a separate upstream issue). pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning"), pytest.mark.filterwarnings( "ignore:coroutine 'ClientCreatorContext.__aexit__' was never awaited:RuntimeWarning" ), # s3fs finalizers can fail when sessions are garbage collected without being entered pytest.mark.filterwarnings( "ignore:Exception ignored in.*finalize object.*:pytest.PytestUnraisableExceptionWarning" ), ] fsspec = pytest.importorskip("fsspec") s3fs = pytest.importorskip("s3fs") requests = pytest.importorskip("requests") # Skip this module entirely when moto is absent; the server itself comes from the shared # `moto_server` fixture in tests/conftest.py. pytest.importorskip("moto") botocore = pytest.importorskip("botocore") # ### amended from s3fs ### # test_bucket_name = "test" secure_bucket_name = "test-secure" @pytest.fixture def endpoint_url(moto_server: str) -> str: """Endpoint of the shared session-scoped moto server (see tests/conftest.py). A fixture rather than a module-level constant because the server binds an ephemeral port, so the endpoint is only known once the server is running.""" return moto_server def get_boto3_client(endpoint_url: str) -> botocore.client.BaseClient: # NB: we use the sync botocore client for setup session = botocore.session.Session() # Prevent IllegalLocationConstraintException by explicitly setting region to # "us-east-1", which does not require configuring LocationConstraint during # bucket creation. (It is, in fact, forbidden for that region.) Necessary # in the face of "ambient" AWS configuration in a development environment # where the default region might be configured differently. return session.create_client("s3", endpoint_url=endpoint_url, region_name="us-east-1") @pytest.fixture(autouse=True) def s3(endpoint_url: str) -> Generator[s3fs.S3FileSystem, None, None]: """ Quoting Martin Durant: pytest-asyncio creates a new event loop for each async test. When an async-mode s3fs instance is made from async, it will be assigned to the loop from which it is made. That means that if you use s3fs again from a subsequent test, you will have the same identical instance, but be running on a different loop - which fails. For the rest: it's very convenient to clean up the state of the store between tests, make sure we start off blank each time. https://github.com/zarr-developers/zarr-python/pull/1785#discussion_r1634856207 """ client = get_boto3_client(endpoint_url) client.create_bucket(Bucket=test_bucket_name, ACL="public-read") s3fs.S3FileSystem.clear_instance_cache() s3 = s3fs.S3FileSystem( anon=False, client_kwargs={"endpoint_url": endpoint_url}, # Prevent "AssertionError: Session was never entered" from aiobotocore # at end of test execution. Using clear_instance_cache is insufficient, # although still necessary. skip_instance_cache=True, ) session = sync(s3.set_session()) s3.invalidate_cache() yield s3 requests.post(f"{endpoint_url}/moto-api/reset") client.close() sync(session.close()) # ### end from s3fs ### # async def test_basic(endpoint_url: str) -> None: store = FsspecStore.from_url( f"s3://{test_bucket_name}/foo/spam/", storage_options={"endpoint_url": endpoint_url, "anon": False}, ) assert store.fs.asynchronous assert store.path == f"{test_bucket_name}/foo/spam" assert await _collect_aiterator(store.list()) == () assert not await store.exists("foo") data = b"hello" await store.set("foo", cpu.Buffer.from_bytes(data)) assert await store.exists("foo") buf = await store.get("foo", prototype=default_buffer_prototype()) assert buf is not None assert buf.to_bytes() == data out = await store.get_partial_values( prototype=default_buffer_prototype(), key_ranges=[("foo", OffsetByteRequest(1))] ) assert out[0] is not None assert out[0].to_bytes() == data[1:] class TestFsspecStoreS3(StoreTests[FsspecStore, cpu.Buffer]): store_cls = FsspecStore buffer_cls = cpu.Buffer @pytest.fixture def store_kwargs(self, endpoint_url: str) -> dict[str, str | bool]: try: from fsspec import url_to_fs except ImportError: # before fsspec==2024.3.1 from fsspec.core import url_to_fs fs, path = url_to_fs( f"s3://{test_bucket_name}", endpoint_url=endpoint_url, anon=False, asynchronous=True ) return {"fs": fs, "path": path} @pytest.fixture async def store(self, store_kwargs: dict[str, Any]) -> FsspecStore: return self.store_cls(**store_kwargs) async def get(self, store: FsspecStore, key: str) -> Buffer: # make a new, synchronous instance of the filesystem because this test is run in sync code new_fs = fsspec.filesystem( "s3", endpoint_url=store.fs.endpoint_url, anon=store.fs.anon, asynchronous=False ) return self.buffer_cls.from_bytes(new_fs.cat(f"{store.path}/{key}")) async def set(self, store: FsspecStore, key: str, value: Buffer) -> None: # make a new, synchronous instance of the filesystem because this test is run in sync code new_fs = fsspec.filesystem( "s3", endpoint_url=store.fs.endpoint_url, anon=store.fs.anon, asynchronous=False ) new_fs.write_bytes(f"{store.path}/{key}", value.to_bytes()) def test_store_repr(self, store: FsspecStore) -> None: assert str(store) == "" def test_store_supports_writes(self, store: FsspecStore) -> None: assert store.supports_writes def test_store_supports_listing(self, store: FsspecStore) -> None: assert store.supports_listing async def test_fsspec_store_from_uri(self, store: FsspecStore, endpoint_url: str) -> None: storage_options = { "endpoint_url": endpoint_url, "anon": False, } meta: dict[str, JSON] = { "attributes": {"key": "value"}, "zarr_format": 3, "node_type": "group", } await store.set( "zarr.json", self.buffer_cls.from_bytes(json.dumps(meta).encode()), ) group = await zarr.api.asynchronous.open_group( store=f"s3://{test_bucket_name}", storage_options=storage_options ) assert dict(group.attrs) == {"key": "value"} meta = { "attributes": {"key": "value-2"}, "zarr_format": 3, "node_type": "group", } await store.set( "directory-2/zarr.json", self.buffer_cls.from_bytes(json.dumps(meta).encode()), ) group = await zarr.api.asynchronous.open_group( store=f"s3://{test_bucket_name}/directory-2", storage_options=storage_options ) assert dict(group.attrs) == {"key": "value-2"} meta = { "attributes": {"key": "value-3"}, "zarr_format": 3, "node_type": "group", } await store.set( "directory-3/zarr.json", self.buffer_cls.from_bytes(json.dumps(meta).encode()), ) group = await zarr.api.asynchronous.open_group( store=f"s3://{test_bucket_name}", path="directory-3", storage_options=storage_options ) assert dict(group.attrs) == {"key": "value-3"} @pytest.mark.skipif( parse_version(fsspec.__version__) < parse_version("2024.03.01"), reason="Prior bug in from_upath", ) def test_from_upath(self, endpoint_url: str) -> None: upath = pytest.importorskip("upath") path = upath.UPath( f"s3://{test_bucket_name}/foo/bar/", endpoint_url=endpoint_url, anon=False, asynchronous=True, ) result = FsspecStore.from_upath(path) assert result.fs.endpoint_url == endpoint_url assert result.fs.asynchronous assert result.path == f"{test_bucket_name}/foo/bar" @pytest.mark.skipif( parse_version(fsspec.__version__) < parse_version("2024.03.01"), reason="Prior bug in from_upath", ) def test_from_upath_sync_filesystem(self, endpoint_url: str) -> None: """ A UPath built without ``asynchronous=True`` -- the common case -- yields an async-mode filesystem that keeps the original storage options. """ upath = pytest.importorskip("upath") path = upath.UPath( f"s3://{test_bucket_name}/foo/bar/", endpoint_url=endpoint_url, anon=False, ) assert not path.fs.asynchronous with warnings.catch_warnings(): warnings.simplefilter("error", ZarrUserWarning) result = FsspecStore.from_upath(path) assert result.fs.asynchronous assert result.fs.endpoint_url == endpoint_url assert result.path == f"{test_bucket_name}/foo/bar" async def test_open_group_from_upath(self, endpoint_url: str) -> None: """ Passing a remote UPath to the top-level API works. Regression test for https://github.com/zarr-developers/zarr-python/issues/4244. """ upath = pytest.importorskip("upath") path = upath.UPath( f"s3://{test_bucket_name}/upath-group", endpoint_url=endpoint_url, anon=False, ) group = await zarr.api.asynchronous.open_group(path, mode="w", attributes={"key": "value"}) assert isinstance(group.store_path.store, FsspecStore) reopened = await zarr.api.asynchronous.open_group(path, mode="r") assert dict(reopened.attrs) == {"key": "value"} def test_init_warns_if_fs_asynchronous_is_false(self, endpoint_url: str) -> None: try: from fsspec import url_to_fs except ImportError: # before fsspec==2024.3.1 from fsspec.core import url_to_fs fs, path = url_to_fs( f"s3://{test_bucket_name}", endpoint_url=endpoint_url, anon=False, asynchronous=False ) store_kwargs = {"fs": fs, "path": path} with pytest.warns(ZarrUserWarning, match=r".* was not created with `asynchronous=True`.*"): self.store_cls(**store_kwargs) async def test_empty_nonexistent_path(self, store_kwargs: dict[str, Any]) -> None: # regression test for https://github.com/zarr-developers/zarr-python/pull/2343 store_kwargs["path"] += "/abc" store = await self.store_cls.open(**store_kwargs) assert await store.is_empty("") async def test_delete_dir_unsupported_deletes(self, store: FsspecStore) -> None: store.supports_deletes = False with pytest.raises( NotImplementedError, match="This method is only available for stores that support deletes.", ): await store.delete_dir("test_prefix") # ── Filesystem lifecycle ────────────────────────────────────────────────── async def test_close_marks_store_closed(self, endpoint_url: str) -> None: """close() must succeed and mark the store not-open.""" store = FsspecStore.from_url( f"s3://{test_bucket_name}/lifecycle/", storage_options={"endpoint_url": endpoint_url, "anon": False}, ) await store.set("probe", cpu.Buffer.from_bytes(b"x")) store.close() assert not store._is_open def array_roundtrip(store: FsspecStore) -> None: """ Round trip an array using a Zarr store Args: store: FsspecStore """ data = np.ones((3, 3)) arr = zarr.create_array(store=store, overwrite=True, data=data) assert isinstance(arr, Array) # Read set values arr2 = zarr.open_array(store=store) assert isinstance(arr2, Array) np.testing.assert_array_equal(arr[:], data) @pytest.mark.parametrize( ("root", "key", "expected"), [ # `"/"` as root collapses so that bare-key backends (notably # ReferenceFileSystem) get the right key. Regression test for # https://github.com/zarr-developers/zarr-python/issues/3922 . ("/", "zarr.json", "zarr.json"), ("", "zarr.json", "zarr.json"), # Trailing slashes on the root are stripped before joining. ("foo/", "zarr.json", "foo/zarr.json"), ("foo", "zarr.json", "foo/zarr.json"), # Leading slashes on the root are preserved -- absolute filesystem # paths must stay absolute. Regression test for the titiler-xarray # breakage that #3924 introduced when `normalize_path` was applied to # `FsspecStore.path`. ("/home/runner/data.zarr", "zarr.json", "/home/runner/data.zarr/zarr.json"), ("/home/runner/data.zarr/", "zarr.json", "/home/runner/data.zarr/zarr.json"), # Multi-segment keys. ("/home/foo", "a/b/zarr.json", "/home/foo/a/b/zarr.json"), ("", "a/b/zarr.json", "a/b/zarr.json"), # Trailing slash on the result is stripped (relevant when key is ""). ("/home/foo", "", "/home/foo"), ], ) def test_dereference_path(root: str, key: str, expected: str) -> None: """Verify the contract `_dereference_path` provides for `FsspecStore`. `FsspecStore.path` is stored verbatim; the join with a key must collapse a sentinel `"/"` root, strip trailing slashes, and preserve leading slashes on absolute paths. """ from zarr.storage._utils import _dereference_path assert _dereference_path(root, key) == expected async def test_fsspec_store_open_group_via_reference_filesystem() -> None: """End-to-end regression test for https://github.com/zarr-developers/zarr-python/issues/3922 . ``ReferenceFileSystem`` keys its refs by bare strings like ``"zarr.json"``. The bug was that ``FsspecStore(fs=ref_fs, path="/")`` produced ``"//zarr.json"`` at the join site and failed to find the entry, raising ``GroupNotFoundError``. This test pins ``path="/"`` explicitly to keep coverage even if the default value changes later. """ import json from fsspec.implementations.reference import ReferenceFileSystem group_json = json.dumps({"zarr_format": 3, "node_type": "group", "attributes": {}}) fs = ReferenceFileSystem( fo={"version": 1, "refs": {"zarr.json": group_json}}, asynchronous=True, ) store = FsspecStore(fs=fs, path="/", read_only=True) group = await zarr.api.asynchronous.open_group(store, mode="r") assert group.metadata.zarr_format == 3 async def test_fsspec_store_read_array_chunk_via_reference_filesystem() -> None: """End-to-end regression test that exercises the byte-range read path against ``ReferenceFileSystem``. Beyond opening a group (covered by ``test_fsspec_store_open_group_via_reference_filesystem``), this test constructs a small zarr v3 array whose chunk lives in the refs dict and reads it through the store. Path-handling bugs on the byte-range fetch path (used by kerchunk-style virtualization) would surface here rather than at metadata-open time. """ import json import numpy as np from fsspec.implementations.reference import ReferenceFileSystem # Construct a minimal v3 zarr: a single 1-D uint8 array of length 4 with # one chunk of size 4. The chunk bytes are little-endian uint8s 1..4. array_meta = json.dumps( { "zarr_format": 3, "node_type": "array", "shape": [4], "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, "data_type": "uint8", "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, "fill_value": 0, "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], "attributes": {}, } ) chunk_bytes = bytes([1, 2, 3, 4]) refs: dict[str, str] = { "zarr.json": array_meta, # ReferenceFileSystem accepts raw bytes via base64 encoding or # latin-1-decoded strings; latin-1 round-trips bytes 1:1. "c/0": chunk_bytes.decode("latin-1"), } fs = ReferenceFileSystem( fo={"version": 1, "refs": refs}, asynchronous=True, ) store = FsspecStore(fs=fs, path="/", read_only=True) array = await zarr.api.asynchronous.open_array(store=store, mode="r") data = await array.getitem(slice(None)) np.testing.assert_array_equal(data, np.array([1, 2, 3, 4], dtype="uint8")) @pytest.mark.parametrize("path", ["", "/"]) async def test_fsspec_store_list_nested_via_reference_filesystem(path: str) -> None: """Listing below the root of a ``ReferenceFileSystem``-backed store. ``FsspecStore.from_url("reference://", ...)`` gives the store a path of ``""``, and the tests above use ``"/"``. Listing a nested prefix used to ask the filesystem for ``"/a"`` or ``"//a"``, which ``ReferenceFileSystem`` does not find, so nested groups listed as empty. """ import json from fsspec.implementations.reference import ReferenceFileSystem group_json = json.dumps({"zarr_format": 3, "node_type": "group", "attributes": {}}) refs = {"zarr.json": group_json, "a/zarr.json": group_json, "a/b/zarr.json": group_json} fs = ReferenceFileSystem(fo={"version": 1, "refs": refs}, asynchronous=True) store = FsspecStore(fs=fs, path=path, read_only=True) assert sorted(await _collect_aiterator(store.list_dir(""))) == ["a", "zarr.json"] assert sorted(await _collect_aiterator(store.list_dir("a"))) == ["b", "zarr.json"] assert sorted(await _collect_aiterator(store.list_prefix("a/"))) == [ "a/b/zarr.json", "a/zarr.json", ] group = await zarr.api.asynchronous.open_group(store, mode="r") subgroup = await group.getitem("a") assert [name async for name in subgroup.group_keys()] == ["b"] @pytest.mark.skipif( parse_version(fsspec.__version__) < parse_version("2024.12.0"), reason="No AsyncFileSystemWrapper", ) def test_wrap_sync_filesystem(tmp_path: pathlib.Path) -> None: """The local fs is not async so we should expect it to be wrapped automatically""" from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper store = FsspecStore.from_url(f"file://{tmp_path}", storage_options={"auto_mkdir": True}) assert isinstance(store.fs, AsyncFileSystemWrapper) assert store.fs.async_impl array_roundtrip(store) @pytest.mark.skipif( parse_version(fsspec.__version__) >= parse_version("2024.12.0"), reason="No AsyncFileSystemWrapper", ) def test_wrap_sync_filesystem_raises(tmp_path: pathlib.Path) -> None: """The local fs is not async so we should expect it to be wrapped automatically""" with pytest.raises(ImportError, match="The filesystem .*"): FsspecStore.from_url(f"file://{tmp_path}", storage_options={"auto_mkdir": True}) @pytest.mark.skipif( parse_version(fsspec.__version__) < parse_version("2024.12.0"), reason="No AsyncFileSystemWrapper", ) def test_no_wrap_async_filesystem(endpoint_url: str) -> None: """An async fs should not be wrapped automatically; fsspec's s3 filesystem is such an fs""" from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper store = FsspecStore.from_url( f"s3://{test_bucket_name}/foo/spam/", storage_options={"endpoint_url": endpoint_url, "anon": False, "asynchronous": True}, read_only=False, ) assert not isinstance(store.fs, AsyncFileSystemWrapper) assert store.fs.async_impl array_roundtrip(store) @pytest.mark.skipif( parse_version(fsspec.__version__) < parse_version("2024.12.0"), reason="No AsyncFileSystemWrapper", ) def test_open_fsmap_file(tmp_path: pathlib.Path) -> None: min_fsspec_with_async_wrapper = parse_version("2024.12.0") current_version = parse_version(fsspec.__version__) fs = fsspec.filesystem("file", auto_mkdir=True) mapper = fs.get_mapper(tmp_path) if current_version < min_fsspec_with_async_wrapper: # Expect ImportError for older versions with pytest.raises( ImportError, match=r"The filesystem .* is synchronous, and the required AsyncFileSystemWrapper is not available.*", ): array_roundtrip(mapper) else: # Newer versions should work array_roundtrip(mapper) @pytest.mark.skipif( parse_version(fsspec.__version__) < parse_version("2024.12.0"), reason="No AsyncFileSystemWrapper", ) def test_open_fsmap_file_raises(tmp_path: pathlib.Path) -> None: fsspec = pytest.importorskip("fsspec.implementations.local") fs = fsspec.LocalFileSystem(auto_mkdir=False) mapper = fs.get_mapper(tmp_path) with pytest.raises(FileNotFoundError, match="No such file or directory: .*"): array_roundtrip(mapper) @pytest.mark.parametrize("asynchronous", [True, False]) def test_open_fsmap_s3(asynchronous: bool, endpoint_url: str) -> None: s3_filesystem = s3fs.S3FileSystem( asynchronous=asynchronous, endpoint_url=endpoint_url, anon=False ) mapper = s3_filesystem.get_mapper(f"s3://{test_bucket_name}/map/foo/") array_roundtrip(mapper) def test_open_s3map_raises(endpoint_url: str) -> None: with pytest.raises(TypeError, match="Unsupported type for store_like:.*"): zarr.open(store=0, mode="w", shape=(3, 3)) s3_filesystem = s3fs.S3FileSystem(asynchronous=True, endpoint_url=endpoint_url, anon=False) mapper = s3_filesystem.get_mapper(f"s3://{test_bucket_name}/map/foo/") with pytest.raises( ValueError, match="'path' was provided but is not used for FSMap store_like objects" ): zarr.open(store=mapper, path="bar", mode="w", shape=(3, 3)) with pytest.raises( TypeError, match="'storage_options' is only used when the store is passed as an FSSpec URI string.", ): zarr.open(store=mapper, storage_options={"anon": True}, mode="w", shape=(3, 3)) async def test_list_dir_http_yields_only_children(tmp_path: pathlib.Path) -> None: """list_dir over HTTP yields each direct child once, by bare name. An HTTP listing is scraped from an HTML index page, whose links include the site root, the parent, in-page anchors, queries, "./" relative links, and directories with a trailing "/". Regression test for https://github.com/zarr-developers/zarr-python/issues/3575, where the site-root link surfaced as a group member named "". """ pytest.importorskip("aiohttp") links = [ "/", "../", "./", "#", "#usage", "?sort=name", "a/", "a", "b", "./d", ".zgroup", "/group/c/", ] group = tmp_path / "group" group.mkdir() (group / "index.html").write_text( "" + "".join(f'{link}' for link in links) + "" ) class QuietHandler(http.server.SimpleHTTPRequestHandler): def log_message(self, format: str, *args: Any) -> None: pass handler = functools.partial(QuietHandler, directory=str(tmp_path)) with http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) as server: threading.Thread(target=server.serve_forever, daemon=True).start() store = FsspecStore.from_url(f"http://127.0.0.1:{server.server_port}/group") try: observed = await _collect_aiterator(store.list_dir("")) finally: store.close() server.shutdown() assert sorted(observed) == [".zgroup", "a", "b", "c", "d"] async def test_close_does_not_close_filesystem_session() -> None: """close() must not touch the filesystem's session. fsspec caches and shares filesystem instances across callers, so the session is not the store's to close. HTTP is used because its aiohttp session is observably closed for good; s3fs transparently reconnects, which would hide a regression. No request is issued — set_session() only constructs the session. """ pytest.importorskip("aiohttp") store = FsspecStore.from_url("http://example.com/a") session = await store.fs.set_session() store.close() assert not session.closed async def test_close_does_not_break_a_sibling_store() -> None: """Closing one store must not close a session another store is using. Two stores from different URLs on one host are handed the same cached filesystem; a store that closed it on close() would take the sibling's session down too. This is the regression guard for that bug. """ pytest.importorskip("aiohttp") s1 = FsspecStore.from_url("http://example.com/a") s2 = FsspecStore.from_url("http://example.com/b") session = await s2.fs.set_session() s1.close() assert not session.closed @pytest.mark.skipif( parse_version(fsspec.__version__) < parse_version("2024.12.0"), reason="No AsyncFileSystemWrapper", ) def test_from_mapper_wraps_sync_filesystem(tmp_path: pathlib.Path) -> None: """from_mapper() with a sync fs wraps it in an AsyncFileSystemWrapper.""" import fsspec as _fsspec from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper fs = _fsspec.filesystem("file", auto_mkdir=True) mapper = fs.get_mapper(str(tmp_path)) store = FsspecStore.from_mapper(mapper) assert isinstance(store.fs, AsyncFileSystemWrapper) @pytest.mark.skipif( parse_version(fsspec.__version__) < parse_version("2024.12.0"), reason="No AsyncFileSystemWrapper", ) def test_with_read_only_shares_filesystem(tmp_path: pathlib.Path) -> None: """with_read_only() returns a store sharing the source's filesystem.""" source = FsspecStore.from_url(f"file://{tmp_path}", storage_options={"auto_mkdir": False}) derived = source.with_read_only(read_only=True) assert derived.fs is source.fs assert derived.read_only assert not source.read_only def test_make_async_preserves_unserializable_storage_options() -> None: """A sync instance of an async filesystem whose storage options hold objects that cannot round-trip through JSON (e.g. an Azure credential) must still convert. See https://github.com/zarr-developers/zarr-python/issues/4220 """ pytest.importorskip("aiohttp") credential = object() # stand-in for e.g. azure.identity.DefaultAzureCredential sync_fs = fsspec.filesystem("http", client_kwargs={"auth": credential}) assert sync_fs.async_impl assert not sync_fs.asynchronous async_fs = _make_async(sync_fs) assert async_fs.asynchronous assert async_fs.client_kwargs["auth"] is credential @pytest.mark.parametrize("asynchronous", [True, False]) def test_make_async(asynchronous: bool, endpoint_url: str) -> None: s3_filesystem = s3fs.S3FileSystem( asynchronous=asynchronous, endpoint_url=endpoint_url, anon=False ) fs = _make_async(s3_filesystem) assert fs.asynchronous @pytest.mark.skipif( parse_version(fsspec.__version__) < parse_version("2024.12.0"), reason="No AsyncFileSystemWrapper", ) async def test_delete_dir_wrapped_filesystem(tmp_path: Path) -> None: from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper from fsspec.implementations.local import LocalFileSystem wrapped_fs = AsyncFileSystemWrapper(LocalFileSystem(auto_mkdir=True)) store = FsspecStore(wrapped_fs, read_only=False, path=f"{tmp_path}/test/path") assert isinstance(store.fs, AsyncFileSystemWrapper) assert store.fs.asynchronous await store.set("zarr.json", cpu.Buffer.from_bytes(b"root")) await store.set("foo-bar/zarr.json", cpu.Buffer.from_bytes(b"root")) await store.set("foo/zarr.json", cpu.Buffer.from_bytes(b"bar")) await store.set("foo/c/0", cpu.Buffer.from_bytes(b"chunk")) await store.delete_dir("foo") assert await store.exists("zarr.json") assert await store.exists("foo-bar/zarr.json") assert not await store.exists("foo/zarr.json") assert not await store.exists("foo/c/0") @pytest.mark.skipif( parse_version(fsspec.__version__) < parse_version("2024.12.0"), reason="No AsyncFileSystemWrapper", ) async def test_with_read_only_auto_mkdir(tmp_path: Path) -> None: """ Test that creating a read-only copy of a store backed by the local file system does not error if auto_mkdir is False. """ store_w = FsspecStore.from_url(f"file://{tmp_path}", storage_options={"auto_mkdir": False}) _ = store_w.with_read_only() @pytest.mark.skipif( parse_version(fsspec.__version__) < parse_version("2024.12.0"), reason="No AsyncFileSystemWrapper", ) async def test_memory_scheme() -> None: """Test that the "memory" scheme creates a `MemoryFileSystem`-backed store""" store = await make_store("memory://test") assert isinstance(store, FsspecStore) assert store.fs.protocol == "memory" zarr-developers-zarr-python-a2f2415/tests/test_store/test_fsspec_get_ranges.py000066400000000000000000000105411527014137600301150ustar00rootroot00000000000000# tests/test_store/test_fsspec_get_ranges.py """Lightweight integration tests for FsspecStore.get_ranges using MemoryFileSystem. These don't need moto/s3 — they exercise the new method against an in-process fsspec MemoryFileSystem wrapped in the async wrapper. """ from __future__ import annotations import pytest from packaging.version import parse as parse_version from zarr.abc.store import RangeByteRequest from zarr.core.buffer import Buffer, default_buffer_prototype from zarr.storage import FsspecStore from zarr.storage._fsspec import _make_async fsspec = pytest.importorskip("fsspec") # AsyncFileSystemWrapper (needed to wrap a sync MemoryFileSystem) landed in fsspec 2024.12.0. # Older versions are pinned by the min-deps CI job, so skip the whole file there. pytestmark = pytest.mark.skipif( parse_version(fsspec.__version__) < parse_version("2024.12.0"), reason="No AsyncFileSystemWrapper", ) @pytest.fixture def memory_store() -> FsspecStore: """An FsspecStore backed by fsspec MemoryFileSystem (wrapped async).""" from fsspec.implementations.memory import MemoryFileSystem # Each test gets a clean filesystem; MemoryFileSystem is a singleton per target_options, # so clear state explicitly. fs: MemoryFileSystem = MemoryFileSystem() fs.store.clear() fs.pseudo_dirs.clear() async_fs = _make_async(fs) return FsspecStore(fs=async_fs, path="/root") async def _write(store: FsspecStore, key: str, data: bytes) -> None: buf = default_buffer_prototype().buffer.from_bytes(data) await store.set(key, buf) async def test_get_ranges_happy_path(memory_store: FsspecStore) -> None: blob = bytes(i % 256 for i in range(1024)) await _write(memory_store, "blob", blob) proto = default_buffer_prototype() ranges = [ RangeByteRequest(0, 10), RangeByteRequest(100, 110), RangeByteRequest(500, 520), ] groups: list[list[tuple[int, Buffer | None]]] = [ list(group) async for group in memory_store.get_ranges("blob", ranges, prototype=proto) ] flat: dict[int, bytes] = {} for group in groups: for idx, buf in group: assert buf is not None flat[idx] = buf.to_bytes() assert flat[0] == blob[0:10] assert flat[1] == blob[100:110] assert flat[2] == blob[500:520] async def test_get_ranges_missing_key_raises(memory_store: FsspecStore) -> None: """A request against a missing key raises BaseExceptionGroup containing FileNotFoundError.""" proto = default_buffer_prototype() agen = memory_store.get_ranges("does-not-exist", [RangeByteRequest(0, 10)], prototype=proto) with pytest.RaisesGroup(pytest.RaisesExc(FileNotFoundError)): await anext(agen) async def test_get_ranges_forwards_coalescing_kwargs(memory_store: FsspecStore) -> None: """`max_gap_bytes=-1` forces no merging; we should see three groups for three ranges.""" blob = bytes(i % 256 for i in range(1024)) await _write(memory_store, "blob", blob) proto = default_buffer_prototype() ranges = [ RangeByteRequest(0, 10), RangeByteRequest(11, 20), # adjacent: would merge under defaults RangeByteRequest(21, 30), ] groups: list[list[tuple[int, Buffer | None]]] = [ list(group) async for group in memory_store.get_ranges( "blob", ranges, prototype=proto, max_gap_bytes=-1 ) ] # With merging disabled, every range becomes its own one-tuple group. assert sorted(len(g) for g in groups) == [1, 1, 1] async def test_get_ranges_mixed_range_types(memory_store: FsspecStore) -> None: """Covers RangeByteRequest, OffsetByteRequest, SuffixByteRequest, and None in one call.""" from zarr.abc.store import ByteRequest, OffsetByteRequest, SuffixByteRequest blob = bytes(i % 256 for i in range(512)) await _write(memory_store, "mixed", blob) proto = default_buffer_prototype() ranges: list[ByteRequest | None] = [ RangeByteRequest(0, 10), OffsetByteRequest(500), SuffixByteRequest(12), None, ] flat: dict[int, bytes] = {} async for group in memory_store.get_ranges("mixed", ranges, prototype=proto): for idx, buf in group: assert buf is not None flat[idx] = buf.to_bytes() assert flat[0] == blob[0:10] assert flat[1] == blob[500:] assert flat[2] == blob[-12:] assert flat[3] == blob zarr-developers-zarr-python-a2f2415/tests/test_store/test_get_ranges.py000066400000000000000000000152441527014137600265570ustar00rootroot00000000000000# tests/test_store/test_get_ranges.py """Tests for `Store.get_ranges` — the ABC default implementation and wrapper delegation. `Store.get_ranges` is defined on the ABC with a default implementation built on `coalesced_get(self.get, ...)`, so every store inherits a working version. These tests cover that inherited path and the explicit delegation in `WrapperStore` (which ensures wrapped stores' optimized overrides are honored). Store-specific overrides (e.g. `FsspecStore`) have their own test modules. """ from __future__ import annotations from typing import TYPE_CHECKING import pytest from zarr.abc.store import RangeByteRequest from zarr.core.buffer import default_buffer_prototype from zarr.storage import MemoryStore, ZipStore from zarr.storage._wrapper import WrapperStore if TYPE_CHECKING: from collections.abc import AsyncIterator, Sequence from pathlib import Path from zarr.abc.store import ByteRequest from zarr.core.buffer import Buffer, BufferPrototype async def _write(store: MemoryStore, key: str, data: bytes) -> None: buf = default_buffer_prototype().buffer.from_bytes(data) await store.set(key, buf) async def test_memory_store_inherits_get_ranges_from_abc() -> None: """MemoryStore doesn't override `get_ranges`; the ABC default must work end-to-end.""" store = MemoryStore() blob = bytes(i % 256 for i in range(512)) await _write(store, "blob", blob) ranges = [RangeByteRequest(0, 10), RangeByteRequest(100, 110)] proto = default_buffer_prototype() flat: dict[int, bytes] = {} async for group in store.get_ranges("blob", ranges, prototype=proto): for idx, buf in group: assert buf is not None flat[idx] = buf.to_bytes() assert flat[0] == blob[0:10] assert flat[1] == blob[100:110] async def test_memory_store_get_ranges_missing_key_raises() -> None: """A missing key on a default-impl store raises BaseExceptionGroup containing FileNotFoundError.""" store = MemoryStore() proto = default_buffer_prototype() agen = store.get_ranges("does-not-exist", [RangeByteRequest(0, 10)], prototype=proto) with pytest.RaisesGroup(pytest.RaisesExc(FileNotFoundError)): await anext(agen) def test_get_ranges_sync_reads_multiple_ranges() -> None: """The synchronous `get_ranges_sync` on a sync-capable store returns each requested range, mirroring the async `get_ranges` happy path.""" import asyncio store = MemoryStore() blob = bytes(i % 256 for i in range(512)) asyncio.run(_write(store, "blob", blob)) ranges = [RangeByteRequest(0, 10), RangeByteRequest(100, 110)] proto = default_buffer_prototype() flat: dict[int, bytes] = {} for idx, buf in store.get_ranges_sync("blob", ranges, prototype=proto): assert buf is not None flat[idx] = buf.to_bytes() assert flat[0] == blob[0:10] assert flat[1] == blob[100:110] def test_get_ranges_sync_missing_key_raises() -> None: """A missing key makes `get_ranges_sync` raise a BaseExceptionGroup containing FileNotFoundError — the same contract as async `get_ranges`, so callers handle a deleted shard uniformly across sync and async paths.""" store = MemoryStore() proto = default_buffer_prototype() with pytest.RaisesGroup(pytest.RaisesExc(FileNotFoundError)): store.get_ranges_sync("does-not-exist", [RangeByteRequest(0, 10)], prototype=proto) def test_get_ranges_sync_on_non_sync_store_raises_type_error(tmp_path: Path) -> None: """`get_ranges_sync` requires the store to support synchronous reads (`SupportsGetSync`); a non-sync store raises TypeError rather than silently falling back.""" store = ZipStore(tmp_path / "store.zip", mode="w") proto = default_buffer_prototype() with pytest.raises(TypeError, match="does not support synchronous reads"): store.get_ranges_sync("k", [RangeByteRequest(0, 10)], prototype=proto) async def test_wrapper_store_delegates_get_ranges() -> None: """WrapperStore.get_ranges must delegate to the wrapped store, not fall back to the default.""" class CountingMemoryStore(MemoryStore): """Tallies get_ranges invocations so we can assert delegation.""" get_ranges_calls: int = 0 async def get_ranges( self, key: str, byte_ranges: Sequence[ByteRequest | None], *, prototype: BufferPrototype, max_concurrency: int = 10, max_gap_bytes: int = 1 << 20, max_coalesced_bytes: int = 16 << 20, ) -> AsyncIterator[Sequence[tuple[int, Buffer | None]]]: type(self).get_ranges_calls += 1 async for group in super().get_ranges( key, byte_ranges, prototype=prototype, max_concurrency=max_concurrency, max_gap_bytes=max_gap_bytes, max_coalesced_bytes=max_coalesced_bytes, ): yield group inner = CountingMemoryStore() blob = b"x" * 100 await _write(inner, "k", blob) wrapped = WrapperStore(inner) proto = default_buffer_prototype() groups: list[list[tuple[int, Buffer | None]]] = [ list(group) async for group in wrapped.get_ranges("k", [RangeByteRequest(0, 5)], prototype=proto) ] assert CountingMemoryStore.get_ranges_calls == 1 assert len(groups) == 1 assert groups[0][0][0] == 0 async def test_wrapper_store_forwards_coalescing_kwargs() -> None: """Coalescing kwargs flow through WrapperStore to the wrapped store's get_ranges.""" class SpyMemoryStore(MemoryStore): last_max_gap_bytes: int | None = None async def get_ranges( self, key: str, byte_ranges: Sequence[ByteRequest | None], *, prototype: BufferPrototype, max_concurrency: int = 10, max_gap_bytes: int = 1 << 20, max_coalesced_bytes: int = 16 << 20, ) -> AsyncIterator[Sequence[tuple[int, Buffer | None]]]: type(self).last_max_gap_bytes = max_gap_bytes async for group in super().get_ranges( key, byte_ranges, prototype=prototype, max_concurrency=max_concurrency, max_gap_bytes=max_gap_bytes, max_coalesced_bytes=max_coalesced_bytes, ): yield group inner = SpyMemoryStore() await _write(inner, "k", b"y" * 100) wrapped = WrapperStore(inner) proto = default_buffer_prototype() async for _ in wrapped.get_ranges( "k", [RangeByteRequest(0, 5)], prototype=proto, max_gap_bytes=-1 ): pass assert SpyMemoryStore.last_max_gap_bytes == -1 zarr-developers-zarr-python-a2f2415/tests/test_store/test_latency.py000066400000000000000000000146701527014137600261020ustar00rootroot00000000000000from __future__ import annotations import time from unittest.mock import patch import numpy as np import pytest import zarr from zarr.abc.store import RangeByteRequest from zarr.core.buffer import default_buffer_prototype from zarr.core.codec_pipeline import FusedCodecPipeline from zarr.core.config import config as zarr_config from zarr.storage import MemoryStore from zarr.testing.store import LatencyStore async def test_latency_store_with_read_only_round_trip() -> None: """ Ensure that LatencyStore.with_read_only returns another LatencyStore with the requested read_only state, preserves latency configuration, and does not change the original wrapper. """ base = await MemoryStore.open() # Start from a read-only underlying store ro_base = base.with_read_only(read_only=True) latency_ro = LatencyStore(ro_base, get_latency=0.01, set_latency=0.02) assert latency_ro.read_only assert latency_ro.get_latency == pytest.approx(0.01) assert latency_ro.set_latency == pytest.approx(0.02) buf = default_buffer_prototype().buffer.from_bytes(b"abcd") # Cannot write through the read-only wrapper with pytest.raises( ValueError, match="store was opened in read-only mode and does not support writing" ): await latency_ro.set("key", buf) # Create a writable wrapper from the read-only one writer = latency_ro.with_read_only(read_only=False) assert isinstance(writer, LatencyStore) assert not writer.read_only # Latency configuration is preserved assert writer.get_latency == latency_ro.get_latency assert writer.set_latency == latency_ro.set_latency # Writes via the writable wrapper succeed await writer.set("key", buf) out = await writer.get("key", prototype=default_buffer_prototype()) assert out is not None assert out.to_bytes() == buf.to_bytes() # Creating a read-only copy from the writable wrapper works and is enforced reader = writer.with_read_only(read_only=True) assert isinstance(reader, LatencyStore) assert reader.read_only with pytest.raises( ValueError, match="store was opened in read-only mode and does not support writing" ): await reader.set("other", buf) # The original read-only wrapper remains read-only assert latency_ro.read_only @pytest.mark.parametrize( ("get_latency", "set_latency"), [ (0.01, 0.02), ((0.1, 0.05), (0.2, 0.01)), ], ids=["scalar", "distribution"], ) def test_with_store_preserves_latency_config( get_latency: float | tuple[float, float], set_latency: float | tuple[float, float] ) -> None: """Derived stores (e.g. via `with_read_only`) keep the raw latency config — a `(loc, scale)` distribution must not collapse to one sampled float.""" store = LatencyStore(MemoryStore(), get_latency=get_latency, set_latency=set_latency) derived = store.with_read_only(True) assert derived._get_latency == store._get_latency assert derived._set_latency == store._set_latency def test_sync_methods_inject_latency(monkeypatch: pytest.MonkeyPatch) -> None: """`get_sync`/`set_sync` sleep the configured latency on the calling thread before delegating to the wrapped store.""" sleeps: list[float] = [] monkeypatch.setattr(time, "sleep", sleeps.append) store = LatencyStore(MemoryStore(), get_latency=0.123, set_latency=0.456) buf = default_buffer_prototype().buffer.from_bytes(b"abcd") store.set_sync("key", buf) assert sleeps == [pytest.approx(0.456)] out = store.get_sync("key", prototype=default_buffer_prototype()) assert out is not None assert out.to_bytes() == b"abcd" assert sleeps == [pytest.approx(0.456), pytest.approx(0.123)] async def test_get_ranges_pays_latency_per_fetch() -> None: """`get_ranges` routes through the coalescing default built on `self.get`, so each merged fetch pays the configured latency instead of bypassing it via WrapperStore delegation. Two ranges further apart than `max_gap_bytes` cannot coalesce -> exactly two `get` calls.""" proto = default_buffer_prototype() inner = MemoryStore() await inner.set("blob", proto.buffer.from_bytes(bytes(4 << 20))) store = LatencyStore(inner, get_latency=0.0) requests = [RangeByteRequest(0, 10), RangeByteRequest(2 << 20, (2 << 20) + 10)] results: list[tuple[int, object]] = [] with patch.object(store, "get", wraps=store.get) as get_spy: async for group in store.get_ranges("blob", requests, prototype=proto): results.extend(group) assert get_spy.await_count == 2 assert sorted(idx for idx, _ in results) == [0, 1] for _, buf in results: assert buf is not None assert len(buf) == 10 # type: ignore[arg-type] async def test_get_partial_values_routes_through_get() -> None: """`get_partial_values` issues one `self.get` per key-range so each fetch pays the configured latency instead of bypassing it via WrapperStore delegation.""" proto = default_buffer_prototype() inner = MemoryStore() await inner.set("blob", proto.buffer.from_bytes(b"0123456789")) store = LatencyStore(inner, get_latency=0.0) with patch.object(store, "get", wraps=store.get) as get_spy: results = await store.get_partial_values( proto, [("blob", RangeByteRequest(0, 4)), ("blob", None)] ) assert get_spy.await_count == 2 assert results[0] is not None assert results[0].to_bytes() == b"0123" assert results[1] is not None assert results[1].to_bytes() == b"0123456789" def test_latency_store_engages_fused_sync_path() -> None: """A LatencyStore wrapping a sync-capable store must take the fused sync fast path: reads go through the inner store's `get_sync`, not the async fallback.""" inner = MemoryStore() store = LatencyStore(inner, get_latency=0.0, set_latency=0.0) with zarr_config.set({"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"}): arr = zarr.create_array( store=store, shape=(8,), chunks=(4,), dtype="uint8", compressors=None, fill_value=0, ) assert isinstance(arr._async_array.codec_pipeline, FusedCodecPipeline) data = np.arange(8, dtype="uint8") arr[:] = data with patch.object(inner, "get_sync", wraps=inner.get_sync) as get_sync_spy: np.testing.assert_array_equal(arr[:], data) assert get_sync_spy.call_count == 2 # one per chunk zarr-developers-zarr-python-a2f2415/tests/test_store/test_local.py000066400000000000000000000420731527014137600255330ustar00rootroot00000000000000from __future__ import annotations import asyncio import inspect import io import os import pathlib import re import time from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any import numpy as np import pytest import zarr import zarr.storage._local from zarr import create_array from zarr.core.buffer import Buffer, cpu from zarr.storage import LocalStore from zarr.storage._local import _RETRY_DELAYS, _atomic_write, _move_with_retry from zarr.testing.store import StoreTests from zarr.testing.utils import assert_bytes_equal if TYPE_CHECKING: from collections.abc import AsyncIterator, Awaitable, Callable, Iterator _LOCAL_STORE_FILE = zarr.storage._local.__file__ _ASYNC_CODE_FLAGS = inspect.CO_COROUTINE | inspect.CO_ASYNC_GENERATOR try: import coverage as _coverage except ImportError: # pragma: no cover _COVERAGE_DIR = None else: # coverage.py canonicalizes a source file's path (os.path.realpath, hence os.lstat) the # first time its tracer sees code from that file, and it does so on whatever thread is # running that code. Such calls carry coverage's own frames and are not LocalStore's. _COVERAGE_DIR = os.path.dirname(_coverage.__file__) + os.sep # The syscall-level entry points that pathlib / os.path / shutil helpers bottom out in. # Patching these, rather than each Path method, catches a blocking call no matter which # helper made it. The list is deliberately wider than what LocalStore uses today. _FILESYSTEM_CALLS: tuple[tuple[Any, str], ...] = ( (os, "stat"), (os, "lstat"), (os, "access"), (os, "scandir"), (os, "listdir"), (os, "mkdir"), (os, "rmdir"), (os, "unlink"), (os, "remove"), (os, "link"), (os, "rename"), (os, "replace"), (os, "utime"), (os, "open"), (os, "fsync"), (io, "open"), # On Windows these are builtins (nt._path_isfile and friends) that never reach # os.stat, and from Python 3.14 Path.is_file, is_dir and exists call them. (os.path, "isfile"), (os.path, "isdir"), (os.path, "islink"), (os.path, "exists"), (os.path, "lexists"), ) @dataclass class _FilesystemCalls: """What the patched filesystem entry points saw from LocalStore code during one test.""" off_loop: set[tuple[str, str]] = field(default_factory=set) """``(outermost LocalStore function, op)`` pairs made from a thread with no running loop.""" on_loop: list[str] = field(default_factory=list) """Calls made on an event loop's thread from inside a LocalStore coroutine: violations.""" def record(self, op: str) -> None: frame = inspect.currentframe() innermost = outermost = None while frame is not None: if _COVERAGE_DIR is not None and frame.f_code.co_filename.startswith(_COVERAGE_DIR): return # the coverage tracer resolving a filename, not LocalStore doing I/O if frame.f_code.co_filename == _LOCAL_STORE_FILE: if innermost is None: innermost = frame outermost = frame frame = frame.f_back if outermost is None or innermost is None: return # not LocalStore's doing (pytest, tmp_path, the test body, ...) try: asyncio.get_running_loop() except RuntimeError: # A worker thread, such as the one asyncio.to_thread uses: blocking is fine here. self.off_loop.add((outermost.f_code.co_name, op)) return if not outermost.f_code.co_flags & _ASYNC_CODE_FLAGS: return # a synchronous LocalStore method: its caller chose to block the loop site = ( f"LocalStore.{outermost.f_code.co_name} called {op} at _local.py:{innermost.f_lineno}" ) if site not in self.on_loop: self.on_loop.append(site) async def _collect(keys: AsyncIterator[str]) -> list[str]: return [key async for key in keys] class TestLocalStore(StoreTests[LocalStore, cpu.Buffer]): store_cls = LocalStore buffer_cls = cpu.Buffer async def test_delete_dir_on_a_file_raises(self, tmp_path: pathlib.Path) -> None: """`delete_dir` refuses a prefix that names a file rather than a directory.""" store = await LocalStore.open(tmp_path) await store.set("file", self.buffer_cls.from_bytes(b"x")) with pytest.raises(ValueError, match="that is a file"): await store.delete_dir("file") assert await store.exists("file") @pytest.fixture(autouse=True) def filesystem_calls(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[_FilesystemCalls]: """Fail if any LocalStore coroutine does filesystem I/O on the event loop thread. Every async method must hand its filesystem work to ``asyncio.to_thread``; doing it inline stalls every other task sharing the loop. """ calls = _FilesystemCalls() def patch(module: Any, name: str) -> None: original = getattr(module, name) def wrapper(*args: Any, **kwargs: Any) -> Any: calls.record(f"{module.__name__}.{name}") return original(*args, **kwargs) monkeypatch.setattr(module, name, wrapper) for module, name in _FILESYSTEM_CALLS: patch(module, name) yield calls assert not calls.on_loop, "filesystem calls on the event loop thread:\n" + "\n".join( calls.on_loop ) @pytest.mark.parametrize( ("method", "helper"), [ ("open", "_ensure_root"), ("lazy_open", "_ensure_root"), ("get", "_get"), ("get_partial_values", "_get"), ("set", "_put"), ("set_if_not_exists", "_put"), ("exists", "_exists"), ("getsize", "_getsize"), ("delete", "_delete"), ("delete_dir", "_delete_dir"), ("list", "_list_files"), ("list_prefix", "_list_files"), ("list_dir", "_list_dir"), ("clear", "_clear"), ("move", "_move"), ], ) async def test_filesystem_calls_are_observed( self, store: LocalStore, filesystem_calls: _FilesystemCalls, tmp_path_factory: pytest.TempPathFactory, method: str, helper: str, ) -> None: """The detector must see each async method's I/O, or its silence means nothing.""" data = self.buffer_cls.from_bytes(b"x") await self.set(store, "a/b", data) ops: dict[str, Callable[[], Awaitable[object]]] = { "open": lambda: LocalStore.open(store.root), "lazy_open": lambda: LocalStore(store.root).get("a/b"), "get": lambda: store.get("a/b"), "get_partial_values": lambda: store.get_partial_values( cpu.buffer_prototype, [("a/b", None)] ), "set": lambda: store.set("a/c", data), "set_if_not_exists": lambda: store.set_if_not_exists("a/c", data), "exists": lambda: store.exists("a/b"), "getsize": lambda: store.getsize("a/b"), "delete": lambda: store.delete("a/b"), "delete_dir": lambda: store.delete_dir("a"), "list": lambda: _collect(store.list()), "list_prefix": lambda: _collect(store.list_prefix("a")), "list_dir": lambda: _collect(store.list_dir("a")), "clear": store.clear, "move": lambda: store.move(tmp_path_factory.mktemp("dest") / "moved"), } await ops[method]() assert helper in {function for function, _ in filesystem_calls.off_loop} async def test_concurrent_lazy_open(self, store_not_open: LocalStore) -> None: """Concurrent first calls on an unopened store all succeed. Opening now suspends (the root check runs in a thread), so every caller that finds the store closed races to open it; none of them may hit ``Store._open``'s "already open" error. """ data = self.buffer_cls.from_bytes(b"x") keys = [f"k{i}" for i in range(8)] await asyncio.gather( store_not_open.get("missing"), *(store_not_open.set(k, data) for k in keys) ) assert store_not_open._is_open for key in keys: assert_bytes_equal(await store_not_open.get(key), data) async def get(self, store: LocalStore, key: str) -> Buffer: return self.buffer_cls.from_bytes((store.root / key).read_bytes()) async def set(self, store: LocalStore, key: str, value: Buffer) -> None: parent = (store.root / key).parent if not parent.exists(): parent.mkdir(parents=True) (store.root / key).write_bytes(value.to_bytes()) @pytest.fixture def store_kwargs(self, tmp_path: pathlib.Path) -> dict[str, str]: return {"root": str(tmp_path)} def test_store_repr(self, store: LocalStore) -> None: assert str(store) == f"file://{store.root.as_posix()}" def test_store_supports_writes(self, store: LocalStore) -> None: assert store.supports_writes def test_store_supports_listing(self, store: LocalStore) -> None: assert store.supports_listing async def test_empty_with_empty_subdir(self, store: LocalStore) -> None: assert await store.is_empty("") (store.root / "foo/bar").mkdir(parents=True) assert await store.is_empty("") def test_delete_sync_directory(self, store: LocalStore) -> None: """`delete_sync` on a key that is a directory must remove the whole tree. Mirrors the async `delete_dir` behavior: deleting `"foo"` where `"foo"` is a directory containing further nested paths should remove everything under it, not just fail or delete a single file. """ (store.root / "foo" / "bar").mkdir(parents=True) (store.root / "foo" / "bar" / "baz").write_bytes(b"data") store.delete_sync("foo") assert not (store.root / "foo").exists() def test_creates_new_directory(self, tmp_path: pathlib.Path) -> None: target = tmp_path.joinpath("a", "b", "c") assert not target.exists() store = self.store_cls(root=target) zarr.group(store=store) def test_invalid_root_raises(self) -> None: """ Test that a TypeError is raised when a non-str/Path type is used for the `root` argument """ with pytest.raises( TypeError, match=r"'root' must be a string or Path instance. Got an instance of instead.", ): LocalStore(root=0) # type: ignore[arg-type] async def test_get_with_prototype_default(self, store: LocalStore) -> None: """ Ensure that data can be read via ``store.get`` if the prototype keyword argument is unspecified, i.e. set to ``None``. """ data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") key = "c/0" await self.set(store, key, data_buf) observed = await store.get(key, prototype=None) assert_bytes_equal(observed, data_buf) @pytest.mark.parametrize("ndim", [0, 1, 3]) @pytest.mark.parametrize( "destination", ["destination", "foo/bar/destintion", pathlib.Path("foo/bar/destintion")] ) async def test_move( self, tmp_path: pathlib.Path, ndim: int, destination: pathlib.Path | str ) -> None: origin = tmp_path / "origin" if isinstance(destination, str): destination = str(tmp_path / destination) else: destination = tmp_path / destination print(type(destination)) store = await LocalStore.open(root=origin) shape = (4,) * ndim chunks = (2,) * ndim data = np.arange(4**ndim) if ndim > 0: data = data.reshape(*shape) array = create_array(store, data=data, chunks=chunks or "auto") await store.move(destination) assert store.root == pathlib.Path(destination) assert pathlib.Path(destination).exists() assert not origin.exists() assert np.array_equal(array[...], data) store2 = await LocalStore.open(root=origin) with pytest.raises( FileExistsError, match=re.escape(f"Destination root {destination} already exists") ): await store2.move(destination) @pytest.mark.parametrize("exclusive", [True, False]) def test_atomic_write_successful(tmp_path: pathlib.Path, exclusive: bool) -> None: path = tmp_path / "data" with _atomic_write(path, "wb", exclusive=exclusive) as f: f.write(b"abc") assert path.read_bytes() == b"abc" assert list(path.parent.iterdir()) == [path] # no temp files @pytest.mark.parametrize("exclusive", [True, False]) def test_atomic_write_incomplete(tmp_path: pathlib.Path, exclusive: bool) -> None: path = tmp_path / "data" with pytest.raises(RuntimeError): # noqa: PT012 with _atomic_write(path, "wb", exclusive=exclusive) as f: f.write(b"a") raise RuntimeError assert not path.exists() assert list(path.parent.iterdir()) == [] # no temp files def test_atomic_write_non_exclusive_preexisting(tmp_path: pathlib.Path) -> None: path = tmp_path / "data" with path.open("wb") as f: f.write(b"xyz") assert path.read_bytes() == b"xyz" with _atomic_write(path, "wb", exclusive=False) as f: f.write(b"abc") assert path.read_bytes() == b"abc" assert list(path.parent.iterdir()) == [path] # no temp files def test_atomic_write_exclusive_preexisting(tmp_path: pathlib.Path) -> None: path = tmp_path / "data" with path.open("wb") as f: f.write(b"xyz") assert path.read_bytes() == b"xyz" with pytest.raises(FileExistsError): with _atomic_write(path, "wb", exclusive=True) as f: f.write(b"abc") assert path.read_bytes() == b"xyz" assert list(path.parent.iterdir()) == [path] # no temp files def _oserror(winerror: int) -> OSError: """An OSError shaped like the one a failed MoveFileEx produces.""" error = OSError(13, "Access is denied") error.winerror = winerror # type: ignore[attr-defined] return error class _FlakyMove: """A move that raises `error` the first `failures` times it is called.""" def __init__(self, failures: int, error: OSError) -> None: self.failures = failures self.error = error self.attempts = 0 def __call__(self, src: pathlib.Path, dst: pathlib.Path) -> None: self.attempts += 1 if self.attempts <= self.failures: raise self.error src.replace(dst) @pytest.fixture def sleeps(monkeypatch: pytest.MonkeyPatch) -> list[float]: """Record the delays `_move_with_retry` would sleep for instead of sleeping.""" recorded: list[float] = [] monkeypatch.setattr(time, "sleep", recorded.append) return recorded @pytest.mark.parametrize("winerror", [5, 32]) @pytest.mark.parametrize("failures", [1, 2, 3]) def test_move_with_retry_recovers( tmp_path: pathlib.Path, sleeps: list[float], winerror: int, failures: int ) -> None: """A destination that is briefly busy is retried, not reported.""" src = tmp_path / "src" dst = tmp_path / "dst" src.write_bytes(b"abc") move = _FlakyMove(failures, _oserror(winerror)) _move_with_retry(src, dst, move) assert dst.read_bytes() == b"abc" assert move.attempts == failures + 1 assert sleeps == list(_RETRY_DELAYS[:failures]) def test_move_with_retry_gives_up(tmp_path: pathlib.Path, sleeps: list[float]) -> None: """A destination that never frees still raises, after a bounded wait.""" src = tmp_path / "src" dst = tmp_path / "dst" src.write_bytes(b"abc") move = _FlakyMove(len(_RETRY_DELAYS) + 1, _oserror(5)) with pytest.raises(OSError, match="Access is denied"): _move_with_retry(src, dst, move) assert move.attempts == len(_RETRY_DELAYS) + 1 assert sleeps == list(_RETRY_DELAYS) assert sum(sleeps) < 1.0 @pytest.mark.parametrize("winerror", [2, 183, None]) def test_move_with_retry_does_not_retry_other_errors( tmp_path: pathlib.Path, sleeps: list[float], winerror: int | None ) -> None: """Only a busy destination is transient; everything else fails at once. 183 is the case that matters: `ERROR_ALREADY_EXISTS` is how the `exclusive` path reports that a node is already there, and retrying it would overwrite what `_safe_move` refused to touch. """ src = tmp_path / "src" dst = tmp_path / "dst" src.write_bytes(b"abc") error = OSError(17, "boom") if winerror is not None: error.winerror = winerror # type: ignore[attr-defined] move = _FlakyMove(1, error) with pytest.raises(OSError, match="boom"): _move_with_retry(src, dst, move) assert move.attempts == 1 assert sleeps == [] @pytest.mark.parametrize("exclusive", [True, False]) def test_atomic_write_onto_directory( tmp_path: pathlib.Path, sleeps: list[float], exclusive: bool ) -> None: """Writing a key whose destination is a directory fails and leaves no temp file.""" path = tmp_path / "node" path.mkdir() with pytest.raises(OSError), _atomic_write(path, "wb", exclusive=exclusive) as f: f.write(b"abc") assert path.is_dir() assert list(tmp_path.iterdir()) == [path] # no temp files zarr-developers-zarr-python-a2f2415/tests/test_store/test_logging.py000066400000000000000000000153741527014137600260730ustar00rootroot00000000000000from __future__ import annotations import logging from typing import TYPE_CHECKING, TypedDict import pytest import zarr from zarr.core.buffer import Buffer, cpu, default_buffer_prototype from zarr.storage import LocalStore, LoggingStore from zarr.testing.store import StoreTests if TYPE_CHECKING: from pathlib import Path from zarr.abc.store import Store class StoreKwargs(TypedDict): store: LocalStore log_level: str class TestLoggingStore(StoreTests[LoggingStore[LocalStore], cpu.Buffer]): # store_cls is needed to do an isinstance check, so can't be a subscripted generic store_cls = LoggingStore # type: ignore[assignment] buffer_cls = cpu.Buffer async def get(self, store: LoggingStore[LocalStore], key: str) -> Buffer: return self.buffer_cls.from_bytes((store._store.root / key).read_bytes()) async def set(self, store: LoggingStore[LocalStore], key: str, value: Buffer) -> None: parent = (store._store.root / key).parent if not parent.exists(): parent.mkdir(parents=True) (store._store.root / key).write_bytes(value.to_bytes()) @pytest.fixture def store_kwargs(self, tmp_path: Path) -> StoreKwargs: return {"store": LocalStore(str(tmp_path)), "log_level": "DEBUG"} @pytest.fixture def open_kwargs(self, tmp_path: Path) -> dict[str, type[LocalStore] | str]: return {"store_cls": LocalStore, "root": str(tmp_path), "log_level": "DEBUG"} @pytest.fixture def store(self, store_kwargs: StoreKwargs) -> LoggingStore[LocalStore]: return self.store_cls(**store_kwargs) def test_store_supports_writes(self, store: LoggingStore[LocalStore]) -> None: assert store.supports_writes def test_store_supports_listing(self, store: LoggingStore[LocalStore]) -> None: assert store.supports_listing def test_store_repr(self, store: LoggingStore[LocalStore]) -> None: assert f"{store!r}" == f"LoggingStore(LocalStore, 'file://{store._store.root.as_posix()}')" def test_store_str(self, store: LoggingStore[LocalStore]) -> None: assert str(store) == f"logging-file://{store._store.root.as_posix()}" async def test_default_handler( self, local_store: LocalStore, capsys: pytest.CaptureFixture[str] ) -> None: # Store and then remove existing handlers to enter default handler code path handlers = logging.getLogger().handlers[:] for h in handlers: logging.getLogger().removeHandler(h) # Test logs are sent to stdout wrapped = LoggingStore(store=local_store) buffer = default_buffer_prototype().buffer res = await wrapped.set("foo/bar/c/0", buffer.from_bytes(b"\x01\x02\x03\x04")) # type: ignore[func-returns-value] assert res is None captured = capsys.readouterr() assert len(captured) == 2 assert "Calling LocalStore.set" in captured.out assert "Finished LocalStore.set" in captured.out # Restore handlers for h in handlers: logging.getLogger().addHandler(h) def test_is_open_setter_raises(self, store: LoggingStore[LocalStore]) -> None: "Test that a user cannot change `_is_open` without opening the underlying store." with pytest.raises( NotImplementedError, match="LoggingStore must be opened via the `_open` method" ): store._is_open = True async def test_with_read_only_round_trip(self, local_store: LocalStore) -> None: """ Ensure that LoggingStore.with_read_only returns another LoggingStore with the requested read_only state, preserves logging configuration, and does not change the original store. """ # Start from a read-only underlying store ro_store = local_store.with_read_only(read_only=True) wrapped_ro = LoggingStore(store=ro_store, log_level="INFO") assert wrapped_ro.read_only buf = default_buffer_prototype().buffer.from_bytes(b"0123") # Cannot write through the read-only wrapper with pytest.raises( ValueError, match="store was opened in read-only mode and does not support writing" ): await wrapped_ro.set("foo", buf) # Create a writable wrapper writer = wrapped_ro.with_read_only(read_only=False) assert isinstance(writer, LoggingStore) assert not writer.read_only # logging configuration is preserved assert writer.log_level == wrapped_ro.log_level assert writer.log_handler == wrapped_ro.log_handler # Writes via the writable wrapper succeed await writer.set("foo", buf) out = await writer.get("foo", prototype=default_buffer_prototype()) assert out is not None assert out.to_bytes() == buf.to_bytes() # The original wrapper remains read-only assert wrapped_ro.read_only with pytest.raises( ValueError, match="store was opened in read-only mode and does not support writing" ): await wrapped_ro.set("bar", buf) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) async def test_logging_store(store: Store, caplog: pytest.LogCaptureFixture) -> None: wrapped = LoggingStore(store=store, log_level="DEBUG") buffer = default_buffer_prototype().buffer caplog.clear() res = await wrapped.set("foo/bar/c/0", buffer.from_bytes(b"\x01\x02\x03\x04")) # type: ignore[func-returns-value] assert res is None assert len(caplog.record_tuples) == 2 for tup in caplog.record_tuples: assert str(store) in tup[0] assert f"Calling {type(store).__name__}.set" in caplog.record_tuples[0][2] assert f"Finished {type(store).__name__}.set" in caplog.record_tuples[1][2] caplog.clear() keys = [k async for k in wrapped.list()] assert keys == ["foo/bar/c/0"] assert len(caplog.record_tuples) == 2 for tup in caplog.record_tuples: assert str(store) in tup[0] assert f"Calling {type(store).__name__}.list" in caplog.record_tuples[0][2] assert f"Finished {type(store).__name__}.list" in caplog.record_tuples[1][2] @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) async def test_logging_store_counter(store: Store) -> None: wrapped = LoggingStore(store=store, log_level="DEBUG") arr = zarr.create(shape=(10,), store=wrapped, overwrite=True) arr[:] = 1 assert wrapped.counter["set"] == 2 assert wrapped.counter["list"] == 0 assert wrapped.counter["list_dir"] == 0 assert wrapped.counter["list_prefix"] == 0 if store.supports_deletes: assert wrapped.counter["get"] == 0 # 1 if overwrite=False assert wrapped.counter["delete_dir"] == 1 else: assert wrapped.counter["get"] == 1 assert wrapped.counter["delete_dir"] == 0 zarr-developers-zarr-python-a2f2415/tests/test_store/test_memory.py000066400000000000000000000521471527014137600257540ustar00rootroot00000000000000from __future__ import annotations import re from typing import TYPE_CHECKING, Any import numpy as np import numpy.typing as npt import pytest import zarr from zarr.core.buffer import Buffer, cpu, default_buffer_prototype, gpu from zarr.errors import ZarrUserWarning from zarr.storage import GpuMemoryStore, ManagedMemoryStore, MemoryStore from zarr.storage._utils import _join_paths from zarr.testing.store import StoreTests from zarr.testing.utils import gpu_test if TYPE_CHECKING: from zarr.core.common import ZarrFormat # TODO: work out where this warning is coming from and fix it @pytest.mark.filterwarnings( re.escape("ignore:coroutine 'ClientCreatorContext.__aexit__' was never awaited") ) class TestMemoryStore(StoreTests[MemoryStore, cpu.Buffer]): store_cls = MemoryStore buffer_cls = cpu.Buffer async def set(self, store: MemoryStore, key: str, value: Buffer) -> None: store._store_dict[key] = value async def get(self, store: MemoryStore, key: str) -> Buffer: return store._store_dict[key] @pytest.fixture(params=[None, True]) def store_kwargs(self, request: pytest.FixtureRequest) -> dict[str, Any]: kwargs: dict[str, Any] if request.param is True: kwargs = {"store_dict": {}} else: kwargs = {"store_dict": None} return kwargs @pytest.fixture async def store(self, store_kwargs: dict[str, Any]) -> MemoryStore: return self.store_cls(**store_kwargs) def test_store_repr(self, store: MemoryStore) -> None: assert str(store) == f"memory://{id(store._store_dict)}" def test_store_supports_writes(self, store: MemoryStore) -> None: assert store.supports_writes def test_store_supports_listing(self, store: MemoryStore) -> None: assert store.supports_listing async def test_list_prefix(self, store: MemoryStore) -> None: assert True @pytest.mark.parametrize("dtype", ["uint8", "float32", "int64"]) @pytest.mark.parametrize("zarr_format", [2, 3]) async def test_deterministic_size( self, store: MemoryStore, dtype: npt.DTypeLike, zarr_format: ZarrFormat ) -> None: a = zarr.empty( store=store, shape=(3,), chunks=(1000,), dtype=dtype, zarr_format=zarr_format, overwrite=True, ) a[...] = 1 a.resize((1000,)) np.testing.assert_array_equal(a[:3], 1) np.testing.assert_array_equal(a[3:], 0) @pytest.mark.parametrize("method", ["set", "set_sync", "set_if_not_exists"]) async def test_set_does_not_retain_caller_buffer(self, store: MemoryStore, method: str) -> None: """Writing a buffer must not alias the caller's memory. MemoryStore keeps whatever it is handed alive in a dict, so retaining the caller's buffer lets a later mutation of that buffer rewrite data already committed to the store. """ source = np.frombuffer(bytearray(b"\x01\x02\x03\x04"), dtype="B") value = cpu.Buffer.from_array_like(source) if method == "set_sync": store.set_sync("k", value) else: await getattr(store, method)("k", value) source[:] = 0xF # mutate the caller's memory after the write stored = await store.get("k", prototype=default_buffer_prototype()) assert stored is not None assert stored.to_bytes() == b"\x01\x02\x03\x04" @pytest.mark.parametrize( "pipeline", [ "zarr.core.codec_pipeline.BatchedCodecPipeline", "zarr.core.codec_pipeline.FusedCodecPipeline", ], ) @pytest.mark.parametrize(("shape", "chunks"), [((30,), (10,)), ((8,), (4,)), ((4,), (4,))]) def test_write_does_not_alias_source_array( self, pipeline: str, shape: tuple[int], chunks: tuple[int] ) -> None: """Mutating the source array after a write must not corrupt stored chunks. Without compression the encoded buffer is a zero-copy view of the caller's array all the way down to the store, so this covers both the single-chunk and multi-chunk write paths. """ with zarr.config.set({"codec_pipeline.path": pipeline}): array = zarr.create_array( store=MemoryStore(), shape=shape, chunks=chunks, dtype="i4", compressors=None ) source = np.arange(shape[0], dtype="i4") expected = source.copy() array[:] = source source[:] = -1 np.testing.assert_array_equal(array[:], expected) # TODO: fix this warning @pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning") @gpu_test class TestGpuMemoryStore(StoreTests[GpuMemoryStore, gpu.Buffer]): store_cls = GpuMemoryStore buffer_cls = gpu.Buffer async def set(self, store: GpuMemoryStore, key: str, value: gpu.Buffer) -> None: # type: ignore[override] store._store_dict[key] = value async def get(self, store: MemoryStore, key: str) -> Buffer: return store._store_dict[key] @pytest.fixture(params=[None, True]) def store_kwargs(self, request: pytest.FixtureRequest) -> dict[str, Any]: kwargs: dict[str, Any] if request.param is True: kwargs = {"store_dict": {}} else: kwargs = {"store_dict": None} return kwargs @pytest.fixture async def store(self, store_kwargs: dict[str, Any]) -> GpuMemoryStore: return self.store_cls(**store_kwargs) def test_store_repr(self, store: GpuMemoryStore) -> None: assert str(store) == f"gpumemory://{id(store._store_dict)}" def test_store_supports_writes(self, store: GpuMemoryStore) -> None: assert store.supports_writes def test_store_supports_listing(self, store: GpuMemoryStore) -> None: assert store.supports_listing async def test_list_prefix(self, store: GpuMemoryStore) -> None: assert True def test_dict_reference(self, store: GpuMemoryStore) -> None: store_dict: dict[str, Any] = {} result = GpuMemoryStore(store_dict=store_dict) assert result._store_dict is store_dict def test_from_dict(self) -> None: d = { "a": gpu.Buffer.from_bytes(b"aaaa"), "b": cpu.Buffer.from_bytes(b"bbbb"), } msg = "Creating a zarr.buffer.gpu.Buffer with an array that does not support the __cuda_array_interface__ for zero-copy transfers, falling back to slow copy based path" with pytest.warns(ZarrUserWarning, match=msg): result = GpuMemoryStore.from_dict(d) for v in result._store_dict.values(): assert type(v) is gpu.Buffer def test_set_sync_converts_to_gpu_buffer(self, store: GpuMemoryStore) -> None: """`set_sync` must convert its value to a `gpu.Buffer`, mirroring `set`. `GpuMemoryStore`'s invariant is that every stored value is a `gpu.Buffer`. Without this override, the inherited `MemoryStore.set_sync` would store the CPU buffer it was given as-is, breaking that invariant for whichever code path (e.g. the fused pipeline) uses the sync API. """ cpu_value = cpu.Buffer.from_bytes(b"aaaa") msg = "Creating a zarr.buffer.gpu.Buffer with an array that does not support the __cuda_array_interface__ for zero-copy transfers, falling back to slow copy based path" with pytest.warns(ZarrUserWarning, match=msg): store.set_sync("k", cpu_value) assert type(store._store_dict["k"]) is gpu.Buffer class TestManagedMemoryStore(StoreTests[ManagedMemoryStore, cpu.Buffer]): store_cls = ManagedMemoryStore buffer_cls = cpu.Buffer async def set(self, store: ManagedMemoryStore, key: str, value: Buffer) -> None: store._store_dict[_join_paths([store.path, key])] = value async def get(self, store: ManagedMemoryStore, key: str) -> Buffer: return store._store_dict[_join_paths([store.path, key])] @pytest.fixture def store_kwargs(self, request: pytest.FixtureRequest) -> dict[str, Any]: # Use a unique name per test to avoid sharing state between tests # but ensure the name is deterministic for equality tests # Replace '/' with '-' since store names cannot contain '/' # A non-empty path exercises prefix handling; a store with an # unprefixed key in its backing dict would pass these tests # vacuously with path="". sanitized_name = request.node.name.replace("/", "-") return {"name": f"test-{sanitized_name}", "path": "prefix"} @pytest.fixture async def store(self, store_kwargs: dict[str, Any]) -> ManagedMemoryStore: return self.store_cls(**store_kwargs) def test_store_repr(self, store: ManagedMemoryStore) -> None: assert str(store) == _join_paths([f"memory://{store.name}", store.path]) async def test_serializable_store(self, store: ManagedMemoryStore) -> None: """ Test pickling semantics for ManagedMemoryStore. When pickled and unpickled within the same process (where the original store still exists in the registry), the unpickled store reconnects to the same backing dict. """ import pickle # Add some data to the store await store.set("test-key", self.buffer_cls.from_bytes(b"test-value")) # Pickle and unpickle the store pickled = pickle.dumps(store) store2 = pickle.loads(pickled) # The unpickled store should reconnect to the same backing dict assert store2._store_dict is store._store_dict assert store2.name == store.name assert store2.path == store.path assert store2.read_only == store.read_only # The data should be accessible result = await store2.get("test-key") assert result is not None assert result.to_bytes() == b"test-value" async def test_pickle_with_path(self) -> None: """Test that path is preserved through pickle round-trip.""" import pickle store = ManagedMemoryStore(name="pickle-path-test", path="some/path") await store.set("key", self.buffer_cls.from_bytes(b"value")) pickled = pickle.dumps(store) store2 = pickle.loads(pickled) assert store2.path == "some/path" assert store2._store_dict is store._store_dict # Check that operations use the path correctly result = await store2.get("key") assert result is not None assert result.to_bytes() == b"value" def test_pickle_after_gc(self) -> None: """ Test that unpickling after the original store is garbage collected creates a new empty store with the same name (in the same process). """ import gc import pickle # Create a store with a unique name and pickle it store = ManagedMemoryStore(name="gc-pickle-test") store._store_dict["key"] = self.buffer_cls.from_bytes(b"value") pickled = pickle.dumps(store) # Delete the store and garbage collect del store gc.collect() # Unpickling should create a new store with an empty dict store2 = pickle.loads(pickled) assert store2.name == "gc-pickle-test" # The dict is empty because the original was garbage collected assert len(store2._store_dict) == 0 async def test_cross_process_detection(self) -> None: """ Test that unpickling a ManagedMemoryStore in a different process raises an error. This prevents silent data loss when a store is pickled and unpickled in a different process (e.g., with multiprocessing). """ import os store = ManagedMemoryStore(name="cross-process-test") await store.set("key", self.buffer_cls.from_bytes(b"value")) # Get the reduce tuple and modify the state to simulate a different process cls, args, state = store.__reduce__() state["created_pid"] = os.getpid() + 1 # Fake a different process ID # Manually reconstruct what pickle.loads would do # This simulates unpickling data that was pickled in a different process reconstructed = cls(*args) with pytest.raises(RuntimeError, match="was created in process"): reconstructed.__setstate__(state) def test_store_supports_writes(self, store: ManagedMemoryStore) -> None: assert store.supports_writes def test_store_supports_listing(self, store: ManagedMemoryStore) -> None: assert store.supports_listing @pytest.mark.parametrize("dtype", ["uint8", "float32", "int64"]) @pytest.mark.parametrize("zarr_format", [2, 3]) async def test_deterministic_size( self, store: MemoryStore, dtype: npt.DTypeLike, zarr_format: ZarrFormat ) -> None: a = zarr.empty( store=store, shape=(3,), chunks=(1000,), dtype=dtype, zarr_format=zarr_format, overwrite=True, ) a[...] = 1 a.resize((1000,)) np.testing.assert_array_equal(a[:3], 1) np.testing.assert_array_equal(a[3:], 0) def test_from_url(self, store: ManagedMemoryStore) -> None: """Test that from_url creates a store sharing the same dict.""" url = str(store) store2 = ManagedMemoryStore.from_url(url) assert store2._store_dict is store._store_dict def test_from_url_with_path(self, store: ManagedMemoryStore) -> None: """Test that from_url extracts path component from URL.""" # Reconnect to the fixture's dict via its name, but with an empty # path, so appending "/some/path" below yields exactly that path. base = ManagedMemoryStore(name=store.name) url = f"{base}/some/path" store2 = ManagedMemoryStore.from_url(url) assert store2._store_dict is store._store_dict assert store2.path == "some/path" assert str(store2) == url def test_from_url_invalid(self) -> None: """Test that from_url raises ValueError for non-existent store.""" with pytest.raises(ValueError, match="Memory store not found"): ManagedMemoryStore.from_url("memory://nonexistent-store") def test_from_url_not_memory_scheme(self) -> None: """Test that from_url raises ValueError for non-memory URLs.""" with pytest.raises(ValueError, match="Expected a 'memory://' URL"): ManagedMemoryStore.from_url("file:///tmp/test") def test_named_store(self) -> None: """Test that stores can be created with explicit names.""" store = ManagedMemoryStore(name="my-test-store") assert store.name == "my-test-store" assert str(store) == "memory://my-test-store" def test_named_store_shares_dict(self) -> None: """Test that creating a store with the same name shares the dict.""" store1 = ManagedMemoryStore(name="shared-store") store2 = ManagedMemoryStore(name="shared-store") assert store1._store_dict is store2._store_dict assert store1.name == store2.name def test_auto_generated_name(self) -> None: """Test that stores get auto-generated names when none provided.""" store = ManagedMemoryStore() assert store.name is not None assert str(store) == f"memory://{store.name}" def test_with_read_only_shares_dict(self, store: ManagedMemoryStore) -> None: """Test that with_read_only creates a store sharing the same dict.""" store2 = store.with_read_only(True) assert store2._store_dict is store._store_dict assert store2.read_only is True assert store.read_only is False def test_with_read_only_preserves_path(self) -> None: """Test that with_read_only preserves the path.""" store = ManagedMemoryStore(name="path-test", path="some/path") store2 = store.with_read_only(True) assert store2.path == "some/path" assert store2._store_dict is store._store_dict async def test_path_prefix_operations(self) -> None: """Test that store operations use the path prefix correctly.""" store = ManagedMemoryStore(name="prefix-test") store_with_path = ManagedMemoryStore.from_url("memory://prefix-test/subdir") # Write via store_with_path await store_with_path.set("key", self.buffer_cls.from_bytes(b"value")) # The key should be stored with the prefix in the underlying dict assert "subdir/key" in store._store_dict assert "key" not in store._store_dict # Read via store_with_path should work result = await store_with_path.get("key") assert result is not None assert result.to_bytes() == b"value" # Read via store without path should use full key result2 = await store.get("subdir/key") assert result2 is not None assert result2.to_bytes() == b"value" async def test_path_list_operations(self) -> None: """Test that list operations filter by path prefix.""" store = ManagedMemoryStore(name="list-test") # Set up some keys at different paths await store.set("a/key1", self.buffer_cls.from_bytes(b"v1")) await store.set("a/key2", self.buffer_cls.from_bytes(b"v2")) await store.set("b/key3", self.buffer_cls.from_bytes(b"v3")) # Create a store with path "a" store_a = ManagedMemoryStore.from_url("memory://list-test/a") # list() should only return keys under "a", without the "a/" prefix keys = [k async for k in store_a.list()] assert sorted(keys) == ["key1", "key2"] async def test_path_exists(self) -> None: """Test that exists() uses the path prefix.""" store = ManagedMemoryStore(name="exists-test") await store.set("prefix/key", self.buffer_cls.from_bytes(b"value")) store_with_path = ManagedMemoryStore.from_url("memory://exists-test/prefix") assert await store_with_path.exists("key") assert not await store_with_path.exists("prefix/key") def test_path_normalization(self) -> None: """Test that paths are normalized.""" store1 = ManagedMemoryStore(name="norm-test", path="a/b/") store2 = ManagedMemoryStore(name="norm-test", path="/a/b") store3 = ManagedMemoryStore(name="norm-test", path="a//b") assert store1.path == "a/b" assert store2.path == "a/b" assert store3.path == "a/b" def test_name_cannot_contain_slash(self) -> None: """Test that store names cannot contain '/'.""" with pytest.raises(ValueError, match="cannot contain '/'"): ManagedMemoryStore(name="foo/bar") def test_garbage_collection(self) -> None: """Test that the dict is garbage collected when no stores reference it.""" import gc store = ManagedMemoryStore() url = str(store) # URL should resolve while store exists store2 = ManagedMemoryStore.from_url(url) assert store2._store_dict is store._store_dict # Delete both stores del store del store2 gc.collect() # URL should no longer resolve with pytest.raises(ValueError, match="garbage collected"): ManagedMemoryStore.from_url(url) def test_sync_methods_respect_path_prefix(self) -> None: """`get_sync`/`set_sync`/`delete_sync` must prefix keys with `self.path`, exactly like the async `get`/`set`/`delete` methods. `ManagedMemoryStore` used to inherit these from `MemoryStore`, which writes/reads the raw key. Two stores sharing a dict with different `path` values would then cross-talk through the sync API. """ store = ManagedMemoryStore(name="sync-prefix-test", path="subdir") data_buf = self.buffer_cls.from_bytes(b"value") store.set_sync("key", data_buf) assert "subdir/key" in store._store_dict assert "key" not in store._store_dict result = store.get_sync("key") assert result is not None assert result.to_bytes() == b"value" store.delete_sync("key") assert "subdir/key" not in store._store_dict def test_fused_pipeline_respects_path_prefix(self) -> None: """End-to-end regression: the fused pipeline's sync store fast path must write chunks under the store's path prefix. `FusedCodecPipeline` uses `set_sync`/`get_sync` when a store implements the sync protocols. If those methods skip the prefix that the async methods apply, chunk data lands outside `self.path` and a fresh handle re-reading through the prefix silently sees fill values instead. """ with zarr.config.set( {"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"} ): store = ManagedMemoryStore(name="fused-prefix-test", path="subdir") arr = zarr.create_array(store, shape=(4,), chunks=(4,), dtype="uint8", zarr_format=3) arr[:] = np.arange(4, dtype="uint8") bad_keys = [k for k in store._store_dict if not k.startswith("subdir/")] assert bad_keys == [], f"keys written outside the store's path prefix: {bad_keys}" store2 = ManagedMemoryStore.from_url("memory://fused-prefix-test/subdir") arr2 = zarr.open_array(store2, mode="r") np.testing.assert_array_equal(arr2[:], np.arange(4, dtype="uint8")) zarr-developers-zarr-python-a2f2415/tests/test_store/test_object.py000066400000000000000000000122501527014137600257010ustar00rootroot00000000000000import re from pathlib import Path from typing import TypedDict import pytest obstore = pytest.importorskip("obstore") from hypothesis.stateful import ( run_state_machine_as_test, ) from obstore.store import LocalStore, MemoryStore, S3Store from zarr.core.buffer import Buffer, cpu from zarr.core.sync import _collect_aiterator from zarr.storage import ObjectStore from zarr.testing.stateful import ZarrHierarchyStateMachine from zarr.testing.store import StoreTests class StoreKwargs(TypedDict): store: LocalStore read_only: bool class TestObjectStore(StoreTests[ObjectStore[LocalStore], cpu.Buffer]): # store_cls is needed to do an isinstance check, so can't be a subscripted generic store_cls = ObjectStore # type: ignore[assignment] buffer_cls = cpu.Buffer @pytest.fixture def store_kwargs(self, tmp_path: Path) -> StoreKwargs: store = LocalStore(prefix=tmp_path) return {"store": store, "read_only": False} @pytest.fixture def store(self, store_kwargs: StoreKwargs) -> ObjectStore[LocalStore]: return self.store_cls(**store_kwargs) async def get(self, store: ObjectStore[LocalStore], key: str) -> Buffer: assert isinstance(store.store, LocalStore) new_local_store = LocalStore(prefix=store.store.prefix) return self.buffer_cls.from_bytes(obstore.get(new_local_store, key).bytes()) async def set(self, store: ObjectStore[LocalStore], key: str, value: Buffer) -> None: assert isinstance(store.store, LocalStore) new_local_store = LocalStore(prefix=store.store.prefix) obstore.put(new_local_store, key, value.to_bytes()) def test_store_repr(self, store: ObjectStore[LocalStore]) -> None: from fnmatch import fnmatch pattern = "ObjectStore(object_store://LocalStore(*))" assert fnmatch(f"{store!r}", pattern) def test_store_supports_writes(self, store: ObjectStore[LocalStore]) -> None: assert store.supports_writes def test_store_supports_partial_writes(self, store: ObjectStore[LocalStore]) -> None: assert not store.supports_partial_writes def test_store_supports_listing(self, store: ObjectStore[LocalStore]) -> None: assert store.supports_listing def test_store_equal(self, store: ObjectStore[LocalStore]) -> None: """Test store equality""" # Test equality against a different instance type assert store != 0 # Test equality against a different store type new_memory_store = ObjectStore(MemoryStore()) assert store != new_memory_store # Test equality against a read only store assert isinstance(store.store, LocalStore) new_local_store = ObjectStore(LocalStore(prefix=store.store.prefix), read_only=True) assert store != new_local_store # Test two memory stores cannot be equal second_memory_store = ObjectStore(MemoryStore()) assert new_memory_store != second_memory_store def test_store_init_raises(self) -> None: """Test __init__ raises appropriate error for improper store type""" with pytest.raises(TypeError): ObjectStore("path/to/store") # type: ignore[type-var] async def test_store_getsize(self, store: ObjectStore[LocalStore]) -> None: buf = cpu.Buffer.from_bytes(b"\x01\x02\x03\x04") await self.set(store, "key", buf) size = await store.getsize("key") assert size == len(buf) async def test_store_getsize_prefix(self, store: ObjectStore[LocalStore]) -> None: buf = cpu.Buffer.from_bytes(b"\x01\x02\x03\x04") await self.set(store, "c/key1/0", buf) await self.set(store, "c/key2/0", buf) size = await store.getsize_prefix("c/key1") assert size == len(buf) total_size = await store.getsize_prefix("c") assert total_size == len(buf) * 2 @pytest.mark.filterwarnings( re.escape("ignore:datetime.datetime.utcnow() is deprecated:DeprecationWarning") ) async def test_list_dir_ignores_s3_prefix_marker(moto_server: str) -> None: """Ensure obstore's exact-prefix S3 directory marker is not listed as a child.""" boto3 = pytest.importorskip("boto3") bucket = "object-store-prefix-marker" client = boto3.client( "s3", endpoint_url=moto_server, region_name="us-east-1", aws_access_key_id="x", aws_secret_access_key="x", ) client.create_bucket(Bucket=bucket) client.put_object(Bucket=bucket, Key="g/", Body=b"") store = ObjectStore( S3Store( bucket=bucket, endpoint=moto_server, region="us-east-1", access_key_id="x", secret_access_key="x", client_options={"allow_http": True}, virtual_hosted_style_request=False, ) ) assert await _collect_aiterator(store.list_dir("g")) == () assert await _collect_aiterator(store.list_dir("g/")) == () @pytest.mark.slow_hypothesis def test_zarr_hierarchy() -> None: sync_store = ObjectStore(MemoryStore()) def mk_test_instance_sync() -> ZarrHierarchyStateMachine: return ZarrHierarchyStateMachine(sync_store) run_state_machine_as_test(mk_test_instance_sync) # type: ignore[no-untyped-call] zarr-developers-zarr-python-a2f2415/tests/test_store/test_stateful.py000066400000000000000000000036331527014137600262670ustar00rootroot00000000000000# Stateful tests for arbitrary Zarr stores. from collections.abc import Generator import pytest from hypothesis.stateful import ( run_state_machine_as_test, ) import zarr from zarr.abc.store import Store from zarr.storage import LocalStore, ZipStore from zarr.testing.stateful import ZarrHierarchyStateMachine, ZarrStoreStateMachine pytestmark = [ pytest.mark.slow_hypothesis, # TODO: work out where this warning is coming from and fix pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning"), ] @pytest.fixture(autouse=True) def _enable_rectilinear_chunks() -> Generator[None, None, None]: """Enable rectilinear chunks since strategies may generate them.""" with zarr.config.set({"array.rectilinear_chunks": True}): yield @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_zarr_hierarchy(sync_store: Store) -> None: def mk_test_instance_sync() -> ZarrHierarchyStateMachine: return ZarrHierarchyStateMachine(sync_store) if isinstance(sync_store, ZipStore): pytest.skip(reason="ZipStore does not support delete") run_state_machine_as_test(mk_test_instance_sync) # type: ignore[no-untyped-call] def test_zarr_store(sync_store: Store) -> None: def mk_test_instance_sync() -> ZarrStoreStateMachine: return ZarrStoreStateMachine(sync_store) if isinstance(sync_store, ZipStore): pytest.skip(reason="ZipStore does not support delete") if isinstance(sync_store, LocalStore): # This test uses arbitrary keys, which are passed to `set` and `delete`. # It assumes that `set` and `delete` are the only two operations that modify state. # But LocalStore, directories can hang around even after a key is delete-d. pytest.skip(reason="Test isn't suitable for LocalStore.") run_state_machine_as_test(mk_test_instance_sync) # type: ignore[no-untyped-call] zarr-developers-zarr-python-a2f2415/tests/test_store/test_utils.py000066400000000000000000000111441527014137600255740ustar00rootroot00000000000000from __future__ import annotations import sys from unittest.mock import patch import pytest from zarr.abc.store import SuffixByteRequest from zarr.core.buffer.core import default_buffer_prototype from zarr.storage._utils import ParsedStoreUrl, _normalize_byte_range_index, parse_store_url class TestParseStoreUrl: """Tests for parse_store_url.""" def test_memory_url(self) -> None: result = parse_store_url("memory://mystore") assert result == ParsedStoreUrl( scheme="memory", name="mystore", path="", raw="memory://mystore" ) def test_memory_url_with_path(self) -> None: result = parse_store_url("memory://mystore/path/to/data") assert result == ParsedStoreUrl( scheme="memory", name="mystore", path="path/to/data", raw="memory://mystore/path/to/data", ) def test_memory_url_no_name(self) -> None: result = parse_store_url("memory://") assert result.scheme == "memory" assert result.name is None def test_s3_url(self) -> None: result = parse_store_url("s3://bucket/key") assert result == ParsedStoreUrl( scheme="s3", name="bucket", path="key", raw="s3://bucket/key" ) def test_file_url(self) -> None: result = parse_store_url("file:///tmp/test") assert result.scheme == "file" def test_local_absolute_path(self) -> None: result = parse_store_url("/local/path") assert result == ParsedStoreUrl(scheme="", name=None, path="/local/path", raw="/local/path") def test_local_relative_path(self) -> None: result = parse_store_url("relative/path") assert result == ParsedStoreUrl( scheme="", name=None, path="relative/path", raw="relative/path" ) @pytest.mark.parametrize( "url", [ "C:\\Users\\foo", "C:/Users/foo", "D:/data/zarr", "c:/test", ], ) def test_windows_drive_letter(self, url: str) -> None: """On Windows, bare drive-letter paths must be treated as local paths.""" with patch.object(sys, "platform", "win32"): result = parse_store_url(url) assert result.scheme == "" assert result.name is None assert result.path == url assert result.raw == url @pytest.mark.parametrize( "url", [ "file:///C:/Users/foo", "file://C:/Users/foo", ], ) def test_file_url_with_drive_letter_on_windows(self, url: str) -> None: """file:// URLs with drive letters are not treated as bare paths.""" with patch.object(sys, "platform", "win32"): result = parse_store_url(url) assert result.scheme == "file" @pytest.mark.parametrize( "url", [ "C:\\Users\\foo", "C:/Users/foo", ], ) def test_drive_letter_not_special_on_non_windows(self, url: str) -> None: """On non-Windows platforms, drive-letter paths go through urlparse.""" with patch.object(sys, "platform", "linux"): result = parse_store_url(url) # urlparse interprets the drive letter as a scheme assert result.scheme == "c" class TestNormalizeByteRangeIndex: """Tests for _normalize_byte_range_index.""" def test_suffix_larger_than_data_returns_all_bytes(self) -> None: """Regression: SuffixByteRequest with suffix > len(data) must not produce a negative start index that causes numpy to return fewer bytes than available.""" prototype = default_buffer_prototype() data = prototype.buffer.from_bytes(b"hello") # 5 bytes byte_range = SuffixByteRequest(suffix=7) start, stop = _normalize_byte_range_index(data, byte_range) assert start == 0, f"start should be 0 (clamped), got {start}" result = data[start:stop] assert len(result) == 5, f"expected all 5 bytes, got {len(result)}" def test_suffix_exact_length(self) -> None: """SuffixByteRequest with suffix == len(data) returns all bytes.""" prototype = default_buffer_prototype() data = prototype.buffer.from_bytes(b"hello") start, _stop = _normalize_byte_range_index(data, SuffixByteRequest(suffix=5)) assert start == 0 def test_suffix_shorter_than_data(self) -> None: """SuffixByteRequest with suffix < len(data) returns the last n bytes.""" prototype = default_buffer_prototype() data = prototype.buffer.from_bytes(b"hello") start, _stop = _normalize_byte_range_index(data, SuffixByteRequest(suffix=3)) assert start == 2 zarr-developers-zarr-python-a2f2415/tests/test_store/test_wrapper.py000066400000000000000000000144021527014137600261140ustar00rootroot00000000000000from __future__ import annotations from typing import TYPE_CHECKING, Any, TypedDict import pytest from zarr.abc.store import ByteRequest, Store, _store_supports_sync_io from zarr.core.buffer import Buffer from zarr.core.buffer.cpu import Buffer as CPUBuffer from zarr.core.buffer.cpu import buffer_prototype from zarr.storage import LocalStore, MemoryStore, WrapperStore, ZipStore from zarr.testing.store import LatencyStore, StoreTests if TYPE_CHECKING: from pathlib import Path from zarr.core.buffer.core import BufferPrototype class StoreKwargs(TypedDict): store: LocalStore class OpenKwargs(TypedDict): store_cls: type[LocalStore] root: str # TODO: fix this warning @pytest.mark.filterwarnings( "ignore:coroutine 'ClientCreatorContext.__aexit__' was never awaited:RuntimeWarning" ) class TestWrapperStore(StoreTests[WrapperStore[Any], Buffer]): store_cls = WrapperStore buffer_cls = CPUBuffer async def get(self, store: WrapperStore[LocalStore], key: str) -> Buffer: return self.buffer_cls.from_bytes((store._store.root / key).read_bytes()) async def set(self, store: WrapperStore[LocalStore], key: str, value: Buffer) -> None: parent = (store._store.root / key).parent if not parent.exists(): parent.mkdir(parents=True) (store._store.root / key).write_bytes(value.to_bytes()) @pytest.fixture def store_kwargs(self, tmp_path: Path) -> StoreKwargs: return {"store": LocalStore(str(tmp_path))} @pytest.fixture def open_kwargs(self, tmp_path: Path) -> OpenKwargs: return {"store_cls": LocalStore, "root": str(tmp_path)} def test_store_supports_writes(self, store: WrapperStore[LocalStore]) -> None: assert store.supports_writes def test_store_supports_listing(self, store: WrapperStore[LocalStore]) -> None: assert store.supports_listing def test_store_repr(self, store: WrapperStore[LocalStore]) -> None: assert f"{store!r}" == f"WrapperStore(LocalStore, 'file://{store._store.root.as_posix()}')" def test_store_str(self, store: WrapperStore[LocalStore]) -> None: assert str(store) == f"wrapping-file://{store._store.root.as_posix()}" def test_check_writeable(self, store: WrapperStore[LocalStore]) -> None: """ Test _check_writeable() runs without errors. """ store._check_writable() def test_close(self, store: WrapperStore[LocalStore]) -> None: "Test store can be closed" store.close() assert not store._is_open def test_is_open_setter_raises(self, store: WrapperStore[LocalStore]) -> None: """ Test that a user cannot change `_is_open` without opening the underlying store. """ with pytest.raises( NotImplementedError, match="WrapperStore must be opened via the `_open` method" ): store._is_open = True # TODO: work out where warning is coming from and fix @pytest.mark.filterwarnings( "ignore:coroutine 'ClientCreatorContext.__aexit__' was never awaited:RuntimeWarning" ) @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=True) async def test_wrapped_set(store: Store, capsys: pytest.CaptureFixture[str]) -> None: # define a class that prints when it sets class NoisySetter(WrapperStore[Store]): async def set(self, key: str, value: Buffer) -> None: print(f"setting {key}") await super().set(key, value) key = "foo" value = CPUBuffer.from_bytes(b"bar") store_wrapped = NoisySetter(store) await store_wrapped.set(key, value) captured = capsys.readouterr() assert f"setting {key}" in captured.out assert await store_wrapped.get(key, buffer_prototype) == value @pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning") @pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=True) async def test_wrapped_get(store: Store, capsys: pytest.CaptureFixture[str]) -> None: # define a class that prints when it sets class NoisyGetter(WrapperStore[Any]): async def get( self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None ) -> None: print(f"getting {key}") await super().get(key, prototype=prototype, byte_range=byte_range) key = "foo" value = CPUBuffer.from_bytes(b"bar") store_wrapped = NoisyGetter(store) await store_wrapped.set(key, value) await store_wrapped.get(key, buffer_prototype) captured = capsys.readouterr() assert f"getting {key}" in captured.out @pytest.mark.parametrize( ("store_factory", "expected"), [ (lambda tmp: MemoryStore(), True), (lambda tmp: LocalStore(str(tmp)), True), (lambda tmp: WrapperStore(MemoryStore()), True), (lambda tmp: LatencyStore(MemoryStore()), True), (lambda tmp: ZipStore(tmp / "store.zip", mode="w"), False), (lambda tmp: WrapperStore(ZipStore(tmp / "store.zip", mode="w")), False), ], ids=[ "memory", "local", "wrapper-of-memory", "latency-wrapper-of-memory", "zip", "wrapper-of-zip", ], ) def test_supports_sync_io(store_factory: Any, expected: bool, tmp_path: Path | Any) -> None: """`_store_supports_sync_io` is True only for stores implementing the full sync surface (get_sync + set_sync + delete_sync); wrappers forward the wrapped store's capability via `_supports_sync_io`.""" assert _store_supports_sync_io(store_factory(tmp_path)) is expected def test_wrapper_get_sync_without_inner_sync_raises(tmp_path: Any) -> None: store = WrapperStore(ZipStore(tmp_path / "store.zip", mode="w")) with pytest.raises(TypeError, match="does not support synchronous get"): store.get_sync("key") def test_wrapper_set_sync_without_inner_sync_raises(tmp_path: Any) -> None: store = WrapperStore(ZipStore(tmp_path / "store.zip", mode="w")) with pytest.raises(TypeError, match="does not support synchronous set"): store.set_sync("key", CPUBuffer.from_bytes(b"data")) def test_wrapper_delete_sync_without_inner_sync_raises(tmp_path: Any) -> None: store = WrapperStore(ZipStore(tmp_path / "store.zip", mode="w")) with pytest.raises(TypeError, match="does not support synchronous delete"): store.delete_sync("key") zarr-developers-zarr-python-a2f2415/tests/test_store/test_zip.py000066400000000000000000001122331527014137600252370ustar00rootroot00000000000000from __future__ import annotations import gc import io import os import pathlib import pickle import shutil import tempfile import threading import zipfile from concurrent.futures import Future, ThreadPoolExecutor, wait from typing import TYPE_CHECKING, Literal, cast import numpy as np import pytest from hypothesis import settings from hypothesis.stateful import ( RuleBasedStateMachine, initialize, precondition, rule, run_state_machine_as_test, ) import zarr from zarr import create_array from zarr.core.buffer import Buffer, cpu, default_buffer_prototype from zarr.core.sync import sync from zarr.storage import ZipStore from zarr.testing.store import StoreTests if TYPE_CHECKING: from collections.abc import AsyncGenerator, AsyncIterator from pathlib import Path from typing import Any from zarr.storage._zip import ZipStoreAccessModeLiteral # TODO: work out where this is coming from and fix pytestmark = [ pytest.mark.filterwarnings( "ignore:coroutine method 'aclose' of 'ZipStore.list' was never awaited:RuntimeWarning" ) ] async def _drain(keys: AsyncIterator[str]) -> list[str]: return [k async for k in keys] class TestZipStore(StoreTests[ZipStore, cpu.Buffer]): store_cls = ZipStore buffer_cls = cpu.Buffer @pytest.fixture def store_kwargs(self) -> dict[str, str | bool]: fd, temp_path = tempfile.mkstemp() os.close(fd) os.unlink(temp_path) return {"path": temp_path, "mode": "w", "read_only": False} async def get(self, store: ZipStore, key: str) -> Buffer: buf = store._get(key, prototype=default_buffer_prototype()) assert buf is not None return buf async def set(self, store: ZipStore, key: str, value: Buffer) -> None: return store._set(key, value) def test_store_read_only(self, store: ZipStore) -> None: assert not store.read_only async def test_read_only_store_raises(self, store_kwargs: dict[str, Any]) -> None: # we need to create the zipfile in write mode before switching to read mode store = await self.store_cls.open(**store_kwargs) store.close() kwargs = {**store_kwargs, "mode": "a", "read_only": True} store = await self.store_cls.open(**kwargs) assert store._zmode == "a" assert store.read_only # set with pytest.raises(ValueError): await store.set("foo", cpu.Buffer.from_bytes(b"bar")) def test_store_repr(self, store: ZipStore) -> None: assert str(store) == f"zip://{store.path}" def test_store_supports_writes(self, store: ZipStore) -> None: assert store.supports_writes def test_store_supports_listing(self, store: ZipStore) -> None: assert store.supports_listing # TODO: fix this warning @pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning") def test_api_integration(self, store: ZipStore) -> None: root = zarr.open_group(store=store, mode="a") data = np.arange(10000, dtype=np.uint16).reshape(100, 100) z = root.create_array( shape=data.shape, chunks=(10, 10), name="foo", dtype=np.uint16, fill_value=99 ) z[:] = data assert np.array_equal(data, z[:]) # you can overwrite existing chunks but zipfile will issue a warning with pytest.warns(UserWarning, match="Duplicate name: 'foo/c/0/0'"): z[0, 0] = 100 # TODO: assigning an entire chunk to fill value ends up deleting the chunk which is not supported # a work around will be needed here. with pytest.raises(NotImplementedError): z[0:10, 0:10] = 99 bar = root.create_group("bar", attributes={"hello": "world"}) assert "hello" in dict(bar.attrs) # keys cannot be deleted with pytest.raises(NotImplementedError): del root["bar"] store.close() @pytest.mark.parametrize("read_only", [True, False]) async def test_store_open_read_only( self, store_kwargs: dict[str, Any], read_only: bool ) -> None: if read_only: # create an empty zipfile with zipfile.ZipFile(store_kwargs["path"], mode="w"): pass await super().test_store_open_read_only(store_kwargs, read_only) @pytest.mark.parametrize(("zip_mode", "read_only"), [("w", False), ("a", False), ("x", False)]) async def test_zip_open_mode_translation( self, store_kwargs: dict[str, Any], zip_mode: str, read_only: bool ) -> None: kws = {**store_kwargs, "mode": zip_mode} store = await self.store_cls.open(**kws) assert store.read_only == read_only def test_externally_zipped_store(self, tmp_path: Path) -> None: # See: https://github.com/zarr-developers/zarr-python/issues/2757 zarr_path = tmp_path / "foo.zarr" root = zarr.open_group(store=zarr_path, mode="w") root.require_group("foo") foo = root.get_group("foo") foo["bar"] = np.array([1]) shutil.make_archive(str(zarr_path), "zip", zarr_path) zip_path = tmp_path / "foo.zarr.zip" zipped = zarr.open_group(ZipStore(zip_path, mode="r"), mode="r") assert list(zipped.keys()) == list(root.keys()) group = zipped.get_group("foo") assert list(group.keys()) == list(group.keys()) async def test_list_without_explicit_open(self, tmp_path: Path) -> None: # ZipStore.list(), list_dir(), and exists() should auto-open # the zip file just like _get() and _set() do. zip_path = tmp_path / "data.zip" zarr_path = tmp_path / "foo.zarr" root = zarr.open_group(store=zarr_path, mode="w") root["x"] = np.array([1, 2, 3]) shutil.make_archive(str(zarr_path), "zip", zarr_path) shutil.move(f"{zarr_path}.zip", zip_path) store = ZipStore(zip_path, mode="r") assert not store._is_open keys = [k async for k in store.list()] assert len(keys) > 0 store2 = ZipStore(zip_path, mode="r") assert not store2._is_open assert await store2.exists(keys[0]) store3 = ZipStore(zip_path, mode="r") assert not store3._is_open dir_keys = [k async for k in store3.list_dir("")] assert len(dir_keys) > 0 async def test_move(self, tmp_path: Path) -> None: origin = tmp_path / "origin.zip" destination = tmp_path / "some_folder" / "destination.zip" store = await ZipStore.open(path=origin, mode="a") array = create_array(store, data=np.arange(10)) await store.move(str(destination)) assert store.path == destination assert destination.exists() assert not origin.exists() assert np.array_equal(array[...], np.arange(10)) @pytest.mark.parametrize( ("call", "result", "keys"), [ pytest.param( lambda s: s.get("foo", default_buffer_prototype()), b"bar", ["foo"], id="get" ), pytest.param( lambda s: s.get_partial_values(default_buffer_prototype(), [("foo", None)]), [b"bar"], ["foo"], id="get_partial_values", ), pytest.param(lambda s: s.exists("foo"), True, ["foo"], id="exists"), pytest.param( lambda s: s.set("bar", cpu.Buffer.from_bytes(b"x")), None, ["foo", "bar"], id="set", ), pytest.param( lambda s: s.set_if_not_exists("bar", cpu.Buffer.from_bytes(b"x")), None, ["foo", "bar"], id="set_if_not_exists", ), pytest.param( lambda s: s.set_if_not_exists("foo", cpu.Buffer.from_bytes(b"x")), None, ["foo"], id="set_if_not_exists_existing", ), pytest.param(lambda s: s.clear(), None, [], id="clear"), pytest.param(lambda s: s.delete("bar"), None, ["foo"], id="delete"), pytest.param(lambda s: s.delete_dir("bar"), None, ["foo"], id="delete_dir"), pytest.param(lambda s: s.is_empty(""), False, ["foo"], id="is_empty"), ], ) async def test_methods_open_store_on_first_use( self, store_kwargs: dict[str, Any], call: Any, result: Any, keys: list[str] ) -> None: """ Every method works on a store that was constructed but never opened, and sees the entries already in the archive. """ seed = await self.store_cls.open(**store_kwargs) await seed.set("foo", cpu.Buffer.from_bytes(b"bar")) seed.close() store = self.store_cls(**{**store_kwargs, "mode": "a"}) assert not store._is_open out = await call(store) if isinstance(out, list): out = [b.to_bytes() for b in out] elif isinstance(out, Buffer): out = out.to_bytes() assert out == result store.close() with zipfile.ZipFile(store_kwargs["path"]) as zf: assert zf.namelist() == keys @pytest.mark.parametrize("method", ["list", "list_dir"]) async def test_listing_opens_store_on_first_use( self, store_kwargs: dict[str, Any], method: str ) -> None: """ Listing a store that was constructed but never opened opens the archive and yields the entries already in it. """ seed = await self.store_cls.open(**store_kwargs) await seed.set("foo", cpu.Buffer.from_bytes(b"bar")) seed.close() store = self.store_cls(**{**store_kwargs, "mode": "r"}) args = () if method == "list" else ("",) assert [k async for k in getattr(store, method)(*args)] == ["foo"] store.close() @pytest.mark.parametrize("mode", ["w", "x"]) @pytest.mark.parametrize("reopen", ["use_after_close", "move", "pickle"]) async def test_reopen_keeps_entries( self, tmp_path: Path, mode: ZipStoreAccessModeLiteral, reopen: str ) -> None: """ "w" truncates and "x" refuses an existing file; neither may apply when a store that already wrote entries is opened again. """ store = ZipStore(tmp_path / "data.zip", mode=mode) await store.set("foo", cpu.Buffer.from_bytes(b"bar")) if reopen == "use_after_close": store.close() value = await store.get("foo", default_buffer_prototype()) assert value is not None assert value.to_bytes() == b"bar" store.close() elif reopen == "move": await store.move(tmp_path / "moved" / "data.zip") store.close() else: store.close() copy = pickle.loads(pickle.dumps(store)) value = await copy.get("foo", default_buffer_prototype()) assert value is not None assert value.to_bytes() == b"bar" copy.close() with zipfile.ZipFile(store.path) as zf: # type: ignore[arg-type] assert zf.namelist() == ["foo"] async def test_close_blocks_concurrent_reopen( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """ A thread that uses the store while close() is writing the central directory must wait until the archive is closed. """ store = ZipStore(tmp_path / "data.zip", mode="w") await store.set("foo", cpu.Buffer.from_bytes(b"1")) write_end_record = store._zf._write_end_record # type: ignore[attr-defined] writes: list[Future[None]] = [] with ThreadPoolExecutor(max_workers=1) as pool: def write_during_close() -> None: writes.append(pool.submit(sync, store.set("bar", cpu.Buffer.from_bytes(b"2")))) wait(writes, timeout=0.2) write_end_record() monkeypatch.setattr(store._zf, "_write_end_record", write_during_close) store.close() writes[0].result(timeout=5) store.close() with zipfile.ZipFile(store.path) as zf: # type: ignore[arg-type] assert zf.namelist() == ["foo", "bar"] async def test_move_blocks_concurrent_reopen( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """ A thread that uses the store while move() runs must wait for the move, or it reopens the old path between close and reopen. """ origin = tmp_path / "data.zip" destination = tmp_path / "moved" / "data.zip" store = ZipStore(origin, mode="w") await store.set("foo", cpu.Buffer.from_bytes(b"1")) writer = threading.Thread( target=lambda: sync(store.set("bar", cpu.Buffer.from_bytes(b"2"))) ) move_file = shutil.move def write_then_move(src: Any, dst: Any) -> Any: # the store is closed here; start a write before move() reopens it writer.start() writer.join(timeout=0.2) return move_file(src, dst) monkeypatch.setattr(shutil, "move", write_then_move) await store.move(destination) monkeypatch.undo() writer.join(timeout=5) assert not writer.is_alive() store.close() assert not origin.exists() with zipfile.ZipFile(destination) as zf: assert zf.namelist() == ["foo", "bar"] async def test_failed_clear_leaves_store_closed( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """ If clear() fails after closing the archive, the store is closed rather than left open on a closed handle, and the next use clears. """ store = ZipStore(tmp_path / "data.zip", mode="w") await store.set("foo", cpu.Buffer.from_bytes(b"1")) def unavailable(*args: Any, **kwargs: Any) -> None: raise OSError("unavailable") with monkeypatch.context() as m: m.setattr(zipfile, "ZipFile", unavailable) with pytest.raises(OSError, match="unavailable"): await store.clear() assert not store._is_open await store.set("bar", cpu.Buffer.from_bytes(b"2")) store.close() with zipfile.ZipFile(store.path) as zf: # type: ignore[arg-type] assert zf.namelist() == ["bar"] async def test_clear_exclusive_mode_keeps_existing_file(self, tmp_path: Path) -> None: """ A never-opened "x" store has not claimed the file, so clear() must refuse it the way the first open would instead of deleting it. """ path = tmp_path / "data.zip" with zipfile.ZipFile(path, mode="w") as zf: zf.writestr("foo", b"1") store = ZipStore(path, mode="x") with pytest.raises(FileExistsError): await store.clear() with zipfile.ZipFile(path) as zf: assert zf.namelist() == ["foo"] @pytest.mark.parametrize( "use", [ pytest.param(lambda s: s._ensure_open(), id="ensure_open"), pytest.param(lambda s: _drain(s.list_dir("")), id="list_dir"), ], ) async def test_first_use_waits_for_lock(self, tmp_path: Path, use: Any) -> None: """ A thread's first use must wait while another thread holds the lock, then find the archive that thread opened instead of opening it again. """ store = ZipStore(tmp_path / "data.zip", mode="w") with ThreadPoolExecutor(max_workers=1) as pool: with store._lock: first_use = pool.submit(sync, use(store)) wait([first_use], timeout=0.2) assert not store._is_open store._zipfile() first_use.result(timeout=5) store.close() @pytest.mark.parametrize("error", [OSError, KeyboardInterrupt]) async def test_failed_close_blocks_reopen( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, error: type[BaseException] ) -> None: """ If the central directory could not be written, reopening in append mode would start a new archive and drop the entries, so later use raises instead. """ store = ZipStore(tmp_path / "data.zip", mode="w") await store.set("foo", cpu.Buffer.from_bytes(b"1")) def disk_full() -> None: raise error("disk full") monkeypatch.setattr(store._zf, "_write_end_record", disk_full) with pytest.raises(error, match="disk full"): store.close() assert not store._is_open with pytest.raises(RuntimeError, match="closing the archive"): await store.set("bar", cpu.Buffer.from_bytes(b"2")) # clear() replaces the archive, so it cannot drop entries, and it # makes the store usable again await store.clear() await store.set("baz", cpu.Buffer.from_bytes(b"3")) store.close() with zipfile.ZipFile(store.path) as zf: # type: ignore[arg-type] assert zf.namelist() == ["baz"] async def test_unused_copy_of_open_writer_does_not_write(self, tmp_path: Path) -> None: """ A copy of a store that is still writing, like those dask makes while building a graph, leaves the file alone if it is never used, even when it is collected after the original closes. """ store = ZipStore(tmp_path / "data.zip", mode="w") await store.set("foo", cpu.Buffer.from_bytes(b"1")) copy = pickle.loads(pickle.dumps(store)) await store.set("bar", cpu.Buffer.from_bytes(b"2")) store.close() del copy gc.collect() with zipfile.ZipFile(store.path) as zf: # type: ignore[arg-type] assert zf.namelist() == ["foo", "bar"] assert zf.testzip() is None async def test_used_copy_of_open_writer_raises(self, tmp_path: Path) -> None: """ The file has no central directory until the original closes, so a copy that uses it refuses to reopen it instead of starting a new archive after the entries already written. """ store = ZipStore(tmp_path / "data.zip", mode="w") await store.set("foo", cpu.Buffer.from_bytes(b"1")) copy = pickle.loads(pickle.dumps(store)) with pytest.raises(zipfile.BadZipFile, match="not a zip archive"): await copy.get("foo", default_buffer_prototype()) store.close() with zipfile.ZipFile(store.path) as zf: # type: ignore[arg-type] assert zf.namelist() == ["foo"] async def test_failed_clear_keeps_failed_close_guard( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """ Clear() lifts the failed-close guard only once the old file is gone. """ store = ZipStore(tmp_path / "data.zip", mode="w") await store.set("foo", cpu.Buffer.from_bytes(b"1")) def disk_full() -> None: raise OSError("disk full") def locked(path: Any, missing_ok: bool = False) -> None: raise PermissionError("locked") monkeypatch.setattr(store._zf, "_write_end_record", disk_full) with pytest.raises(OSError, match="disk full"): store.close() with monkeypatch.context() as m: m.setattr(pathlib.Path, "unlink", locked) with pytest.raises(PermissionError, match="locked"): await store.clear() with pytest.raises(RuntimeError, match="closing the archive"): await store.set("bar", cpu.Buffer.from_bytes(b"2")) @pytest.mark.parametrize("damage", ["garbage", "missing"]) @pytest.mark.parametrize("close", ["closed", "close_failed"]) async def test_clear_replaces_damaged_file( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, damage: str, close: str ) -> None: """ Clear() replaces the archive even when the file it wrote is no longer a zip or is gone, including after a failed close(). """ path = tmp_path / "data.zip" store = ZipStore(path, mode="w") await store.set("foo", cpu.Buffer.from_bytes(b"1")) if close == "closed": store.close() else: def disk_full() -> None: raise OSError("disk full") monkeypatch.setattr(store._zf, "_write_end_record", disk_full) with pytest.raises(OSError, match="disk full"): store.close() if damage == "garbage": path.write_bytes(b"not a zip") else: path.unlink() await store.clear() await store.set("bar", cpu.Buffer.from_bytes(b"2")) store.close() with zipfile.ZipFile(path) as zf: assert zf.namelist() == ["bar"] async def test_reopen_recreates_deleted_file(self, tmp_path: Path) -> None: """ A file removed between uses is created again, not refused. """ path = tmp_path / "data.zip" store = ZipStore(path, mode="w") await store.set("foo", cpu.Buffer.from_bytes(b"1")) store.close() path.unlink() await store.set("bar", cpu.Buffer.from_bytes(b"2")) store.close() with zipfile.ZipFile(path) as zf: assert zf.namelist() == ["bar"] async def test_first_open_append_on_empty_file(self, tmp_path: Path) -> None: """ Mode "a" on an empty file, e.g. from tempfile.mkstemp, starts an archive in it; the reopen guard applies only to archives this store wrote. """ path = tmp_path / "data.zip" path.touch() store = ZipStore(path, mode="a") await store.set("foo", cpu.Buffer.from_bytes(b"1")) store.close() with zipfile.ZipFile(path) as zf: assert zf.namelist() == ["foo"] @pytest.mark.parametrize("file", ["archive", "missing"]) async def test_clear_read_mode_writable_store(self, tmp_path: Path, file: str) -> None: """ A store opened with mode "r" but read_only=False creates the new archive with "w", since "r" cannot open the file clear() removed, and it does not need to open the old file first. """ path = tmp_path / "data.zip" if file == "archive": with zipfile.ZipFile(path, mode="w") as zf: zf.writestr("foo", b"1") store = ZipStore(path, mode="r", read_only=False) await store.clear() await store.set("bar", cpu.Buffer.from_bytes(b"2")) store.close() with zipfile.ZipFile(path) as zf: assert zf.namelist() == ["bar"] async def test_reopen_reports_directory_at_path(self, tmp_path: Path) -> None: """ A directory where the archive was is reported as an OS error, not as an unfinished archive. """ path = tmp_path / "data.zip" store = ZipStore(path, mode="w") await store.set("foo", cpu.Buffer.from_bytes(b"1")) store.close() path.unlink() path.mkdir() with pytest.raises(OSError): await store.get("foo", default_buffer_prototype()) @pytest.mark.parametrize("mode", ["r", "a"]) async def test_unpickle_state_from_older_release(self, tmp_path: Path, mode: str) -> None: """ A pickle made before _was_opened and _close_failed existed opens the file as a first open would; for "a" on an empty file that starts an archive rather than tripping the reopen guard. """ path = tmp_path / "data.zip" if mode == "r": with zipfile.ZipFile(path, mode="w") as zf: zf.writestr("foo", b"1") else: path.touch() state = ZipStore(path, mode=mode).__getstate__() # type: ignore[arg-type] del state["_was_opened"], state["_close_failed"] store = ZipStore.__new__(ZipStore) store.__setstate__(state) if mode == "a": await store.set("foo", cpu.Buffer.from_bytes(b"1")) assert [k async for k in store.list()] == ["foo"] store.close() async def test_list_does_not_hold_lock_while_iterating(self, tmp_path: Path) -> None: """ Another thread can use the store while a caller is partway through iterating list(). """ store = ZipStore(tmp_path / "data.zip", mode="w") await store.set("foo", cpu.Buffer.from_bytes(b"1")) await store.set("bar", cpu.Buffer.from_bytes(b"2")) # list() is an async generator; the annotation hides aclose() keys = cast("AsyncGenerator[str, None]", store.list()) pool = ThreadPoolExecutor(max_workers=1) try: assert await anext(keys) == "foo" other = pool.submit(sync, _drain(store.list_dir(""))) assert sorted(other.result(timeout=5)) == ["bar", "foo"] finally: await keys.aclose() pool.shutdown() store.close() async def test_move_exclusive_mode_keeps_existing_file(self, tmp_path: Path) -> None: """ A never-opened "x" store has not claimed the file, so move() must refuse it the way the first open would instead of moving it. """ origin = tmp_path / "data.zip" destination = tmp_path / "moved" / "data.zip" with zipfile.ZipFile(origin, mode="w") as zf: zf.writestr("foo", b"1") store = ZipStore(origin, mode="x") with pytest.raises(FileExistsError): await store.move(destination) assert not destination.exists() with zipfile.ZipFile(origin) as zf: assert zf.namelist() == ["foo"] class TestZipStoreFileObj: """ZipStore backed by an open binary file-like object instead of a path.""" @pytest.fixture def zip_bytes(self, tmp_path: Path) -> bytes: path = tmp_path / "data.zip" store = ZipStore(path, mode="w") zarr.create_array(store, data=np.arange(10), chunks=(5,)) store.close() return path.read_bytes() def test_read_from_fileobj(self, zip_bytes: bytes) -> None: # an existing archive can be read through any seekable binary reader store = ZipStore(io.BytesIO(zip_bytes), mode="r") array = zarr.open_array(store, mode="r") assert np.array_equal(array[...], np.arange(10)) assert store.path is None def test_write_to_fileobj(self) -> None: # a writable file object receives the archive; the bytes it holds # after close() are a complete, reopenable zip buffer = io.BytesIO() store = ZipStore(buffer, mode="w", read_only=False) zarr.create_array(store, data=np.arange(4)) store.close() roundtrip = ZipStore(io.BytesIO(buffer.getvalue()), mode="r") array = zarr.open_array(roundtrip, mode="r") assert np.array_equal(array[...], np.arange(4)) async def test_write_after_close_keeps_entries(self) -> None: """ Reopening a file object after close() appends to the archive it already holds instead of starting a new one after it. """ buffer = io.BytesIO() store = ZipStore(buffer, mode="w", read_only=False) await store.set("foo", cpu.Buffer.from_bytes(b"1")) store.close() await store.set("bar", cpu.Buffer.from_bytes(b"2")) store.close() with zipfile.ZipFile(io.BytesIO(buffer.getvalue())) as zf: assert zf.namelist() == ["foo", "bar"] async def test_write_only_reuse_after_close_raises(self, tmp_path: Path) -> None: """ A write-only file object cannot be read back, so reopening it would start a new archive and drop the entries already written. """ path = tmp_path / "data.zip" with path.open("wb") as f: store = ZipStore(f, mode="w", read_only=False) await store.set("foo", cpu.Buffer.from_bytes(b"1")) store.close() try: with pytest.raises(io.UnsupportedOperation, match="readable and seekable"): await store.set("bar", cpu.Buffer.from_bytes(b"2")) finally: # close before the file does, even if the write went through store.close() with zipfile.ZipFile(path) as zf: assert zf.namelist() == ["foo"] async def test_unseekable_reuse_after_close_raises(self) -> None: """ A file object that cannot seek cannot be read back either, so reuse after close() raises the same error as a write-only one. """ buffer = io.BytesIO() pipe = io.BufferedRWPair(buffer, buffer) # readable, not seekable store = ZipStore(pipe, mode="w", read_only=False) # type: ignore[arg-type] await store.set("foo", cpu.Buffer.from_bytes(b"1")) store.close() try: with pytest.raises(io.UnsupportedOperation, match="readable and seekable"): await store.set("bar", cpu.Buffer.from_bytes(b"2")) finally: store.close() async def test_clear_unsupported(self, zip_bytes: bytes) -> None: # clear() requires a filesystem location, so it raises a clear error # for file-object-backed stores store = ZipStore(io.BytesIO(zip_bytes), mode="a", read_only=False) store._sync_open() with pytest.raises(NotImplementedError, match="clear.*file-like"): await store.clear() async def test_move_unsupported(self, zip_bytes: bytes) -> None: # move() requires a filesystem location, so it raises a clear error # for file-object-backed stores store = ZipStore(io.BytesIO(zip_bytes), mode="a", read_only=False) store._sync_open() with pytest.raises(NotImplementedError, match="move.*file-like"): await store.move("elsewhere.zip") def test_invalid_file_object_rejected(self) -> None: # objects without read/seek/tell are rejected at construction, not # deep inside zipfile with pytest.raises(TypeError, match="read/seek/tell"): ZipStore(42, mode="r") # type: ignore[arg-type] @pytest.mark.parametrize("mode", ["w", "a", "x"]) def test_non_iobase_reader_write_modes_rejected(self, zip_bytes: bytes, mode: str) -> None: # readers that are not io.IOBase instances are adapted for reading # only; write modes are rejected at construction with a clear error class MinimalReader: def __init__(self, data: bytes) -> None: self._buffer = io.BytesIO(data) def read(self, size: int, /) -> bytes: return self._buffer.read(size) def seek(self, pos: int, whence: int = 0, /) -> int: return self._buffer.seek(pos, whence) def tell(self) -> int: return self._buffer.tell() with pytest.raises(TypeError, match="opened for reading"): ZipStore(MinimalReader(zip_bytes), mode=mode, read_only=False) # type: ignore[arg-type] def test_fsspec_file(self, tmp_path: Path, zip_bytes: bytes) -> None: # a file opened through fsspec (already an io.IOBase) is used directly; # fsspec's local filesystem stands in for a remote one fsspec = pytest.importorskip("fsspec") path = tmp_path / "fsspec.zip" path.write_bytes(zip_bytes) with fsspec.open(f"local://{path}", "rb") as fileobj: store = ZipStore(fileobj, mode="r") array = zarr.open_array(store, mode="r") assert np.array_equal(array[...], np.arange(10)) assert store.path is None def test_obstore_reader(self, tmp_path: Path, zip_bytes: bytes) -> None: # obstore's ReadableFile is not an io.IOBase and its read() returns a # buffer-protocol object; ZipStore adapts it via _RawReaderAdapter obstore = pytest.importorskip("obstore") from obstore.store import LocalStore as ObstoreLocalStore (tmp_path / "obstore.zip").write_bytes(zip_bytes) reader = obstore.open_reader(ObstoreLocalStore(str(tmp_path)), "obstore.zip") store = ZipStore(reader, mode="r") array = zarr.open_array(store, mode="r") assert np.array_equal(array[...], np.arange(10)) def test_raw_reader_adapter_eof(self) -> None: from zarr.storage._zip import _RawReaderAdapter class MinimalReader: """Non-io.IOBase reader exposing only read/seek/tell, like obstore.""" def __init__(self, data: bytes) -> None: self._buffer = io.BytesIO(data) def read(self, size: int, /) -> bytes: return self._buffer.read(size) def seek(self, pos: int, whence: int = 0, /) -> int: return self._buffer.seek(pos, whence) def tell(self) -> int: return self._buffer.tell() # the adapter must clamp reads to EOF: some readers (obstore < 0.6) # raise on short reads instead of returning fewer bytes data = b"0123456789" adapter = _RawReaderAdapter(MinimalReader(data)) # type: ignore[arg-type] # A read straddling EOF returns only the remaining bytes. adapter.seek(len(data) - 3) buf = bytearray(8) assert adapter.readinto(buf) == 3 assert bytes(buf[:3]) == data[-3:] # A read at EOF returns 0. assert adapter.tell() == len(data) assert adapter.readinto(bytearray(8)) == 0 def test_pickle_fileobj_raises(self, zip_bytes: bytes) -> None: # an open file object cannot be reliably serialized, so pickling a # file-object-backed store raises with a pointer at the alternative store = ZipStore(io.BytesIO(zip_bytes), mode="r") with pytest.raises(TypeError, match="cannot pickle a ZipStore backed by a file-like"): pickle.dumps(store) def test_pickle_path_backed_roundtrip(self, tmp_path: Path, zip_bytes: bytes) -> None: # path-backed stores remain picklable: the path is serialized and the # archive is reopened on unpickling path = tmp_path / "pickled.zip" path.write_bytes(zip_bytes) store = ZipStore(path, mode="r") unpickled = pickle.loads(pickle.dumps(store)) array = zarr.open_array(unpickled, mode="r") assert np.array_equal(array[...], np.arange(10)) def test_str_and_eq(self, zip_bytes: bytes) -> None: # file-object-backed stores stringify with the object repr and # compare equal only when backed by the very same file object fileobj = io.BytesIO(zip_bytes) store = ZipStore(fileobj, mode="r") assert str(store).startswith("zip://<") assert store == ZipStore(fileobj, mode="r") assert store != ZipStore(io.BytesIO(zip_bytes), mode="r") class ZipStoreLifecycleMachine(RuleBasedStateMachine): """Drive a ZipStore through construct / open / write / close transitions. Invariant under test: a constructed ZipStore can always be closed without raising, regardless of whether it was ever opened or did any I/O. This is a property-based generalization of the former example-based regression tests for ZipStore.close() being called on a never-opened store. """ def __init__(self, tmp_path: Path) -> None: super().__init__() self._tmp_path = tmp_path self._counter = 0 self.store: ZipStore | None = None self._opened = False @initialize() def start(self) -> None: self.store = None self._opened = False @precondition(lambda self: self.store is None) @rule() def construct(self) -> None: # Fresh path each time so mode="w" never clobbers a closed archive. self._counter += 1 self.store = ZipStore(self._tmp_path / f"s{self._counter}.zip", mode="w") self._opened = False @precondition(lambda self: self.store is not None and not self._opened) @rule() def open(self) -> None: assert self.store is not None self.store._sync_open() self._opened = True @precondition(lambda self: self.store is not None and not self._opened) @rule() def write(self) -> None: assert self.store is not None # store.set auto-opens the store. sync(self.store.set("a", cpu.Buffer.from_bytes(b"hi"))) self._opened = True @precondition(lambda self: self.store is not None) @rule() def close(self) -> None: assert self.store is not None # The property under test: close() must never raise, even with no # prior open or I/O. self.store.close() self.store = None self._opened = False def test_zipstore_close_lifecycle(tmp_path: Path) -> None: run_state_machine_as_test( # type: ignore[no-untyped-call] lambda: ZipStoreLifecycleMachine(tmp_path), settings=settings(max_examples=50, deadline=None), ) @pytest.mark.parametrize("zarr_format", [2, 3]) def test_resize_needing_deletes_leaves_array_unchanged( tmp_path: Path, zarr_format: Literal[2, 3] ) -> None: """A zip store cannot delete, so a resize that must delete chunks fails before it stores the new metadata.""" path = tmp_path / "a.zip" store = ZipStore(path, mode="w") arr = create_array(store, shape=(20,), chunks=(5,), dtype="i4", zarr_format=zarr_format) arr[:] = np.arange(20) with pytest.raises(NotImplementedError): arr.resize((5,)) assert arr.shape == (20,) store.close() names = zipfile.ZipFile(path).namelist() assert len(names) == len(set(names)) reopened = zarr.open_array(ZipStore(path, mode="r"), mode="r") np.testing.assert_array_equal(reopened[:], np.arange(20)) zarr-developers-zarr-python-a2f2415/tests/test_sync.py000066400000000000000000000114611527014137600232170ustar00rootroot00000000000000import asyncio from collections.abc import AsyncGenerator from unittest.mock import AsyncMock, patch import pytest import zarr from zarr.core.sync import ( SyncError, SyncMixin, _get_executor, _get_lock, _get_loop, cleanup_resources, loop, sync, ) @pytest.fixture(params=[True, False]) def sync_loop(request: pytest.FixtureRequest) -> asyncio.AbstractEventLoop | None: if request.param is True: return _get_loop() else: return None @pytest.fixture def clean_state(): # use this fixture to make sure no existing threads/loops exist in zarr.core.sync cleanup_resources() yield cleanup_resources() def test_get_loop() -> None: # test that calling _get_loop() twice returns the same loop loop = _get_loop() loop2 = _get_loop() assert loop is loop2 def test_get_lock() -> None: # test that calling _get_lock() twice returns the same lock lock = _get_lock() lock2 = _get_lock() assert lock is lock2 def test_sync(sync_loop: asyncio.AbstractEventLoop | None) -> None: foo = AsyncMock(return_value="foo") assert sync(foo(), loop=sync_loop) == "foo" foo.assert_awaited_once() def test_sync_raises(sync_loop: asyncio.AbstractEventLoop | None) -> None: foo = AsyncMock(side_effect=ValueError("foo-bar")) with pytest.raises(ValueError, match="foo-bar"): sync(foo(), loop=sync_loop) foo.assert_awaited_once() def test_sync_timeout() -> None: duration = 0.02 async def foo() -> None: await asyncio.sleep(duration) with pytest.raises(asyncio.TimeoutError): sync(foo(), timeout=duration / 10) def test_sync_raises_if_no_coroutine(sync_loop: asyncio.AbstractEventLoop | None) -> None: def foo() -> str: return "foo" with pytest.raises(TypeError): sync(foo(), loop=sync_loop) # type: ignore[arg-type] @pytest.mark.filterwarnings("ignore:coroutine.*was never awaited") def test_sync_raises_if_loop_is_closed() -> None: loop = _get_loop() foo = AsyncMock(return_value="foo") with patch.object(loop, "is_closed", return_value=True): with pytest.raises(RuntimeError): sync(foo(), loop=loop) foo.assert_not_awaited() @pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning") @pytest.mark.filterwarnings("ignore:coroutine.*was never awaited") def test_sync_raises_if_calling_sync_from_within_a_running_loop( sync_loop: asyncio.AbstractEventLoop | None, ) -> None: def foo() -> str: # technically, this should be an async function but doing that # yields a warning because it is never awaited by the inner function return "foo" async def bar() -> str: return sync(foo(), loop=sync_loop) # type: ignore[arg-type] with pytest.raises(SyncError): sync(bar(), loop=sync_loop) @pytest.mark.filterwarnings("ignore:coroutine.*was never awaited") def test_sync_raises_if_loop_is_invalid_type() -> None: foo = AsyncMock(return_value="foo") with pytest.raises(TypeError): sync(foo(), loop=1) # type: ignore[arg-type] foo.assert_not_awaited() def test_sync_mixin(sync_loop) -> None: class AsyncFoo: def __init__(self) -> None: pass async def foo(self) -> str: return "foo" async def bar(self) -> AsyncGenerator: for i in range(10): yield i class SyncFoo(SyncMixin): def __init__(self, async_foo: AsyncFoo) -> None: self._async_foo = async_foo def foo(self) -> str: return self._sync(self._async_foo.foo()) def bar(self) -> list[int]: return self._sync_iter(self._async_foo.bar()) async_foo = AsyncFoo() foo = SyncFoo(async_foo) assert foo.foo() == "foo" assert foo.bar() == list(range(10)) @pytest.mark.parametrize("workers", [None, 1, 2]) def test_threadpool_executor(clean_state, workers: int | None) -> None: with zarr.config.set({"threading.max_workers": workers}): _ = zarr.zeros(shape=(1,)) # trigger executor creation assert loop != [None] # confirm loop was created if workers is None: # confirm no executor was created if no workers were specified # (this is the default behavior) assert loop[0]._default_executor is None else: # confirm executor was created and attached to loop as the default executor # note: python doesn't have a direct way to get the default executor so we # use the private attribute assert _get_executor() is loop[0]._default_executor assert _get_executor()._max_workers == workers def test_cleanup_resources_idempotent() -> None: _get_executor() # trigger resource creation (iothread, loop, thread-pool) cleanup_resources() cleanup_resources() zarr-developers-zarr-python-a2f2415/tests/test_tree.py000066400000000000000000000061551527014137600232060ustar00rootroot00000000000000import textwrap from typing import Any import pytest import zarr @pytest.mark.parametrize("root_name", [None, "root"]) @pytest.mark.parametrize("atty", [True, False]) @pytest.mark.parametrize("plain", [True, False]) def test_tree(root_name: Any, atty: bool, plain: bool, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr("sys.stdout.isatty", lambda: atty) if atty and not plain: BOPEN = "\x1b[1m" BCLOSE = "\x1b[0m" else: BOPEN = "" BCLOSE = "" g = zarr.group(path=root_name) A = g.create_group("A") B = g.create_group("B") C = B.create_group("C") D = C.create_group("C") A.create_array(name="x", shape=(2), dtype="float64") A.create_array(name="y", shape=(0,), dtype="int8") B.create_array(name="x", shape=(0,), dtype="float64") C.create_array(name="x", shape=(0,), dtype="float64") D.create_array(name="x", shape=(0,), dtype="float64") result = repr(g.tree(plain=plain)) root = root_name or "" expected = textwrap.dedent(f"""\ {BOPEN}/{root}{BCLOSE} ├── {BOPEN}A{BCLOSE} │ ├── {BOPEN}x{BCLOSE} (2,) float64 │ └── {BOPEN}y{BCLOSE} (0,) int8 └── {BOPEN}B{BCLOSE} ├── {BOPEN}C{BCLOSE} │ ├── {BOPEN}C{BCLOSE} │ │ └── {BOPEN}x{BCLOSE} (0,) float64 │ └── {BOPEN}x{BCLOSE} (0,) float64 └── {BOPEN}x{BCLOSE} (0,) float64 """) assert result == expected result = repr(g.tree(level=0, plain=plain)) expected = textwrap.dedent(f"""\ {BOPEN}/{root}{BCLOSE} ├── {BOPEN}A{BCLOSE} └── {BOPEN}B{BCLOSE} """) assert result == expected if not plain: tree = g.tree(plain=False) bundle = tree._repr_mimebundle_() assert "text/plain" in bundle assert "text/html" in bundle assert "A" in bundle["text/html"] assert "x" in bundle["text/html"] assert " None: g = zarr.group() g.create_group("a") g.create_group("b") g.create_group("c") g.create_group("d") g.create_group("e") result = repr(g.tree(max_nodes=3, plain=True)) assert "Truncated at max_nodes=3" in result # Should show exactly 3 nodes (lines with ── connectors). lines = result.strip().split("\n") node_lines = [line for line in lines if "──" in line] assert len(node_lines) == 3 # Full tree should not show truncation message. full = repr(g.tree(max_nodes=500, plain=True)) assert "truncated" not in full def test_tree_html_escaping() -> None: g = zarr.group() g.create_group("") tree = g.tree() bundle = tree._repr_mimebundle_() assert "<img" in bundle["text/html"] assert "" in bundle["text/plain"] def test_expand_not_implemented() -> None: g = zarr.group() with pytest.raises(NotImplementedError): g.tree(expand=True) zarr-developers-zarr-python-a2f2415/tests/test_unified_chunk_grid.py000066400000000000000000003370341527014137600260720ustar00rootroot00000000000000""" Tests for the unified ChunkGrid design (POC). Tests the core ChunkGrid with FixedDimension/VaryingDimension internals, ChunkSpec, serialization round-trips, indexing with rectilinear grids, and end-to-end array creation + read/write. """ from __future__ import annotations import re from functools import partial from typing import TYPE_CHECKING, Any import numpy as np import pytest import zarr from tests.test_metadata.conftest import minimal_metadata_dict_v3 from zarr.core.buffer import default_buffer_prototype from zarr.core.chunk_grids import ( ChunkGrid, ChunkSpec, FixedDimension, VaryingDimension, guess_chunks, ) from zarr.core.common import compress_rle, expand_rle from zarr.core.dtype import UInt8 from zarr.core.metadata.v3 import ( ArrayV3Metadata, RectilinearChunkGridMetadata, RectilinearChunkGridMetadataJSON, RegularChunkGridMetadata, parse_chunk_grid, ) from zarr.errors import BoundsCheckError from zarr.storage import MemoryStore if TYPE_CHECKING: from collections.abc import Callable, Generator from pathlib import Path from zarr.core.common import ChunksLike @pytest.fixture(autouse=True) def _enable_rectilinear_chunks() -> Generator[None, None, None]: """Enable rectilinear chunks for all tests in this module.""" with zarr.config.set({"array.rectilinear_chunks": True}): yield def _edges(grid: ChunkGrid, dim: int) -> tuple[int, ...]: """Extract the per-chunk edge lengths for *dim* from a ChunkGrid.""" d = grid._dimensions[dim] if isinstance(d, FixedDimension): return tuple(d.size for _ in range(d.nchunks)) if isinstance(d, VaryingDimension): return tuple(d.edges) raise TypeError(f"Unexpected dimension type: {type(d)}") # --------------------------------------------------------------------------- # Dimension index_to_chunk bounds tests # --------------------------------------------------------------------------- @pytest.mark.parametrize( ("dim", "index", "match"), [ (VaryingDimension([10, 20, 30], extent=60), 60, "out of bounds"), (VaryingDimension([10, 20, 30], extent=60), 100, "out of bounds"), (FixedDimension(size=10, extent=95), 95, "out of bounds"), (FixedDimension(size=10, extent=95), -1, "Negative"), ], ids=[ "varying-at-extent", "varying-past-extent", "fixed-at-extent", "fixed-negative", ], ) def test_dimension_index_to_chunk_bounds( dim: FixedDimension | VaryingDimension, index: int, match: str ) -> None: """Out-of-bounds or negative indices raise IndexError for both dimension types""" with pytest.raises(IndexError, match=match): dim.index_to_chunk(index) @pytest.mark.parametrize( ("dim", "index", "expected"), [ (VaryingDimension([10, 20, 30], extent=60), 59, 2), (FixedDimension(size=10, extent=95), 94, 9), ], ids=["varying-last-valid", "fixed-last-valid"], ) def test_dimension_index_to_chunk_last_valid( dim: FixedDimension | VaryingDimension, index: int, expected: int ) -> None: """Last valid index maps to the correct chunk for both dimension types""" assert dim.index_to_chunk(index) == expected # --------------------------------------------------------------------------- # Rectilinear feature flag tests # --------------------------------------------------------------------------- _RECTILINEAR_DOC = minimal_metadata_dict_v3( shape=(30, 50), chunk_grid={ "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[10, 20], [25, 25]]}, }, ) @pytest.mark.parametrize( "action", [ lambda: ArrayV3Metadata.from_dict(dict(_RECTILINEAR_DOC)), # type: ignore[arg-type] lambda: ArrayV3Metadata( shape=(30, 50), data_type=UInt8(), chunk_grid=RectilinearChunkGridMetadata(chunk_shapes=((10, 20), (25, 25))), chunk_key_encoding={"name": "default"}, fill_value=0, codecs=[{"name": "bytes"}], attributes=None, dimension_names=None, ).to_buffer_dict(default_buffer_prototype()), lambda: zarr.create_array(MemoryStore(), shape=(30,), chunks=[[10, 20]], dtype="int32"), ], ids=["read", "store", "create_array"], ) def test_rectilinear_feature_flag_blocked(action: Any) -> None: """Reading or storing an array metadata document that declares a rectilinear chunk grid raises ValueError when the feature flag is disabled.""" with zarr.config.set({"array.rectilinear_chunks": False}): with pytest.raises(ValueError, match="experimental and disabled by default"): action() def test_rectilinear_feature_flag_names_stored_array() -> None: """Opening a stored array whose document the flag refuses names the array.""" store = MemoryStore() with zarr.config.set({"array.rectilinear_chunks": True}): zarr.create_array(store, name="r", shape=(30,), chunks=[[10, 20]], dtype="int32") with ( zarr.config.set({"array.rectilinear_chunks": False}), pytest.raises(ValueError, match="experimental and disabled by default") as info, ): zarr.open_array(store, path="r") assert info.value.__notes__ == [f"Array {str(store) + '/r'!r}: nothing was read."] def test_rectilinear_metadata_classes_not_gated() -> None: """The flag gates stored documents, not the chunk grid metadata classes.""" with zarr.config.set({"array.rectilinear_chunks": False}): grid = RectilinearChunkGridMetadata(chunk_shapes=((10, 20), (25, 25))) assert grid.ndim == 2 # --------------------------------------------------------------------------- # FixedDimension tests # --------------------------------------------------------------------------- @pytest.mark.parametrize( ( "size", "extent", "chunk_ix", "expected_nchunks", "expected_chunk_size", "expected_data_size", "expected_offset", ), [ (10, 100, 0, 10, 10, 10, 0), (10, 100, 1, 10, 10, 10, 10), (10, 100, 9, 10, 10, 10, 90), (10, 95, 9, 10, 10, 5, 90), # boundary chunk (10, 0, None, 0, None, None, None), # zero-extent: no chunks, size still >= 1 ], ids=["start", "middle", "end", "boundary", "zero-extent"], ) def test_fixed_dimension( size: int, extent: int, chunk_ix: int | None, expected_nchunks: int, expected_chunk_size: int | None, expected_data_size: int | None, expected_offset: int | None, ) -> None: """FixedDimension properties match expected values for various chunk/extent combinations""" d = FixedDimension(size=size, extent=extent) assert d.nchunks == expected_nchunks if chunk_ix is not None: assert d.chunk_size(chunk_ix) == expected_chunk_size assert d.data_size(chunk_ix) == expected_data_size assert d.chunk_offset(chunk_ix) == expected_offset @pytest.mark.parametrize( ("idx", "expected"), [(0, 0), (9, 0), (10, 1), (25, 2)], ) def test_fixed_dimension_index_to_chunk(idx: int, expected: int) -> None: """FixedDimension.index_to_chunk maps element indices to correct chunk indices""" d = FixedDimension(size=10, extent=100) assert d.index_to_chunk(idx) == expected def test_fixed_dimension_indices_to_chunks() -> None: """FixedDimension.indices_to_chunks vectorizes index-to-chunk mapping over an array""" d = FixedDimension(size=10, extent=100) indices = np.array([0, 5, 10, 15, 99]) np.testing.assert_array_equal(d.indices_to_chunks(indices), [0, 0, 1, 1, 9]) @pytest.mark.parametrize( ("size", "extent", "match"), [ (-1, 100, "size must be >= 1"), (0, 100, "size must be >= 1"), (0, 0, "size must be >= 1"), (10, -1, "extent must be >= 0"), ], ids=["negative-size", "zero-size", "zero-size-zero-extent", "negative-extent"], ) def test_fixed_dimension_rejects_invalid(size: int, extent: int, match: str) -> None: """FixedDimension raises ValueError for a size below 1 or a negative extent.""" with pytest.raises(ValueError, match=match): FixedDimension(size=size, extent=extent) # --------------------------------------------------------------------------- # VaryingDimension tests # --------------------------------------------------------------------------- def test_varying_dimension_construction() -> None: """VaryingDimension stores edges, cumulative sums, nchunks, and extent correctly""" d = VaryingDimension([10, 20, 30], extent=60) assert d.edges == (10, 20, 30) assert d.cumulative == (10, 30, 60) assert d.nchunks == 3 assert d.extent == 60 @pytest.mark.parametrize( ( "chunk_idx", "expected_offset", "expected_size", "expected_data", "expected_chunk_for_first_idx", ), [ (0, 0, 10, 10, 0), (1, 10, 20, 20, 1), (2, 30, 30, 30, 2), ], ) def test_varying_dimension( chunk_idx: int, expected_offset: int, expected_size: int, expected_data: int, expected_chunk_for_first_idx: int, ) -> None: """VaryingDimension chunk_offset, chunk_size, data_size, and index_to_chunk return correct values""" d = VaryingDimension([10, 20, 30], extent=60) assert d.chunk_offset(chunk_idx) == expected_offset assert d.chunk_size(chunk_idx) == expected_size assert d.data_size(chunk_idx) == expected_data assert d.index_to_chunk(expected_offset) == expected_chunk_for_first_idx def test_varying_dimension_indices_to_chunks() -> None: """VaryingDimension.indices_to_chunks vectorizes index-to-chunk mapping over an array""" d = VaryingDimension([10, 20, 30], extent=60) indices = np.array([0, 9, 10, 29, 30, 59]) np.testing.assert_array_equal(d.indices_to_chunks(indices), [0, 0, 1, 1, 2, 2]) @pytest.mark.parametrize( ("edges", "extent", "match"), [ ([], 0, "must not be empty"), ([10, 0, 5], 15, "must be > 0"), ], ids=["empty", "zero-edge"], ) def test_varying_dimension_rejects_invalid(edges: list[int], extent: int, match: str) -> None: """VaryingDimension raises ValueError for empty edges or zero-length edges""" with pytest.raises(ValueError, match=match): VaryingDimension(edges, extent=extent) # --------------------------------------------------------------------------- # ChunkSpec tests # --------------------------------------------------------------------------- @pytest.mark.parametrize( ("slices", "codec_shape", "expected_shape", "expected_boundary"), [ ((slice(0, 10), slice(0, 20)), (10, 20), (10, 20), False), ((slice(90, 95), slice(0, 20)), (10, 20), (5, 20), True), ((slice(10, 10),), (0,), (0,), False), ((slice(0, 10), slice(0, 5)), (10, 10), (10, 5), True), ], ids=["basic", "boundary", "empty-slices", "multidim-boundary"], ) def test_chunk_spec( slices: tuple[slice, ...], codec_shape: tuple[int, ...], expected_shape: tuple[int, ...], expected_boundary: bool, ) -> None: """ChunkSpec reports correct shape and boundary status from slices and codec_shape""" spec = ChunkSpec(slices=slices, codec_shape=codec_shape) assert spec.shape == expected_shape assert spec.is_boundary == expected_boundary # --------------------------------------------------------------------------- # ChunkGrid construction tests # --------------------------------------------------------------------------- @pytest.mark.parametrize( ("array_shape", "chunk_sizes", "expected_regular", "expected_ndim", "expected_chunk_shape"), [ ((100, 200), (10, 20), True, 2, (10, 20)), ((), (), True, 0, ()), ((60, 100), [[10, 20, 30], [25, 25, 25, 25]], False, 2, None), # explicit edge lists stay rectilinear even when uniform (gh-4272), including # uniform edges overhanging the extent after a shrinking resize ((30, 50), [[10, 10, 10], [25, 25]], False, 2, None), ((35,), [[10, 10, 10, 10]], False, 1, None), ], ids=["regular", "zero-dim", "rectilinear", "uniform-edges", "uniform-edges-overhang"], ) def test_chunk_grid_construction( array_shape: tuple[int, ...], chunk_sizes: Any, expected_regular: bool, expected_ndim: int, expected_chunk_shape: tuple[int, ...] | None, ) -> None: """ChunkGrid.from_sizes produces grids with correct regularity, ndim, and chunk_shape""" g = ChunkGrid.from_sizes(array_shape, chunk_sizes) assert g.is_regular == expected_regular assert g.ndim == expected_ndim if expected_chunk_shape is not None: assert g.chunk_shape == expected_chunk_shape else: with pytest.raises(ValueError, match="only available for regular"): _ = g.chunk_shape # --------------------------------------------------------------------------- # ChunkGrid query tests # --------------------------------------------------------------------------- @pytest.mark.parametrize( ("shape", "chunks", "expected_grid_shape"), [ ((100, 200), (10, 20), (10, 10)), ((95, 200), (10, 20), (10, 10)), ((60, 100), [[10, 20, 30], [25, 25, 25, 25]], (3, 4)), ], ids=["regular", "regular-boundary", "rectilinear"], ) def test_chunk_grid_shape( shape: tuple[int, ...], chunks: Any, expected_grid_shape: tuple[int, ...], ) -> None: """ChunkGrid.grid_shape returns the expected number of chunks per dimension""" g = ChunkGrid.from_sizes(shape, chunks) assert g.grid_shape == expected_grid_shape @pytest.mark.parametrize( ( "array_shape", "chunk_sizes", "coords", "expected_shape", "expected_codec_shape", "expected_boundary", ), [ # regular interior ((100, 200), (10, 20), (0, 0), (10, 20), (10, 20), False), # regular boundary ((95, 200), (10, 20), (9, 0), (5, 20), (10, 20), True), # rectilinear ((60, 100), [[10, 20, 30], [25, 25, 25, 25]], (0, 0), (10, 25), (10, 25), False), ((60, 100), [[10, 20, 30], [25, 25, 25, 25]], (1, 0), (20, 25), (20, 25), False), ((60, 100), [[10, 20, 30], [25, 25, 25, 25]], (2, 3), (30, 25), (30, 25), False), ], ids=["regular", "regular-boundary", "rectilinear-0,0", "rectilinear-1,0", "rectilinear-2,3"], ) def test_chunk_grid_getitem( array_shape: tuple[int, ...], chunk_sizes: Any, coords: tuple[int, ...], expected_shape: tuple[int, ...], expected_codec_shape: tuple[int, ...], expected_boundary: bool, ) -> None: """ChunkGrid.__getitem__ returns a ChunkSpec with correct shape, codec_shape, and boundary flag""" g = ChunkGrid.from_sizes(array_shape, chunk_sizes) spec = g[coords] assert spec is not None assert spec.shape == expected_shape assert spec.codec_shape == expected_codec_shape assert spec.is_boundary == expected_boundary @pytest.mark.parametrize( ("array_shape", "chunk_sizes", "coords"), [ ((100, 200), (10, 20), (99, 0)), ((60, 100), [[10, 20, 30], [25, 25, 25, 25]], (3, 0)), ], ids=["regular-oob", "rectilinear-oob"], ) def test_chunk_grid_getitem_oob( array_shape: tuple[int, ...], chunk_sizes: Any, coords: tuple[int, ...] ) -> None: """Out-of-bounds chunk coordinates return None""" g = ChunkGrid.from_sizes(array_shape, chunk_sizes) assert g[coords] is None def test_chunk_grid_getitem_slices() -> None: """ChunkSpec.slices reflect the correct start/stop for a rectilinear chunk""" g = ChunkGrid.from_sizes((60, 100), [[10, 20, 30], [25, 25, 25, 25]]) spec = g[(1, 2)] assert spec is not None assert spec.slices == (slice(10, 30, 1), slice(50, 75, 1)) # -- all_chunk_coords tests -- @pytest.mark.parametrize( ("array_shape", "chunk_sizes", "origin", "selection_shape", "expected_coords"), [ # rectilinear grid ( (60, 100), [[10, 20, 30], [50, 50]], None, None, [(0, 0), (0, 1), (1, 0), (1, 1), (2, 0), (2, 1)], ), ((60, 100), [[10, 20, 30], [50, 50]], (1, 0), None, [(1, 0), (1, 1), (2, 0), (2, 1)]), ((60, 100), [[10, 20, 30], [50, 50]], None, (2, 1), [(0, 0), (1, 0)]), ((60, 100), [[10, 20, 30], [50, 50]], (1, 1), (2, 1), [(1, 1), (2, 1)]), # regular grid ((30, 40), (10, 20), (2, 1), None, [(2, 1)]), ((30, 40), (10, 20), None, (0, 0), []), ((60, 80), (20, 20), (0, 2), (3, 1), [(0, 2), (1, 2), (2, 2)]), ], ids=[ "all", "with-origin", "with-sel-shape", "origin+sel", "last-chunk", "zero-sel", "single-dim", ], ) def test_all_chunk_coords( array_shape: tuple[int, ...], chunk_sizes: Any, origin: tuple[int, ...] | None, selection_shape: tuple[int, ...] | None, expected_coords: list[tuple[int, ...]], ) -> None: """all_chunk_coords yields the expected coordinates with optional origin and selection_shape""" g = ChunkGrid.from_sizes(array_shape, chunk_sizes) kwargs: dict[str, Any] = {} if origin is not None: kwargs["origin"] = origin if selection_shape is not None: kwargs["selection_shape"] = selection_shape assert list(g.all_chunk_coords(**kwargs)) == expected_coords def test_chunk_grid_get_nchunks() -> None: """get_nchunks returns the total number of chunks across all dimensions""" g = ChunkGrid.from_sizes((60, 100), [[10, 20, 30], [50, 50]]) assert g.get_nchunks() == 6 def test_chunk_grid_iter() -> None: """Iterating a ChunkGrid yields the correct number of ChunkSpec objects""" g = ChunkGrid.from_sizes((30, 40), (10, 20)) specs = list(g) assert len(specs) == 6 assert all(isinstance(s, ChunkSpec) for s in specs) # --------------------------------------------------------------------------- # RLE tests # --------------------------------------------------------------------------- @pytest.mark.parametrize( ("compressed", "expected"), [ ([[10, 3]], [10, 10, 10]), ([[10, 2], [20, 1]], [10, 10, 20]), ([[True, 2], [3, 1]], [1, 1, 3]), ], ) def test_rle_expand(compressed: list[Any], expected: list[int]) -> None: """RLE-encoded edges expand correctly""" assert expand_rle(compressed) == expected @pytest.mark.parametrize( ("original", "expected"), [ ([10, 10, 10], [[10, 3]]), ([10, 10, 20], [[10, 2], 20]), ([5], [5]), ([10, 20, 30], [10, 20, 30]), ], ) def test_rle_compress(original: list[int], expected: list[Any]) -> None: """compress_rle produces the expected RLE encoding for various input sequences""" assert compress_rle(original) == expected def test_rle_roundtrip() -> None: """compress_rle followed by expand_rle recovers the original sequence""" original = [10, 10, 10, 20, 20, 30] compressed = compress_rle(original) assert expand_rle(compressed) == original @pytest.mark.parametrize( ("rle_input", "match"), [ ([0], "Chunk edge length must be >= 1"), ([-5], "Chunk edge length must be >= 1"), ([[0, 3]], "Chunk edge length must be >= 1"), ([[-10, 2]], "Chunk edge length must be >= 1"), ([[5, 0]], "RLE repeat count must be >= 1"), ([[5, -1]], "RLE repeat count must be >= 1"), ([[5, 2, 1]], r"RLE entries must be an integer or \[size, count\], got \[5, 2, 1\]"), ], ids=[ "zero-edge", "negative-edge", "zero-rle-size", "negative-rle-size", "zero-rle-count", "negative-rle-count", "rle-entry-of-three", ], ) def test_rle_expand_rejects_invalid(rle_input: list[Any], match: str) -> None: """expand_rle raises ValueError for zero/negative edge lengths or repeat counts""" with pytest.raises(ValueError, match=match): expand_rle(rle_input) @pytest.mark.parametrize( ("rle_input", "match"), [ ([10.5], "Chunk edge length must be an int, got 10.5"), ([10.0], "Chunk edge length must be an int, got 10.0"), ([[10.0, 3]], "Chunk edge length must be an int, got 10.0"), (["10"], "Chunk edge length must be an int, got '10'"), ([[10, 3.5]], "RLE repeat count must be an int, got 3.5"), ([[10, 3.0]], "RLE repeat count must be an int, got 3.0"), ([np.int64(10)], "Chunk edge length must be an int, got np.int64(10)"), ([[10, np.int64(3)]], "RLE repeat count must be an int, got np.int64(3)"), ], ids=[ "fractional-edge", "float-edge", "float-rle-size", "string-edge", "fractional-count", "float-count", "numpy-int-edge", "numpy-int-count", ], ) def test_rle_expand_rejects_non_int(rle_input: list[Any], match: str) -> None: """expand_rle reads `int`s (and `bool`s) only, not floats or NumPy integers.""" with pytest.raises(TypeError, match=re.escape(match)): expand_rle(rle_input) @pytest.mark.parametrize( ("rle_input", "match"), [ ([0], "chunk edge length must be >= 1"), ([10.5], "chunk edge length must be an int,"), ([[5, 0]], "RLE repeat count must be >= 1"), ([[5, 2, 1]], r"RLE entries must be an integer or \[size, count\]"), ], ids=["zero-edge", "fractional-edge", "zero-rle-count", "rle-entry-of-three"], ) def test_rle_expand_names_dimension(rle_input: list[Any], match: str) -> None: """Given the dimension `axis` the edges belong to, every error of expand_rle names it.""" with pytest.raises((TypeError, ValueError), match=f"^Dimension 2: {match}"): expand_rle(rle_input, axis=2) # --------------------------------------------------------------------------- # Serialization tests # --------------------------------------------------------------------------- def test_serialization_error_non_regular_chunk_shape() -> None: """Accessing chunk_shape on a non-regular grid raises ValueError.""" grid = ChunkGrid.from_sizes((60, 100), [[10, 20, 30], [25, 25, 25, 25]]) with pytest.raises(ValueError, match="only available for regular"): grid.chunk_shape # noqa: B018 def test_serialization_error_zero_extent_rectilinear() -> None: """RectilinearChunkGridMetadata rejects empty edge tuples.""" with pytest.raises(ValueError, match="has no chunk edges"): RectilinearChunkGridMetadata(chunk_shapes=((),)) def test_serialization_unknown_name_parse() -> None: """Parsing metadata with an unknown chunk grid name raises ValueError""" with pytest.raises(ValueError, match="Unknown chunk grid"): parse_chunk_grid({"name": "hexagonal", "configuration": {}}) def test_from_metadata_unknown_chunk_grid_type() -> None: """ChunkGrid.from_metadata raises TypeError for unrecognised chunk grid metadata.""" from unittest.mock import MagicMock from zarr.core.metadata.v3 import ArrayV3Metadata mock_meta = MagicMock(spec=ArrayV3Metadata) mock_meta.chunk_grid = MagicMock() # not Regular or Rectilinear with pytest.raises(TypeError, match="Unknown chunk grid metadata type"): ChunkGrid.from_metadata(mock_meta) def test_from_sizes_rejects_empty_edge_list() -> None: """ChunkGrid.from_sizes raises ValueError when a dimension has an empty edge list.""" with pytest.raises(ValueError, match="edges must not be empty"): ChunkGrid.from_sizes((10,), ([],)) # --------------------------------------------------------------------------- # Spec compliance tests # --------------------------------------------------------------------------- def test_spec_kind_inline_required_on_deserialize() -> None: """Deserialization requires kind: 'inline'.""" data: dict[str, Any] = { "name": "rectilinear", "configuration": {"chunk_shapes": [[10, 20], [15, 15]]}, } with pytest.raises(ValueError, match="requires a 'kind' field"): parse_chunk_grid(data) def test_spec_kind_unknown_rejected() -> None: """Unsupported rectilinear chunk grid kind raises ValueError on parse""" data: dict[str, Any] = { "name": "rectilinear", "configuration": {"kind": "reference", "chunk_shapes": [[10, 20], [15, 15]]}, } with pytest.raises(ValueError, match="Unsupported rectilinear chunk grid kind"): parse_chunk_grid(data) def test_spec_integer_shorthand_per_dimension() -> None: """A bare integer in chunk_shapes means repeat until >= extent.""" data: dict[str, Any] = { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [4, [1, 2, 3]]}, } meta = parse_chunk_grid(data) assert isinstance(meta, RectilinearChunkGridMetadata) g = ChunkGrid.from_sizes((6, 6), meta.chunk_shapes) assert _edges(g, 0) == (4, 4) assert _edges(g, 1) == (1, 2, 3) def test_spec_mixed_rle_and_bare_integers() -> None: """An array can mix bare integers and [value, count] RLE pairs.""" data: dict[str, Any] = { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[[1, 3], 3]]}, } meta = parse_chunk_grid(data) assert isinstance(meta, RectilinearChunkGridMetadata) g = ChunkGrid.from_sizes((6,), meta.chunk_shapes) assert _edges(g, 0) == (1, 1, 1, 3) def test_spec_overflow_chunks_allowed() -> None: """Edge sum >= extent is valid (overflow chunks permitted).""" data: dict[str, Any] = { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[4, 4, 4]]}, } meta = parse_chunk_grid(data) assert isinstance(meta, RectilinearChunkGridMetadata) g = ChunkGrid.from_sizes((6,), meta.chunk_shapes) assert _edges(g, 0) == (4, 4, 4) def test_spec_example() -> None: """The full example from the spec README.""" data: dict[str, Any] = { "name": "rectilinear", "configuration": { "kind": "inline", "chunk_shapes": [ 4, [1, 2, 3], [[4, 2]], [[1, 3], 3], [4, 4, 4], ], }, } meta = parse_chunk_grid(data) assert isinstance(meta, RectilinearChunkGridMetadata) g = ChunkGrid.from_sizes((6, 6, 6, 6, 6), meta.chunk_shapes) assert _edges(g, 0) == (4, 4) assert _edges(g, 1) == (1, 2, 3) assert _edges(g, 2) == (4, 4) assert _edges(g, 3) == (1, 1, 1, 3) assert _edges(g, 4) == (4, 4, 4) # --------------------------------------------------------------------------- # parse_chunk_grid validation tests # --------------------------------------------------------------------------- def test_parse_chunk_grid_varying_extent_mismatch_raises() -> None: """Reconstructing a ChunkGrid with mismatched extents raises ValueError""" g = ChunkGrid.from_sizes((60, 100), [[10, 20, 30], [50, 50]]) with pytest.raises(ValueError, match="extent"): ChunkGrid( dimensions=tuple( dim.with_extent(ext) for dim, ext in zip(g._dimensions, (100, 100), strict=True) ) ) def test_parse_chunk_grid_varying_extent_match_ok() -> None: """Reconstructing a ChunkGrid with matching extents succeeds""" g = ChunkGrid.from_sizes((60, 100), [[10, 20, 30], [50, 50]]) g2 = ChunkGrid( dimensions=tuple( dim.with_extent(ext) for dim, ext in zip(g._dimensions, (60, 100), strict=True) ) ) assert g2._dimensions[0].extent == 60 @pytest.mark.parametrize( ("chunk_shapes", "array_shape", "match"), [ ([[10, 20, 30], [25, 25]], (100, 50), "extent 100 exceeds sum of edges 60"), ([[50, 50], [10, 20]], (100, 50), "extent 50 exceeds sum of edges 30"), ], ids=["first-dim-mismatch", "second-dim-mismatch"], ) def test_parse_chunk_grid_rectilinear_extent_mismatch_raises( chunk_shapes: list[list[int]], array_shape: tuple[int, ...], match: str ) -> None: """Rectilinear grid raises ValueError when array extent exceeds sum of edges""" data: dict[str, Any] = { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": chunk_shapes}, } meta = parse_chunk_grid(data) assert isinstance(meta, RectilinearChunkGridMetadata) with pytest.raises(ValueError, match=match): ChunkGrid.from_sizes(array_shape, meta.chunk_shapes) def test_parse_chunk_grid_rectilinear_extent_match_passes() -> None: """Rectilinear grid with matching extents parses and builds successfully""" data: dict[str, Any] = { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[10, 20, 30], [25, 25]]}, } meta = parse_chunk_grid(data) assert isinstance(meta, RectilinearChunkGridMetadata) g = ChunkGrid.from_sizes((60, 50), meta.chunk_shapes) assert g.grid_shape == (3, 2) def test_parse_chunk_grid_rectilinear_ndim_mismatch_raises() -> None: """Mismatched ndim between array shape and chunk_sizes raises ValueError""" data: dict[str, Any] = { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[10, 20], [25, 25]]}, } meta = parse_chunk_grid(data) assert isinstance(meta, RectilinearChunkGridMetadata) with pytest.raises(ValueError, match="3 dimensions but chunk_sizes has 2"): ChunkGrid.from_sizes((30, 50, 100), meta.chunk_shapes) def test_parse_chunk_grid_rectilinear_rle_extent_validated() -> None: """RLE-encoded edges are expanded before validation.""" data: dict[str, Any] = { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[[10, 5]], [[25, 2]]]}, } meta = parse_chunk_grid(data) assert isinstance(meta, RectilinearChunkGridMetadata) g = ChunkGrid.from_sizes((50, 50), meta.chunk_shapes) assert g.grid_shape == (5, 2) with pytest.raises(ValueError, match="extent 100 exceeds sum of edges 50"): ChunkGrid.from_sizes((100, 50), meta.chunk_shapes) def test_parse_chunk_grid_varying_dimension_extent_mismatch_on_chunkgrid_input() -> None: """ChunkGrid constructor rejects VaryingDimension with extent exceeding sum of edges""" g = ChunkGrid.from_sizes((60, 50), [[10, 20, 30], [25, 25]]) with pytest.raises(ValueError, match="less than"): ChunkGrid( dimensions=tuple( dim.with_extent(ext) for dim, ext in zip(g._dimensions, (100, 50), strict=True) ) ) # --------------------------------------------------------------------------- # Rectilinear indexing tests # --------------------------------------------------------------------------- def test_basic_indexer_rectilinear() -> None: """BasicIndexer produces correct projections for a full-slice rectilinear selection""" from zarr.core.indexing import BasicIndexer g = ChunkGrid.from_sizes((60, 100), [[10, 20, 30], [50, 50]]) indexer = BasicIndexer( selection=(slice(None), slice(None)), shape=(60, 100), chunk_grid=g, ) projections = list(indexer) assert len(projections) == 6 p0 = projections[0] assert p0.chunk_coords == (0, 0) assert p0.chunk_selection == (slice(0, 10, 1), slice(0, 50, 1)) p1 = projections[2] assert p1.chunk_coords == (1, 0) assert p1.chunk_selection == (slice(0, 20, 1), slice(0, 50, 1)) def test_basic_indexer_int_selection() -> None: """BasicIndexer with integer selection maps to the correct chunk and local offset""" from zarr.core.indexing import BasicIndexer g = ChunkGrid.from_sizes((60, 100), [[10, 20, 30], [50, 50]]) indexer = BasicIndexer( selection=(15, slice(None)), shape=(60, 100), chunk_grid=g, ) projections = list(indexer) assert len(projections) == 2 assert projections[0].chunk_coords == (1, 0) assert projections[0].chunk_selection == (5, slice(0, 50, 1)) def test_basic_indexer_slice_subset() -> None: """BasicIndexer with partial slices spans the expected chunk dimensions""" from zarr.core.indexing import BasicIndexer g = ChunkGrid.from_sizes((60, 100), [[10, 20, 30], [50, 50]]) indexer = BasicIndexer( selection=(slice(5, 35), slice(0, 50)), shape=(60, 100), chunk_grid=g, ) projections = list(indexer) chunk_coords_dim0 = sorted({p.chunk_coords[0] for p in projections}) assert chunk_coords_dim0 == [0, 1, 2] def test_orthogonal_indexer_rectilinear() -> None: """OrthogonalIndexer produces the expected number of projections for a rectilinear grid""" from zarr.core.indexing import OrthogonalIndexer g = ChunkGrid.from_sizes((60, 100), [[10, 20, 30], [50, 50]]) indexer = OrthogonalIndexer( selection=(slice(None), slice(None)), shape=(60, 100), chunk_grid=g, ) projections = list(indexer) assert len(projections) == 6 def test_oob_block_raises_bounds_check_error() -> None: """Out-of-bounds block index should raise BoundsCheckError, not IndexError.""" store = MemoryStore() a = zarr.create_array(store, shape=(30,), chunks=[[10, 20]], dtype="int32") with pytest.raises(BoundsCheckError): a.get_block_selection((2,)) # --------------------------------------------------------------------------- # End-to-end tests # --------------------------------------------------------------------------- @pytest.mark.parametrize( ("shape", "chunks", "expected_regular"), [ ((100, 200), (10, 20), True), ((60, 100), [[10, 20, 30], [50, 50]], False), ], ids=["regular", "rectilinear"], ) def test_e2e_create_array( tmp_path: Path, shape: tuple[int, ...], chunks: Any, expected_regular: bool ) -> None: """End-to-end array creation sets correct regularity and ndim on chunk_grid""" arr = zarr.create_array( store=tmp_path / "arr.zarr", shape=shape, chunks=chunks, dtype="float32", ) assert ChunkGrid.from_metadata(arr.metadata).is_regular == expected_regular assert ChunkGrid.from_metadata(arr.metadata).ndim == len(shape) @pytest.mark.parametrize( ("shape", "chunks", "grid_type_name", "grid_name"), [ ((100, 200), (10, 20), "RegularChunkGridMetadata", "regular"), ((60, 100), [[10, 20, 30], [50, 50]], "RectilinearChunkGridMetadata", "rectilinear"), ], ids=["regular", "rectilinear"], ) def test_e2e_chunk_grid_serializes( tmp_path: Path, shape: tuple[int, ...], chunks: Any, grid_type_name: str, grid_name: str ) -> None: """Array metadata serializes chunk_grid with the correct type and name""" from zarr.core.metadata.v3 import ( ArrayV3Metadata, RectilinearChunkGridMetadata, RegularChunkGridMetadata, ) grid_type = ( RegularChunkGridMetadata if grid_type_name == "RegularChunkGridMetadata" else RectilinearChunkGridMetadata ) arr = zarr.create_array( store=tmp_path / "arr.zarr", shape=shape, chunks=chunks, dtype="float32", ) assert isinstance(arr.metadata, ArrayV3Metadata) assert isinstance(arr.metadata.chunk_grid, grid_type) d = arr.metadata.to_dict() chunk_grid_dict = d["chunk_grid"] assert isinstance(chunk_grid_dict, dict) assert chunk_grid_dict["name"] == grid_name def test_e2e_chunk_grid_name_roundtrip_preserves_rectilinear(tmp_path: Path) -> None: """A rectilinear grid with uniform edges stays 'rectilinear' through to_dict/from_dict.""" from zarr.core.metadata.v3 import ArrayV3Metadata, RectilinearChunkGridMetadata meta_dict: dict[str, Any] = { "zarr_format": 3, "node_type": "array", "shape": [100, 100], "chunk_grid": { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[[50, 2]], [[25, 4]]]}, }, "chunk_key_encoding": {"name": "default"}, "data_type": "float32", "fill_value": 0.0, "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], } meta = ArrayV3Metadata.from_dict(meta_dict) assert isinstance(meta.chunk_grid, RectilinearChunkGridMetadata) d = meta.to_dict() chunk_grid_dict = d["chunk_grid"] assert isinstance(chunk_grid_dict, dict) assert chunk_grid_dict["name"] == "rectilinear" def test_e2e_chunk_grid_name_regular_from_dict(tmp_path: Path) -> None: """A 'regular' chunk grid name is preserved through from_dict.""" from zarr.core.metadata.v3 import ArrayV3Metadata, RegularChunkGridMetadata meta_dict: dict[str, Any] = { "zarr_format": 3, "node_type": "array", "shape": [100, 100], "chunk_grid": { "name": "regular", "configuration": {"chunk_shape": [50, 25]}, }, "chunk_key_encoding": {"name": "default"}, "data_type": "float32", "fill_value": 0.0, "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], } meta = ArrayV3Metadata.from_dict(meta_dict) assert isinstance(meta.chunk_grid, RegularChunkGridMetadata) d = meta.to_dict() chunk_grid_dict = d["chunk_grid"] assert isinstance(chunk_grid_dict, dict) assert chunk_grid_dict["name"] == "regular" # --------------------------------------------------------------------------- # Input syntax determines grid kind (gh-4272) # --------------------------------------------------------------------------- @pytest.mark.parametrize( ("shape", "chunks", "expected_type"), [ # scalar specs (including numpy integers and the -1 sentinel) produce a regular grid ((30,), (10,), RegularChunkGridMetadata), ((30,), 10, RegularChunkGridMetadata), ((30,), (np.int64(10),), RegularChunkGridMetadata), ((30,), -1, RegularChunkGridMetadata), # explicit per-chunk lists stay rectilinear even when the sizes are uniform ((30,), [[10, 10, 10]], RectilinearChunkGridMetadata), # ... or uniform with a short tail (the append-oriented time-series layout of gh-4272) ((168 * 13 + 24,), [[168] * 13 + [24]], RectilinearChunkGridMetadata), # genuinely varying edges are rectilinear too ((60,), [[10, 20, 30]], RectilinearChunkGridMetadata), ], ids=[ "scalar-tuple", "scalar-int", "scalar-numpy-int", "full-span-sentinel", "explicit-uniform", "explicit-uniform-short-tail", "explicit-varying", ], ) def test_chunk_grid_kind_follows_input_syntax( tmp_path: Path, shape: tuple[int, ...], chunks: Any, expected_type: type ) -> None: """The stored grid kind is decided by the input syntax, not the chunk sizes: scalar specs produce a regular grid, while explicit per-chunk lists produce a rectilinear grid even when the sizes happen to describe a regular one (gh-4272).""" from zarr.core.metadata.v3 import ArrayV3Metadata arr = zarr.create_array(store=tmp_path / "arr.zarr", shape=shape, chunks=chunks, dtype="int32") assert isinstance(arr.metadata, ArrayV3Metadata) assert isinstance(arr.metadata.chunk_grid, expected_type) def test_mixed_scalar_and_list_dims_keep_shorthand(tmp_path: Path) -> None: """In a mixed spec, a scalar dimension serializes as the bare-int step-size shorthand while an explicit list keeps its per-chunk edges.""" from zarr.core.metadata.v3 import ArrayV3Metadata arr = zarr.create_array( store=tmp_path / "arr.zarr", shape=(30, 100), chunks=(5, [10, 20, 70]), dtype="int32" ) assert isinstance(arr.metadata, ArrayV3Metadata) grid = arr.metadata.chunk_grid assert isinstance(grid, RectilinearChunkGridMetadata) assert grid.chunk_shapes == (5, (10, 20, 70)) # The chunk/shard spec matrix for `from_array` round-trip tests. Every kind of # spec `create_array` accepts should appear here, so that any change to how # specs are normalized or stored is automatically checked against the "keep" # path of `from_array` as well. FROM_ARRAY_KEEP_CASES = [ pytest.param((30, 40), (10, 20), None, None, id="regular"), pytest.param((40, 40), (5, 10), (10, 20), None, id="regular-sharded"), pytest.param((60, 100), [[10, 20, 30], [50, 50]], None, None, id="rectilinear"), pytest.param((9, 30), (3, [10, 20]), None, None, id="rectilinear-mixed-bare-int"), pytest.param((3,), [[3]], None, None, id="rectilinear-uniform-list"), pytest.param((100,), (10,), [[50, 50]], None, id="rectilinear-sharded"), # A shrinking resize legally leaves trailing edges beyond the extent in the # stored grid; the "keep" path must preserve them verbatim rather than # re-validating the stored grid as if it were a user-supplied spec. pytest.param((25,), [[5, 10, 10]], None, (18,), id="rectilinear-shrunk"), pytest.param((100,), (10,), [[50, 50]], (80,), id="rectilinear-sharded-shrunk"), ] @pytest.mark.parametrize("write_data", [True, False]) @pytest.mark.parametrize(("shape", "chunks", "shards", "resize_to"), FROM_ARRAY_KEEP_CASES) def test_from_array_keep_roundtrips_chunk_grid( shape: tuple[int, ...], chunks: Any, shards: Any, resize_to: tuple[int, ...] | None, write_data: bool, ) -> None: """For every chunk/shard spec accepted by `create_array` — including grids later shrunk by `resize`, whose stored edges legally overhang the extent — `from_array` with the default "keep" parameters reproduces the source's stored chunk grid and codecs exactly: the grid kind is preserved (gh-4272), uniform dimensions keep their bare-int shorthand instead of being expanded per chunk, and sharding survives — including under a rectilinear shard grid. `.info` is defined for every kind of source, and the default data copy works for every kind of grid.""" from zarr.core.metadata.v3 import ArrayV3Metadata src = zarr.create_array( MemoryStore(), shape=shape, chunks=chunks, shards=shards, dtype="uint16" ) src[:] = np.arange(np.prod(shape), dtype="uint16").reshape(shape) assert src.info is not None if resize_to is None: # The array is fully written, so every chunk is initialized, whatever # the grid kind and sharding layout. (After a shrink the count reflects # declared chunk sizes, which overhang the extent, so it is only # asserted for the unresized cases.) assert src.nchunks_initialized == src.nchunks else: src.resize(resize_to) dst = zarr.from_array(MemoryStore(), data=src, name="0", write_data=write_data) assert isinstance(src.metadata, ArrayV3Metadata) assert isinstance(dst.metadata, ArrayV3Metadata) assert dst.metadata.chunk_grid == src.metadata.chunk_grid assert dst.metadata.codecs == src.metadata.codecs if write_data: np.testing.assert_array_equal(dst[:], src[:]) def test_info_reports_variable_shard_shape_for_rectilinear_shard_grid() -> None: """A sharded array whose shard grid is rectilinear has no uniform shard shape. `.info` must still present it as sharded — shard shape rendered as ``, and the initialized count labeled as shards, not chunks.""" arr = zarr.create_array( MemoryStore(), shape=(100,), chunks=(10,), shards=[[50, 50]], dtype="int32" ) arr[:] = 1 assert "Shard shape : " in repr(arr.info) info = arr.info_complete() assert "Shards Initialized : 2" in repr(info) assert arr.nchunks_initialized == 10 def test_nchunks_initialized_counts_per_shard_for_rectilinear_shard_grid() -> None: """With a rectilinear shard grid the chunk count per shard varies, so the initialized-chunk count is summed per stored shard rather than derived from a uniform multiplier.""" arr = zarr.create_array( MemoryStore(), shape=(90,), chunks=(10,), shards=[[60, 30]], dtype="int32" ) assert arr.nchunks_initialized == 0 arr[60:90] = 1 # initializes only the second shard, which holds 3 chunks assert arr.nchunks_initialized == 3 arr[0:10] = 1 # initializes the first shard, which holds 6 chunks assert arr.nchunks_initialized == 9 def test_nchunks_initialized_regular_sharded_multiply_matches_per_shard_sum() -> None: """For a regular shard grid the count comes from an O(1) multiply (initialized shards x chunks per shard); it must agree with summing each initialized shard's chunk count individually, as done for rectilinear shard grids.""" arr = zarr.create_array(MemoryStore(), shape=(100,), chunks=(5,), shards=(20,), dtype="int32") arr[0:20] = 1 arr[40:60] = 1 # two initialized shards, four chunks each assert arr._nshards_initialized == 2 assert arr.nchunks_initialized == 2 * 4 arr[:] = 1 assert arr.nchunks_initialized == arr.nchunks == 20 def test_iter_shard_regions_bounds_check() -> None: """An out-of-bounds selection fails loudly and consistently across the two sibling shard-iteration APIs — these regions are the safe concurrent-write partitions, so a miscomputed selection must not silently shrink.""" arr = zarr.create_array(MemoryStore(), shape=(30,), chunks=(10,), dtype="uint8") with pytest.raises(IndexError, match="Invalid selection shape"): list(arr._iter_shard_keys(origin=(1,), selection_shape=(5,))) with pytest.raises(IndexError, match="Invalid selection shape"): list(arr._iter_shard_regions(origin=(1,), selection_shape=(5,))) @pytest.mark.parametrize( ("shape", "chunks"), [ ((30, 100), (5, [10, 20, 70])), ((100, 30), ([10, 20, 70], 5)), # uniform edges are still rectilinear, whether given as lists or as # stored metadata (gh-4272) ((30, 100), ([10, 10, 10], 50)), # (metadata is built lazily: constructing it needs the rectilinear flag) ((30, 100), partial(RectilinearChunkGridMetadata, chunk_shapes=((10, 10, 10), 50))), ], ids=["int-first", "list-first", "uniform-list", "uniform-metadata"], ) def test_rectilinear_chunks_gates( shape: tuple[int, ...], chunks: ChunksLike | Callable[[], ChunksLike] ) -> None: """The v2 and chunks+shards gates recognize every rectilinear spec as rectilinear, regardless of which dimension carries the sequence and whether its edges happen to be uniform.""" chunks = chunks() if callable(chunks) else chunks with pytest.raises(ValueError, match="Rectilinear chunks with sharding is not supported"): zarr.create_array(MemoryStore(), shape=shape, chunks=chunks, shards=(10, 10), dtype="uint8") with pytest.raises(ValueError, match="Zarr format 2 does not support rectilinear chunk grids"): zarr.create_array(MemoryStore(), shape=shape, chunks=chunks, zarr_format=2, dtype="uint8") with pytest.raises(ValueError, match="Zarr format 2 does not support rectilinear chunk grids"): zarr.create(store=MemoryStore(), shape=shape, chunks=chunks, zarr_format=2, dtype="uint8") @pytest.mark.parametrize( ("chunks", "expected"), [ (None, None), (0, None), ((), None), ([], None), (False, None), (np.int64(0), None), (np.False_, None), (np.array(0), None), (np.array(False), None), (np.array([0]), None), (np.array([]), None), (np.array(7), (7, 7)), (np.array([5, 3]), (5, 3)), ], ids=repr, ) def test_legacy_create_v2_chunks( chunks: ChunksLike | None, expected: tuple[int, ...] | None ) -> None: """The legacy `zarr.create` Zarr format 2 path reads a falsy `chunks` as not given and chunks automatically (`expected` is `None`). It never tests the truth value of a numpy array with more than one element, which has none: that array is the chunk shape.""" shape = (2**12, 2**12) arr = zarr.create(store=MemoryStore(), shape=shape, chunks=chunks, dtype="int32", zarr_format=2) assert arr.chunks == (expected or guess_chunks(shape, 4).chunk_shape) assert all(type(c) is int for c in arr.chunks) def test_legacy_create_v2_chunks_all_zero_array_raises() -> None: """A numpy `chunks` array with more than one element is given even when every element is 0, so the legacy Zarr format 2 path rejects its 0 sizes instead of chunking automatically.""" chunks = np.array([0, 0]) with pytest.raises(ValueError, match="Chunk size must be positive or -1, got 0"): zarr.create( store=MemoryStore(), shape=(2**12, 2**12), chunks=chunks, dtype="int32", zarr_format=2 ) def test_from_array_keep_preserves_all_bare_int_rectilinear_grid() -> None: """A rectilinear grid using the bare-int shorthand on every dimension can only arrive from externally written metadata — `create_array` never produces one — and behaves identically to a regular grid. The "keep" path stores the source grid verbatim, so even this grid keeps its kind.""" import json from zarr.core.buffer import default_buffer_prototype from zarr.core.metadata.v3 import ArrayV3Metadata from zarr.core.sync import sync meta = ArrayV3Metadata.from_dict( { "zarr_format": 3, "node_type": "array", "shape": [90, 40], "chunk_grid": { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [30, 20]}, }, "chunk_key_encoding": {"name": "default"}, "data_type": "int32", "fill_value": 0, "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], } ) store = MemoryStore() buf = default_buffer_prototype().buffer.from_bytes(json.dumps(meta.to_dict()).encode()) sync(store.set("a/zarr.json", buf)) src = zarr.open_array(store, path="a") dst = zarr.from_array(MemoryStore(), data=src, name="0", write_data=False) assert isinstance(dst.metadata, ArrayV3Metadata) assert dst.metadata.chunk_grid == meta.chunk_grid def test_chunks_raises_for_nonsharded_rectilinear_grid() -> None: """`.chunks` has no uniform value for a non-sharded rectilinear grid.""" arr = zarr.create_array(MemoryStore(), shape=(30,), chunks=[[10, 20]], dtype="int32") with pytest.raises(NotImplementedError, match="regular chunk grids"): _ = arr.chunks def test_shards_raises_for_rectilinear_shard_grid() -> None: """`.shards` has no uniform value when the shard grid is rectilinear.""" arr = zarr.create_array( MemoryStore(), shape=(100,), chunks=(10,), shards=[[50, 50]], dtype="int32" ) with pytest.raises(NotImplementedError, match="regular chunk grids"): _ = arr.shards def test_from_array_keep_is_o1_in_chunk_count() -> None: """`chunks="keep"` passes uniform dimensions through as bare-int shorthand rather than expanding one entry per chunk, so copying array metadata is O(ndim), not O(nchunks): this completes instantly despite ~2**60 chunks along the first dimension (and would hang if the shorthand were expanded).""" src = zarr.create_array(MemoryStore(), shape=(2**62, 30), chunks=(3, [10, 20]), dtype="uint8") dst = zarr.from_array(MemoryStore(), data=src, name="0", write_data=False) grid = dst.metadata.chunk_grid assert isinstance(grid, RectilinearChunkGridMetadata) assert grid.chunk_shapes == (3, (10, 20)) @pytest.mark.parametrize( "rect_chunks", [[[10, 10, 10]], partial(RectilinearChunkGridMetadata, chunk_shapes=((10, 10, 10),))], ids=["list", "metadata"], ) def test_resize_uniform_rectilinear_appends_edge( rect_chunks: ChunksLike | Callable[[], ChunksLike], ) -> None: """Growing an explicitly rectilinear array whose edges look regular appends a new edge chunk, while the same sizes declared as a scalar extend the uniform pattern instead (gh-4272), so an append-only workload writing the grown region touches exactly one new chunk.""" from zarr.core.metadata.v3 import ArrayV3Metadata rect_chunks = rect_chunks() if callable(rect_chunks) else rect_chunks rect_store: dict[str, Any] = {} rect = zarr.create_array(store=rect_store, shape=(30,), chunks=rect_chunks, dtype="int32") assert isinstance(rect.metadata.chunk_grid, RectilinearChunkGridMetadata) rect.resize((45,)) assert isinstance(rect.metadata, ArrayV3Metadata) rect_grid = rect.metadata.chunk_grid assert isinstance(rect_grid, RectilinearChunkGridMetadata) assert rect_grid.chunk_shapes == ((10, 10, 10, 15),) assert rect.nchunks == 4 # the appended region is exactly the new edge chunk rect[30:45] = 1 assert sorted(k for k in rect_store if k.startswith("c/")) == ["c/3"] reg_store: dict[str, Any] = {} reg = zarr.create_array(store=reg_store, shape=(30,), chunks=(10,), dtype="int32") reg.resize((45,)) assert isinstance(reg.metadata, ArrayV3Metadata) reg_grid = reg.metadata.chunk_grid assert isinstance(reg_grid, RegularChunkGridMetadata) assert reg_grid.chunk_shape == (10,) assert reg.nchunks == 5 # the same write straddles two chunks of the extended uniform pattern reg[30:45] = 1 assert sorted(k for k in reg_store if k.startswith("c/")) == ["c/3", "c/4"] # --------------------------------------------------------------------------- # Sharding compatibility tests # --------------------------------------------------------------------------- def test_sharding_accepts_rectilinear_outer_grid() -> None: """ShardingCodec.validate should not reject rectilinear outer grids.""" from zarr.codecs.sharding import ShardingCodec from zarr.core.dtype import Float32 from zarr.core.metadata.v3 import RectilinearChunkGridMetadata codec = ShardingCodec(chunk_shape=(5, 5)) grid_meta = RectilinearChunkGridMetadata(chunk_shapes=((10, 20, 30), (50, 50))) codec.validate( shape=(60, 100), dtype=Float32(), chunk_grid=grid_meta, ) def test_sharding_rejects_non_divisible_rectilinear() -> None: """Rectilinear shard sizes not divisible by inner chunk_shape should raise.""" from zarr.codecs.sharding import ShardingCodec from zarr.core.dtype import Float32 from zarr.core.metadata.v3 import RectilinearChunkGridMetadata codec = ShardingCodec(chunk_shape=(5, 5)) grid_meta = RectilinearChunkGridMetadata(chunk_shapes=((10, 20, 17), (50, 50))) with pytest.raises(ValueError, match="divisible"): codec.validate( shape=(47, 100), dtype=Float32(), chunk_grid=grid_meta, ) def test_sharding_accepts_divisible_rectilinear() -> None: """Rectilinear shard sizes all divisible by inner chunk_shape should pass.""" from zarr.codecs.sharding import ShardingCodec from zarr.core.dtype import Float32 from zarr.core.metadata.v3 import RectilinearChunkGridMetadata codec = ShardingCodec(chunk_shape=(5, 5)) grid_meta = RectilinearChunkGridMetadata(chunk_shapes=((10, 20, 30), (50, 50))) codec.validate( shape=(60, 100), dtype=Float32(), chunk_grid=grid_meta, ) def test_sharding_rejects_non_divisible_among_repeated_edges() -> None: """Shard validation catches a non-divisible edge even among many repeated valid ones.""" from zarr.codecs.sharding import ShardingCodec from zarr.core.dtype import Float32 from zarr.core.metadata.v3 import RectilinearChunkGridMetadata # edges (10, 10, 7) — 7 is not divisible by 5 codec = ShardingCodec(chunk_shape=(5,)) grid_meta = RectilinearChunkGridMetadata(chunk_shapes=((10, 10, 7),)) with pytest.raises(ValueError, match="divisible"): codec.validate(shape=(27,), dtype=Float32(), chunk_grid=grid_meta) def test_sharding_accepts_all_repeated_divisible_edges() -> None: """Shard validation passes when all distinct edges are divisible by inner chunk size.""" from zarr.codecs.sharding import ShardingCodec from zarr.core.dtype import Float32 from zarr.core.metadata.v3 import RectilinearChunkGridMetadata # edges (10, 10, 20, 10) — unique values {10, 20}, both divisible by 5 codec = ShardingCodec(chunk_shape=(5,)) grid_meta = RectilinearChunkGridMetadata(chunk_shapes=((10, 10, 20, 10),)) codec.validate(shape=(50,), dtype=Float32(), chunk_grid=grid_meta) # --------------------------------------------------------------------------- # Edge cases # --------------------------------------------------------------------------- def test_edge_case_chunk_grid_boundary_getitem() -> None: """ChunkGrid with boundary FixedDimension via direct construction.""" g = ChunkGrid(dimensions=(FixedDimension(10, 95), FixedDimension(20, 40))) spec = g[(9, 1)] assert spec is not None assert spec.shape == (5, 20) assert spec.codec_shape == (10, 20) assert spec.is_boundary def test_edge_case_chunk_grid_boundary_iter() -> None: """Iterating a boundary grid yields correct boundary ChunkSpecs.""" g = ChunkGrid(dimensions=(FixedDimension(10, 25),)) specs = list(g) assert len(specs) == 3 assert specs[0].shape == (10,) assert specs[1].shape == (10,) assert specs[2].shape == (5,) assert specs[2].is_boundary assert not specs[0].is_boundary def test_edge_case_chunk_grid_boundary_shape() -> None: """shape property with boundary extent.""" g = ChunkGrid(dimensions=(FixedDimension(10, 95),)) assert g.grid_shape == (10,) # -- Zero-size and zero-extent -- @pytest.mark.parametrize("size", [1, 10], ids=["size-1", "size-10"]) def test_fixed_dimension_zero_extent(size: int) -> None: """A zero-length axis has zero chunks and behaves like an empty grid.""" d = FixedDimension(size=size, extent=0) assert d.nchunks == 0 assert d.ngridcells == 0 assert d.data_size(0) == 0 assert d.with_extent(0) == d assert d.with_extent(3) == FixedDimension(size=size, extent=3) empty = np.array([], dtype=np.intp) np.testing.assert_array_equal(d.indices_to_chunks(empty), empty) with pytest.raises(IndexError): d.index_to_chunk(0) g = ChunkGrid(dimensions=(d,)) assert g[0] is None assert list(g) == [] # -- 0-d grid -- def test_0d_grid_getitem() -> None: """0-d grid has exactly one chunk at coords ().""" g = ChunkGrid.from_sizes((), ()) spec = g[()] assert spec is not None assert spec.shape == () assert spec.codec_shape == () assert not spec.is_boundary def test_0d_grid_iter() -> None: """0-d grid iteration yields a single ChunkSpec.""" g = ChunkGrid.from_sizes((), ()) specs = list(g) assert len(specs) == 1 def test_0d_grid_all_chunk_coords() -> None: """0-d grid has one chunk coord: the empty tuple.""" g = ChunkGrid.from_sizes((), ()) coords = list(g.all_chunk_coords()) assert coords == [()] def test_0d_grid_nchunks() -> None: """0-d grid reports exactly one chunk""" g = ChunkGrid.from_sizes((), ()) assert g.get_nchunks() == 1 # -- parse_chunk_grid edge cases -- def test_parse_chunk_grid_preserves_varying_extent() -> None: """parse_chunk_grid does not overwrite VaryingDimension extent.""" g = ChunkGrid.from_sizes((60, 100), [[10, 20, 30], [50, 50]]) assert isinstance(g._dimensions[0], VaryingDimension) assert g._dimensions[0].extent == 60 g2 = ChunkGrid( dimensions=tuple( dim.with_extent(ext) for dim, ext in zip(g._dimensions, (60, 100), strict=True) ) ) assert isinstance(g2._dimensions[0], VaryingDimension) assert g2._dimensions[0].extent == 60 def test_parse_chunk_grid_rebinds_fixed_extent() -> None: """parse_chunk_grid updates FixedDimension extent from array shape.""" g = ChunkGrid.from_sizes((100, 200), (10, 20)) assert g._dimensions[0].extent == 100 g2 = ChunkGrid( dimensions=tuple( dim.with_extent(ext) for dim, ext in zip(g._dimensions, (50, 100), strict=True) ) ) assert isinstance(g2._dimensions[0], FixedDimension) assert g2._dimensions[0].extent == 50 assert g2.grid_shape == (5, 5) # -- ChunkGrid.__getitem__ validation -- def test_getitem_int_1d_regular() -> None: """Integer indexing works for 1-d regular grids.""" g = ChunkGrid.from_sizes((100,), (10,)) spec = g[0] assert spec is not None assert spec.shape == (10,) assert spec.slices == (slice(0, 10, 1),) spec = g[9] assert spec is not None assert spec.shape == (10,) def test_getitem_int_1d_rectilinear() -> None: """Integer indexing works for 1-d rectilinear grids.""" g = ChunkGrid.from_sizes((100,), [[20, 30, 50]]) spec = g[0] assert spec is not None assert spec.shape == (20,) spec = g[1] assert spec is not None assert spec.shape == (30,) spec = g[2] assert spec is not None assert spec.shape == (50,) @pytest.mark.parametrize( ("shape", "chunks", "match"), [ ((), (), "Expected 0 coordinate.*got 1"), ((100, 200), (10, 20), "Expected 2 coordinate.*got 1"), ], ids=["0d", "2d"], ) def test_getitem_int_ndim_mismatch_raises( shape: tuple[int, ...], chunks: tuple[int, ...], match: str ) -> None: """Integer indexing on a multi-dim or 0-d grid raises ValueError for ndim mismatch""" g = ChunkGrid.from_sizes(shape, chunks) with pytest.raises(ValueError, match=match): g[0] @pytest.mark.parametrize( "index", [(10,), (99,), (-1,)], ids=["oob-10", "oob-99", "negative"], ) def test_getitem_oob_returns_none(index: tuple[int, ...]) -> None: """Out-of-bounds or negative chunk indices return None""" g = ChunkGrid.from_sizes((100,), (10,)) assert g[index] is None # -- Rectilinear with zero-nchunks FixedDimension -- def test_zero_nchunks_fixed_dim_in_rectilinear() -> None: """A rectilinear grid with a 0-extent FixedDimension still has valid size.""" g = ChunkGrid( dimensions=( VaryingDimension([10, 20], extent=30), FixedDimension(size=10, extent=0), ) ) assert g.grid_shape == (2, 0) # -- VaryingDimension data_size -- def test_varying_dim_data_size_equals_chunk_size() -> None: """For VaryingDimension, data_size == chunk_size (no padding).""" d = VaryingDimension([10, 20, 5], extent=35) for i in range(3): assert d.data_size(i) == d.chunk_size(i) # --------------------------------------------------------------------------- # OrthogonalIndexer rectilinear tests # --------------------------------------------------------------------------- def test_orthogonal_int_array_selection_rectilinear() -> None: """Integer array selection with rectilinear grid must produce correct chunk-local selections.""" from zarr.core.indexing import OrthogonalIndexer g = ChunkGrid.from_sizes((60, 100), [[10, 20, 30], [50, 50]]) indexer = OrthogonalIndexer( selection=(np.array([5, 15, 35]), slice(None)), shape=(60, 100), chunk_grid=g, ) projections = list(indexer) chunk_coords = [p.chunk_coords for p in projections] assert chunk_coords == [(0, 0), (0, 1), (1, 0), (1, 1), (2, 0), (2, 1)] def test_orthogonal_bool_array_selection_rectilinear() -> None: """Boolean array selection with rectilinear grid produces correct chunk projections.""" from zarr.core.indexing import OrthogonalIndexer g = ChunkGrid.from_sizes((60, 100), [[10, 20, 30], [50, 50]]) mask = np.zeros(60, dtype=bool) mask[5] = True mask[15] = True mask[35] = True indexer = OrthogonalIndexer( selection=(mask, slice(None)), shape=(60, 100), chunk_grid=g, ) projections = list(indexer) assert len(projections) == 6 chunk_coords = [p.chunk_coords for p in projections] assert (0, 0) in chunk_coords assert (1, 0) in chunk_coords assert (2, 0) in chunk_coords assert (0, 1) in chunk_coords assert (1, 1) in chunk_coords assert (2, 1) in chunk_coords def test_orthogonal_advanced_indexing_produces_correct_projections() -> None: """Verify OrthogonalIndexer produces correct chunk projections for advanced indexing with VaryingDimension.""" from zarr.core.indexing import OrthogonalIndexer g = ChunkGrid.from_sizes((60, 100), [[10, 20, 30], [50, 50]]) indexer = OrthogonalIndexer( selection=(np.array([5, 15]), slice(None)), shape=(60, 100), chunk_grid=g, ) projections = list(indexer) assert len(projections) == 4 coords = [p.chunk_coords for p in projections] assert (0, 0) in coords assert (0, 1) in coords assert (1, 0) in coords assert (1, 1) in coords # --------------------------------------------------------------------------- # Full pipeline rectilinear tests (helpers) # --------------------------------------------------------------------------- def _make_1d(tmp_path: Path) -> tuple[zarr.Array[Any], np.ndarray[Any, Any]]: a = np.arange(30, dtype="int32") z = zarr.create_array( store=tmp_path / "arr1d.zarr", shape=(30,), chunks=[[5, 10, 15]], dtype="int32", ) z[:] = a return z, a def _make_2d(tmp_path: Path) -> tuple[zarr.Array[Any], np.ndarray[Any, Any]]: a = np.arange(6000, dtype="int32").reshape(60, 100) z = zarr.create_array( store=tmp_path / "arr2d.zarr", shape=(60, 100), chunks=[[10, 20, 30], [25, 25, 25, 25]], dtype="int32", ) z[:] = a return z, a # --- Basic selection --- def test_pipeline_basic_selection_1d(tmp_path: Path) -> None: """1D rectilinear basic selections match numpy for ints, slices, and full-array reads""" z, a = _make_1d(tmp_path) sels: list[Any] = [0, 4, 5, 14, 15, 29, -1, slice(None), slice(3, 18), slice(0, 0)] for sel in sels: np.testing.assert_array_equal(z[sel], a[sel], err_msg=f"sel={sel}") def test_pipeline_basic_selection_1d_strided(tmp_path: Path) -> None: """1D rectilinear strided slice selections match numpy""" z, a = _make_1d(tmp_path) for sel in [slice(None, None, 2), slice(1, 25, 3), slice(0, 30, 7)]: np.testing.assert_array_equal(z[sel], a[sel], err_msg=f"sel={sel}") def test_pipeline_basic_selection_2d(tmp_path: Path) -> None: """2D rectilinear basic selections match numpy across chunk boundaries""" z, a = _make_2d(tmp_path) selections: list[Any] = [ 42, -1, (9, 24), (10, 25), (30, 50), (59, 99), slice(None), (slice(5, 35), slice(20, 80)), (slice(0, 10), slice(0, 25)), (slice(10, 10), slice(None)), (slice(None, None, 3), slice(None, None, 7)), ] for sel in selections: np.testing.assert_array_equal(z[sel], a[sel], err_msg=f"sel={sel}") # --- Orthogonal selection --- def test_pipeline_orthogonal_selection_1d_bool(tmp_path: Path) -> None: """1D boolean orthogonal indexing on rectilinear arrays matches numpy""" z, a = _make_1d(tmp_path) ix = np.zeros(30, dtype=bool) ix[[0, 4, 5, 14, 15, 29]] = True np.testing.assert_array_equal(z.oindex[ix], a[ix]) def test_pipeline_orthogonal_selection_1d_int(tmp_path: Path) -> None: """1D integer and negative-index orthogonal selection on rectilinear arrays matches numpy""" z, a = _make_1d(tmp_path) ix = np.array([0, 4, 5, 14, 15, 29]) np.testing.assert_array_equal(z.oindex[ix], a[ix]) ix_neg = np.array([0, -1, -15, -25]) np.testing.assert_array_equal(z.oindex[ix_neg], a[ix_neg]) def test_pipeline_orthogonal_selection_2d_bool(tmp_path: Path) -> None: """2D boolean orthogonal selection on rectilinear arrays matches numpy""" z, a = _make_2d(tmp_path) ix0 = np.zeros(60, dtype=bool) ix0[[0, 9, 10, 29, 30, 59]] = True ix1 = np.zeros(100, dtype=bool) ix1[[0, 24, 25, 49, 50, 99]] = True np.testing.assert_array_equal(z.oindex[ix0, ix1], a[np.ix_(ix0, ix1)]) def test_pipeline_orthogonal_selection_2d_int(tmp_path: Path) -> None: """2D integer orthogonal selection on rectilinear arrays matches numpy""" z, a = _make_2d(tmp_path) ix0 = np.array([0, 9, 10, 29, 30, 59]) ix1 = np.array([0, 24, 25, 49, 50, 99]) np.testing.assert_array_equal(z.oindex[ix0, ix1], a[np.ix_(ix0, ix1)]) def test_pipeline_orthogonal_selection_2d_mixed(tmp_path: Path) -> None: """2D mixed int-array and slice orthogonal selection on rectilinear arrays matches numpy""" z, a = _make_2d(tmp_path) ix = np.array([0, 9, 10, 29, 30, 59]) np.testing.assert_array_equal(z.oindex[ix, slice(25, 75)], a[np.ix_(ix, np.arange(25, 75))]) np.testing.assert_array_equal( z.oindex[slice(10, 30), ix[:4]], a[np.ix_(np.arange(10, 30), ix[:4])] ) # --- Coordinate (vindex) selection --- def test_pipeline_coordinate_selection_1d(tmp_path: Path) -> None: """1D coordinate (vindex) selection on rectilinear arrays matches numpy""" z, a = _make_1d(tmp_path) ix = np.array([0, 4, 5, 14, 15, 29]) np.testing.assert_array_equal(z.vindex[ix], a[ix]) def test_pipeline_coordinate_selection_2d(tmp_path: Path) -> None: """2D coordinate (vindex) selection on rectilinear arrays matches numpy""" z, a = _make_2d(tmp_path) r = np.array([0, 9, 10, 29, 30, 59]) c = np.array([0, 24, 25, 49, 50, 99]) np.testing.assert_array_equal(z.vindex[r, c], a[r, c]) def test_pipeline_coordinate_selection_2d_bool_mask(tmp_path: Path) -> None: """2D boolean mask vindex selection on rectilinear arrays matches numpy""" z, a = _make_2d(tmp_path) mask = a > 3000 np.testing.assert_array_equal(z.vindex[mask], a[mask]) # --- Block selection --- def test_pipeline_block_selection_1d(tmp_path: Path) -> None: """1D block selection on rectilinear arrays returns correct chunk data""" z, a = _make_1d(tmp_path) np.testing.assert_array_equal(z.blocks[0], a[0:5]) np.testing.assert_array_equal(z.blocks[1], a[5:15]) np.testing.assert_array_equal(z.blocks[2], a[15:30]) np.testing.assert_array_equal(z.blocks[-1], a[15:30]) np.testing.assert_array_equal(z.blocks[0:2], a[0:15]) np.testing.assert_array_equal(z.blocks[1:3], a[5:30]) np.testing.assert_array_equal(z.blocks[:], a[:]) def test_pipeline_block_selection_2d(tmp_path: Path) -> None: """2D block selection on rectilinear arrays returns correct chunk data""" z, a = _make_2d(tmp_path) np.testing.assert_array_equal(z.blocks[0, 0], a[0:10, 0:25]) np.testing.assert_array_equal(z.blocks[1, 2], a[10:30, 50:75]) np.testing.assert_array_equal(z.blocks[2, 3], a[30:60, 75:100]) np.testing.assert_array_equal(z.blocks[-1, -1], a[30:60, 75:100]) np.testing.assert_array_equal(z.blocks[0:2, 1:3], a[0:30, 25:75]) np.testing.assert_array_equal(z.blocks[:, :], a[:, :]) def test_pipeline_set_block_selection_1d(tmp_path: Path) -> None: """Writing via 1D block selection on rectilinear arrays persists correctly""" z, a = _make_1d(tmp_path) val = np.full(10, -1, dtype="int32") z.blocks[1] = val a[5:15] = val np.testing.assert_array_equal(z[:], a) def test_pipeline_set_block_selection_2d(tmp_path: Path) -> None: """Writing via 2D block selection on rectilinear arrays persists correctly""" z, a = _make_2d(tmp_path) val = np.full((30, 50), -99, dtype="int32") z.blocks[0:2, 1:3] = val a[0:30, 25:75] = val np.testing.assert_array_equal(z[:], a) def test_pipeline_block_selection_slice_stop_at_nchunks(tmp_path: Path) -> None: """Block slice with stop == nchunks exercises the dim_len fallback.""" z, a = _make_1d(tmp_path) np.testing.assert_array_equal(z.blocks[1:3], a[5:30]) np.testing.assert_array_equal(z.blocks[0:10], a[:]) def test_pipeline_block_selection_slice_stop_at_nchunks_2d(tmp_path: Path) -> None: """Same fallback test for 2D rectilinear arrays.""" z, a = _make_2d(tmp_path) np.testing.assert_array_equal(z.blocks[2:3, 3:4], a[30:60, 75:100]) np.testing.assert_array_equal(z.blocks[0:99, 0:99], a[:, :]) # --- Set coordinate selection --- def test_pipeline_set_coordinate_selection_1d(tmp_path: Path) -> None: """Writing via 1D coordinate selection on rectilinear arrays persists correctly""" z, a = _make_1d(tmp_path) ix = np.array([0, 4, 5, 14, 15, 29]) val = np.full(len(ix), -7, dtype="int32") z.vindex[ix] = val a[ix] = val np.testing.assert_array_equal(z[:], a) def test_pipeline_set_coordinate_selection_2d(tmp_path: Path) -> None: """Writing via 2D coordinate selection on rectilinear arrays persists correctly""" z, a = _make_2d(tmp_path) r = np.array([0, 9, 10, 29, 30, 59]) c = np.array([0, 24, 25, 49, 50, 99]) val = np.full(len(r), -42, dtype="int32") z.vindex[r, c] = val a[r, c] = val np.testing.assert_array_equal(z[:], a) # --- Set selection --- def test_pipeline_set_basic_selection(tmp_path: Path) -> None: """Writing via basic slice selection on rectilinear arrays persists correctly""" z, a = _make_2d(tmp_path) new_data = np.full((20, 50), -1, dtype="int32") z[5:25, 10:60] = new_data a[5:25, 10:60] = new_data np.testing.assert_array_equal(z[:], a) def test_pipeline_set_orthogonal_selection(tmp_path: Path) -> None: """Writing via orthogonal selection on rectilinear arrays persists correctly""" z, a = _make_2d(tmp_path) rows = np.array([0, 10, 30]) cols = np.array([0, 25, 50, 75]) val = np.full((3, 4), -99, dtype="int32") z.oindex[rows, cols] = val a[np.ix_(rows, cols)] = val np.testing.assert_array_equal(z[:], a) # --- Higher dimensions --- def test_pipeline_3d_array(tmp_path: Path) -> None: """3D rectilinear array write and read-back match numpy""" shape = (12, 20, 15) chunk_shapes = [[4, 8], [5, 5, 10], [5, 10]] a = np.arange(int(np.prod(shape)), dtype="int32").reshape(shape) z = zarr.create_array( store=tmp_path / "arr3d.zarr", shape=shape, chunks=chunk_shapes, dtype="int32", ) z[:] = a np.testing.assert_array_equal(z[:], a) np.testing.assert_array_equal(z[2:10, 3:18, 4:14], a[2:10, 3:18, 4:14]) def test_pipeline_1d_single_chunk(tmp_path: Path) -> None: """Single-chunk rectilinear array write and read-back match numpy""" a = np.arange(20, dtype="int32") z = zarr.create_array( store=tmp_path / "arr1c.zarr", shape=(20,), chunks=[[20]], dtype="int32", ) z[:] = a np.testing.assert_array_equal(z[:], a) # --- Persistence roundtrip --- def test_pipeline_persistence_roundtrip(tmp_path: Path) -> None: """Rectilinear array survives close and reopen with correct data""" _, a = _make_2d(tmp_path) z2 = zarr.open_array(store=tmp_path / "arr2d.zarr", mode="r") assert not ChunkGrid.from_metadata(z2.metadata).is_regular np.testing.assert_array_equal(z2[:], a) # --- Highly irregular chunks --- def test_pipeline_highly_irregular_chunks(tmp_path: Path) -> None: """Highly irregular chunk sizes produce correct write and partial-read results""" shape = (100, 100) chunk_shapes = [[5, 10, 15, 20, 50], [100]] a = np.arange(10000, dtype="int32").reshape(shape) z = zarr.create_array( store=tmp_path / "irreg.zarr", shape=shape, chunks=chunk_shapes, dtype="int32", ) z[:] = a np.testing.assert_array_equal(z[:], a) np.testing.assert_array_equal(z[3:97, 10:90], a[3:97, 10:90]) # --- API validation --- def test_pipeline_v2_rejects_rectilinear(tmp_path: Path) -> None: """Creating a rectilinear array with zarr_format=2 raises ValueError""" with pytest.raises(ValueError, match="Zarr format 2"): zarr.create_array( store=tmp_path / "v2.zarr", shape=(30,), chunks=[[10, 20]], dtype="int32", zarr_format=2, ) def test_pipeline_sharding_rejects_rectilinear_chunks_with_shards(tmp_path: Path) -> None: """Rectilinear chunks (inner) with sharding is not supported.""" with pytest.raises(ValueError, match="Rectilinear chunks with sharding"): zarr.create_array( store=tmp_path / "shard.zarr", shape=(60, 100), chunks=[[10, 20, 30], [25, 25, 25, 25]], shards=(30, 50), dtype="int32", ) def test_pipeline_rectilinear_shards_roundtrip(tmp_path: Path) -> None: """Rectilinear shards with uniform inner chunks: full write/read roundtrip.""" data = np.arange(120 * 100, dtype="int32").reshape(120, 100) arr = zarr.create_array( store=tmp_path / "rect_shards.zarr", shape=(120, 100), chunks=(10, 10), shards=[[60, 40, 20], [50, 50]], dtype="int32", ) arr[:] = data result = arr[:] np.testing.assert_array_equal(result, data) def test_pipeline_rectilinear_shards_partial_read(tmp_path: Path) -> None: """Partial reads across rectilinear shard boundaries.""" data = np.arange(120 * 100, dtype="float64").reshape(120, 100) arr = zarr.create_array( store=tmp_path / "rect_shards.zarr", shape=(120, 100), chunks=(10, 10), shards=[[60, 40, 20], [50, 50]], dtype="float64", ) arr[:] = data result = arr[50:70, 40:60] np.testing.assert_array_equal(result, data[50:70, 40:60]) def test_pipeline_rectilinear_shards_validates_divisibility(tmp_path: Path) -> None: """Inner chunk_shape must divide every shard's dimensions.""" with pytest.raises(ValueError, match="divisible"): zarr.create_array( store=tmp_path / "bad.zarr", shape=(120, 100), chunks=(10, 10), shards=[[60, 45, 15], [50, 50]], dtype="int32", ) def test_pipeline_nchunks(tmp_path: Path) -> None: """Rectilinear array reports the correct total number of chunks""" z, _ = _make_2d(tmp_path) assert ChunkGrid.from_metadata(z.metadata).get_nchunks() == 12 def test_pipeline_parse_chunk_grid_regular_from_dict() -> None: """parse_chunk_grid constructs a regular grid from a metadata dict.""" d: dict[str, Any] = {"name": "regular", "configuration": {"chunk_shape": [10, 20]}} meta = parse_chunk_grid(d) assert isinstance(meta, RegularChunkGridMetadata) g = ChunkGrid.from_sizes((100, 200), tuple(meta.chunk_shape)) assert g.is_regular assert g.chunk_shape == (10, 20) assert g.grid_shape == (10, 10) assert g.get_nchunks() == 100 # --------------------------------------------------------------------------- # VaryingDimension boundary tests # --------------------------------------------------------------------------- @pytest.mark.parametrize( ("edges", "extent", "chunk_idx", "expected_data_size"), [ ([10, 20, 30], 50, 0, 10), ([10, 20, 30], 50, 1, 20), ([10, 20, 30], 50, 2, 20), ([10, 20, 30], 60, 2, 30), ([10, 20, 30], 31, 0, 10), ([10, 20, 30], 31, 1, 20), ([10, 20, 30], 31, 2, 1), ], ids=[ "interior-0", "interior-1", "boundary-clipped", "exact-no-clip", "single-element-boundary-0", "single-element-boundary-1", "single-element-boundary-2", ], ) def test_varying_dimension_boundary_data_size( edges: list[int], extent: int, chunk_idx: int, expected_data_size: int ) -> None: """VaryingDimension.data_size clips correctly at boundary chunks""" d = VaryingDimension(edges, extent=extent) assert d.data_size(chunk_idx) == expected_data_size def test_varying_dimension_boundary_extent_parameter() -> None: """VaryingDimension preserves extent and full chunk_size even when extent < sum of edges""" d = VaryingDimension([10, 20, 30], extent=50) assert d.extent == 50 assert d.chunk_size(2) == 30 def test_varying_dimension_extent_exceeds_sum_rejected() -> None: """VaryingDimension rejects extent greater than sum of edges""" with pytest.raises(ValueError, match="exceeds sum of edges"): VaryingDimension([10, 20], extent=50) def test_varying_dimension_negative_extent_rejected() -> None: """VaryingDimension rejects negative extent""" with pytest.raises(ValueError, match="must be >= 0"): VaryingDimension([10, 20], extent=-1) def test_varying_dimension_zero_extent() -> None: """VaryingDimension with extent=0 has zero active chunks but retains all grid cells.""" d = VaryingDimension([10, 20], extent=0) assert d.nchunks == 0 assert d.ngridcells == 2 # No chunks overlap [0, 0), so the grid is structurally non-empty but logically empty g = ChunkGrid(dimensions=(d,)) assert g.grid_shape == (0,) assert list(g) == [] def test_varying_dimension_boundary_chunk_spec() -> None: """ChunkGrid with a boundary VaryingDimension produces correct ChunkSpec.""" g = ChunkGrid(dimensions=(VaryingDimension([10, 20, 30], extent=50),)) spec = g[(2,)] assert spec is not None assert spec.codec_shape == (30,) assert spec.shape == (20,) assert spec.is_boundary is True def test_varying_dimension_interior_chunk_spec() -> None: """Interior VaryingDimension chunk has matching codec_shape and shape with no boundary""" g = ChunkGrid(dimensions=(VaryingDimension([10, 20, 30], extent=50),)) spec = g[(0,)] assert spec is not None assert spec.codec_shape == (10,) assert spec.shape == (10,) assert spec.is_boundary is False # --------------------------------------------------------------------------- # Multiple overflow chunks tests # --------------------------------------------------------------------------- def test_overflow_multiple_chunks_past_extent() -> None: """Edges past extent are structural; nchunks counts active only.""" g = ChunkGrid.from_sizes((50,), [[10, 20, 30, 40]]) d = g._dimensions[0] assert d.ngridcells == 4 assert d.nchunks == 3 assert d.data_size(0) == 10 assert d.data_size(1) == 20 assert d.data_size(2) == 20 assert d.chunk_size(2) == 30 def test_overflow_chunk_spec_past_extent_is_oob() -> None: """Chunk entirely past the extent is out of bounds (not active).""" g = ChunkGrid.from_sizes((50,), [[10, 20, 30, 40]]) spec = g[(3,)] assert spec is None def test_overflow_chunk_spec_partial() -> None: """ChunkSpec for a partially-overflowing chunk clips correctly.""" g = ChunkGrid.from_sizes((50,), [[10, 20, 30, 40]]) spec = g[(2,)] assert spec is not None assert spec.shape == (20,) assert spec.codec_shape == (30,) assert spec.is_boundary is True assert spec.slices == (slice(30, 50, 1),) def test_overflow_chunk_sizes() -> None: """chunk_sizes only includes active chunks.""" g = ChunkGrid.from_sizes((50,), [[10, 20, 30, 40]]) assert g.chunk_sizes == ((10, 20, 20),) def test_overflow_multidim() -> None: """Overflow in multiple dimensions simultaneously.""" g = ChunkGrid.from_sizes((45, 100), [[10, 20, 30], [40, 40, 40]]) assert g.chunk_sizes == ((10, 20, 15), (40, 40, 20)) spec = g[(2, 2)] assert spec is not None assert spec.shape == (15, 20) assert spec.codec_shape == (30, 40) def test_overflow_index_to_chunk_near_extent() -> None: """Index lookup near and at the extent boundary.""" d = VaryingDimension([10, 20, 30, 40], extent=50) assert d.index_to_chunk(29) == 1 assert d.index_to_chunk(30) == 2 assert d.index_to_chunk(49) == 2 # --------------------------------------------------------------------------- # Boundary indexing tests # --------------------------------------------------------------------------- @pytest.mark.parametrize( ( "dim", "mask", "dim_len", "expected_chunk_ix", "expected_sel_len", "expected_first_two", "expected_third", ), [ ( FixedDimension(size=5, extent=7), np.array([False, False, False, False, False, True, True]), 7, 1, 5, (np.True_, np.True_), np.False_, ), ( VaryingDimension([5, 10], extent=7), np.array([False, False, False, False, False, True, True]), 7, 1, 10, (np.True_, np.True_), np.False_, ), ], ids=["fixed-boundary", "varying-boundary"], ) def test_bool_indexer_boundary( dim: FixedDimension | VaryingDimension, mask: np.ndarray[Any, Any], dim_len: int, expected_chunk_ix: int, expected_sel_len: int, expected_first_two: tuple[Any, Any], expected_third: Any, ) -> None: """BoolArrayDimIndexer pads to codec size for boundary chunks.""" from zarr.core.indexing import BoolArrayDimIndexer indexer = BoolArrayDimIndexer(mask, dim_len, dim) projections = list(indexer) assert len(projections) == 1 p = projections[0] assert p.dim_chunk_ix == expected_chunk_ix sel = p.dim_chunk_sel assert isinstance(sel, np.ndarray) assert sel.shape[0] == expected_sel_len assert sel[0] is expected_first_two[0] assert sel[1] is expected_first_two[1] assert sel[2] is expected_third def test_bool_indexer_no_padding_interior() -> None: """No padding needed for interior chunks.""" from zarr.core.indexing import BoolArrayDimIndexer dim = FixedDimension(size=5, extent=10) mask = np.array([True, False, False, False, False, False, False, False, False, False]) indexer = BoolArrayDimIndexer(mask, 10, dim) projections = list(indexer) assert len(projections) == 1 p = projections[0] assert p.dim_chunk_ix == 0 sel = p.dim_chunk_sel assert isinstance(sel, np.ndarray) assert sel.shape[0] == 5 def test_slice_indexer_varying_boundary() -> None: """SliceDimIndexer clips to data_size at boundary for VaryingDimension.""" from zarr.core.indexing import SliceDimIndexer dim = VaryingDimension([5, 10], extent=7) indexer = SliceDimIndexer(slice(None), 7, dim) projections = list(indexer) assert len(projections) == 2 assert projections[0].dim_chunk_sel == slice(0, 5, 1) assert projections[1].dim_chunk_sel == slice(0, 2, 1) def test_int_array_indexer_varying_boundary() -> None: """IntArrayDimIndexer handles indices near boundary correctly.""" from zarr.core.indexing import IntArrayDimIndexer dim = VaryingDimension([5, 10], extent=7) indices = np.array([6]) indexer = IntArrayDimIndexer(indices, 7, dim) projections = list(indexer) assert len(projections) == 1 assert projections[0].dim_chunk_ix == 1 sel = projections[0].dim_chunk_sel assert isinstance(sel, np.ndarray) np.testing.assert_array_equal(sel, [1]) @pytest.mark.parametrize( "dim", [FixedDimension(size=2, extent=10), VaryingDimension([5, 5], extent=10)], ids=["fixed", "varying"], ) def test_slice_indexer_empty_slice_at_boundary(dim: FixedDimension | VaryingDimension) -> None: """SliceDimIndexer yields no projections for an empty slice at the dimension boundary.""" from zarr.core.indexing import SliceDimIndexer indexer = SliceDimIndexer(slice(10, 10), 10, dim) projections = list(indexer) assert len(projections) == 0 def test_orthogonal_indexer_varying_boundary_advanced() -> None: """OrthogonalIndexer with advanced indexing uses per-chunk chunk_size.""" from zarr.core.indexing import OrthogonalIndexer g = ChunkGrid( dimensions=( VaryingDimension([5, 10], extent=7), FixedDimension(size=4, extent=8), ) ) indexer = OrthogonalIndexer( selection=(np.array([0, 6]), slice(None)), shape=(7, 8), chunk_grid=g, ) projections = list(indexer) assert len(projections) == 4 coords = {p.chunk_coords for p in projections} assert coords == {(0, 0), (0, 1), (1, 0), (1, 1)} # --------------------------------------------------------------------------- # update_shape tests # --------------------------------------------------------------------------- def test_update_shape_no_change() -> None: """update_shape with the same shape preserves edges unchanged""" grid = ChunkGrid.from_sizes((60, 50), [[10, 20, 30], [25, 25]]) new_grid = grid.update_shape((60, 50)) assert _edges(new_grid, 0) == (10, 20, 30) assert _edges(new_grid, 1) == (25, 25) def test_update_shape_grow_single_dim() -> None: """Growing a single dimension appends a new edge chunk""" grid = ChunkGrid.from_sizes((60, 50), [[10, 20, 30], [25, 25]]) new_grid = grid.update_shape((80, 50)) assert _edges(new_grid, 0) == (10, 20, 30, 20) assert _edges(new_grid, 1) == (25, 25) def test_update_shape_grow_multiple_dims() -> None: """Growing multiple dimensions appends correctly sized edge chunks""" grid = ChunkGrid.from_sizes((30, 50), [[10, 20], [20, 30]]) new_grid = grid.update_shape((45, 65)) assert _edges(new_grid, 0) == (10, 20, 15) assert _edges(new_grid, 1) == (20, 30, 15) def test_update_shape_shrink_single_dim() -> None: """Shrinking a single dimension reduces nchunks while preserving edges""" grid = ChunkGrid.from_sizes((100, 50), [[10, 20, 30, 40], [25, 25]]) new_grid = grid.update_shape((35, 50)) assert _edges(new_grid, 0) == (10, 20, 30, 40) assert new_grid._dimensions[0].nchunks == 3 assert _edges(new_grid, 1) == (25, 25) def test_update_shape_shrink_to_single_chunk() -> None: """Shrinking to fit within the first chunk reduces nchunks to 1""" grid = ChunkGrid.from_sizes((60, 50), [[10, 20, 30], [25, 25]]) new_grid = grid.update_shape((5, 50)) assert _edges(new_grid, 0) == (10, 20, 30) assert new_grid._dimensions[0].nchunks == 1 assert _edges(new_grid, 1) == (25, 25) def test_update_shape_shrink_multiple_dims() -> None: """Shrinking multiple dimensions reduces nchunks in each dimension""" grid = ChunkGrid.from_sizes((40, 60), [[10, 10, 15, 5], [20, 25, 15]]) new_grid = grid.update_shape((25, 35)) assert _edges(new_grid, 0) == (10, 10, 15, 5) assert new_grid._dimensions[0].nchunks == 3 assert _edges(new_grid, 1) == (20, 25, 15) assert new_grid._dimensions[1].nchunks == 2 def test_update_shape_dimension_mismatch_error() -> None: """update_shape raises ValueError when new shape has different ndim""" grid = ChunkGrid.from_sizes((30, 70), [[10, 20], [30, 40]]) with pytest.raises(ValueError, match="dimensions"): grid.update_shape((30, 70, 100)) def test_update_shape_boundary_cases() -> None: """update_shape handles grow-one-dim and shrink-both-dims edge cases correctly""" grid = ChunkGrid.from_sizes((60, 40), [[10, 20, 30], [15, 25]]) new_grid = grid.update_shape((60, 65)) assert _edges(new_grid, 0) == (10, 20, 30) assert _edges(new_grid, 1) == (15, 25, 25) grid2 = ChunkGrid.from_sizes((60, 50), [[10, 20, 30], [15, 25, 10]]) new_grid2 = grid2.update_shape((30, 40)) assert _edges(new_grid2, 0) == (10, 20, 30) assert new_grid2._dimensions[0].nchunks == 2 assert _edges(new_grid2, 1) == (15, 25, 10) assert new_grid2._dimensions[1].nchunks == 2 def test_update_shape_regular_preserves_extents(tmp_path: Path) -> None: """Resize a regular array -- chunk_grid extents must match new shape.""" z = zarr.create_array( store=tmp_path / "regular.zarr", shape=(100,), chunks=(10,), dtype="int32", ) z[:] = np.arange(100, dtype="int32") z.resize(50) assert z.shape == (50,) assert ChunkGrid.from_metadata(z.metadata)._dimensions[0].extent == 50 # --------------------------------------------------------------------------- # update_shape boundary tests # --------------------------------------------------------------------------- def test_update_shape_shrink_creates_boundary() -> None: """Shrinking extent into a chunk creates a boundary with clipped data_size""" grid = ChunkGrid.from_sizes((60,), [[10, 20, 30]]) new_grid = grid.update_shape((45,)) dim = new_grid._dimensions[0] assert isinstance(dim, VaryingDimension) assert dim.edges == (10, 20, 30) assert dim.extent == 45 assert dim.chunk_size(2) == 30 assert dim.data_size(2) == 15 def test_update_shape_shrink_to_exact_boundary() -> None: """Shrinking to an exact chunk boundary reduces nchunks without partial data""" grid = ChunkGrid.from_sizes((60,), [[10, 20, 30]]) new_grid = grid.update_shape((30,)) dim = new_grid._dimensions[0] assert isinstance(dim, VaryingDimension) assert dim.edges == (10, 20, 30) assert dim.nchunks == 2 assert dim.ngridcells == 3 assert dim.extent == 30 assert dim.data_size(1) == 20 def test_update_shape_shrink_chunk_spec() -> None: """After shrink, ChunkSpec reflects boundary correctly.""" grid = ChunkGrid.from_sizes((60,), [[10, 20, 30]]) new_grid = grid.update_shape((45,)) spec = new_grid[(2,)] assert spec is not None assert spec.codec_shape == (30,) assert spec.shape == (15,) assert spec.is_boundary is True def test_update_shape_parse_chunk_grid_rebinds_extent() -> None: """parse_chunk_grid re-binds VaryingDimension extent to array shape.""" g = ChunkGrid.from_sizes((60,), [[10, 20, 30]]) g2 = ChunkGrid( dimensions=tuple( dim.with_extent(ext) for dim, ext in zip(g._dimensions, (50,), strict=True) ) ) dim = g2._dimensions[0] assert isinstance(dim, VaryingDimension) assert dim.extent == 50 assert dim.data_size(2) == 20 # --------------------------------------------------------------------------- # Resize rectilinear tests # --------------------------------------------------------------------------- async def test_async_resize_grow() -> None: """Async resize grow appends new edge chunks and preserves existing data""" store = zarr.storage.MemoryStore() arr = await zarr.api.asynchronous.create_array( store=store, shape=(30, 40), chunks=[[10, 20], [20, 20]], dtype="i4", zarr_format=3, ) data = np.arange(30 * 40, dtype="i4").reshape(30, 40) await arr.setitem(slice(None), data) await arr.resize((50, 60)) assert arr.shape == (50, 60) assert _edges(ChunkGrid.from_metadata(arr.metadata), 0) == (10, 20, 20) assert _edges(ChunkGrid.from_metadata(arr.metadata), 1) == (20, 20, 20) result = await arr.getitem((slice(0, 30), slice(0, 40))) np.testing.assert_array_equal(result, data) async def test_async_resize_shrink() -> None: """Async resize shrink truncates data to the new shape""" store = zarr.storage.MemoryStore() arr = await zarr.api.asynchronous.create_array( store=store, shape=(60, 50), chunks=[[10, 20, 30], [25, 25]], dtype="f4", zarr_format=3, ) data = np.arange(60 * 50, dtype="f4").reshape(60, 50) await arr.setitem(slice(None), data) await arr.resize((25, 30)) assert arr.shape == (25, 30) result = await arr.getitem(slice(None)) np.testing.assert_array_equal(result, data[:25, :30]) def test_sync_resize_grow() -> None: """Sync resize grow expands the array and preserves existing data""" store = zarr.storage.MemoryStore() arr = zarr.create_array( store=store, shape=(20, 30), chunks=[[8, 12], [10, 20]], dtype="u1", zarr_format=3, ) data = np.arange(20 * 30, dtype="u1").reshape(20, 30) arr[:] = data arr.resize((35, 45)) assert arr.shape == (35, 45) np.testing.assert_array_equal(arr[:20, :30], data) def test_sync_resize_shrink() -> None: """Sync resize shrink truncates the array and returns correct data""" store = zarr.storage.MemoryStore() arr = zarr.create_array( store=store, shape=(40, 50), chunks=[[10, 15, 15], [20, 30]], dtype="i2", zarr_format=3, ) data = np.arange(40 * 50, dtype="i2").reshape(40, 50) arr[:] = data arr.resize((15, 30)) assert arr.shape == (15, 30) np.testing.assert_array_equal(arr[:], data[:15, :30]) # --------------------------------------------------------------------------- # Append rectilinear tests # --------------------------------------------------------------------------- async def test_append_first_axis() -> None: """Appending along axis 0 grows the array and concatenates data correctly""" store = zarr.storage.MemoryStore() arr = await zarr.api.asynchronous.create_array( store=store, shape=(30, 20), chunks=[[10, 20], [10, 10]], dtype="i4", zarr_format=3, ) initial = np.arange(30 * 20, dtype="i4").reshape(30, 20) await arr.setitem(slice(None), initial) append_data = np.arange(30 * 20, 45 * 20, dtype="i4").reshape(15, 20) await arr.append(append_data, axis=0) assert arr.shape == (45, 20) result = await arr.getitem(slice(None)) np.testing.assert_array_equal(result, np.vstack([initial, append_data])) async def test_append_second_axis() -> None: """Appending along axis 1 grows the array and concatenates data correctly""" store = zarr.storage.MemoryStore() arr = await zarr.api.asynchronous.create_array( store=store, shape=(20, 30), chunks=[[10, 10], [10, 20]], dtype="f4", zarr_format=3, ) initial = np.arange(20 * 30, dtype="f4").reshape(20, 30) await arr.setitem(slice(None), initial) append_data = np.arange(20 * 30, 20 * 45, dtype="f4").reshape(20, 15) await arr.append(append_data, axis=1) assert arr.shape == (20, 45) result = await arr.getitem(slice(None)) np.testing.assert_array_equal(result, np.hstack([initial, append_data])) def test_sync_append() -> None: """Sync append grows the array and preserves both initial and appended data""" store = zarr.storage.MemoryStore() arr = zarr.create_array( store=store, shape=(20, 20), chunks=[[8, 12], [7, 13]], dtype="u2", zarr_format=3, ) initial = np.arange(20 * 20, dtype="u2").reshape(20, 20) arr[:] = initial append_data = np.arange(20 * 20, 25 * 20, dtype="u2").reshape(5, 20) arr.append(append_data, axis=0) assert arr.shape == (25, 20) np.testing.assert_array_equal(arr[:20, :], initial) np.testing.assert_array_equal(arr[20:, :], append_data) async def test_multiple_appends() -> None: """Multiple sequential appends accumulate data correctly""" store = zarr.storage.MemoryStore() arr = await zarr.api.asynchronous.create_array( store=store, shape=(10, 10), chunks=[[3, 7], [4, 6]], dtype="i4", zarr_format=3, ) initial = np.arange(10 * 10, dtype="i4").reshape(10, 10) await arr.setitem(slice(None), initial) all_data = [initial] for i in range(3): chunk = np.full((5, 10), i + 100, dtype="i4") await arr.append(chunk, axis=0) all_data.append(chunk) assert arr.shape == (25, 10) result = await arr.getitem(slice(None)) np.testing.assert_array_equal(result, np.vstack(all_data)) async def test_append_with_partial_edge_chunks() -> None: """Appending data that creates partial edge chunks preserves all data""" store = zarr.storage.MemoryStore() arr = await zarr.api.asynchronous.create_array( store=store, shape=(25, 30), chunks=[[10, 15], [12, 18]], dtype="f8", zarr_format=3, ) initial = np.random.default_rng(42).random((25, 30)) await arr.setitem(slice(None), initial) append_data = np.random.default_rng(43).random((10, 30)) await arr.append(append_data, axis=0) assert arr.shape == (35, 30) result = np.asarray(await arr.getitem(slice(None))) np.testing.assert_array_almost_equal(result, np.vstack([initial, append_data])) async def test_append_small_data() -> None: """Appending a small amount of data smaller than a chunk works correctly""" store = zarr.storage.MemoryStore() arr = await zarr.api.asynchronous.create_array( store=store, shape=(20, 20), chunks=[[8, 12], [7, 13]], dtype="i4", zarr_format=3, ) data = np.arange(20 * 20, dtype="i4").reshape(20, 20) await arr.setitem(slice(None), data) small = np.full((3, 20), 999, dtype="i4") await arr.append(small, axis=0) assert arr.shape == (23, 20) result = await arr.getitem((slice(20, 23), slice(None))) np.testing.assert_array_equal(result, small) # --------------------------------------------------------------------------- # V2 regression tests # --------------------------------------------------------------------------- def test_v2_create_and_readback(tmp_path: Path) -> None: """Basic V2 array: create, write, read back.""" data = np.arange(60, dtype="float64").reshape(6, 10) a = zarr.create_array( store=tmp_path / "v2.zarr", shape=data.shape, chunks=(3, 5), dtype=data.dtype, zarr_format=2, ) a[:] = data np.testing.assert_array_equal(a[:], data) def test_v2_chunk_grid_is_regular(tmp_path: Path) -> None: """V2 chunk_grid produces a regular ChunkGrid with FixedDimensions.""" a = zarr.create_array( store=tmp_path / "v2.zarr", shape=(20, 30), chunks=(10, 15), dtype="int32", zarr_format=2, ) grid = ChunkGrid.from_metadata(a.metadata) assert grid.is_regular assert grid.chunk_shape == (10, 15) assert grid.grid_shape == (2, 2) assert all(isinstance(d, FixedDimension) for d in grid._dimensions) def test_v2_boundary_chunks(tmp_path: Path) -> None: """V2 boundary chunks: codec buffer size stays full, data is clipped.""" a = zarr.create_array( store=tmp_path / "v2.zarr", shape=(25,), chunks=(10,), dtype="int32", zarr_format=2, ) grid = ChunkGrid.from_metadata(a.metadata) assert grid._dimensions[0].nchunks == 3 assert grid._dimensions[0].chunk_size(2) == 10 assert grid._dimensions[0].data_size(2) == 5 def test_v2_slicing_with_boundary(tmp_path: Path) -> None: """V2 array slicing across boundary chunks returns correct data.""" data = np.arange(25, dtype="int32") a = zarr.create_array( store=tmp_path / "v2.zarr", shape=(25,), chunks=(10,), dtype="int32", zarr_format=2, ) a[:] = data np.testing.assert_array_equal(a[18:25], data[18:25]) np.testing.assert_array_equal(a[:], data) def test_v2_metadata_roundtrip(tmp_path: Path) -> None: """V2 metadata survives store close and reopen.""" store_path = tmp_path / "v2.zarr" data = np.arange(12, dtype="float32").reshape(3, 4) a = zarr.create_array( store=store_path, shape=data.shape, chunks=(2, 2), dtype=data.dtype, zarr_format=2, ) a[:] = data b = zarr.open_array(store=store_path, mode="r") assert b.metadata.zarr_format == 2 assert b.chunks == (2, 2) assert ChunkGrid.from_metadata(b.metadata).chunk_shape == (2, 2) np.testing.assert_array_equal(b[:], data) def test_v2_chunk_spec_via_grid(tmp_path: Path) -> None: """ChunkSpec from V2 grid has correct slices and codec_shape.""" a = zarr.create_array( store=tmp_path / "v2.zarr", shape=(15, 20), chunks=(10, 10), dtype="int32", zarr_format=2, ) grid = ChunkGrid.from_metadata(a.metadata) spec = grid[(0, 0)] assert spec is not None assert spec.shape == (10, 10) assert spec.codec_shape == (10, 10) spec = grid[(1, 1)] assert spec is not None assert spec.shape == (5, 10) assert spec.codec_shape == (10, 10) # --------------------------------------------------------------------------- # ChunkSizes tests # --------------------------------------------------------------------------- @pytest.mark.parametrize( ("shape", "chunks", "expected"), [ ((100, 80), (30, 40), ((30, 30, 30, 10), (40, 40))), ((90, 80), (30, 40), ((30, 30, 30), (40, 40))), ((60, 100), [[10, 20, 30], [50, 50]], ((10, 20, 30), (50, 50))), ((10,), (10,), ((10,),)), ], ids=["regular", "regular-exact", "rectilinear", "single-chunk"], ) def test_chunk_sizes( shape: tuple[int, ...], chunks: Any, expected: tuple[tuple[int, ...], ...] ) -> None: """chunk_sizes returns the per-dimension tuple of actual data sizes""" grid = ChunkGrid.from_sizes(shape, chunks) assert grid.chunk_sizes == expected def test_array_read_chunk_sizes_regular() -> None: """Regular array exposes correct read_chunk_sizes and write_chunk_sizes""" store = zarr.storage.MemoryStore() arr = zarr.create_array( store=store, shape=(100, 80), chunks=(30, 40), dtype="i4", zarr_format=3 ) assert arr.read_chunk_sizes == ((30, 30, 30, 10), (40, 40)) assert arr.write_chunk_sizes == ((30, 30, 30, 10), (40, 40)) def test_array_read_chunk_sizes_rectilinear() -> None: """Rectilinear array exposes correct read_chunk_sizes and write_chunk_sizes""" store = zarr.storage.MemoryStore() arr = zarr.create_array( store=store, shape=(60, 100), chunks=[[10, 20, 30], [50, 50]], dtype="i4", zarr_format=3 ) assert arr.read_chunk_sizes == ((10, 20, 30), (50, 50)) assert arr.write_chunk_sizes == ((10, 20, 30), (50, 50)) def test_array_sharded_chunk_sizes() -> None: """Sharded array read_chunk_sizes reflects inner chunks and write_chunk_sizes reflects shards""" store = zarr.storage.MemoryStore() arr = zarr.create_array( store=store, shape=(120, 80), chunks=(60, 40), shards=(120, 80), dtype="i4", zarr_format=3, ) assert arr.read_chunk_sizes == ((60, 60), (40, 40)) assert arr.write_chunk_sizes == ((120,), (80,)) # --------------------------------------------------------------------------- # Info display test # --------------------------------------------------------------------------- def test_chunk_grid_repr_regular() -> None: """ChunkGrid repr shows uniform chunk sizes and array shape for regular grids.""" grid = ChunkGrid.from_sizes((100, 200), (10, 20)) r = repr(grid) assert r == "ChunkGrid(chunk_sizes=(10, 20), array_shape=(100, 200))" def test_chunk_grid_repr_rectilinear() -> None: """ChunkGrid repr shows per-chunk edge tuples for rectilinear dimensions.""" grid = ChunkGrid.from_sizes((30,), ([10, 20],)) r = repr(grid) assert "(10, 20)" in r assert "(30,)" in r def test_info_display_rectilinear() -> None: """Array.info should not crash for rectilinear grids.""" store = zarr.storage.MemoryStore() arr = zarr.create_array( store=store, shape=(30,), chunks=[[10, 20]], dtype="i4", zarr_format=3, ) info = arr.info text = repr(info) assert "" in text assert "Array" in text # --------------------------------------------------------------------------- # nchunks tests # --------------------------------------------------------------------------- @pytest.mark.parametrize( ("shape", "chunks", "expected"), [ ((30,), [[10, 20]], 2), ((30, 40), [[10, 20], [15, 25]], 4), ], ids=["1d", "2d"], ) def test_nchunks_rectilinear( shape: tuple[int, ...], chunks: list[list[int]], expected: int ) -> None: """Array.nchunks reports correct total chunk count for rectilinear arrays""" store = MemoryStore() a = zarr.create_array(store, shape=shape, chunks=chunks, dtype="int32") assert a.nchunks == expected # --------------------------------------------------------------------------- # iter_chunk_regions test # --------------------------------------------------------------------------- def test_iter_chunk_regions_rectilinear() -> None: """_iter_chunk_regions should work for rectilinear arrays.""" from zarr.core.array import _iter_chunk_regions store = MemoryStore() a = zarr.create_array(store, shape=(30,), chunks=[[10, 20]], dtype="int32") regions = list(_iter_chunk_regions(a)) assert len(regions) == 2 assert regions[0] == (slice(0, 10, 1),) assert regions[1] == (slice(10, 30, 1),) # --------------------------------------------------------------------------- # RectilinearChunkGridMetadata metadata object tests (already parametrized) # --------------------------------------------------------------------------- @pytest.mark.parametrize( ("json_input", "expected_chunk_shapes"), [ ( { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [4, 8]}, }, (4, 8), ), ( { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[1, 2, 3], [10, 20]]}, }, ((1, 2, 3), (10, 20)), ), ( { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[[4, 3]], [10, 20]]}, }, ((4, 4, 4), (10, 20)), ), ( { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[[1, 3], 3], [5]]}, }, ((1, 1, 1, 3), (5,)), ), ( { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [4, [10, 20]]}, }, (4, (10, 20)), ), ], ) def test_rectilinear_from_dict( json_input: RectilinearChunkGridMetadataJSON, expected_chunk_shapes: tuple[int | tuple[int, ...], ...], ) -> None: """RectilinearChunkGridMetadata.from_dict correctly parses all spec forms.""" grid = RectilinearChunkGridMetadata.from_dict(json_input) assert grid.chunk_shapes == expected_chunk_shapes @pytest.mark.parametrize( ("chunk_shapes", "expected_json_shapes"), [ ((4, 8), [4, 8]), (((4,), (8,)), [[4], [8]]), (((10, 20), (5, 5)), [[10, 20], [[5, 2]]]), (((4, 4, 4), (10, 20)), [[[4, 3]], [10, 20]]), ((4, (10, 20)), [4, [10, 20]]), ], ) def test_rectilinear_to_dict( chunk_shapes: tuple[int | tuple[int, ...], ...], expected_json_shapes: list[Any], ) -> None: """RectilinearChunkGridMetadata.to_dict serializes back to spec-compliant JSON.""" grid = RectilinearChunkGridMetadata(chunk_shapes=chunk_shapes) result = grid.to_dict() assert result["name"] == "rectilinear" assert result["configuration"]["kind"] == "inline" assert list(result["configuration"]["chunk_shapes"]) == expected_json_shapes @pytest.mark.parametrize( "json_input", [ {"name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [4, 8]}}, { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[1, 2, 3], [10, 20]]}, }, { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [[[4, 3]], [[5, 2]]]}, }, ], ) def test_rectilinear_roundtrip(json_input: RectilinearChunkGridMetadataJSON) -> None: """from_dict -> to_dict -> from_dict produces the same grid.""" grid1 = RectilinearChunkGridMetadata.from_dict(json_input) grid2 = RectilinearChunkGridMetadata.from_dict(grid1.to_dict()) assert grid1.chunk_shapes == grid2.chunk_shapes # --------------------------------------------------------------------------- # Hypothesis property tests # --------------------------------------------------------------------------- pytest.importorskip("hypothesis") import hypothesis.strategies as st from hypothesis import event, given, settings from tests.conftest import declared_chunk_data_sizes from zarr.testing.strategies import _rectilinear_chunks @st.composite def rectilinear_arrays_st( draw: st.DrawFn, ) -> tuple[zarr.Array[Any], np.ndarray[Any, Any], list[int | list[int]]]: """Generate a rectilinear zarr array with random data, shape, and chunks, with the `chunks=` it was created with.""" from zarr.storage import MemoryStore ndim = draw(st.integers(min_value=1, max_value=3)) shape = draw(st.tuples(*[st.integers(min_value=2, max_value=20) for _ in range(ndim)])) chunk_shapes = draw(_rectilinear_chunks(shape=shape)) event(f"ndim={ndim}, shape={shape}") a = np.arange(int(np.prod(shape)), dtype="int32").reshape(shape) store = MemoryStore() z = zarr.create_array(store=store, shape=shape, chunks=chunk_shapes, dtype="int32") z[:] = a return z, a, chunk_shapes @settings(deadline=None, max_examples=50) @given(data=st.data()) def test_property_block_indexing_rectilinear(data: st.DataObject) -> None: """Property test: block indexing on rectilinear arrays matches numpy.""" z, a, chunks = data.draw(rectilinear_arrays_st()) for dim in range(a.ndim): # The block extents come from the declaration, not from zarr's grid code. sizes = declared_chunk_data_sizes(chunks[dim], a.shape[dim]) block_ix = data.draw(st.integers(min_value=0, max_value=len(sizes) - 1)) sel = [slice(None)] * a.ndim start = sum(sizes[:block_ix]) sel[dim] = slice(start, start + sizes[block_ix]) block_sel: list[slice | int] = [slice(None)] * a.ndim block_sel[dim] = block_ix np.testing.assert_array_equal( z.blocks[tuple(block_sel)], a[tuple(sel)], err_msg=f"dim={dim}, block={block_ix}", ) zarr-developers-zarr-python-a2f2415/tests/test_v2.py000066400000000000000000000336551527014137600226030ustar00rootroot00000000000000import json import pickle from pathlib import Path from typing import Any, Literal import numcodecs.errors import numcodecs.pickles import numpy as np import pytest from numcodecs import Delta, Pickle, VLenUTF8, Zlib from numcodecs.blosc import Blosc from numcodecs.zstd import Zstd import zarr import zarr.core.buffer import zarr.storage from zarr import config from zarr.abc.store import Store from zarr.core.buffer.core import default_buffer_prototype from zarr.core.dtype import FixedLengthUTF32, VariableLengthUTF8 from zarr.core.dtype.npy.bytes import NullTerminatedBytes from zarr.core.dtype.npy.structured import Struct from zarr.core.dtype.wrapper import ZDType from zarr.core.group import Group from zarr.core.sync import sync from zarr.errors import PickleCodecDisabledError, ZarrDeprecationWarning from zarr.storage import MemoryStore, StorePath @pytest.fixture async def store() -> StorePath: return StorePath(await MemoryStore.open()) def test_simple(store: StorePath) -> None: data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) a = zarr.create_array( store / "simple_v2", zarr_format=2, shape=data.shape, chunks=(16, 16), dtype=data.dtype, fill_value=0, ) a[:, :] = data assert np.array_equal(data, a[:, :]) def test_codec_pipeline() -> None: # https://github.com/zarr-developers/zarr-python/issues/2243 store = MemoryStore() array = zarr.create( store=store, shape=(1,), dtype="i4", zarr_format=2, filters=[Delta(dtype="i4").get_config()], compressor=Blosc().get_config(), ) array[:] = 1 result = array[:] expected = np.ones(1) np.testing.assert_array_equal(result, expected) @pytest.mark.parametrize( ("dtype", "expected_dtype", "fill_value", "fill_value_json"), [ ("|S1", "|S1", b"X", "WA=="), ("|V1", "|V1", b"X", "WA=="), ("|V10", "|V10", b"X", "WAAAAAAAAAAAAA=="), ], ) async def test_v2_encode_decode( dtype: str, expected_dtype: str, fill_value: bytes, fill_value_json: str ) -> None: store = zarr.storage.MemoryStore() g = zarr.group(store=store, zarr_format=2) g.create_array( name="foo", shape=(3,), chunks=(3,), dtype=dtype, fill_value=fill_value, compressor=None ) result = await store.get("foo/.zarray", zarr.core.buffer.default_buffer_prototype()) assert result is not None serialized = json.loads(result.to_bytes()) expected = { "chunks": [3], "compressor": None, "dtype": expected_dtype, "fill_value": fill_value_json, "filters": None, "order": "C", "shape": [3], "zarr_format": 2, "dimension_separator": ".", } assert serialized == expected data = zarr.open_array(store=store, path="foo")[:] np.testing.assert_equal(data, np.full((3,), b"X", dtype=dtype)) data = zarr.open_array(store=store, path="foo")[:] np.testing.assert_equal(data, np.full((3,), b"X", dtype=dtype)) @pytest.mark.parametrize( ("dtype", "value"), [ (NullTerminatedBytes(length=1), b"Y"), (FixedLengthUTF32(length=1), "Y"), (VariableLengthUTF8(), "Y"), ], ) def test_v2_encode_decode_with_data(dtype: ZDType[Any, Any], value: str) -> None: expected = np.full((3,), value, dtype=dtype.to_native_dtype()) a = zarr.create( shape=(3,), zarr_format=2, dtype=dtype, ) a[:] = expected data = a[:] np.testing.assert_equal(data, expected) @pytest.mark.parametrize("filters", [[], [Delta(dtype=" None: array_fixture = [42] with config.set({"array.order": order}): arr = zarr.create(shape=1, dtype=" None: """ Test that passing compressor=None results in no compressor. Also test that the default value of the compressor parameter does produce a compressor. """ g = zarr.open(store, mode="w", zarr_format=2) assert isinstance(g, Group) arr = g.create_array("one", dtype="i8", shape=(1,), chunks=(1,), compressor=None) assert arr.async_array.compressor is None assert not (arr.filters) arr = g.create_array("two", dtype="i8", shape=(1,), chunks=(1,)) assert arr.async_array.compressor is not None assert not (arr.filters) arr = g.create_array("three", dtype="i8", shape=(1,), chunks=(1,), compressor=Zstd()) assert arr.async_array.compressor is not None assert not (arr.filters) with pytest.raises(ValueError): g.create_array( "four", dtype="i8", shape=(1,), chunks=(1,), compressor=None, compressors=None ) @pytest.mark.parametrize("numpy_order", ["C", "F"]) @pytest.mark.parametrize("zarr_order", ["C", "F"]) def test_v2_non_contiguous(numpy_order: Literal["C", "F"], zarr_order: Literal["C", "F"]) -> None: """ Make sure zarr v2 arrays save data using the memory order given to the zarr array, not the memory order of the original numpy array. """ store = MemoryStore() arr = zarr.create_array( store, shape=(10, 8), chunks=(3, 3), fill_value=np.nan, dtype="float64", zarr_format=2, filters=None, compressors=None, overwrite=True, order=zarr_order, ) # Non-contiguous write, using numpy memory order a = np.arange(arr.shape[0] * arr.shape[1]).reshape(arr.shape, order=numpy_order) arr[6:9, 3:6] = a[6:9, 3:6] # The slice on the RHS is important np.testing.assert_array_equal(arr[6:9, 3:6], a[6:9, 3:6]) buf = sync(store.get("2.1", default_buffer_prototype())) assert buf is not None np.testing.assert_array_equal( a[6:9, 3:6], np.frombuffer(buf.to_bytes(), dtype="float64").reshape((3, 3), order=zarr_order), ) # After writing and reading from zarr array, order should be same as zarr order sub_arr = arr[6:9, 3:6] assert isinstance(sub_arr, np.ndarray) if zarr_order == "F": assert (sub_arr).flags.f_contiguous else: assert (sub_arr).flags.c_contiguous # Contiguous write store = MemoryStore() arr = zarr.create_array( store, shape=(10, 8), chunks=(3, 3), fill_value=np.nan, dtype="float64", zarr_format=2, compressors=None, filters=None, overwrite=True, order=zarr_order, ) a = np.arange(9).reshape((3, 3), order=numpy_order) arr[6:9, 3:6] = a np.testing.assert_array_equal(arr[6:9, 3:6], a) # After writing and reading from zarr array, order should be same as zarr order sub_arr = arr[6:9, 3:6] assert isinstance(sub_arr, np.ndarray) if zarr_order == "F": assert (sub_arr).flags.f_contiguous else: assert (sub_arr).flags.c_contiguous def test_default_compressor_deprecation_warning() -> None: with pytest.warns(ZarrDeprecationWarning, match="default_compressor is deprecated"): zarr.storage.default_compressor = "zarr.codecs.zstd.ZstdCodec()" # type: ignore[attr-defined] @pytest.mark.parametrize("fill_value", [None, (b"", 0, 0.0)], ids=["no_fill", "fill"]) def test_structured_dtype_roundtrip(fill_value: float | bytes, tmp_path: Path) -> None: a = np.array( [(b"aaa", 1, 4.2), (b"bbb", 2, 8.4), (b"ccc", 3, 12.6)], dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")], ) array_path = tmp_path / "data.zarr" za = zarr.create( shape=(3,), store=array_path, chunks=(2,), fill_value=fill_value, zarr_format=2, dtype=a.dtype, ) if fill_value is not None: assert (np.array([fill_value] * a.shape[0], dtype=a.dtype) == za[:]).all() za[...] = a za = zarr.open_array(store=array_path) assert (a == za[:]).all() @pytest.mark.parametrize( ( "fill_value", "dtype", "expected_result", ), [ ( ("Alice", 30), np.dtype([("name", "U10"), ("age", "i4")]), np.array([("Alice", 30)], dtype=[("name", "U10"), ("age", "i4")])[0], ), ( ["Bob", 25], np.dtype([("name", "U10"), ("age", "i4")]), np.array([("Bob", 25)], dtype=[("name", "U10"), ("age", "i4")])[0], ), ( b"\x01\x00\x00\x00\x02\x00\x00\x00", np.dtype([("x", " None: zdtype = Struct.from_native_dtype(dtype) result = zdtype.cast_scalar(fill_value) assert result.dtype == expected_result.dtype assert result == expected_result if isinstance(expected_result, np.void): for name in expected_result.dtype.names or []: assert result[name] == expected_result[name] @pytest.mark.parametrize("fill_value", [None, b"x"], ids=["no_fill", "fill"]) def test_other_dtype_roundtrip(fill_value: bytes | None, tmp_path: Path) -> None: a = np.array([b"a\0\0", b"bb", b"ccc"], dtype="V7") array_path = tmp_path / "data.zarr" za = zarr.create( shape=(3,), store=array_path, chunks=(2,), fill_value=fill_value, zarr_format=2, dtype=a.dtype, ) if fill_value is not None: assert (np.array([fill_value] * a.shape[0], dtype=a.dtype) == za[:]).all() za[...] = a za = zarr.open_array(store=array_path) assert (a == za[:]).all() def _pickle_compressed_array() -> zarr.Array[Any]: # Writing with the pickle codec is allowed without opting in, only decoding is gated. arr = zarr.create_array( store={}, shape=(3,), dtype=VariableLengthUTF8(), # type: ignore[arg-type] zarr_format=2, filters=[VLenUTF8()], compressors=Pickle(), fill_value="", ) arr[:] = np.array(["a", "bb", "ccc"]) return arr @pytest.fixture def allow_numcodecs_unpickle(monkeypatch: pytest.MonkeyPatch) -> None: """ Opt in to numcodecs' own gate on ``Pickle.decode``. numcodecs refuses to unpickle unless the process opts in (GHSA-jqc8-qjfv-5m6r). zarr's opt-in is separate, so tests that read pickled data after opting in to zarr also need this. """ monkeypatch.setattr(numcodecs.pickles, "ALLOW_UNSAFE_PICKLE_DECODE", True) def test_pickle_codec_read_disabled_by_default() -> None: """Reading data encoded with the pickle codec fails unless the user opts in.""" arr = _pickle_compressed_array() with pytest.raises( PickleCodecDisabledError, match=r"array\.allow_unsafe_pickle_decode" ) as excinfo: arr[:] # numcodecs has its own opt-in; the message must name it so users are not left guessing assert "ALLOW_UNSAFE_PICKLE_DECODE" in str(excinfo.value) def test_pickle_codec_read_does_not_unpickle() -> None: """A malicious chunk is not unpickled when the pickle codec is disabled.""" class Payload: def __reduce__(self) -> tuple[Any, ...]: return (exec, ("raise AssertionError('payload was unpickled')",)) meta = { "zarr_format": 2, "shape": [1], "chunks": [1], "dtype": "|O", "compressor": {"id": "pickle"}, "fill_value": None, "filters": [{"id": "vlen-utf8"}], "order": "C", } buffer_cls = default_buffer_prototype().buffer store = { ".zarray": buffer_cls.from_bytes(json.dumps(meta).encode()), "0": buffer_cls.from_bytes(pickle.dumps(Payload())), } arr = zarr.open_array(store=MemoryStore(store), mode="r", zarr_format=2) with pytest.raises(PickleCodecDisabledError): arr[:] def test_pickle_codec_zarr_opt_in_does_not_bypass_numcodecs( monkeypatch: pytest.MonkeyPatch, ) -> None: """Opting in to zarr does not disable numcodecs' own gate on ``Pickle.decode``.""" monkeypatch.setattr(numcodecs.pickles, "ALLOW_UNSAFE_PICKLE_DECODE", False) arr = _pickle_compressed_array() with ( config.set({"array.allow_unsafe_pickle_decode": True}), pytest.raises(numcodecs.errors.UnsafeDecodeError), ): arr[:] @pytest.mark.usefixtures("allow_numcodecs_unpickle") def test_pickle_codec_read_enabled_via_config() -> None: arr = _pickle_compressed_array() with config.set({"array.allow_unsafe_pickle_decode": True}): np.testing.assert_array_equal(arr[:], np.array(["a", "bb", "ccc"])) @pytest.mark.usefixtures("allow_numcodecs_unpickle") @pytest.mark.parametrize(("env_value", "allowed"), [("True", True), ("false", False)]) def test_pickle_codec_read_enabled_via_env_var( monkeypatch: pytest.MonkeyPatch, env_value: str, allowed: bool ) -> None: """ The opt-in can be set with an environment variable. Only a value that parses to ``True`` enables it: donfig keeps ``false`` as a (truthy) string, which must not enable it. """ arr = _pickle_compressed_array() monkeypatch.setenv("ZARR_ARRAY__ALLOW_UNSAFE_PICKLE_DECODE", env_value) config.refresh() if allowed: np.testing.assert_array_equal(arr[:], np.array(["a", "bb", "ccc"])) else: with pytest.raises(PickleCodecDisabledError): arr[:] def test_pickle_opt_in_does_not_affect_other_codecs() -> None: """Arrays that do not use the pickle codec are read without opting in.""" assert config.get("array.allow_unsafe_pickle_decode") is False arr = zarr.create_array( store={}, shape=(3,), dtype=" None: # Use packaging.Version so we transparently handle hatch-vcs dev suffixes # like "3.2.2.dev30+gdc5e1825" that appear on any source build past the # latest v* tag — Version.major returns 3 for that string. parsed = Version(zarr.__version__) assert parsed.major >= 3, ( f"zarr.__version__={zarr.__version__!r} is not on the v3 (or newer) " f"release line. If this fires on a from-source build, check that " f"[tool.hatch] version.raw-options.git_describe_command in " f"pyproject.toml still includes ``--match v*`` so the " f"``zarr_metadata-v*`` subpackage tags are excluded. See PR #3994." ) zarr-developers-zarr-python-a2f2415/tests/test_zarr.py000066400000000000000000000012621527014137600232170ustar00rootroot00000000000000import pytest import zarr def test_exports() -> None: """ Ensure that everything in __all__ can be imported. """ from zarr import __all__ for export in __all__: getattr(zarr, export) def test_print_debug_info(capsys: pytest.CaptureFixture[str]) -> None: """ Ensure that print_debug_info does not raise an error """ from importlib.metadata import version from zarr import __version__, print_debug_info print_debug_info() captured = capsys.readouterr() # test that at least some of what we expect is # printed out assert f"zarr: {__version__}" in captured.out assert f"numpy: {version('numpy')}" in captured.out zarr-developers-zarr-python-a2f2415/uv.lock000066400000000000000000025437601527014137600210120ustar00rootroot00000000000000version = 1 revision = 5 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.15'", "python_full_version < '3.15'", ] [[package]] name = "aiobotocore" version = "3.9.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, { name = "aioitertools" }, { name = "botocore" }, { name = "jmespath" }, { name = "multidict" }, { name = "python-dateutil" }, { name = "wrapt" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a5/95/4f7e0453d5e81bdb1b40ecf09f135bb88e12ce27fdb3efa125022dcca5d7/aiobotocore-3.9.2.tar.gz", hash = "sha256:8e32238c5bd77717ab1ac90077ff51ab141d025528115ccc7cc9b50b2d176e5f", size = 524343, upload-time = "2026-10-01T02:01:59.522Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/09/ab/c8f982c4dfaab49200e31820a4ffe6275356f7dcaaa1024735d8f8b4e048/aiobotocore-3.9.2-py3-none-any.whl", hash = "sha256:363b4892423b272eb84afbb58000da0be6ccd0cf8a68b98438c957d05b841efc", size = 102571, upload-time = "2026-10-01T02:01:57.883Z" }, ] [[package]] name = "aiohappyeyeballs" version = "2.6.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/26/30/f84a107a9c4331c14b2b586036f40965c128aa4fee4dda5d3d51cb14ad54/aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", size = 22760, upload-time = "2025-03-12T01:42:48.764Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8", size = 15265, upload-time = "2025-03-12T01:42:47.083Z" }, ] [[package]] name = "aiohttp" version = "3.14.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohappyeyeballs" }, { name = "aiosignal" }, { name = "attrs" }, { name = "frozenlist" }, { name = "multidict" }, { name = "propcache" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, { name = "yarl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/93/2f/6a91adaa2dc26877d6ed2f54c0370c8910f019db7d77c5c6a194611e93ea/aiohttp-3.14.4.tar.gz", hash = "sha256:831fc5bd39ec2517851e348f613ddb5447a47cf4b71cb09845af7ad7ed45d8f9", size = 8077839, upload-time = "2026-10-05T17:43:48.309Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/22/02/db5935d45347fa8ede56b777930740fb0e72bbb4473ddb6ff8541f6f2afa/aiohttp-3.14.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:55b0b335982b0db7117dff7f6a8e3aad43c4105f9e392ca04f9fc8d958a95cce", size = 817689, upload-time = "2026-10-05T17:38:50.69Z" }, { url = "https://files.pythonhosted.org/packages/e6/56/f00aa46127e6342f019d567853eeeb932e04fb9b9424b5fb981ad68f4178/aiohttp-3.14.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d0d8e435582b0d53009dac99e26a3d2be932ed7c9dbfcfc70f3fce1cf943bd5", size = 549599, upload-time = "2026-10-05T17:38:52.525Z" }, { url = "https://files.pythonhosted.org/packages/3f/bc/825c09b09962253a2103e4a88c04374ee939c0c8ea31c270daa0308efb07/aiohttp-3.14.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bf734016932d68e1324cfb638cac7a0866a83933a33badf15e836a7e24efb8c0", size = 548132, upload-time = "2026-10-05T17:38:54.629Z" }, { url = "https://files.pythonhosted.org/packages/40/80/8a1bc8036540fe7cb38219cc5c0aa67d32e9068165725e8199ea8200b08b/aiohttp-3.14.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:322e3d741b3133bcbb981a41d4f9b8ac3545c18de49c662c2b1f4e1c770b98ad", size = 1898485, upload-time = "2026-10-05T17:38:56.439Z" }, { url = "https://files.pythonhosted.org/packages/3c/fe/65b464d8520ace0d65b4ca69bc9abe175f842c8ce8d1522d363d9669903f/aiohttp-3.14.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:15a3bb5f90a4e515071f3617a45dfe26560beb834a6b6e108cdf4bc9e9719ec3", size = 1875334, upload-time = "2026-10-05T17:38:58.536Z" }, { url = "https://files.pythonhosted.org/packages/be/51/9c583be502720e927560a8414f4dfb7c7de41945e1a30ccf7003b93c6f85/aiohttp-3.14.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6c044e52df1a466af82cad3518c92ea80cc4ddcfea86966ff083bb34bab29bf4", size = 1937772, upload-time = "2026-10-05T17:39:00.376Z" }, { url = "https://files.pythonhosted.org/packages/a2/3d/b56512df7ea9cbb5f59099a6b375088dd578356a5a3fb27198ab82e19832/aiohttp-3.14.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d613d7d51bf06fe9a5e3aab86ce06d835bb0ceb1a31ea9e8766df4a403ae62a6", size = 2066220, upload-time = "2026-10-05T17:39:02.54Z" }, { url = "https://files.pythonhosted.org/packages/11/6c/a7a42701d9ea319140a3d44a24ec72a00b0b46d1b899b26d8a705e65bdd8/aiohttp-3.14.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5e93ab6e6330dce40f5b43996f1fc5f2543296d263a5e4b59ff09e73870b7d8", size = 1922895, upload-time = "2026-10-05T17:39:04.638Z" }, { url = "https://files.pythonhosted.org/packages/77/04/f8337bd824ca3fe3968d34dc5037ace1c4872551c4621f8266239311d301/aiohttp-3.14.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:20134dc8e68c67e7478124d516449428684c0545b2d3faa575691c8bcbd56678", size = 1712589, upload-time = "2026-10-05T17:39:06.671Z" }, { url = "https://files.pythonhosted.org/packages/fe/a6/d20e2c940b592d7c4a21690889dbff64fe0ca3facd9190696368694e1488/aiohttp-3.14.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f0daef5ef012369dddc391c61f604111424aa9a96313a69fb899018b09d1b01e", size = 1862251, upload-time = "2026-10-05T17:39:08.523Z" }, { url = "https://files.pythonhosted.org/packages/f5/e4/d50edd5ae5d024b380a8e892e333840bb22967a9eb9c8492108a403f678a/aiohttp-3.14.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:19290a108de0718b73bc69b86e788975c6a59855c671ebe2a98a07a6f968328f", size = 1895155, upload-time = "2026-10-05T17:39:10.305Z" }, { url = "https://files.pythonhosted.org/packages/c3/d3/f36f2931f807ccf8437ec15e8f8bfaa4748d4792639401cc767a866abaa0/aiohttp-3.14.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1494877625baeffab669f4a25a09ec4eea721b49e35363081a2a8e4231fe141f", size = 1906623, upload-time = "2026-10-05T17:39:12.411Z" }, { url = "https://files.pythonhosted.org/packages/5d/0a/94821b711493142bf5e11c27b0d7502776778f7792c77ecc6647357d20c1/aiohttp-3.14.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:a779ce4bf099ce4ad2cdfed62f1ca536d435f53237c54ff40172fddada24d1b5", size = 1698932, upload-time = "2026-10-05T17:39:14.446Z" }, { url = "https://files.pythonhosted.org/packages/d9/76/3d5a0d28362f0053c1c457024749911932481e8fd105fddf8305b1f93c81/aiohttp-3.14.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:0b4cec9fca876e2d4f4c6cb32b177390132d04618867689f43fefabf3f5c3980", size = 1934528, upload-time = "2026-10-05T17:39:16.354Z" }, { url = "https://files.pythonhosted.org/packages/a7/51/9784a21dc05d2e0f6acaf864f16db53dab0fbd7b0753cef168601eacb012/aiohttp-3.14.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:217707313e23aedde9bea67cf8365c7b138b549ae179121ef3f476cf67e3173a", size = 1901984, upload-time = "2026-10-05T17:39:18.299Z" }, { url = "https://files.pythonhosted.org/packages/44/4f/f98c88703e6d620243aa08473a946ca4e3756b6288690e38f0d9f9c1d007/aiohttp-3.14.4-cp312-cp312-win32.whl", hash = "sha256:18a9fb9a3f6e6e63f4d27f9adeef313a2d3c69a84d31170a844623824e722534", size = 480278, upload-time = "2026-10-05T17:39:20.206Z" }, { url = "https://files.pythonhosted.org/packages/b7/bc/8555013f84b53e4fdde423131b641af50e822c6d7d9118b24c52b31f6900/aiohttp-3.14.4-cp312-cp312-win_amd64.whl", hash = "sha256:69dcb02d33dfe415d5ee342cc63b7d320efcaf4b761701c7bd7bfb91c715481a", size = 506317, upload-time = "2026-10-05T17:39:22.706Z" }, { url = "https://files.pythonhosted.org/packages/21/3f/ca5bad7bfdaa7ddca053a044a1c7c0d4f3ad9d1f12f96dcab10bfb9c50e7/aiohttp-3.14.4-cp312-cp312-win_arm64.whl", hash = "sha256:d36b0263e7c2fbf1750b9f35d4e7e48b0442d8c45e4a89e3e74168821fb014bb", size = 487683, upload-time = "2026-10-05T17:39:24.482Z" }, { url = "https://files.pythonhosted.org/packages/23/ec/7fee140da700ec3f5aa5e0fd2319f67be5e86de197e2384d27e5e6209e0e/aiohttp-3.14.4-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:264b5a7568306590c44ae4a0237ad1715e35d447ea44ebf324e4c8fd9f78f67a", size = 539114, upload-time = "2026-10-05T17:39:26.322Z" }, { url = "https://files.pythonhosted.org/packages/4e/04/930c01ce9e186e787046433d7dc5b45b188885f8406c14b58435b4431974/aiohttp-3.14.4-cp313-cp313-android_24_x86_64.whl", hash = "sha256:17e5d7c7775d8dd8e894c7ef6ebfa26509de36868a9a6c7c4fcfaf6a1bf43d60", size = 553514, upload-time = "2026-10-05T17:39:28.104Z" }, { url = "https://files.pythonhosted.org/packages/2c/d7/983e3cea81c07e95f35d34da4d1e182b5c05489c49b7560b198a83f0da79/aiohttp-3.14.4-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:e91dc8fe9cbd052d16f7f1267e285cf8bbac4d2194bd5741f05ef201c77ed09e", size = 521381, upload-time = "2026-10-05T17:39:30.419Z" }, { url = "https://files.pythonhosted.org/packages/0e/2d/80baed11659093883f51245755105557d3253e45bd0b693d70df0059d68b/aiohttp-3.14.4-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8df6157d31703972aec9e3b93963f039ff06318301e30b007c3e941ff8a7ac42", size = 528347, upload-time = "2026-10-05T17:39:32.344Z" }, { url = "https://files.pythonhosted.org/packages/c8/18/bfda255c311096c08dab26dc65f3d219f090df37dc5f5f6345f8c3aefc1f/aiohttp-3.14.4-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:220912b351549dc736c59d104dc443fa615753102bb778a4e98f86e4d94bff6a", size = 539914, upload-time = "2026-10-05T17:39:34.265Z" }, { url = "https://files.pythonhosted.org/packages/51/fc/3b6ca6117c0c42481d719411fff7b5d3970ebd906543d3dc1ba40fe2462f/aiohttp-3.14.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2efa42b39bb3f524d5eca3760639b584afdd1143a6b047fda29910ffb75e4993", size = 813997, upload-time = "2026-10-05T17:39:36.061Z" }, { url = "https://files.pythonhosted.org/packages/b5/f6/86e3e02a9712aadc1ae0a1076bb2d1adf3d9addfe5542a28190c8f9b6d9b/aiohttp-3.14.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7e7356059c8262d2c1fd95242e2b7b70fbc6eee3135bf470049cbec3e968b5b0", size = 548058, upload-time = "2026-10-05T17:39:38.109Z" }, { url = "https://files.pythonhosted.org/packages/85/7e/6b9c2e271419d40ca76589613b3c7ab185fd3050a407bd2565b6aa46f695/aiohttp-3.14.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fac247d0cc956d732df1211ac2d8a9999ec8dc1a92a90401de9d52e4ead546f7", size = 545933, upload-time = "2026-10-05T17:39:40.193Z" }, { url = "https://files.pythonhosted.org/packages/14/fc/33d6e24a5f8ac32423230088c415e2a1c6efee23d902b62f20506631e40b/aiohttp-3.14.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e2eb0f8b03b4f2bd154ed8cf7c1a119a1ad04808c258bf58f90138c83b8c6d5", size = 1888265, upload-time = "2026-10-05T17:39:42.182Z" }, { url = "https://files.pythonhosted.org/packages/8d/e8/2eca1d468d83a514621d1280f9992d08649191db2028c9236b4edfa7c723/aiohttp-3.14.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7fbe827d27e0e369fd7bb88ceeaa5b46ea006e73e015b9dd77e3723a53162293", size = 1872300, upload-time = "2026-10-05T17:39:44.195Z" }, { url = "https://files.pythonhosted.org/packages/d6/f6/5872983de71b0214acdfb86822f5cca295b39be4832ec589fa224aee0f6c/aiohttp-3.14.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a70387146d03af9f047c629b71aa27d9594b3b31ec4c9c91e1a73d0d8bf38e0e", size = 1926213, upload-time = "2026-10-05T17:39:46.199Z" }, { url = "https://files.pythonhosted.org/packages/ee/65/ebfefa11a546c33c717af63c67868408ecf0b33b23a9d75669af95485407/aiohttp-3.14.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0d0934926fc65744e2fdd44ce68b2d79d5ee608a1e23f0596b35dd696519bb7c", size = 2056609, upload-time = "2026-10-05T17:39:48.221Z" }, { url = "https://files.pythonhosted.org/packages/69/c0/70667e82ec041f4122a17afce711bbcf94a71bec5888b69a49aed8817be2/aiohttp-3.14.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff4b366510c733974adb7a4102c094e4a6308e6f369315a4cf14d2d0a619cf82", size = 1914740, upload-time = "2026-10-05T17:39:50.304Z" }, { url = "https://files.pythonhosted.org/packages/bf/86/559e9dfea676206b20737d7ccb5b3716bff5e58d2a8d53682e5301724cd1/aiohttp-3.14.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:85fac7c99e0ac3dcbd6ae1c33774e0603bfc732ddb967a6fce4a731103c1888e", size = 1703390, upload-time = "2026-10-05T17:39:52.385Z" }, { url = "https://files.pythonhosted.org/packages/fc/91/c98897d4cf23e120b2f3980042835cb34c86bbe3c653f3eaacf4bf986349/aiohttp-3.14.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef1e9c3b4a2300023a5dc865889ce532cfaf6798ba4ef78fbbefe738a9d23efa", size = 1852642, upload-time = "2026-10-05T17:39:54.391Z" }, { url = "https://files.pythonhosted.org/packages/14/13/0258f352cb2c236f358a5b57d138e15fa2566e17accc0758d968dbf2ceb6/aiohttp-3.14.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:97e65916588be0f952b3307aaa8460c0771fa98dbea2dd27dabc9f5d84d53c3b", size = 1890616, upload-time = "2026-10-05T17:39:56.466Z" }, { url = "https://files.pythonhosted.org/packages/64/3c/f43e294ce1a06d1234a97bd862c1aa31acfa1a0eea3786c22fd0dc1ea731/aiohttp-3.14.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:840775cb39a9424f9edf142133a1192263ac5bd79117ad2a16602cef6a0c1e9d", size = 1896638, upload-time = "2026-10-05T17:39:58.585Z" }, { url = "https://files.pythonhosted.org/packages/9f/f7/ce1e3a85b73d05037d729d3d32da7d6959b9e433435e9abd46c2fb38410f/aiohttp-3.14.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:54f9256430ca040d58affbd551513de13a5cff7d065204fc52b9a1b0af09a049", size = 1693003, upload-time = "2026-10-05T17:40:00.698Z" }, { url = "https://files.pythonhosted.org/packages/1d/72/45eb900d6a9ee046728f5589f3d07f5d78f049b5fb84f48881fa4174d5df/aiohttp-3.14.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:6112d5931bf12c776188cab81b82bbb17d86e2e1381f2ade6edc43e6b240f0cb", size = 1927730, upload-time = "2026-10-05T17:40:02.943Z" }, { url = "https://files.pythonhosted.org/packages/b2/43/a83ff79dc8375953a1f00b808a21c82590f4e68304fecccdcc42a25fb7d8/aiohttp-3.14.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:facc6df39047c4732cbd885bb62e40df20885fcd709fc399e11f81b45ce0fff9", size = 1891177, upload-time = "2026-10-05T17:40:04.989Z" }, { url = "https://files.pythonhosted.org/packages/5c/ba/fbdc41608d4e2fb6084d960a41b392fa82beb5b126c2466de549b5c67fc8/aiohttp-3.14.4-cp313-cp313-win32.whl", hash = "sha256:7968634fa3a967a2bc0b9aeba1d5a1955501dcd7d0de44b96f7ebfb662797735", size = 479943, upload-time = "2026-10-05T17:40:07.402Z" }, { url = "https://files.pythonhosted.org/packages/35/51/d75629750e705832fd5cd52516a82943ed58b6e09c30e14830f5d87452fa/aiohttp-3.14.4-cp313-cp313-win_amd64.whl", hash = "sha256:c0a894b0265d139cd4c2c8bd4643822cc289a4a5331e21e810881bd000ffb2dc", size = 506209, upload-time = "2026-10-05T17:40:09.49Z" }, { url = "https://files.pythonhosted.org/packages/02/64/66452d2ffcc13e5e5ee4ba4a968656fa4a98c29ed5ff0205d4fde02b2943/aiohttp-3.14.4-cp313-cp313-win_arm64.whl", hash = "sha256:da16c3037178e72589d91de59536d271dffc92c2be2b47c38a45538cba54fd29", size = 487163, upload-time = "2026-10-05T17:40:11.526Z" }, { url = "https://files.pythonhosted.org/packages/d2/fd/b653b1f969e2e949fe4726af7a8043e8ef117f202bed7757ac8564e077a1/aiohttp-3.14.4-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:749bfc43bb1df71bbe6ed215044d5412d786698fc5771d2037436ca49622f3c9", size = 540738, upload-time = "2026-10-05T17:40:14.109Z" }, { url = "https://files.pythonhosted.org/packages/90/b5/eea49a0bb697deffac5165d707ec53a24748d6720b3af77a544107b49573/aiohttp-3.14.4-cp314-cp314-android_24_x86_64.whl", hash = "sha256:c113e4b489d711f606ca278cd8dc89179746244ea841aa0638b1b134600afad0", size = 554494, upload-time = "2026-10-05T17:40:16.231Z" }, { url = "https://files.pythonhosted.org/packages/11/ea/cf8f4efbbfa8572d9965cd43b3c6c84c581aa006772e5ce3ab4020027e9b/aiohttp-3.14.4-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:50969522bc7022026a2c7968e67da5c789f6b716b988edac55dc630390efd5df", size = 522653, upload-time = "2026-10-05T17:40:18.879Z" }, { url = "https://files.pythonhosted.org/packages/a3/24/9f6721f0b52323789cb1041ffc3afa729462d9925607e733a065b6fc5e17/aiohttp-3.14.4-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:cbf92cd7a1c800cee244558dd5630abb45c1c1093ef7b0d546a99f526ff7a9eb", size = 529697, upload-time = "2026-10-05T17:40:21.296Z" }, { url = "https://files.pythonhosted.org/packages/9c/a4/399debc3c2ca0ab88a08e3082b4c4f62d0f66978af60243c63f2345ef337/aiohttp-3.14.4-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:fc76be08ff407fba717a769aff04ca1b4bf9ecfdcb5caadef84a563fdbbd146d", size = 540526, upload-time = "2026-10-05T17:40:23.548Z" }, { url = "https://files.pythonhosted.org/packages/a2/97/90379813b18b795bccdeb3a5a85e9b93ac2c69f9c2a5d0faba59da3ad6d1/aiohttp-3.14.4-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:9b416f995b68a921069e6ec61fa07f2ac3ff033d92e97a69f673af89f5bb954d", size = 818866, upload-time = "2026-10-05T17:40:25.842Z" }, { url = "https://files.pythonhosted.org/packages/78/70/824fd0dac819652f1df895b3f621a75bf0a3050c30e38d8d2316e34ab821/aiohttp-3.14.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2c9585101cfca10e74a07cc0ef2bdc6d90616e6750926b20456f429d0608f740", size = 549666, upload-time = "2026-10-05T17:40:28.523Z" }, { url = "https://files.pythonhosted.org/packages/8c/45/eafb2dd200dd2b770ed25a2dc5731c4ba5d58ab3fa8e2e82494bb946808e/aiohttp-3.14.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3c4dcf611b095e7983421f49d01ffbed7ed574891533d606ea115fc23e7c4e90", size = 548961, upload-time = "2026-10-05T17:40:30.613Z" }, { url = "https://files.pythonhosted.org/packages/a7/bc/6c733a1c71300293471ac7ef1dffd12b5b4dc9a43c53461224b2b8a7f667/aiohttp-3.14.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f8f61a9b5fa56c58e3441f9dc64137f786232d40fb89127190d0e0f84e1e5865", size = 1883165, upload-time = "2026-10-05T17:40:33.062Z" }, { url = "https://files.pythonhosted.org/packages/37/34/885ddbd351679a31141b25e0d1a5e4f67738806e564c858b399709b61239/aiohttp-3.14.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d43e538dfc85b705fd832620d050970ef5613b3869a7169b44a554b9dab9eee9", size = 1844119, upload-time = "2026-10-05T17:40:35.266Z" }, { url = "https://files.pythonhosted.org/packages/cc/ef/0a64dac53c11d569851efb3e7eddb43551df72b2c172ebb34cccc93bcd3f/aiohttp-3.14.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4135c4bb4b7682a7633fadc7170ea969d843f83aa7224e70111a20675105c985", size = 1930222, upload-time = "2026-10-05T17:40:37.527Z" }, { url = "https://files.pythonhosted.org/packages/69/5f/774792ed8f973e0d234fcb06b0f0bc9f29e9a0a0dd847f6418c2904be7d9/aiohttp-3.14.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9055324f156a94fdf6dc9a825ae60f9a2947b88f591ea81ee80826633de62ae7", size = 2038842, upload-time = "2026-10-05T17:40:39.815Z" }, { url = "https://files.pythonhosted.org/packages/cb/05/e00f35581c97656d9e864edaa03292382c620dd1d9c836a4f73a2279acc4/aiohttp-3.14.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fb22c8e4b60385a9b414c70c6e8b2d0f4b23ff7fc4bb572ceb58297b228169f1", size = 1887845, upload-time = "2026-10-05T17:40:42.359Z" }, { url = "https://files.pythonhosted.org/packages/ca/2e/07e3eff5b4c1b082a2eaf0ffb508b409b21ea1ddc9fdc57f3eabb51469a1/aiohttp-3.14.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3b6945559a260112742ed58cde2165488d2fc51aeea34afbaa99d9371b43e1b5", size = 1702533, upload-time = "2026-10-05T17:40:44.569Z" }, { url = "https://files.pythonhosted.org/packages/75/44/816844b7f876e6f8fee4ecb088269fcd92241da712e4814a4d624eda13a9/aiohttp-3.14.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8f4078a90b174f15efb2c8725e8720c40dbd3b2cf6350dbc085c964a938d9de7", size = 1849058, upload-time = "2026-10-05T17:40:47.818Z" }, { url = "https://files.pythonhosted.org/packages/ae/bd/163242e12464a1f3e58a77e28beca949652ddce84abdd5500a91c70a1e0d/aiohttp-3.14.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:f433c5654e32f72de3ef074d715eb5c7604eb6d28a5c2f3c9e5056f20fe6b306", size = 1854600, upload-time = "2026-10-05T17:40:50.401Z" }, { url = "https://files.pythonhosted.org/packages/f2/59/5c1ae708a89524eddfd9c727b87cae26b5c50c3c76683b910ac06151ead0/aiohttp-3.14.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:871035fe79c636b6bc126ed78df37d72b830dfc751612cb4b1040360edf2719f", size = 1896565, upload-time = "2026-10-05T17:40:52.625Z" }, { url = "https://files.pythonhosted.org/packages/37/45/95300a7de69241fe47d3f838f3fb582c712782e172d447ff650a013ea673/aiohttp-3.14.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2617132524f66338f6c1d218cd16081576aefc40df4f2bcba338a99fbf67fec1", size = 1693092, upload-time = "2026-10-05T17:40:54.862Z" }, { url = "https://files.pythonhosted.org/packages/ff/33/01a8e61d48d0094121092c74125df14c0f8f10ee3149bfe68b6f869409a9/aiohttp-3.14.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d8bc326133272deee8ef337581550450e4bb735c0520be893b950a1f5baaf41a", size = 1917073, upload-time = "2026-10-05T17:40:57.656Z" }, { url = "https://files.pythonhosted.org/packages/9e/d6/1a4f24624cda6686d5b927e26d5d1ae3ebb692faf88e93017fab4b391fd0/aiohttp-3.14.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3ec57223e261b7fa9979c12dbce009ca53439be76cac3af27f860d3e4b1d6f7a", size = 1872656, upload-time = "2026-10-05T17:41:00.123Z" }, { url = "https://files.pythonhosted.org/packages/be/be/ed286fcfa2d22061ae021b568c790123505cd482df35e23023339ecaed96/aiohttp-3.14.4-cp314-cp314-win32.whl", hash = "sha256:7242043e71fc449a19a47d92f4e9fd1e0ae8d3f88488982381f422bf6d651218", size = 485461, upload-time = "2026-10-05T17:41:02.686Z" }, { url = "https://files.pythonhosted.org/packages/c0/74/e040e6f65f4da305777f1adb3023d47401693489e4a337c983b52d2a0a6b/aiohttp-3.14.4-cp314-cp314-win_amd64.whl", hash = "sha256:019587c7b3a44917dd61bcb312fd5c034f341dc9851b83e7b41b65e08b68ca0f", size = 512147, upload-time = "2026-10-05T17:41:04.98Z" }, { url = "https://files.pythonhosted.org/packages/2e/5e/bddb33ae15fdd5a68a10a986587aa9916c418a712fdd110d3b3ae9b51407/aiohttp-3.14.4-cp314-cp314-win_arm64.whl", hash = "sha256:60dc71d38db0988f37e78f6ad4a010a2f6355cfac3fff5f130cf5ef2231f3f0d", size = 494685, upload-time = "2026-10-05T17:41:07.366Z" }, { url = "https://files.pythonhosted.org/packages/22/2e/7b884014b40573f919fc3a528e5028bc37b249e31a93928ba61bdd25c960/aiohttp-3.14.4-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:674f53ab9d2794e2dd767619a3384bf1006f8d075e43c19d449a645f8ff641c2", size = 855696, upload-time = "2026-10-05T17:41:09.514Z" }, { url = "https://files.pythonhosted.org/packages/5f/e6/b6e2b4f580ab6ae39241bb9182aca60e6e1915cdad0148faaa04db3f366b/aiohttp-3.14.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:6d243221f73e5ac546c2ed9255100423e4bc092b08b944f23498e8cf3096c26c", size = 567931, upload-time = "2026-10-05T17:41:11.685Z" }, { url = "https://files.pythonhosted.org/packages/63/99/ae8e3a1a18adcbf636839d18b3cccc51007836ccdae5209381c5cf82d1ae/aiohttp-3.14.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0daa016eb74e7888ed5f9ceec6eea3158999c1c4a115ef5cbe88baa8d924f94c", size = 567421, upload-time = "2026-10-05T17:41:13.836Z" }, { url = "https://files.pythonhosted.org/packages/63/38/c5d4bb58be72b967a4090d2deeb38571f8c992852936c51dbbbeae49db1c/aiohttp-3.14.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70c031a622aa20add05ebd234b90adaf6bf51c1c88cc305ec760721af83308c3", size = 2054173, upload-time = "2026-10-05T17:41:16.098Z" }, { url = "https://files.pythonhosted.org/packages/eb/87/3c8a9c7670f56d478a75c5bb84b336fb61a67b908e36c30a4a96ed8c8e86/aiohttp-3.14.4-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7df363fa7c38945a896e12c69af4a1a56cbd4152791d9e41fd0bac396b72de38", size = 1918296, upload-time = "2026-10-05T17:41:18.478Z" }, { url = "https://files.pythonhosted.org/packages/49/97/bd00e499c0a1505b44f5e23b9739034ea9be792ad42537f2894211ba1c07/aiohttp-3.14.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:02a508ba5ad91c4d730523584c0adc65882274bba5e37c33bc2f475e37577256", size = 2027513, upload-time = "2026-10-05T17:41:20.874Z" }, { url = "https://files.pythonhosted.org/packages/23/1d/a3666a7c224f86a881f18d673a8ffdc92fa4c35274fb4361822f8f0dc566/aiohttp-3.14.4-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0cb3edece6fe6a944eaf97d803b88f606da6bc1600b1894c2969e78ffdc1fc29", size = 2152686, upload-time = "2026-10-05T17:41:23.181Z" }, { url = "https://files.pythonhosted.org/packages/46/7e/f4204b8c4959c46f6367c182afbbacef08a1b53874281912c52fbd80151c/aiohttp-3.14.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21ce1223b461bcc5fb389a7204001578f1c0b85c2da60a84d915ebb021294245", size = 2006431, upload-time = "2026-10-05T17:41:25.776Z" }, { url = "https://files.pythonhosted.org/packages/33/43/091e69d42967cbea833ceb355c5457f8f70748fe08b88c9d91491b3fafb8/aiohttp-3.14.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:35d5432f0dc1c8b71307e9112461b52f336da225e344283eec4779222c0a1dff", size = 1789101, upload-time = "2026-10-05T17:41:28.095Z" }, { url = "https://files.pythonhosted.org/packages/47/7b/b57cbe533c01098c5d30471f057e7c3c19f281122f756acc9c716b636e2c/aiohttp-3.14.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d6ec364493057c118d08292f7cd60865965c209167d8657b45643a26aef77495", size = 1974472, upload-time = "2026-10-05T17:41:30.499Z" }, { url = "https://files.pythonhosted.org/packages/9f/44/79acce03611450e4ed8594f96f1cea40d70bc84fb4e918f94d7d9454a4af/aiohttp-3.14.4-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:70492983b7aed9e61bda82fe1ce80bcba9199610886443133783a288638923c0", size = 1956340, upload-time = "2026-10-05T17:41:33.009Z" }, { url = "https://files.pythonhosted.org/packages/da/bb/36e6cbd6720e3377a02f8709aec6fe80de729ef41e78ba01f13d50363c73/aiohttp-3.14.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:a77776673331577147e9d25cd7833a72e8891b47556336cff06ba1569e488b99", size = 1965771, upload-time = "2026-10-05T17:41:35.506Z" }, { url = "https://files.pythonhosted.org/packages/69/0f/879735ac70ce5e6852916eed51c706a517b205150a8d384f9fa60933a3fc/aiohttp-3.14.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:73799000dd6fd5247094fd8e6d3e1eb871e41aab5cc35dff9d9f654cd230bc24", size = 1780297, upload-time = "2026-10-05T17:41:38.284Z" }, { url = "https://files.pythonhosted.org/packages/01/5d/7c030ca5b6baf3b02eccd11afc3d58f0b35e407031d5cc78cae9c24921c0/aiohttp-3.14.4-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:27b370fb532e707e8c96d04fa0c96301fb7dba2d50aaca57f6c456cdfffec5bd", size = 2001374, upload-time = "2026-10-05T17:41:40.888Z" }, { url = "https://files.pythonhosted.org/packages/80/1e/ddacbb8141d727034c51dd192d17749521d5b30b03f84aa034325396089e/aiohttp-3.14.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:884e0d1c707b6217fee3080b9b4029eee7551af788ee761010df19be158414b7", size = 1968531, upload-time = "2026-10-05T17:41:43.62Z" }, { url = "https://files.pythonhosted.org/packages/3a/08/2214f823701eebc3e57db141faeae76df7bfd415f254cc7c1692258f228d/aiohttp-3.14.4-cp314-cp314t-win32.whl", hash = "sha256:f2e3d34edb151d3e27f93a8338e2da33e2c7261cd4c879ce5f5fe280cc190ea5", size = 504126, upload-time = "2026-10-05T17:41:46.321Z" }, { url = "https://files.pythonhosted.org/packages/91/a1/90e9b6fd23310b4e82d0c0a37b5ecc2944053350473c57533c1da9647235/aiohttp-3.14.4-cp314-cp314t-win_amd64.whl", hash = "sha256:88cf889e51092537fe44adfbff96c2a22d6479d6f92b97b3d413fbfd0a4decf4", size = 531494, upload-time = "2026-10-05T17:41:48.849Z" }, { url = "https://files.pythonhosted.org/packages/97/ab/5b501472c92b9b1dc53742591d6210dccb262962f4721299300d3a9f664b/aiohttp-3.14.4-cp314-cp314t-win_arm64.whl", hash = "sha256:ca450c6d42fda7c0bfaf0e200f32f77c5f2c960a33be1e14d197c84588b7f5f6", size = 510544, upload-time = "2026-10-05T17:41:51.358Z" }, { url = "https://files.pythonhosted.org/packages/0f/77/ef165a197850573ea07c8d77bf5f4854a1db12fdb20e1cf0b218abf025fc/aiohttp-3.14.4-cp315-cp315-android_24_arm64_v8a.whl", hash = "sha256:3b10799df72f66c30935bc661cb89e5ac2918debfc575e350e226fbaa846cbb7", size = 540737, upload-time = "2026-10-05T17:41:53.761Z" }, { url = "https://files.pythonhosted.org/packages/e6/aa/1663f64d1316b3be9d3f6fafa02b8d888070171a6544cba515e82f3af595/aiohttp-3.14.4-cp315-cp315-android_24_x86_64.whl", hash = "sha256:7ffd7f9dd39caccc6b2f68ec0339435ecad2b20f6ea2e2b3de91bbe464ac7137", size = 554600, upload-time = "2026-10-05T17:41:56.183Z" }, { url = "https://files.pythonhosted.org/packages/98/f7/d4d535855123c1a774a798643e75fdfbac62c047b0c0729d6505d09df444/aiohttp-3.14.4-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:a0a7a227353f7693f73878ad14cd2623cafadd656ababcd20e2adfb7225172ea", size = 523184, upload-time = "2026-10-05T17:41:58.624Z" }, { url = "https://files.pythonhosted.org/packages/0a/5f/18a8d1cd2958a0b6164b047c18ccb192f2123bb70f1c0a145f6f01f858ad/aiohttp-3.14.4-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1c43b6af8e4708e8b3812a9ee4f79895207e12d555b7f6d70deb838e0011e12d", size = 530342, upload-time = "2026-10-05T17:42:00.96Z" }, { url = "https://files.pythonhosted.org/packages/7f/c4/d1dd2b1fcf8a3192a431375b287a8762ebfd22195ab5b201d3fac30e3a7a/aiohttp-3.14.4-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:97416b2b997e5e9b817e322b2f0a89b76c54cde22471f51ac17ac15e10901ba0", size = 540629, upload-time = "2026-10-05T17:42:03.324Z" }, { url = "https://files.pythonhosted.org/packages/37/51/d1868f3459f618f6b68a6ef549d8b3f76aa616fa083298911fd3ae65d6b9/aiohttp-3.14.4-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:ee2a5aa31e507c17841d592364f2148393b17f83bc2766e9211e1eb31ea72a70", size = 819347, upload-time = "2026-10-05T17:42:05.713Z" }, { url = "https://files.pythonhosted.org/packages/84/e6/53179b721b67e0bd20974d02eca97b5691559e8ce666ba23a65aecda6b7f/aiohttp-3.14.4-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:5d5d9843c34866e986dd24eef713452609e304550c471320ce7b54b3cfb1391e", size = 549825, upload-time = "2026-10-05T17:42:08.093Z" }, { url = "https://files.pythonhosted.org/packages/00/0f/d67ddf7a43d4899176954511b6c048328abec1a5df4901f811a3b408a4fc/aiohttp-3.14.4-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:efcbf93e31b52c21665b6ebc489eb6fdb0dfd86b592edc677b5cffdfa7b11677", size = 549315, upload-time = "2026-10-05T17:42:10.766Z" }, { url = "https://files.pythonhosted.org/packages/9d/db/42f372c7f8e259505f239cb49db75f23548deaec56d22c68b4866f1c3fe2/aiohttp-3.14.4-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f1a79977794592dcf6485ed964ec2448db527a1bc0c72ec89663658dad3a739", size = 1892181, upload-time = "2026-10-05T17:42:13.194Z" }, { url = "https://files.pythonhosted.org/packages/23/0d/8b1900216302a94f69b41bf0786bb992a6e67d703f4698e335c2442db615/aiohttp-3.14.4-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:515e7ac6d27509dc3ba2c35fb88970b00b9e7b07ee7cc947c8733dcd54ef5def", size = 1883513, upload-time = "2026-10-05T17:42:15.704Z" }, { url = "https://files.pythonhosted.org/packages/7b/5e/15c014313e567fff931ec4a5f81683cc40251257791fbd607ec37c0dad80/aiohttp-3.14.4-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:185154ffa3b54765b5de96f464f41cfa476f8815a03854ae5e0fe4d48fb06e9c", size = 1933283, upload-time = "2026-10-05T17:42:18.241Z" }, { url = "https://files.pythonhosted.org/packages/f5/b3/bf7dc39a450218efb1d5c692bac7dc85ef56da282e889bee594f3267211e/aiohttp-3.14.4-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6aece59f77959628a22cd2ffe48b102cda9611d680e090f204385b3998a1190f", size = 2041054, upload-time = "2026-10-05T17:42:20.728Z" }, { url = "https://files.pythonhosted.org/packages/b6/03/ffba18d7aaedb5eafac3f41f0c26621ebb0a740bb0115ea2fae1911d902b/aiohttp-3.14.4-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7205aa6589f1ad1306c6994ff2e1ab5efabacf938f9ffd7682fb225c197b06a", size = 1895494, upload-time = "2026-10-05T17:42:23.518Z" }, { url = "https://files.pythonhosted.org/packages/45/f4/68123365b86826eee82bdceb7fb4c429204cb3a104d6b7ae44bc6a543fcb/aiohttp-3.14.4-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f38c7b3ce60d55ac0f2a9fd2edc194789d043fefea15cfb7dcb40b7d5f561146", size = 1714098, upload-time = "2026-10-05T17:42:26.471Z" }, { url = "https://files.pythonhosted.org/packages/b1/db/b13c0b214afc571782e8667f3b9e0bbed6843397be90fa35acf2f3ecfbb0/aiohttp-3.14.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:b0a1274ee4ee6203c15a341cce87f9f3df8151d5496047b3608bb62bd969cffc", size = 1857111, upload-time = "2026-10-05T17:42:29.169Z" }, { url = "https://files.pythonhosted.org/packages/6b/7f/4f5d3cee3c23b57119aebba5174962fed0bc493f37f38edab2f5730411b6/aiohttp-3.14.4-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:69ce2407951516f413adf3132b88451eababd710ce631f83c81a9603a3640253", size = 1892123, upload-time = "2026-10-05T17:42:31.786Z" }, { url = "https://files.pythonhosted.org/packages/5e/2e/5a6a1f014c4e17d58d4e935d0dd64e612f4eaa2f33cca2475b48ca5e3fb5/aiohttp-3.14.4-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:b84d3d057c60eb8e0c2b457a1203e868d82709d3f0c272a4975dfd88c2414939", size = 1898809, upload-time = "2026-10-05T17:42:34.34Z" }, { url = "https://files.pythonhosted.org/packages/de/f3/d1cca5311a70fcc1c7c2082e02044578a69bcb10686db33a322df9fb849e/aiohttp-3.14.4-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:4ecc84f69c9e217984f6877e832e4786a74567432c0bf2b7512919fbdfe18ade", size = 1704814, upload-time = "2026-10-05T17:42:37.047Z" }, { url = "https://files.pythonhosted.org/packages/0a/cb/cd58eae366148a5b9b1baa233542151154f2b72cd32daa39fd9db97d0c0d/aiohttp-3.14.4-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:878f2d7950238b2b0a8cbb88054444a125f0cfbc77f54c811aca0483132a9be3", size = 1918390, upload-time = "2026-10-05T17:42:39.674Z" }, { url = "https://files.pythonhosted.org/packages/e3/d8/f5e6c4939432734033048ba631503edf4a1e693ee5e261773e169803bbe4/aiohttp-3.14.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:6ff9d3a0e6935f1cb0a6685f9f4b3f0d3e406456d7a434a03cf309b8cd5d7a2e", size = 1878321, upload-time = "2026-10-05T17:42:42.528Z" }, { url = "https://files.pythonhosted.org/packages/2b/71/462957d6b5d4eb208f78e67745c41480bad2e738aa7caf1d9b07536920c1/aiohttp-3.14.4-cp315-cp315-win32.whl", hash = "sha256:e07dfd7cd360f20be26dc2487ad4c7f21b0b39fdd593e9ebf37d2424ad52defb", size = 485536, upload-time = "2026-10-05T17:42:45.223Z" }, { url = "https://files.pythonhosted.org/packages/ba/6e/48f59266a8b8c6420894ff59b8e74175252da67df85967b96997e96b7092/aiohttp-3.14.4-cp315-cp315-win_amd64.whl", hash = "sha256:921fc4f1ad549091bfc39cb5e93925b04f307f83517756a3111843851130efce", size = 512087, upload-time = "2026-10-05T17:42:47.717Z" }, { url = "https://files.pythonhosted.org/packages/25/f5/46263b23cee4f312e28aaabbdec2e074c807700a12c9dfa18beff461f035/aiohttp-3.14.4-cp315-cp315-win_arm64.whl", hash = "sha256:5be8af106f96fd625f6c264fa9faaaa9e0104089684d9015d2fa5b6eba6407c7", size = 494709, upload-time = "2026-10-05T17:42:50.615Z" }, { url = "https://files.pythonhosted.org/packages/77/8d/e3b1d406c95a8fe436ae2e5f01ee9591196c6c22c44c645b24f20d73eaea/aiohttp-3.14.4-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:e69727ec04a4608200ff32ec29478299df7657e2eb78c416ae338d0ca6662596", size = 856014, upload-time = "2026-10-05T17:42:53.425Z" }, { url = "https://files.pythonhosted.org/packages/5d/fc/e9e78d1097830040bc0a4b8af073379f964064bfa7f0a852d774f96f3168/aiohttp-3.14.4-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:f8b658dd3ef2ebd7708b311ed96ff7cb5edaca001a30018a091504ef492bcb1c", size = 567724, upload-time = "2026-10-05T17:42:56.416Z" }, { url = "https://files.pythonhosted.org/packages/e6/ad/c813a705a1da1c111c30723b52e1efb67763e9f0b3b54a418cb419616e7a/aiohttp-3.14.4-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:3749e018123df1205161a73ba96d336e17e9cf6b947295e53a43571de420e7b1", size = 567888, upload-time = "2026-10-05T17:42:59.048Z" }, { url = "https://files.pythonhosted.org/packages/c2/1f/7676f46aa24554a45686853c61552d02d67ade2b4c51adfe8b2ec4fdecd2/aiohttp-3.14.4-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f28a3c4fca436b2e594090e7cd22558f5e93a30ca18516c436de982da6fc25a", size = 2054356, upload-time = "2026-10-05T17:43:01.781Z" }, { url = "https://files.pythonhosted.org/packages/44/79/b9ef7d7442061e0a3125990d6021230b30361a972f7b8ebfb011042877ce/aiohttp-3.14.4-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e099aae21246992a1d84303ff262262d054c474e46cace33b0d5a69810c71877", size = 1937678, upload-time = "2026-10-05T17:43:04.492Z" }, { url = "https://files.pythonhosted.org/packages/5c/98/5927a31540c8e6d0542effead5bd5c56e749a9d99d84e902fb80ae906ef8/aiohttp-3.14.4-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fafb521891e646d9cf5da89f01c75e7159d08be7254a024149630a698be28251", size = 2026705, upload-time = "2026-10-05T17:43:07.353Z" }, { url = "https://files.pythonhosted.org/packages/62/bd/1be001ed97dcfa57a292188c38979fb78be795b960b50f02660587bffa06/aiohttp-3.14.4-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1d385db2cd154bce0b3dd451cf2a52bec91ebb7be2c3a5d06b14b4391631d807", size = 2147075, upload-time = "2026-10-05T17:43:10.141Z" }, { url = "https://files.pythonhosted.org/packages/0b/f7/d59628ccedd7c86db4c1f904467d1c80fed6cc10a1831ed5d75ff12e5a71/aiohttp-3.14.4-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ae8a244889d8a53549851bf595b9f382f64acca57e8762da1ecb26ba786828f7", size = 2005016, upload-time = "2026-10-05T17:43:12.848Z" }, { url = "https://files.pythonhosted.org/packages/3c/dc/472db3ab1daafa12a61d7f83314f85f502b99c143c78d66554a5e24361fd/aiohttp-3.14.4-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:681dde68ff8d8d5e7d5abcad4feb45bfb457a218310c919825e21fc36cac3d0b", size = 1796363, upload-time = "2026-10-05T17:43:15.562Z" }, { url = "https://files.pythonhosted.org/packages/08/61/6f9834b131b9a42e30d68c469c196033d3cae2d8fe340d4668f362c4fd1f/aiohttp-3.14.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:b91a6441fcaed88ca7bca67726b189332df3ceb6ac0dfc98dcb9654bb723288c", size = 1974240, upload-time = "2026-10-05T17:43:18.884Z" }, { url = "https://files.pythonhosted.org/packages/b5/2f/b8eb2a276685ef1da7f7bc15348404b8b9ee12b8f32f2cd8895385cd1589/aiohttp-3.14.4-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:f15dbbac49bc15eaff95c0b8166525c4050996af578c7be141e145f40ee2cf24", size = 1973387, upload-time = "2026-10-05T17:43:21.661Z" }, { url = "https://files.pythonhosted.org/packages/5b/32/ad559b2d6817e85f7cb294d8641e975e64b526f4da98ffe941c9af67a047/aiohttp-3.14.4-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:7abae043e87692d1c2adda1fab501bb0612d28c4a23eb082005c1e684f792dac", size = 1968766, upload-time = "2026-10-05T17:43:24.544Z" }, { url = "https://files.pythonhosted.org/packages/60/67/4a16e398140bdabb363428b9f439af3f77706a6df8bea6e626bbde53c2ab/aiohttp-3.14.4-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:21f4624fb1051fc1aa10d57a636a80e829f7d0417462d6f1ce380b08fe1a904c", size = 1788193, upload-time = "2026-10-05T17:43:27.467Z" }, { url = "https://files.pythonhosted.org/packages/a9/38/17b326da7f52e530a675d789ade3e727a1b3db9959722f6049b9c392321a/aiohttp-3.14.4-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:2e8dc3a0fdaf9d12b954d3d06cc4c355d91b17fb6c671715b593624ca4c379f8", size = 1999645, upload-time = "2026-10-05T17:43:30.32Z" }, { url = "https://files.pythonhosted.org/packages/53/dc/91b87961250010e05eb3adabaf0d98e4229ee7543147c265e834922e1b0b/aiohttp-3.14.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:1410b6f6e3d52ea7c02c950496522ce6e85d5543bf154bc80a6141e505251cfa", size = 1970998, upload-time = "2026-10-05T17:43:33.364Z" }, { url = "https://files.pythonhosted.org/packages/72/66/9bcf2c2db7b75a3b51c11470ceb67941d028727d674276ff5d5120adc87e/aiohttp-3.14.4-cp315-cp315t-win32.whl", hash = "sha256:a65189a89f3e621f7c27a968b5131bbca29126e2b8517dc72b205747a39d4c96", size = 504036, upload-time = "2026-10-05T17:43:36.212Z" }, { url = "https://files.pythonhosted.org/packages/e2/20/349db4af12dbc16070d32028d30c2067123b4520f3dcb2d9013524efeb4f/aiohttp-3.14.4-cp315-cp315t-win_amd64.whl", hash = "sha256:2b2c95f5f769eec92b552969db1f79ea0282058ce5131ed0f2fa540b4af3f14e", size = 531284, upload-time = "2026-10-05T17:43:39.362Z" }, { url = "https://files.pythonhosted.org/packages/26/21/544c711bc5fad611278ebd153bdcab1bffce8b8581f8ec9096b275948a3c/aiohttp-3.14.4-cp315-cp315t-win_arm64.whl", hash = "sha256:54209fff79346ee1ef0d5cbf92fa80a5bb37396406bb4d932e5107f9c29b2d3f", size = 510623, upload-time = "2026-10-05T17:43:42.15Z" }, { url = "https://files.pythonhosted.org/packages/2b/27/5e1f8446545be98e9416af59c396b1ed1f9f3294afb0a40ee3a709889273/aiohttp-3.14.4-py3-none-any.whl", hash = "sha256:5c6758ba62aea282c537179cfc8474a90f2b5f7b089cc5ff66d8920d86a9bfdd", size = 279487, upload-time = "2026-10-05T17:43:44.905Z" }, ] [[package]] name = "aioitertools" version = "0.13.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/fd/3c/53c4a17a05fb9ea2313ee1777ff53f5e001aefd5cc85aa2f4c2d982e1e38/aioitertools-0.13.0.tar.gz", hash = "sha256:620bd241acc0bbb9ec819f1ab215866871b4bbd1f73836a55f799200ee86950c", size = 19322, upload-time = "2025-11-06T22:17:07.609Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/10/a1/510b0a7fadc6f43a6ce50152e69dbd86415240835868bb0bd9b5b88b1e06/aioitertools-0.13.0-py3-none-any.whl", hash = "sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be", size = 24182, upload-time = "2025-11-06T22:17:06.502Z" }, ] [[package]] name = "aiosignal" version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "frozenlist" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, ] [[package]] name = "alabaster" version = "1.0.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/a6/f8/d9c74d0daf3f742840fd818d69cfae176fa332022fd44e3469487d5a9420/alabaster-1.0.0.tar.gz", hash = "sha256:c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e", size = 24210, upload-time = "2024-07-26T18:15:03.762Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b", size = 13929, upload-time = "2024-07-26T18:15:02.05Z" }, ] [[package]] name = "annotated-doc" version = "0.0.4" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, ] [[package]] name = "annotated-types" version = "0.7.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] [[package]] name = "antlr4-python3-runtime" version = "4.13.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/33/5f/2cdf6f7aca3b20d3f316e9f505292e1f256a32089bd702034c29ebde6242/antlr4_python3_runtime-4.13.2.tar.gz", hash = "sha256:909b647e1d2fc2b70180ac586df3933e38919c85f98ccc656a96cd3f25ef3916", size = 117467, upload-time = "2024-08-03T19:00:12.757Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/89/03/a851e84fcbb85214dc637b6378121ef9a0dd61b4c65264675d8a5c9b1ae7/antlr4_python3_runtime-4.13.2-py3-none-any.whl", hash = "sha256:fe3835eb8d33daece0e799090eda89719dbccee7aa39ef94eed3818cafa5a7e8", size = 144462, upload-time = "2024-08-03T19:00:11.134Z" }, ] [[package]] name = "ast-serialize" version = "0.6.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/58/ad/0d70a3a2d6e01968d985415259e8ec7ad3f777903f9b1c1f3c8c44642c60/ast_serialize-0.6.0.tar.gz", hash = "sha256:aadd3ffcf4858c9726bf3515f7b199c7eadbe504f96028e4a87172c0da65a8fe", size = 61489, upload-time = "2026-06-30T20:02:55.555Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/3f/12/3e5f575f156555547c250a8b0d1347517a3a20fc7f4492e9703a69d4f45e/ast_serialize-0.6.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:a7520b672827885bafeae7501f684d14d47d17e5f45256f9df547686cca52264", size = 1177640, upload-time = "2026-06-30T20:02:06.708Z" }, { url = "https://files.pythonhosted.org/packages/a2/a4/921a9e27951627983b0f368859ea00f8330a551dc0bf4c2fdcb11855a98b/ast_serialize-0.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a14191beec7e0c078d2fc1f6edc0aee88bcd4db9f18e1bc9f8052b559c22dddc", size = 1168111, upload-time = "2026-06-30T20:02:08.366Z" }, { url = "https://files.pythonhosted.org/packages/00/69/950cf404de7b8782cf95e5c1237e25e2aa46177b287f39f9eeddf481fd6f/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32ef62ec34cf6be20ad77d4799556638fbdf187f3ae10698dfb20ef9f2c89516", size = 1227656, upload-time = "2026-06-30T20:02:09.843Z" }, { url = "https://files.pythonhosted.org/packages/4c/a8/46f8f6a6479d9d2273980957bb091a506c55f5b95d3c029ee58518a78407/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:13b7769970a39983b0adf2f38917b1cd3b8946f76df045756c3d741bc689f089", size = 1227706, upload-time = "2026-06-30T20:02:11.367Z" }, { url = "https://files.pythonhosted.org/packages/b7/b9/9ac415bda0a40e49eab8fea3b2741c19c98bb84d57d62c4cfc6230eb67be/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6f7a408601bb3edaefb3bc67a4c01f5235e3253653b6a5729a2ee2382b35341c", size = 1431705, upload-time = "2026-06-30T20:02:12.737Z" }, { url = "https://files.pythonhosted.org/packages/e5/06/8807115d441444879f7561b5eede5ac18fc80392f11826d61ccf31f503b1/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8670bfa51208a2c0c8d138928e40e998fab158f9200d53bb80c088b5b8eda7b8", size = 1249533, upload-time = "2026-06-30T20:02:14.571Z" }, { url = "https://files.pythonhosted.org/packages/3e/c0/c2ba82ef9618650357d9421a1fdb27ffec862a7f57e8e2de82a3ccd11e12/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4826809eb8597a8cd59fd924b6d7c285b8969a1e0007e2cb652cab62376270f", size = 1252619, upload-time = "2026-06-30T20:02:16.219Z" }, { url = "https://files.pythonhosted.org/packages/0f/a7/fa31d52dd4102cede29fb9634e98d214129b2783b4f95528c6dc6a8f6587/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:577a6c189068686869f5f1ddc38363f3ae1808a4753b577266f9202071a7bb66", size = 1242983, upload-time = "2026-06-30T20:02:17.813Z" }, { url = "https://files.pythonhosted.org/packages/b1/20/ddf742b5ad3c4bafd3466f2265037cfd99bc1b9a5ee46a5d58c90d523242/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:085de7f62dc9cc247eb01e965a362707d1d90b1d89a82c5bf78301a60a3c417b", size = 1296148, upload-time = "2026-06-30T20:02:19.146Z" }, { url = "https://files.pythonhosted.org/packages/24/cb/9f6f217cce8b3b632c5568b478d195a35e79dce4dbe309438cb89ba6ea4f/ast_serialize-0.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9f8a8b78b13173de6a9ec22111d9be674874cd5bdccda04f14ae5ebc2bef403a", size = 1403826, upload-time = "2026-06-30T20:02:20.696Z" }, { url = "https://files.pythonhosted.org/packages/2d/f8/9d16d4f0107a183924425cc0e7618d8bf76f96b45afa9ff19f924ed1ad57/ast_serialize-0.6.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f2ff3baffc3a29c1f15bc9098aa0c09763410262d5e6cef42116f7356c184554", size = 1502943, upload-time = "2026-06-30T20:02:22.034Z" }, { url = "https://files.pythonhosted.org/packages/80/dd/bbc1c38756350dddf7e24acae1c9482ef42051c267417e019aecc1ed4075/ast_serialize-0.6.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:0067b25fce104eaae5b88383de9ab803faeb671831e14ca698b771b356e2600f", size = 1497632, upload-time = "2026-06-30T20:02:23.517Z" }, { url = "https://files.pythonhosted.org/packages/42/7e/9daffefcf5b97e6bb4c3e0b3c024c1aee9722f23d3cf7cd2ff80d6fb4a40/ast_serialize-0.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c617417f9cbb0cb144f6283c3cbe0d2e0f01beaf9f608f662b21191058a626ec", size = 1448858, upload-time = "2026-06-30T20:02:24.889Z" }, { url = "https://files.pythonhosted.org/packages/e5/1f/f9baaab81a677ea0af7d2458cac2f94ebcc85958f8a3c15ba9d9e5dab653/ast_serialize-0.6.0-cp314-cp314t-win32.whl", hash = "sha256:5337cb256dcea3df9288205213d1601581536526b8f4da44b6974f1180f3252a", size = 1052600, upload-time = "2026-06-30T20:02:26.263Z" }, { url = "https://files.pythonhosted.org/packages/9e/1f/41b535866519512d8cf6669cb2cff7823b7672bb6279c0333b4ff89d7d9f/ast_serialize-0.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d947e45cafc4b09bd7528917fa84c517654a43de173c79785574b7b3068ac24", size = 1095570, upload-time = "2026-06-30T20:02:27.639Z" }, { url = "https://files.pythonhosted.org/packages/50/64/e472fe3e3a2d33d874b987e8518aedf24562919e3b6161a4fa1797e89c0f/ast_serialize-0.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:6e15ec740436e1a0d62de848641abe5f3a2f89a7f94907d534795ac91bbacf14", size = 1067267, upload-time = "2026-06-30T20:02:28.949Z" }, { url = "https://files.pythonhosted.org/packages/52/19/ac8348ae8711c9b5ae834634f635780cab62a0f5e6f988882e048b89c2ae/ast_serialize-0.6.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:093cb8bb91b720d8523580498d031791bb1bbaa048599c3d21085d380e11a596", size = 1185367, upload-time = "2026-06-30T20:02:30.427Z" }, { url = "https://files.pythonhosted.org/packages/c1/f6/ec7ec652c51db77c2f61d8573338e13e4704303265ccc658cb4031d9f354/ast_serialize-0.6.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:e61580a69faf47e3689795367ed211f2a10fd741478cc0f36a0f128793360aad", size = 1178657, upload-time = "2026-06-30T20:02:31.964Z" }, { url = "https://files.pythonhosted.org/packages/6f/02/613a7534a41d0122f37d1e0c64aa8ac78bfb831f8c92f6db057a311abb3c/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:305802f2ce2a7c4e87835078ea85c58b586ddda8095b92fe2ead9364ae19c80a", size = 1238620, upload-time = "2026-06-30T20:02:33.664Z" }, { url = "https://files.pythonhosted.org/packages/4d/21/087957bba486242afc52f49b2d9e21c9dad00289356cf9efe67084015a9d/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c7b8b8f0c42f752ea00b2b7d7c090b3f80d9c1c5c75cadf16423790a0cc74081", size = 1236075, upload-time = "2026-06-30T20:02:34.936Z" }, { url = "https://files.pythonhosted.org/packages/82/04/78128bbb170071c2c72a210a181f1c00e11cc1cec60a8beef747b07f9201/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd5b91b9e6f2356ace3a556963b0cd783b395fbbb0bb17b4defc283415466e77", size = 1441348, upload-time = "2026-06-30T20:02:36.245Z" }, { url = "https://files.pythonhosted.org/packages/64/64/62fb99d6faf199b4c3e5b08a07136e9a0d7664bb249c6de3670e5b63e9b6/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4d6ef91590258ada18909b9caea344dac4de2013906b035473cd674a43f4b790", size = 1258580, upload-time = "2026-06-30T20:02:37.53Z" }, { url = "https://files.pythonhosted.org/packages/ca/87/b4d6c38e0ccd5e85dc54cecdf933a152c60b28fe5d993a6d8a72fa6d5896/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcbed41e9386059fc0261d602445ede0976c2ecec2939688bcbcb9ed0b6f28b7", size = 1261693, upload-time = "2026-06-30T20:02:39.123Z" }, { url = "https://files.pythonhosted.org/packages/0e/4b/3676ca2191f39bafb75f93f99b2f429ec464586158fece2165f3572805dc/ast_serialize-0.6.0-cp39-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:cdc4e6f930b9090c2f92c9036ad12ffb8e6e44d4a5ba06f1458a05d60f203f7b", size = 1252517, upload-time = "2026-06-30T20:02:40.511Z" }, { url = "https://files.pythonhosted.org/packages/f3/58/494ef8c4b4acb2f4a265ac934caf45f792a08fe27d6b853de35ad991941a/ast_serialize-0.6.0-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:897ac47b5637be41c0c07061c8a912fafa967ef1dc73fa115e4bfa70882a093b", size = 1304843, upload-time = "2026-06-30T20:02:41.961Z" }, { url = "https://files.pythonhosted.org/packages/b1/f2/13736d920ab3d49bbee80ef1a277dd7b7aaf3b3545efd9d2a8114fe05525/ast_serialize-0.6.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c4af9a1386166e40ed01464991806f89038a2d89782576c7774876fa77034e32", size = 1413698, upload-time = "2026-06-30T20:02:44.179Z" }, { url = "https://files.pythonhosted.org/packages/a8/5a/e046f3899e2acba4677d7427b76431443a1aa1a0e583dfb05b55b69d55cf/ast_serialize-0.6.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:c901adbd750029b9ac4ad3d6aa56853e0ad4875119fbf52b7b8298afc223828b", size = 1512209, upload-time = "2026-06-30T20:02:45.584Z" }, { url = "https://files.pythonhosted.org/packages/cc/c7/e42aaca7bb2d22a7c06d5a8c7930086c5a334e93d716e6fa5e6647a4515f/ast_serialize-0.6.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae22a366b752ab4496191525b78b097b5b72d531752e3c1dd7e383a8f2c8a1a", size = 1508464, upload-time = "2026-06-30T20:02:46.942Z" }, { url = "https://files.pythonhosted.org/packages/95/93/5524a3dc6c3f593de3228ed9cbef73afa047625b7000ec21b7f58e6eb4d4/ast_serialize-0.6.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4ed29121da8b3fdc291002801a1de0f76248fa07dce89157a5f277842cf6126e", size = 1457164, upload-time = "2026-06-30T20:02:48.294Z" }, { url = "https://files.pythonhosted.org/packages/4f/c0/36a6ffb4d653cf621427b4c4928671f53ad800c453474de2b82564a44ad9/ast_serialize-0.6.0-cp39-abi3-pyemscripten_2026_0_wasm32.whl", hash = "sha256:b1dac4e09d341c1300ba69cdcbe62867b32a8c75d90db9bf4d083bec3b039f0b", size = 863014, upload-time = "2026-06-30T20:02:49.742Z" }, { url = "https://files.pythonhosted.org/packages/09/c7/7d5ad8b49e1278e1c2a1e0274bd7850560b3f09313aa00c13bc8d5544792/ast_serialize-0.6.0-cp39-abi3-win32.whl", hash = "sha256:82c312a7844d2fdeb4d5c48bd3d215bf940dafd4704e1a9bcf252a99010a99b1", size = 1063165, upload-time = "2026-06-30T20:02:50.98Z" }, { url = "https://files.pythonhosted.org/packages/47/ae/6710c14ecb276031cf10249f6adf5a59e2d3fdb3b5183bd59f70524067ee/ast_serialize-0.6.0-cp39-abi3-win_amd64.whl", hash = "sha256:113b58346f9ceb664352032770caca817d4a3c86f611c6088e6ef65ddaa70f0e", size = 1101444, upload-time = "2026-06-30T20:02:52.554Z" }, { url = "https://files.pythonhosted.org/packages/66/40/c53deb2cd0c9b0fb636d24d9f40924cf2e65028e6b20b10cd5c1eeb2c730/ast_serialize-0.6.0-cp39-abi3-win_arm64.whl", hash = "sha256:ccd132fe8db56f61fe743b1f644d01b8d65b83248a8da506f3132bda86d6ed5e", size = 1072965, upload-time = "2026-06-30T20:02:54.097Z" }, ] [[package]] name = "astor" version = "0.8.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/5a/21/75b771132fee241dfe601d39ade629548a9626d1d39f333fde31bc46febe/astor-0.8.1.tar.gz", hash = "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e", size = 35090, upload-time = "2019-12-10T01:50:35.51Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c3/88/97eef84f48fa04fbd6750e62dcceafba6c63c81b7ac1420856c8dcc0a3f9/astor-0.8.1-py2.py3-none-any.whl", hash = "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5", size = 27488, upload-time = "2019-12-10T01:50:33.628Z" }, ] [[package]] name = "astroid" version = "4.3.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/8d/7e/7c85d2b8549730e089bd984678a7efb64c510a5f09b4f0d9987a8354a80c/astroid-4.3.3.tar.gz", hash = "sha256:d03854b09d92c08e18d8e7d9185d393961186ed0747136d8fcb2d1c008a504ec", size = 441522, upload-time = "2026-09-30T06:03:04.601Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/23/be/54ffbdd679e2ea9f36f1923edd867c39fc1190b3810f111c8282a3f0b608/astroid-4.3.3-py3-none-any.whl", hash = "sha256:366c99c2907b6407869b1bbfd8abb10f3aea5818309e9c93c9b9c93ecf14bc4f", size = 287404, upload-time = "2026-09-30T06:03:02.038Z" }, ] [[package]] name = "attrs" version = "26.1.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] [[package]] name = "aws-sam-translator" version = "1.109.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "boto3" }, { name = "jsonschema" }, { name = "pydantic" }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e0/09/f62aa8d076f6ba85080ec6291e61af345e9be0daf8a4094101555e054ec7/aws_sam_translator-1.109.0.tar.gz", hash = "sha256:0c5e60223ae8434ce0c6bdb9a491d69ba3ec97e15c0d825d3803f7806382d804", size = 369016, upload-time = "2026-04-08T23:34:32.535Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/89/29/db13205af6bbebdc8dae9dd603ef97ee10a23cd8a3e26d9de728948b2e33/aws_sam_translator-1.109.0-py3-none-any.whl", hash = "sha256:9a6376e7c6d4fee173342b8b557035a8e3ec36e795e175e870411c8e4238873d", size = 432447, upload-time = "2026-04-08T23:34:30.881Z" }, ] [[package]] name = "aws-xray-sdk" version = "2.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "wrapt" }, ] sdist = { url = "https://files.pythonhosted.org/packages/14/25/0cbd7a440080def5e6f063720c3b190a25f8aa2938c1e34415dc18241596/aws_xray_sdk-2.15.0.tar.gz", hash = "sha256:794381b96e835314345068ae1dd3b9120bd8b4e21295066c37e8814dbb341365", size = 76315, upload-time = "2025-10-29T20:59:45Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ef/c3/f30a7a63e664acc7c2545ca0491b6ce8264536e0e5cad3965f1d1b91e960/aws_xray_sdk-2.15.0-py2.py3-none-any.whl", hash = "sha256:422d62ad7d52e373eebb90b642eb1bb24657afe03b22a8df4a8b2e5108e278a3", size = 103228, upload-time = "2025-10-29T21:00:24.12Z" }, ] [[package]] name = "babel" version = "2.18.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d", size = 9959554, upload-time = "2026-02-01T12:30:56.078Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, ] [[package]] name = "backrefs" version = "7.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/5e/a7/a7dd63622beef68cc0d3c3c36d472e143dd95443d5ebf14cd1a5b4dfbf11/backrefs-7.0.tar.gz", hash = "sha256:4989bb9e1e99eb23647c7160ed51fb21d0b41b5d200f2d3017da41e023097e82", size = 7012453, upload-time = "2026-04-28T16:28:04.215Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d4/39/39a31d7eae729ea14ed10c3ccef79371197177b9355a86cb3525709e8502/backrefs-7.0-py310-none-any.whl", hash = "sha256:b57cd227ea556b0aed3dc9b8da4628db4eabc0402c6d7fcfc69283a93955f7e9", size = 380824, upload-time = "2026-04-28T16:27:55.647Z" }, { url = "https://files.pythonhosted.org/packages/c9/b5/9302644225ba7dfa934a2ff2b9c7bb85701313a90dddb3dfaf693fa5bae2/backrefs-7.0-py311-none-any.whl", hash = "sha256:a0fa7360c63509e9e077e174ef4e6d3c21c8db94189b9d957289ae6d794b9475", size = 392626, upload-time = "2026-04-28T16:27:57.42Z" }, { url = "https://files.pythonhosted.org/packages/36/da/87912ddec6e06feffbaa3d7aa18fc6352bee2e8f1fee185d7d1690f8f4e8/backrefs-7.0-py312-none-any.whl", hash = "sha256:ca42ce6a49ace3d75684dfa9937f3373902a63284ecb385ce36d15e5dcb41c12", size = 398537, upload-time = "2026-04-28T16:27:58.913Z" }, { url = "https://files.pythonhosted.org/packages/00/bb/90ba423612b6aa0adccc6b1874bcd4a9b44b660c0c16f346611e00f64ac3/backrefs-7.0-py313-none-any.whl", hash = "sha256:f2c52955d631b9e1ac4cd56209f0a3a946d592b98e7790e77699339ae01c102a", size = 400491, upload-time = "2026-04-28T16:28:00.928Z" }, { url = "https://files.pythonhosted.org/packages/3e/5c/fb93d3092640a24dfb7bd7727a24016d7c01774ca013e60efd3f683c8002/backrefs-7.0-py314-none-any.whl", hash = "sha256:a6448b28180e3ca01134c9cf09dcebafad8531072e09903c5451748a05f24bc9", size = 412349, upload-time = "2026-04-28T16:28:02.412Z" }, ] [[package]] name = "blinker" version = "1.9.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, ] [[package]] name = "boto3" version = "1.43.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b7/65/47670987f2f9e181397872c7ee6415b7b95156d711b7eab6c55f66e575bc/boto3-1.43.0.tar.gz", hash = "sha256:80d44a943ef90aba7958ab31d30c155c198acc8a9581b5846b3878b2c8951086", size = 113143, upload-time = "2026-04-29T22:07:49.084Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b3/a0/3e6a0b1c1ea6bec76f71473727ef27abf3cd40e9709b3ebcbfbcfaae6f79/boto3-1.43.0-py3-none-any.whl", hash = "sha256:8ebe03754a4b73a5cb6ec2f14cca03ac33bd4760d0adea53da4724845130258b", size = 140497, upload-time = "2026-04-29T22:07:46.216Z" }, ] [[package]] name = "botocore" version = "1.43.106" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/11/b9/10ca68d0092895d5ea60f485a61a9840d5aff9d732c66ed60da53a20b1d4/botocore-1.43.106.tar.gz", hash = "sha256:006870b3b4e40547232ad12c3bb4faec91bbbe0659aafaa3b7fa48a112c4ee97", size = 16271399, upload-time = "2026-09-30T19:36:35.836Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a8/eb/c51d3b54dfaa9267a687359d946dc2a1ca6a27fb4d381773e59b7bd72bf4/botocore-1.43.106-py3-none-any.whl", hash = "sha256:c1fb8818f9957cb5037f73db34ac1a12ba43562e31d6e7f9cb7e5fc64e1dba78", size = 15965506, upload-time = "2026-09-30T19:36:32.625Z" }, ] [[package]] name = "cairocffi" version = "1.7.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi" }, ] sdist = { url = "https://files.pythonhosted.org/packages/70/c5/1a4dc131459e68a173cbdab5fad6b524f53f9c1ef7861b7698e998b837cc/cairocffi-1.7.1.tar.gz", hash = "sha256:2e48ee864884ec4a3a34bfa8c9ab9999f688286eb714a15a43ec9d068c36557b", size = 88096, upload-time = "2024-06-18T10:56:06.741Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/93/d8/ba13451aa6b745c49536e87b6bf8f629b950e84bd0e8308f7dc6883b67e2/cairocffi-1.7.1-py3-none-any.whl", hash = "sha256:9803a0e11f6c962f3b0ae2ec8ba6ae45e957a146a004697a1ac1bbf16b073b3f", size = 75611, upload-time = "2024-06-18T10:55:59.489Z" }, ] [[package]] name = "cairosvg" version = "2.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cairocffi" }, { name = "cssselect2" }, { name = "defusedxml" }, { name = "pillow" }, { name = "tinycss2" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c6/80/db62c0a96d2e55282c83524f6b1d02f09c7fd7f612e93bf83e30de1dc75c/cairosvg-2.9.1.tar.gz", hash = "sha256:861bc28ad97ce4f537d50eb3d6ee97a7afcccec9c61ac25c4e7d073fe409aec7", size = 41256, upload-time = "2026-09-07T10:35:09.563Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/41/51/8041c2e70649e5b7f2a0aedbbbd0609ac099cfaa0cbde2014279c9c05756/cairosvg-2.9.1-py3-none-any.whl", hash = "sha256:f91c5628e834be024a0ed4544d76261cd84016a4c73bcdf26c386495825c05a1", size = 46165, upload-time = "2026-09-07T10:35:07.952Z" }, ] [[package]] name = "cast-value-rs" version = "0.4.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/56/a4/634d1917ac49ea69291471e87eee9c1aa45d71d6da9b2cd5e1bfe82729de/cast_value_rs-0.4.2.tar.gz", hash = "sha256:fd621b8dd4f7e93bbffdb119882d85e2731c2f0fb1d30de85f29e8e04c044822", size = 86862, upload-time = "2026-08-13T09:16:59.199Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/67/c7/9db59d415df2d1c9f905c4da951fe023c6259a162584ef68bd81c171d448/cast_value_rs-0.4.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6d8e1c6d7b0fec6b1060f83867d20ebd0cb1ac6fa2954cd82f418a31a96ac504", size = 494354, upload-time = "2026-08-13T09:14:41.627Z" }, { url = "https://files.pythonhosted.org/packages/8f/53/10ece0b4ba4a0c156674c2db7e5d05ce9b04efe72e191417546e111d645c/cast_value_rs-0.4.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8c953321ff7954e0fe1b25c1baf96f0a9cae0961b8a56ce63dc7b94e2ea7d85c", size = 458463, upload-time = "2026-08-13T09:14:43.371Z" }, { url = "https://files.pythonhosted.org/packages/91/0b/e50ac2eb271eb6c34402b4abe317c795bc7d650d332d0bf422ae868aa2cd/cast_value_rs-0.4.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68492eef745691c80754cd027e216dbd5f7a4ce2e07fddfded3ee59bd947afb5", size = 487628, upload-time = "2026-08-13T09:14:44.86Z" }, { url = "https://files.pythonhosted.org/packages/a9/0f/69befc7905af2e73a2739b26444787cd528331c041f37235f3853bf15f75/cast_value_rs-0.4.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fbf145ed6f724fa7f5ddc9715f78e0d450ac141fc05299c83b2e7d627793ddaf", size = 555538, upload-time = "2026-08-13T09:14:46.305Z" }, { url = "https://files.pythonhosted.org/packages/4a/1b/4105fabec3268139bda3555422808211ec29428e17fa56566c1af7957c04/cast_value_rs-0.4.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73fb23256dda16e4e9f49bfedebe0e068f00ece85091b526c102618f66144216", size = 650896, upload-time = "2026-08-13T09:14:48.049Z" }, { url = "https://files.pythonhosted.org/packages/83/2a/452cf37675af02c3ac8bfa01fc962cb744a1392c51bccfb6d1faca423067/cast_value_rs-0.4.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8184135c78ebcabb904574a1b3bc5704c131140feae1b58d65c888fe08b46029", size = 561036, upload-time = "2026-08-13T09:14:49.431Z" }, { url = "https://files.pythonhosted.org/packages/d4/87/edb215d17da304a6b36489fc51b8eb785c5b041c2fcc5f200f16adde3133/cast_value_rs-0.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:686ea86aad1ffebd0b6793549043108600593a85032457eb88a3a0513a4100f5", size = 549187, upload-time = "2026-08-13T09:14:50.801Z" }, { url = "https://files.pythonhosted.org/packages/63/82/0f9963ef0bcb434cd1244cb2620123c2d71763d429d017ceb98fbb6faf48/cast_value_rs-0.4.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d6c425d2803e13408755a0022faa9efd0ab0becdf3573a1a8a7af64b1adee4ba", size = 591027, upload-time = "2026-08-13T09:14:52.281Z" }, { url = "https://files.pythonhosted.org/packages/90/97/b489f8048491f2ece21bf99d5a1fea3bbfaf3cd0e9f555ac4a0352c11d81/cast_value_rs-0.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:92556f1bb9d6b222736d967228565ec745c640c29bc291d1cb730a1011d3dcac", size = 665201, upload-time = "2026-08-13T09:14:53.769Z" }, { url = "https://files.pythonhosted.org/packages/ce/9f/fabffd9788f0b2d3f49ca4582b8859ab58b929b1b8837ec0be1263399279/cast_value_rs-0.4.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1b0881c23a8f5ff1dc6c70542179f2fc9191e0b7fa7a02f65316fb48904cd8d1", size = 831085, upload-time = "2026-08-13T09:14:55.244Z" }, { url = "https://files.pythonhosted.org/packages/b5/66/ca653bf9b51eb624615623611db28ec9428a351259fcf5625c5520faf759/cast_value_rs-0.4.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:8a67648bf47d9b740388efaf6f05185f22dea783e97eba895efdd1dbafe8e3cc", size = 788462, upload-time = "2026-08-13T09:14:56.666Z" }, { url = "https://files.pythonhosted.org/packages/f1/54/3a9280f791c8765a54f7814a010d2f460c7d5f5483be3e5634f4d9b38d75/cast_value_rs-0.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:decf297bc576675fff2139885fb88f84844e27b99e5d28eaa01909db7a327c4b", size = 753055, upload-time = "2026-08-13T09:14:58.302Z" }, { url = "https://files.pythonhosted.org/packages/06/26/09f4618dbab2de8ea427e817c349b8391b4d559316de899e9b54dde6143e/cast_value_rs-0.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:79a8c13af39bf65014dd6567dc5fa7c42dfda246e4335ab52814f3dd6fb42cac", size = 442068, upload-time = "2026-08-13T09:15:00.414Z" }, { url = "https://files.pythonhosted.org/packages/fc/49/7def5933625ff3c98e7d20cbd7de0aa1e83098ffa425a9df087764657767/cast_value_rs-0.4.2-cp312-cp312-win_arm64.whl", hash = "sha256:b8af7de3639271ad9e62aba4d15ddd22819c706bd7c7859663fdbb022397bdad", size = 397846, upload-time = "2026-08-13T09:15:02.022Z" }, { url = "https://files.pythonhosted.org/packages/6f/fc/1f48ae9ea1f10caf3bbc31b288d08137835d8965188f6ddcccdb3c7d398e/cast_value_rs-0.4.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:297445abc64891a2e62af93e4f35bb280f874ac548646d0a6b04a86d45e636a5", size = 494394, upload-time = "2026-08-13T09:15:03.604Z" }, { url = "https://files.pythonhosted.org/packages/7f/f3/d633218979cd8839fcfbade64a277c0ff9d8ff59d21bbd8a59f1467085b4/cast_value_rs-0.4.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b8b6e0781b4ce61a559cc050d836a52157bd30b1f59d63fdeee79fbc19ab7e57", size = 458677, upload-time = "2026-08-13T09:15:04.947Z" }, { url = "https://files.pythonhosted.org/packages/97/e4/96d72ccf5e00dd7af7bf55c96313df2ffba1f34a224f10430716232775c0/cast_value_rs-0.4.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8399dc89570c01537c07adc03fc2fc0e43a517be8a198e2f9e3f8a70dd003be0", size = 487289, upload-time = "2026-08-13T09:15:06.493Z" }, { url = "https://files.pythonhosted.org/packages/93/30/18313de8c1d3826e9a6abd883b7a7981b19391f674dcd52ff2635d19f8ee/cast_value_rs-0.4.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:939417d72c9945ce24eb959e6b2cd32f7aa7d0a50a06a0fac484056b10b650dd", size = 555221, upload-time = "2026-08-13T09:15:08.28Z" }, { url = "https://files.pythonhosted.org/packages/e9/1f/ed22b2e536ccbd4c5b234d869fc8f478347a45f7b940273023a0d6013134/cast_value_rs-0.4.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:101318e594ee998a5ccbc5ea7ed045cbd9fd22d45cac467c745fbd30309f1343", size = 650972, upload-time = "2026-08-13T09:15:10.103Z" }, { url = "https://files.pythonhosted.org/packages/d7/d2/39f9ff9b678dfb01d206aec6be0f486137d5e9669ebadf0d1f356d2a7324/cast_value_rs-0.4.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d23bac26b2c2fe6a2368d10fbb072a71671f6e0d4d8cf9e893f62ed3530bbd1b", size = 560869, upload-time = "2026-08-13T09:15:11.602Z" }, { url = "https://files.pythonhosted.org/packages/60/18/f63e8378db66b5a99e96c1422c68ba02a4e35de34355b90cf6d169d99ae6/cast_value_rs-0.4.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9d799a20d7517974d61d7c4dd54e5d7ef439e726012c087bada31bc644b1535", size = 548885, upload-time = "2026-08-13T09:15:13.18Z" }, { url = "https://files.pythonhosted.org/packages/98/af/12349007b670b11e16d3be51c29ab2398550b2653dae692a5d788d3dda87/cast_value_rs-0.4.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b9f0f04815efcc47592474e91f61f24810139396696f0690f3d0bf0e6ff3bb4a", size = 590846, upload-time = "2026-08-13T09:15:14.67Z" }, { url = "https://files.pythonhosted.org/packages/a3/cf/6a39df104923bc397ae58f90e21d1e87edd62d17f24fe3d6777e6c4d3e5f/cast_value_rs-0.4.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e69f81dab0b47c74c1f30c79198c412938b6c41510b8124dbd08fb36d13a8807", size = 665064, upload-time = "2026-08-13T09:15:16.527Z" }, { url = "https://files.pythonhosted.org/packages/36/df/0a9ac5bae4e1d3b7b4a79795054d822c26bdbc3bedad71fe13e8ce666c4b/cast_value_rs-0.4.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:59e15ec495e8707447180e0111e65e083464da60236edd5c3387efedce3da997", size = 830909, upload-time = "2026-08-13T09:15:18.088Z" }, { url = "https://files.pythonhosted.org/packages/2f/7d/9e99c969dd76087dbf731667982d8bb6b390d29c8f37c09dd330657b0487/cast_value_rs-0.4.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f9e787397588fabb28a146a28159150f44e8ef85a25abe364f989044f1b7159d", size = 788217, upload-time = "2026-08-13T09:15:19.706Z" }, { url = "https://files.pythonhosted.org/packages/21/a9/5d7410d37be26bed1dc740c885a0c7fdb213b93e9b13ac76fb3282aa50ea/cast_value_rs-0.4.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c66ca6ad21758993830a9f4d24263065727bf767e4524d79ded0c0b61e3da9d", size = 752817, upload-time = "2026-08-13T09:15:21.197Z" }, { url = "https://files.pythonhosted.org/packages/65/10/9c18cb01104bb211c9b0d9a956fae3f1a40f743c35bbc6763d0114ec36a7/cast_value_rs-0.4.2-cp313-cp313-win_amd64.whl", hash = "sha256:26fe305d5bd35d21c5ee72c5651892895447e175ba3019a24713c565d965c0ef", size = 442025, upload-time = "2026-08-13T09:15:22.575Z" }, { url = "https://files.pythonhosted.org/packages/64/49/42a0b48a71aca3ebb0abd0948fa9d484723c3562af30a8d6904b874abb2f/cast_value_rs-0.4.2-cp313-cp313-win_arm64.whl", hash = "sha256:9ce6678fc4b1bdc459178225347acd8019f1d22bca1e13ab86cc2a3e4ecb2f46", size = 397871, upload-time = "2026-08-13T09:15:23.906Z" }, { url = "https://files.pythonhosted.org/packages/be/e9/baf39cd3991962705ffd055fce071e23e7707f62df3fb44af2ffe0e84d11/cast_value_rs-0.4.2-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:cefdcfd4d2b95e3b35fd98b3d3d865be9b3ef1bc1264db3bc1b8c50c5e8ac999", size = 495672, upload-time = "2026-08-13T09:15:25.168Z" }, { url = "https://files.pythonhosted.org/packages/ed/9f/7ac6bf95d160f3fd89020aa952ffb4a49216b85a3fda02f84bc2c2612a2f/cast_value_rs-0.4.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cbcec2cfd199c1e404f5bda71b273b4a7470762f4b55c1c21f09d60d2762e339", size = 458480, upload-time = "2026-08-13T09:15:26.647Z" }, { url = "https://files.pythonhosted.org/packages/65/e5/46c2985ccbae41c8140f04f2acc9080761f10e8d2f8101c46083e632f812/cast_value_rs-0.4.2-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c224a9b30c2521cd06eac4bb1ac3860f41c902695e6f806344bc039d7a8a1d93", size = 486807, upload-time = "2026-08-13T09:15:28.129Z" }, { url = "https://files.pythonhosted.org/packages/18/13/97851c45b275fbc9d42b5d5a340fd5ea430525b60c2eacfe6a36399d69ef/cast_value_rs-0.4.2-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1eaa7c3d5fe418a74928f78514f9baeaeb8e31b93db96b1761edd6144f0d2eff", size = 555335, upload-time = "2026-08-13T09:15:29.648Z" }, { url = "https://files.pythonhosted.org/packages/7f/b3/c84c1e820786ddb062b162112b80e5dfc1b5b54b11125618f38c1e18d075/cast_value_rs-0.4.2-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bddec46b36f29964c8d4bbe82b8270575fbe1c0fb41c404bb3da1778f390a24", size = 650347, upload-time = "2026-08-13T09:15:31.044Z" }, { url = "https://files.pythonhosted.org/packages/60/38/52b4cc31570384e0286e28a03164a357d53a8aba7421ef83e500843d5449/cast_value_rs-0.4.2-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab3fd77144d26e2da2a113f862c05ea1cf465df347daf5deb6cf1689253fddfa", size = 561498, upload-time = "2026-08-13T09:15:32.485Z" }, { url = "https://files.pythonhosted.org/packages/41/c8/313e1d016b0adfccd1ead2bb28d2c94313ab62534bd6e11db19a480d3b55/cast_value_rs-0.4.2-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4affe8668a2f17764ed978c2dddb2b49554f36a71389e6928a8a5523c1a4173", size = 548676, upload-time = "2026-08-13T09:15:33.912Z" }, { url = "https://files.pythonhosted.org/packages/81/8b/a11f8e514598ebfc084507b0db390ecbdc6bfcffbf91ae666e0de93e6c25/cast_value_rs-0.4.2-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a67316b00ca01a78865d222badef2b1cb6c60ad5ebd360bf4fa3b224e87b8ffa", size = 590803, upload-time = "2026-08-13T09:15:35.384Z" }, { url = "https://files.pythonhosted.org/packages/94/1e/ed86da6fe69eff312cc83e348cf4715d4168ab2699783febd24bfd477cd8/cast_value_rs-0.4.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9946054c77d7c5df579e9e99ebebd4d8a220cb04a6af1b6467112dfe07eee72d", size = 664118, upload-time = "2026-08-13T09:15:36.797Z" }, { url = "https://files.pythonhosted.org/packages/0b/01/58a3eadff1233bd98d5809e2f6b5dc6d1b8c4c770664145f57fd0bdce756/cast_value_rs-0.4.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:95b064458233a7ef0525481eccfd2d37e86620d877ab1535ae809d37355cf131", size = 830686, upload-time = "2026-08-13T09:15:38.645Z" }, { url = "https://files.pythonhosted.org/packages/e6/40/a234d9d06013a0bee08f662ab083b3a55decfa2dbb315094fdf06693d425/cast_value_rs-0.4.2-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:d7de068a1dcc65d1aaf4e93228198b588776bd25b62ef42478ab7efe5245a6f3", size = 787835, upload-time = "2026-08-13T09:15:40.078Z" }, { url = "https://files.pythonhosted.org/packages/d6/72/93f906345d03972de52e3e74dd4ed12b56aab53bec85cc96020690ba6dbe/cast_value_rs-0.4.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae0cbdd44290a74e34519eb990b33bf0f8e96f0fde95ffc799913ff1aab27719", size = 753039, upload-time = "2026-08-13T09:15:41.702Z" }, { url = "https://files.pythonhosted.org/packages/e3/a5/47b6db4f68da086db3f3768775f545b9b7f660cab8f61039c91de1f813c4/cast_value_rs-0.4.2-cp314-cp314-win32.whl", hash = "sha256:d438517530d3a8d6bc810ab6dee831c48b45287435513fea1b0476b5d4fda07d", size = 387092, upload-time = "2026-08-13T09:15:43.218Z" }, { url = "https://files.pythonhosted.org/packages/d3/9c/5f71a5e94875bb999646227a583792dbbe9a2b257df0d961e8a2da879f9e/cast_value_rs-0.4.2-cp314-cp314-win_amd64.whl", hash = "sha256:7e7bb9faa4820b7eb0d99074fd61efff3a7392e72b6abe4b41c9d6b6d14fe457", size = 442126, upload-time = "2026-08-13T09:15:44.623Z" }, { url = "https://files.pythonhosted.org/packages/95/1c/8e7546a044da8e92eafbb12594169301cc94e2a071fd3e34204595d0a05f/cast_value_rs-0.4.2-cp314-cp314-win_arm64.whl", hash = "sha256:1f57e762a06102d61e940548a6ab841e05ad1f51f1c4e2aa26bc27a8c59a5abc", size = 398200, upload-time = "2026-08-13T09:15:46.117Z" }, { url = "https://files.pythonhosted.org/packages/a0/09/527646deeb622e5a597e4271e9e06d9d7cd24d64de77b8cb4ce250128c18/cast_value_rs-0.4.2-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:230f6440db74d2d11e513060d0e72aa3f897546465bc778d30aaeac90f8e733e", size = 485197, upload-time = "2026-08-13T09:15:47.622Z" }, { url = "https://files.pythonhosted.org/packages/b9/79/aaa713cb151a902439b670a4774ea76d0a5dadf1f03954f6a34af2a6cb1e/cast_value_rs-0.4.2-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:37538170dc1fedbfd62dd56670e443b881474e7eda7f583a3c989c83a9ff15fc", size = 538669, upload-time = "2026-08-13T09:15:49.148Z" }, { url = "https://files.pythonhosted.org/packages/d3/b1/2cf4f4f13f74caaa4dab4ddf4924fdeeffb39920d5ec30a80112aa041efa/cast_value_rs-0.4.2-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86d00b9cef0d38d0c5631ba63270c38c5d7c85aa87c609644f204919f939f0c9", size = 649341, upload-time = "2026-08-13T09:15:50.614Z" }, { url = "https://files.pythonhosted.org/packages/39/8b/1faa6b27cda3dd793635c7945118e49188b6714d6bf23bb072efd718e467/cast_value_rs-0.4.2-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:89434eebb7a12f66e99382e08c32648f055e24d9fce66274adef6620af2aebbd", size = 555253, upload-time = "2026-08-13T09:15:52.252Z" }, { url = "https://files.pythonhosted.org/packages/8b/c2/d31f177025b51a3fda70c5888b92f5fd30ec53bc2df2c046fc9a2717cd36/cast_value_rs-0.4.2-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:704171aec75f417f5a70a3eb5f107fbfc79183620093c9d9074290920a5981f1", size = 547964, upload-time = "2026-08-13T09:15:53.669Z" }, { url = "https://files.pythonhosted.org/packages/64/99/521c2d6347ca39deef237909dfee61c6d2cb806dcc20d74636e209bc646e/cast_value_rs-0.4.2-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2add2599d299436ec161661a8e119a69742c763a0936e542db92b1ce9d6e0c04", size = 574778, upload-time = "2026-08-13T09:15:55.479Z" }, { url = "https://files.pythonhosted.org/packages/f0/70/35c7cb37839454aeb93b9576b42daebd8249dd59e3834cd1be020b4b0660/cast_value_rs-0.4.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:da5df79a926bdcc1f5494612f621027970a9702eb49d556068025166d4c0ace2", size = 662461, upload-time = "2026-08-13T09:15:56.97Z" }, { url = "https://files.pythonhosted.org/packages/24/4f/a71bfaea1286bd12cc912733d650bd89a07c4430ee715202551c6e3eb0b0/cast_value_rs-0.4.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:5552e3c80d43de7a8a821793cb57da0dd3720a377051a5bf86bcabe094e87777", size = 814743, upload-time = "2026-08-13T09:15:58.652Z" }, { url = "https://files.pythonhosted.org/packages/94/a3/b04e0f42dafeae30604abca079cce303ebb69ba24f3c4debbf211ecc781c/cast_value_rs-0.4.2-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:73978c6dd2ce45a05a59ec7090df8b3f2a579ee86db9b1c7df21ede17f32189e", size = 782340, upload-time = "2026-08-13T09:16:00.135Z" }, { url = "https://files.pythonhosted.org/packages/3a/6c/913de75054092de796509dc04b92fc6ac1bf83dc86901f1f482667dead16/cast_value_rs-0.4.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:45d630592afb18880d4df0d7afb28b8cfc0ba6aa742d8db115f616df284b453a", size = 754068, upload-time = "2026-08-13T09:16:01.572Z" }, ] [[package]] name = "certifi" version = "2026.10.19" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" }, ] [[package]] name = "cffi" version = "2.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pycparser", marker = "implementation_name != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230, upload-time = "2025-09-08T23:23:00.879Z" }, { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043, upload-time = "2025-09-08T23:23:02.231Z" }, { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" }, { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" }, { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" }, { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" }, { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" }, { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" }, { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" }, { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909, upload-time = "2025-09-08T23:23:14.32Z" }, { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402, upload-time = "2025-09-08T23:23:15.535Z" }, { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780, upload-time = "2025-09-08T23:23:16.761Z" }, { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320, upload-time = "2025-09-08T23:23:18.087Z" }, { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487, upload-time = "2025-09-08T23:23:19.622Z" }, { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" }, { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" }, { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" }, { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" }, { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" }, { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" }, { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328, upload-time = "2025-09-08T23:23:44.61Z" }, { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650, upload-time = "2025-09-08T23:23:45.848Z" }, { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687, upload-time = "2025-09-08T23:23:47.105Z" }, { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773, upload-time = "2025-09-08T23:23:29.347Z" }, { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013, upload-time = "2025-09-08T23:23:30.63Z" }, { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" }, { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" }, { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" }, { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" }, { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" }, { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" }, { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487, upload-time = "2025-09-08T23:23:40.423Z" }, { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726, upload-time = "2025-09-08T23:23:41.742Z" }, { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, ] [[package]] name = "cfn-lint" version = "1.51.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aws-sam-translator" }, { name = "jsonpatch" }, { name = "networkx" }, { name = "pyyaml" }, { name = "regex" }, { name = "sympy" }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5e/ea/bc4954dcbff3ecb500e2593e4b64d0e9a552973249b2a27dd22eee3fb5b0/cfn_lint-1.51.0.tar.gz", hash = "sha256:05d2a59708c99363afe3af6ac7325de95a5b37f8eef7728f41ae567d088a61f1", size = 4088652, upload-time = "2026-05-12T20:34:25.712Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/48/12/016d6b4a43bb7eb2e430770e04325ef159d73d4183dce94c6fa6dc144902/cfn_lint-1.51.0-py3-none-any.whl", hash = "sha256:116d4f9c7c7d039e69c01c31fe9ff309ff79f0884c859ea92b353507903dd89e", size = 6065464, upload-time = "2026-05-12T20:34:22.885Z" }, ] [[package]] name = "charset-normalizer" version = "3.4.7" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", size = 311328, upload-time = "2026-04-02T09:26:24.331Z" }, { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", size = 208061, upload-time = "2026-04-02T09:26:25.568Z" }, { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", size = 229031, upload-time = "2026-04-02T09:26:26.865Z" }, { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a", size = 225239, upload-time = "2026-04-02T09:26:28.044Z" }, { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", size = 216589, upload-time = "2026-04-02T09:26:29.239Z" }, { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb", size = 202733, upload-time = "2026-04-02T09:26:30.5Z" }, { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1", size = 212652, upload-time = "2026-04-02T09:26:31.709Z" }, { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15", size = 211229, upload-time = "2026-04-02T09:26:33.282Z" }, { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5", size = 203552, upload-time = "2026-04-02T09:26:34.845Z" }, { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d", size = 230806, upload-time = "2026-04-02T09:26:36.152Z" }, { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7", size = 212316, upload-time = "2026-04-02T09:26:37.672Z" }, { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464", size = 227274, upload-time = "2026-04-02T09:26:38.93Z" }, { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", size = 218468, upload-time = "2026-04-02T09:26:40.17Z" }, { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c", size = 148460, upload-time = "2026-04-02T09:26:41.416Z" }, { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6", size = 159330, upload-time = "2026-04-02T09:26:42.554Z" }, { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d", size = 147828, upload-time = "2026-04-02T09:26:44.075Z" }, { url = "https://files.pythonhosted.org/packages/c1/3b/66777e39d3ae1ddc77ee606be4ec6d8cbd4c801f65e5a1b6f2b11b8346dd/charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063", size = 309627, upload-time = "2026-04-02T09:26:45.198Z" }, { url = "https://files.pythonhosted.org/packages/2e/4e/b7f84e617b4854ade48a1b7915c8ccfadeba444d2a18c291f696e37f0d3b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c", size = 207008, upload-time = "2026-04-02T09:26:46.824Z" }, { url = "https://files.pythonhosted.org/packages/c4/bb/ec73c0257c9e11b268f018f068f5d00aa0ef8c8b09f7753ebd5f2880e248/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66", size = 228303, upload-time = "2026-04-02T09:26:48.397Z" }, { url = "https://files.pythonhosted.org/packages/85/fb/32d1f5033484494619f701e719429c69b766bfc4dbc61aa9e9c8c166528b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18", size = 224282, upload-time = "2026-04-02T09:26:49.684Z" }, { url = "https://files.pythonhosted.org/packages/fa/07/330e3a0dda4c404d6da83b327270906e9654a24f6c546dc886a0eb0ffb23/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd", size = 215595, upload-time = "2026-04-02T09:26:50.915Z" }, { url = "https://files.pythonhosted.org/packages/e3/7c/fc890655786e423f02556e0216d4b8c6bcb6bdfa890160dc66bf52dee468/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215", size = 201986, upload-time = "2026-04-02T09:26:52.197Z" }, { url = "https://files.pythonhosted.org/packages/d8/97/bfb18b3db2aed3b90cf54dc292ad79fdd5ad65c4eae454099475cbeadd0d/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859", size = 211711, upload-time = "2026-04-02T09:26:53.49Z" }, { url = "https://files.pythonhosted.org/packages/6f/a5/a581c13798546a7fd557c82614a5c65a13df2157e9ad6373166d2a3e645d/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8", size = 210036, upload-time = "2026-04-02T09:26:54.975Z" }, { url = "https://files.pythonhosted.org/packages/8c/bf/b3ab5bcb478e4193d517644b0fb2bf5497fbceeaa7a1bc0f4d5b50953861/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5", size = 202998, upload-time = "2026-04-02T09:26:56.303Z" }, { url = "https://files.pythonhosted.org/packages/e7/4e/23efd79b65d314fa320ec6017b4b5834d5c12a58ba4610aa353af2e2f577/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832", size = 230056, upload-time = "2026-04-02T09:26:57.554Z" }, { url = "https://files.pythonhosted.org/packages/b9/9f/1e1941bc3f0e01df116e68dc37a55c4d249df5e6fa77f008841aef68264f/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6", size = 211537, upload-time = "2026-04-02T09:26:58.843Z" }, { url = "https://files.pythonhosted.org/packages/80/0f/088cbb3020d44428964a6c97fe1edfb1b9550396bf6d278330281e8b709c/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48", size = 226176, upload-time = "2026-04-02T09:27:00.437Z" }, { url = "https://files.pythonhosted.org/packages/6a/9f/130394f9bbe06f4f63e22641d32fc9b202b7e251c9aef4db044324dac493/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a", size = 217723, upload-time = "2026-04-02T09:27:02.021Z" }, { url = "https://files.pythonhosted.org/packages/73/55/c469897448a06e49f8fa03f6caae97074fde823f432a98f979cc42b90e69/charset_normalizer-3.4.7-cp313-cp313-win32.whl", hash = "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e", size = 148085, upload-time = "2026-04-02T09:27:03.192Z" }, { url = "https://files.pythonhosted.org/packages/5d/78/1b74c5bbb3f99b77a1715c91b3e0b5bdb6fe302d95ace4f5b1bec37b0167/charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", hash = "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110", size = 158819, upload-time = "2026-04-02T09:27:04.454Z" }, { url = "https://files.pythonhosted.org/packages/68/86/46bd42279d323deb8687c4a5a811fd548cb7d1de10cf6535d099877a9a9f/charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", hash = "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b", size = 147915, upload-time = "2026-04-02T09:27:05.971Z" }, { url = "https://files.pythonhosted.org/packages/97/c8/c67cb8c70e19ef1960b97b22ed2a1567711de46c4ddf19799923adc836c2/charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0", size = 309234, upload-time = "2026-04-02T09:27:07.194Z" }, { url = "https://files.pythonhosted.org/packages/99/85/c091fdee33f20de70d6c8b522743b6f831a2f1cd3ff86de4c6a827c48a76/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a", size = 208042, upload-time = "2026-04-02T09:27:08.749Z" }, { url = "https://files.pythonhosted.org/packages/87/1c/ab2ce611b984d2fd5d86a5a8a19c1ae26acac6bad967da4967562c75114d/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b", size = 228706, upload-time = "2026-04-02T09:27:09.951Z" }, { url = "https://files.pythonhosted.org/packages/a8/29/2b1d2cb00bf085f59d29eb773ce58ec2d325430f8c216804a0a5cd83cbca/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41", size = 224727, upload-time = "2026-04-02T09:27:11.175Z" }, { url = "https://files.pythonhosted.org/packages/47/5c/032c2d5a07fe4d4855fea851209cca2b6f03ebeb6d4e3afdb3358386a684/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e", size = 215882, upload-time = "2026-04-02T09:27:12.446Z" }, { url = "https://files.pythonhosted.org/packages/2c/c2/356065d5a8b78ed04499cae5f339f091946a6a74f91e03476c33f0ab7100/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae", size = 200860, upload-time = "2026-04-02T09:27:13.721Z" }, { url = "https://files.pythonhosted.org/packages/0c/cd/a32a84217ced5039f53b29f460962abb2d4420def55afabe45b1c3c7483d/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18", size = 211564, upload-time = "2026-04-02T09:27:15.272Z" }, { url = "https://files.pythonhosted.org/packages/44/86/58e6f13ce26cc3b8f4a36b94a0f22ae2f00a72534520f4ae6857c4b81f89/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b", size = 211276, upload-time = "2026-04-02T09:27:16.834Z" }, { url = "https://files.pythonhosted.org/packages/8f/fe/d17c32dc72e17e155e06883efa84514ca375f8a528ba2546bee73fc4df81/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356", size = 201238, upload-time = "2026-04-02T09:27:18.229Z" }, { url = "https://files.pythonhosted.org/packages/6a/29/f33daa50b06525a237451cdb6c69da366c381a3dadcd833fa5676bc468b3/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab", size = 230189, upload-time = "2026-04-02T09:27:19.445Z" }, { url = "https://files.pythonhosted.org/packages/b6/6e/52c84015394a6a0bdcd435210a7e944c5f94ea1055f5cc5d56c5fe368e7b/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46", size = 211352, upload-time = "2026-04-02T09:27:20.79Z" }, { url = "https://files.pythonhosted.org/packages/8c/d7/4353be581b373033fb9198bf1da3cf8f09c1082561e8e922aa7b39bf9fe8/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44", size = 227024, upload-time = "2026-04-02T09:27:22.063Z" }, { url = "https://files.pythonhosted.org/packages/30/45/99d18aa925bd1740098ccd3060e238e21115fffbfdcb8f3ece837d0ace6c/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72", size = 217869, upload-time = "2026-04-02T09:27:23.486Z" }, { url = "https://files.pythonhosted.org/packages/5c/05/5ee478aa53f4bb7996482153d4bfe1b89e0f087f0ab6b294fcf92d595873/charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10", size = 148541, upload-time = "2026-04-02T09:27:25.146Z" }, { url = "https://files.pythonhosted.org/packages/48/77/72dcb0921b2ce86420b2d79d454c7022bf5be40202a2a07906b9f2a35c97/charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f", size = 159634, upload-time = "2026-04-02T09:27:26.642Z" }, { url = "https://files.pythonhosted.org/packages/c6/a3/c2369911cd72f02386e4e340770f6e158c7980267da16af8f668217abaa0/charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246", size = 148384, upload-time = "2026-04-02T09:27:28.271Z" }, { url = "https://files.pythonhosted.org/packages/94/09/7e8a7f73d24dba1f0035fbbf014d2c36828fc1bf9c88f84093e57d315935/charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24", size = 330133, upload-time = "2026-04-02T09:27:29.474Z" }, { url = "https://files.pythonhosted.org/packages/8d/da/96975ddb11f8e977f706f45cddd8540fd8242f71ecdb5d18a80723dcf62c/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79", size = 216257, upload-time = "2026-04-02T09:27:30.793Z" }, { url = "https://files.pythonhosted.org/packages/e5/e8/1d63bf8ef2d388e95c64b2098f45f84758f6d102a087552da1485912637b/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960", size = 234851, upload-time = "2026-04-02T09:27:32.44Z" }, { url = "https://files.pythonhosted.org/packages/9b/40/e5ff04233e70da2681fa43969ad6f66ca5611d7e669be0246c4c7aaf6dc8/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4", size = 233393, upload-time = "2026-04-02T09:27:34.03Z" }, { url = "https://files.pythonhosted.org/packages/be/c1/06c6c49d5a5450f76899992f1ee40b41d076aee9279b49cf9974d2f313d5/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e", size = 223251, upload-time = "2026-04-02T09:27:35.369Z" }, { url = "https://files.pythonhosted.org/packages/2b/9f/f2ff16fb050946169e3e1f82134d107e5d4ae72647ec8a1b1446c148480f/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1", size = 206609, upload-time = "2026-04-02T09:27:36.661Z" }, { url = "https://files.pythonhosted.org/packages/69/d5/a527c0cd8d64d2eab7459784fb4169a0ac76e5a6fc5237337982fd61347e/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44", size = 220014, upload-time = "2026-04-02T09:27:38.019Z" }, { url = "https://files.pythonhosted.org/packages/7e/80/8a7b8104a3e203074dc9aa2c613d4b726c0e136bad1cc734594b02867972/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e", size = 218979, upload-time = "2026-04-02T09:27:39.37Z" }, { url = "https://files.pythonhosted.org/packages/02/9a/b759b503d507f375b2b5c153e4d2ee0a75aa215b7f2489cf314f4541f2c0/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3", size = 209238, upload-time = "2026-04-02T09:27:40.722Z" }, { url = "https://files.pythonhosted.org/packages/c2/4e/0f3f5d47b86bdb79256e7290b26ac847a2832d9a4033f7eb2cd4bcf4bb5b/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0", size = 236110, upload-time = "2026-04-02T09:27:42.33Z" }, { url = "https://files.pythonhosted.org/packages/96/23/bce28734eb3ed2c91dcf93abeb8a5cf393a7b2749725030bb630e554fdd8/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e", size = 219824, upload-time = "2026-04-02T09:27:43.924Z" }, { url = "https://files.pythonhosted.org/packages/2c/6f/6e897c6984cc4d41af319b077f2f600fc8214eb2fe2d6bcb79141b882400/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb", size = 233103, upload-time = "2026-04-02T09:27:45.348Z" }, { url = "https://files.pythonhosted.org/packages/76/22/ef7bd0fe480a0ae9b656189ec00744b60933f68b4f42a7bb06589f6f576a/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe", size = 225194, upload-time = "2026-04-02T09:27:46.706Z" }, { url = "https://files.pythonhosted.org/packages/c5/a7/0e0ab3e0b5bc1219bd80a6a0d4d72ca74d9250cb2382b7c699c147e06017/charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0", size = 159827, upload-time = "2026-04-02T09:27:48.053Z" }, { url = "https://files.pythonhosted.org/packages/7a/1d/29d32e0fb40864b1f878c7f5a0b343ae676c6e2b271a2d55cc3a152391da/charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c", size = 174168, upload-time = "2026-04-02T09:27:49.795Z" }, { url = "https://files.pythonhosted.org/packages/de/32/d92444ad05c7a6e41fb2036749777c163baf7a0301a040cb672d6b2b1ae9/charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d", size = 153018, upload-time = "2026-04-02T09:27:51.116Z" }, { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, ] [[package]] name = "click" version = "8.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bb/63/f9e1ea081ce35720d8b92acde70daaedace594dc93b693c869e0d5910718/click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2", size = 328061, upload-time = "2026-04-22T15:11:27.506Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", size = 110502, upload-time = "2026-04-22T15:11:25.044Z" }, ] [[package]] name = "colorama" version = "0.4.6" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] [[package]] name = "coverage" version = "7.16.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/2f/55/d1eaf3e73781174340a00dc1ba2aee8a65f82fadb18e2797b192b6b3925b/coverage-7.16.2.tar.gz", hash = "sha256:ca64d9f1f384f151b9511bec01126072acd2f313439f8ed015a22d8790aab6fa", size = 971999, upload-time = "2026-09-27T12:29:01.118Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/5e/2c/f8296c63c5d542f3d21aed685e56b7031a419037d155bb3382fc0940d249/coverage-7.16.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:218d742afca2b5ad5ca759e93eddedfbcc6eadf8322f080dcefc40b7bd4e2d48", size = 223969, upload-time = "2026-09-27T12:26:16.753Z" }, { url = "https://files.pythonhosted.org/packages/90/23/6f3dcb1423a0d43216e402ea1746e4a7c7c44f38896b97dd573790f56a40/coverage-7.16.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a9a638be322a8d76a41cdb17781c7f82aaee6a66493d8ffb7e2c09ee22423d99", size = 224328, upload-time = "2026-09-27T12:26:18.15Z" }, { url = "https://files.pythonhosted.org/packages/ac/7d/8f3b6dc920e3fc6732f7678785a2091db439f186afbec30dbf2214d9b1f7/coverage-7.16.2-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:724bd0f1e81856b35e59fc98cf7b4e544a3cb662e4e0864dca73d4326ee9d808", size = 255832, upload-time = "2026-09-27T12:26:19.799Z" }, { url = "https://files.pythonhosted.org/packages/d1/36/6c45f15be4eca4ac1062c6a55a323286494c99726a7e58951fe85967ac08/coverage-7.16.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5375ebd99038021b35e99dc88255022912c06565d316212f4a576e4b08d30f5d", size = 258571, upload-time = "2026-09-27T12:26:21.199Z" }, { url = "https://files.pythonhosted.org/packages/34/fb/b54cbeba3ad89082c2e441278681859e538322cc34b84b2af7ebff00080f/coverage-7.16.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a076277ca9f5750cc230f0f578ebd2620cec60255b25707361699fef6fb465c", size = 259680, upload-time = "2026-09-27T12:26:22.822Z" }, { url = "https://files.pythonhosted.org/packages/6e/a2/0dc65ec3d61930e1e4c2e371763b15eb4290896eb343a12d5d3091308116/coverage-7.16.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:58d4a54c6ea672afef66d49be922a2c69826c5ae1a42a9cd94f0c9c2bacdf800", size = 261945, upload-time = "2026-09-27T12:26:24.336Z" }, { url = "https://files.pythonhosted.org/packages/d6/93/5fad7a61f2c14e08e98946fc31c1c7ffc1195061bf3fdc351db3be77a863/coverage-7.16.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0dcbcfcc059117284c603ff8cb61a65872512882f84a8cf0339241f7f7c2f148", size = 256191, upload-time = "2026-09-27T12:26:25.89Z" }, { url = "https://files.pythonhosted.org/packages/2d/47/74e5de9227b939ece9f64e729645ddc4296bea10dbfa98721c1333c8be2e/coverage-7.16.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:afdf43b72ef3876c1fe66423b91466e37877c9e81e8cec70542b7e8525b9d1b7", size = 257610, upload-time = "2026-09-27T12:26:27.35Z" }, { url = "https://files.pythonhosted.org/packages/13/fe/2cf28d40b43645d1b72388fe3ee7f7c747533a6a9557bb8c24a7ae74fe1a/coverage-7.16.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9acc7f7ec4a1b5f89bd929fde5b8a714f6fafdc6cc18725413d510aa082b47ad", size = 255757, upload-time = "2026-09-27T12:26:28.949Z" }, { url = "https://files.pythonhosted.org/packages/d7/3d/7c149fd99fc8bbc39c80db5e688d1d39fd040be2ecb78b8335a51a55b9c0/coverage-7.16.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:80d3f7b48d43ee8fc5e8707a8adb43d743a5a1a85256c25a24f9d6d0e2238fa6", size = 259822, upload-time = "2026-09-27T12:26:30.515Z" }, { url = "https://files.pythonhosted.org/packages/e6/3f/b283fce09d5995e227bd8e513358dd7471bedc0f78abc85a925ebdb0a2f6/coverage-7.16.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:126d1af8804d7224421fe991ff65d3ce649081560df7a98b1a5ffff07f9923bd", size = 255325, upload-time = "2026-09-27T12:26:32.037Z" }, { url = "https://files.pythonhosted.org/packages/bf/91/f3325edf0c4223fb1fe1532b8dbef2a1d2f729459a9a7d1a44d073bae534/coverage-7.16.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c19cd6d025c1673f22afcd22c7df8a662d779e05d8e3fa6820c22afb895b0206", size = 257191, upload-time = "2026-09-27T12:26:33.525Z" }, { url = "https://files.pythonhosted.org/packages/c4/89/21eb5e83ecf2eed523c4eb3d65ae513cd082c8fd1b6deb34c4cb6c332f97/coverage-7.16.2-cp312-cp312-win32.whl", hash = "sha256:152877cdc8a07264882cfcd503ba56a3ef6cba56a70e8c70f6eb8ffd7384789a", size = 226034, upload-time = "2026-09-27T12:26:35.021Z" }, { url = "https://files.pythonhosted.org/packages/db/de/e3ad6d864c0833624b4f1f9b53f9e58e116c945e5e965c3f1e172c5e84cd/coverage-7.16.2-cp312-cp312-win_amd64.whl", hash = "sha256:e6c52d3307824ff93b39efd99e4185d557db40bd841452abfb32e5d9151ca162", size = 226569, upload-time = "2026-09-27T12:26:36.604Z" }, { url = "https://files.pythonhosted.org/packages/3e/c1/bccc58ebe5489cc70628f635c1932fd371f5d7da850dbcf960f95f4c4afc/coverage-7.16.2-cp312-cp312-win_arm64.whl", hash = "sha256:a678c0b6b22086ec2427359d22e37445d4a792f5fdbbc744112c7dade65cad02", size = 226372, upload-time = "2026-09-27T12:26:38.406Z" }, { url = "https://files.pythonhosted.org/packages/f0/f6/8eb4f220ef24f84fb27d852d4f9bf83e0c73ec1a4a08dd9a87e3f4529739/coverage-7.16.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1a37c6e478cf687e1aa30a593d19c92c02fad9d122b51ab73f51b8dc7a0c0fc9", size = 223993, upload-time = "2026-09-27T12:26:40.164Z" }, { url = "https://files.pythonhosted.org/packages/40/23/d4bbaf0c154e0b0c2b5264890dbf6ef098dcb50ec8f2469be9490d191660/coverage-7.16.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0993d0e90858c03943d3cb152e068a20dd4707924deec84dd2230261baae3b1b", size = 224368, upload-time = "2026-09-27T12:26:41.762Z" }, { url = "https://files.pythonhosted.org/packages/7f/48/fc1e88fd571ec5cb38150b7f89f7696ca1bdf9920e01432febb69774cc85/coverage-7.16.2-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bb2fc905bbf4e6b7f40806ea79e31515abf6349594cdf0adf27c4215f0463204", size = 255358, upload-time = "2026-09-27T12:26:43.442Z" }, { url = "https://files.pythonhosted.org/packages/1d/56/6785397d07c29c8e70fbb9a07e97d062b43c21ffc5f12385917847f09f63/coverage-7.16.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4358b9c8c0125b460407f3017c6cce8156e904b32772c5630d27112f52bdbfe5", size = 257955, upload-time = "2026-09-27T12:26:45.725Z" }, { url = "https://files.pythonhosted.org/packages/27/3b/c8cdd07721e5f99abd81cea970d971997f99bf158c0b85f51bd284179c8b/coverage-7.16.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f15254427c9b33eedac4f198eaf9e356eb4f6214551afb43da6194a2c088ad7", size = 259183, upload-time = "2026-09-27T12:26:47.208Z" }, { url = "https://files.pythonhosted.org/packages/9b/11/606b192fe43d32574ec6238549d48de588fdcc18485682a5ec0a8ac357f2/coverage-7.16.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9a75a4704ff640e46170042eec1f984385a121227c505d5a16ad8e495f452541", size = 261325, upload-time = "2026-09-27T12:26:49.084Z" }, { url = "https://files.pythonhosted.org/packages/67/90/eea481f8b0305ceeb33f081a5f47e298391dbd1b589de0c4b3b3aa50d3f2/coverage-7.16.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:14253fc7bb15749b849795a06f5d3b6d8bc3fb8a4b5ddc341faf7a89dce205fc", size = 255531, upload-time = "2026-09-27T12:26:50.509Z" }, { url = "https://files.pythonhosted.org/packages/6b/be/dedbf9aea1457b120c27ac10b8fc2a357f37fa2b54c3e7286d42980a0a2a/coverage-7.16.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:921415102a90637fcc2e3f169f61dad7699ecf690e8639fc21b813acbedc0967", size = 257329, upload-time = "2026-09-27T12:26:52.005Z" }, { url = "https://files.pythonhosted.org/packages/fa/cb/b25c19d5bb2bd0f2e4e27fe8e2ffcae80c7a91ae181c0dc749ed60e9b1a4/coverage-7.16.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cce2bc991293f15cc4084ca116827b5900c5f34e1a54dfe83f10ab5c43162eb7", size = 255294, upload-time = "2026-09-27T12:26:53.634Z" }, { url = "https://files.pythonhosted.org/packages/5f/a2/892c5c5f4ad44b7b2ca009aee705191f3f268f15052244f2f9e3539b2e35/coverage-7.16.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:e1fa594c887365b69745f25a416806e61085dd07b94c9eae68a6e20730629b23", size = 259444, upload-time = "2026-09-27T12:26:55.243Z" }, { url = "https://files.pythonhosted.org/packages/ed/99/a562537deba0a3e370182ae71c149be796c39d8087365f17a09188f27145/coverage-7.16.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:11e597173af1dc33d5f8a7332ada544199269a223af1ee1770ddd5e245ad0fe8", size = 255111, upload-time = "2026-09-27T12:26:56.851Z" }, { url = "https://files.pythonhosted.org/packages/2d/20/854ec68641a9b3362ff068a32dfa41637299761617ef253791dbade6fc76/coverage-7.16.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3e7f99698ba3a7d13988bdd984b7ebf13af4dbe2166dc8502eef90d77603b0a4", size = 256882, upload-time = "2026-09-27T12:26:58.41Z" }, { url = "https://files.pythonhosted.org/packages/db/0d/748e4518b0ac0f9ff2687c248a6e5f8c0737306e709372632a2556f84443/coverage-7.16.2-cp313-cp313-win32.whl", hash = "sha256:f80bd9f9633eafc73d0a913ba2645c96ba58bba1befc30590f7c0fbfde59d865", size = 226045, upload-time = "2026-09-27T12:26:59.983Z" }, { url = "https://files.pythonhosted.org/packages/31/fa/6e46edba66a183fe4d99d4bb52c173287e9b8dddabe0888d24cb8210e580/coverage-7.16.2-cp313-cp313-win_amd64.whl", hash = "sha256:8be099e979fc42559328a21828281b4578304191ae46ed4e80a407048a82eee6", size = 226581, upload-time = "2026-09-27T12:27:01.494Z" }, { url = "https://files.pythonhosted.org/packages/1b/d9/9ef6845367600b336ff75d000444a0d32497d6972c833141bd39356abf68/coverage-7.16.2-cp313-cp313-win_arm64.whl", hash = "sha256:28ff850182a67d117990fa2ce5ea1032836d8c9630dae867e8bdd3bff4533b79", size = 226409, upload-time = "2026-09-27T12:27:03.116Z" }, { url = "https://files.pythonhosted.org/packages/59/4c/577fc0803dab4155dcf808faffbdd7b159256781c0874a8586e17b81b149/coverage-7.16.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4ee546b9e4872ffa194bf07ac87bfa1202ebb824d0795dc1ef22f175545ca90a", size = 224138, upload-time = "2026-09-27T12:27:05.141Z" }, { url = "https://files.pythonhosted.org/packages/75/9e/e3785ba3ecba2bd11efc74bfe2801ca4b78c4480b15a375648d809a59da3/coverage-7.16.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a2fac6895eb299a2e52d7bbb8fb3903502b9da8d3f5309ceb16ec40c646b58ee", size = 224445, upload-time = "2026-09-27T12:27:06.805Z" }, { url = "https://files.pythonhosted.org/packages/f0/d0/963ff22d3fd27117da3b8cc442f5bdc91196f783321e1a8ff0ec43476772/coverage-7.16.2-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:57ff3783f99d75a1e81dd56a9737eb5665e6736a5d93258ba596b6dcad8fd05b", size = 256112, upload-time = "2026-09-27T12:27:08.43Z" }, { url = "https://files.pythonhosted.org/packages/a8/d4/a306940c81c6ae759e82fff27d20b7fdc6896e422b821f51313cce212b6c/coverage-7.16.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:35f37886699cb9abd29958247d718628d5bc6f39e623dff66a09e546c42a7e03", size = 258727, upload-time = "2026-09-27T12:27:09.927Z" }, { url = "https://files.pythonhosted.org/packages/b9/a3/d3d99d93b02517087aa05bc0cf2d04d372956b849e5443e059079901429b/coverage-7.16.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0fd7a86fdda7cb6d616d178654bd0ad6bc0f3f33c2e478aa598500a1a9e34eda", size = 259909, upload-time = "2026-09-27T12:27:11.55Z" }, { url = "https://files.pythonhosted.org/packages/08/44/39dd599181726758dd185ae4dc0c0ab3aeabf7ca70e68e145060feeaaa16/coverage-7.16.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ac0f3b379c94acc2f7dce5f5f0b24d44fa1cc6a509717ef83dfee07450c2117c", size = 262481, upload-time = "2026-09-27T12:27:13.17Z" }, { url = "https://files.pythonhosted.org/packages/99/e8/91ee43f6ded411460c359d7e1aebde4d6fd8f00a2e5394182d9d212eb23c/coverage-7.16.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7d0732c83746bc24123c581a85d9dd96b70ddb538c9076020aa1a041790361e9", size = 256012, upload-time = "2026-09-27T12:27:14.91Z" }, { url = "https://files.pythonhosted.org/packages/11/8c/e9499ddc33197bd7eabcb1118ca81756fc874457b324e2b479a4804b2ad2/coverage-7.16.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b451c68218c150f616bc9649783ec8de76a59792c759b43aa0c9c0466a465e4", size = 257898, upload-time = "2026-09-27T12:27:16.588Z" }, { url = "https://files.pythonhosted.org/packages/5f/6e/c081cb5991a0afba99f9c4ad6c74a5fce9513a38ddc64e3e6680c6fed9af/coverage-7.16.2-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a56ac4fa5a75c7e182e8f62600cfb4aff43c5ed7356a034f3557659c3bec1d90", size = 255938, upload-time = "2026-09-27T12:27:18.19Z" }, { url = "https://files.pythonhosted.org/packages/b2/42/1c3d819e8f9b6eb01c2fe90874d67a8882adb9507e0bbb09361ed131ea89/coverage-7.16.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4cc4f73aa3fabc36e32046d6cd2971405948d8a903636508a3d3b2f9128b3a95", size = 260368, upload-time = "2026-09-27T12:27:19.903Z" }, { url = "https://files.pythonhosted.org/packages/19/4f/d70eac07901fd587b6ab05e659b52afe13959992aa5113bf6cce059cc572/coverage-7.16.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:723dcdab91357159b722935b500ee8abc0a66c8c432e1e9fabf4cc7598952de8", size = 255620, upload-time = "2026-09-27T12:27:21.621Z" }, { url = "https://files.pythonhosted.org/packages/34/5e/6d87af88317d3d9a9b18a9ca1bc1673eb516917f296e579d0d4a55cb3490/coverage-7.16.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5397e21a90dde0e9c6896b77ded8f0be26b66f8b22b33aed41f6043ed95d55e6", size = 257521, upload-time = "2026-09-27T12:27:23.358Z" }, { url = "https://files.pythonhosted.org/packages/79/bb/90c2641170d2fa1a6757b3f8450ba2740197317b0ddd749e9604b914e886/coverage-7.16.2-cp314-cp314-win32.whl", hash = "sha256:848893e1d361448c113dc2f0913503522a6f7be231d0e38333d2a22d9698a011", size = 226248, upload-time = "2026-09-27T12:27:25.153Z" }, { url = "https://files.pythonhosted.org/packages/30/08/d8d0478bb02c8eb0ae20a496fc80c40fcf4d3450bd184300d682ba2d28a6/coverage-7.16.2-cp314-cp314-win_amd64.whl", hash = "sha256:5a27b731c171e43dc8b5f32b76a5051dde2ec9b9366c87028f08a7088ebc2c7b", size = 226732, upload-time = "2026-09-27T12:27:26.907Z" }, { url = "https://files.pythonhosted.org/packages/32/3f/0001da22155b0a8ce063ec0f7e64ecbe17b373f306e7a74435f6d6accb72/coverage-7.16.2-cp314-cp314-win_arm64.whl", hash = "sha256:1c569a9fd25505f1cd6bea90588818f90373ce90e2632e2cacf19ddbd6e14fdb", size = 226645, upload-time = "2026-09-27T12:27:28.588Z" }, { url = "https://files.pythonhosted.org/packages/d7/85/6d8813aff9b8b8586691a9d33c43c5604f7227622574da7cdc3d91a86861/coverage-7.16.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d93db87adb6b1c1b408dce4763314b55d76a9f589e96783a84ac9e7689e48bdf", size = 224871, upload-time = "2026-09-27T12:27:30.32Z" }, { url = "https://files.pythonhosted.org/packages/5c/70/444f3a4981ac2cda40fdcf4cc9b56a4e1a33c222abeb33e51ed3e3eb2a6b/coverage-7.16.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:aa62c85046473959c13ba9edca9dc90a77d5c1095b1ba313556314d77fe5b036", size = 225123, upload-time = "2026-09-27T12:27:32.33Z" }, { url = "https://files.pythonhosted.org/packages/d0/c1/980681cd7b33eb66ac835044116ef0a92e11fcc7bdd866cc89d10b1130b9/coverage-7.16.2-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db76506aa5416081f3e8974ae0f7965c58ada0bb0ef7339ac86099588dbb20d3", size = 265049, upload-time = "2026-09-27T12:27:34.085Z" }, { url = "https://files.pythonhosted.org/packages/b2/e3/87679875c33bb2191f0f05544a1cc9adcc940fe0c35443a10f2df753dde5/coverage-7.16.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a0f2285329dac10ab08f79cb11f5692c497018e6c7c511f95e6fd63a70b8f831", size = 267717, upload-time = "2026-09-27T12:27:36.025Z" }, { url = "https://files.pythonhosted.org/packages/76/64/5d372776d6eb523d4e93bafba2253f96984e3b18261c4cc56a50863c6d0d/coverage-7.16.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:382d3346d56b0eec1b793d53a4c88799c8053f516aa3a8d7c44315696954bacf", size = 270047, upload-time = "2026-09-27T12:27:37.96Z" }, { url = "https://files.pythonhosted.org/packages/be/c1/44082ff0cbf9f97d0043f57970a71204097ec7ba606361a9fd2065393669/coverage-7.16.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:648352b94507179d82637292e7ae8802508d95f78e2f00a705a50b6c48011681", size = 271360, upload-time = "2026-09-27T12:27:39.766Z" }, { url = "https://files.pythonhosted.org/packages/b8/17/9a215efe25b5e0ecc87c89dbe525c4a87d14d87c8c0c7316ef140a5f6f3e/coverage-7.16.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fb2bde05838fffae1a1bf75e5d411a6cac3e4e9bb97e6640fed8cd47888b33f0", size = 264736, upload-time = "2026-09-27T12:27:42.072Z" }, { url = "https://files.pythonhosted.org/packages/a2/da/7f0a31af8e448107d4d32844bd684757f51ea907bc0c68c8fd537b2123ff/coverage-7.16.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6a75180829efb8ae62b4aded25be6ddca1c888d138d2d82e21d93bfbd88f41cb", size = 267600, upload-time = "2026-09-27T12:27:43.85Z" }, { url = "https://files.pythonhosted.org/packages/dd/a4/3bfecbd3366b775bacdcb3330394d356cf384b5d8f5b2146ac4b14b252b5/coverage-7.16.2-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:99704f73721e23859112072d522076e11c31744fc96b5652e5dd2018aa4359f7", size = 264767, upload-time = "2026-09-27T12:27:45.768Z" }, { url = "https://files.pythonhosted.org/packages/b8/3f/5d62163732d87e4a0c4710a0eab30f0fd6a2d480112abe2029f014fe8c9d/coverage-7.16.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:29309ccc86b7f33df7db12813c299f215bbbc470ed6292d0bedd63ffae1ebf64", size = 269026, upload-time = "2026-09-27T12:27:47.787Z" }, { url = "https://files.pythonhosted.org/packages/49/4d/8e4579f225426535085a9be371cc75e3b026d058d679b80affbdfb4c3ef0/coverage-7.16.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:30c1b65d529e46569899fadca59e4a87c1faf2886923f1307ba61e654d4f3c20", size = 264147, upload-time = "2026-09-27T12:27:49.681Z" }, { url = "https://files.pythonhosted.org/packages/d1/36/ef1f77e2c3f7bb03c2b13b9a2006f88700fdd75535ef158d70049f425c1c/coverage-7.16.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dcf4bc2aab4e16b1c4c0c2005918f23a7dd5d7821ddae82caed9e3342dc2fcce", size = 266376, upload-time = "2026-09-27T12:27:51.551Z" }, { url = "https://files.pythonhosted.org/packages/be/79/0cb2bf4428830dec971c718c2c841a039c084415c99e67281f5a72841aab/coverage-7.16.2-cp314-cp314t-win32.whl", hash = "sha256:a9cd3de0a5bfe7b0e21ee10e1a14e3d61bf52efc88217ab1d95d6ace6970bd46", size = 226585, upload-time = "2026-09-27T12:27:53.945Z" }, { url = "https://files.pythonhosted.org/packages/3c/f9/da17121c16667fd84998e972200ae226a41540f6ea4795776c6d99e8976f/coverage-7.16.2-cp314-cp314t-win_amd64.whl", hash = "sha256:611a44e5229a59d7483ce830160e1a0e85f700562c7a5651c7c63fb8f4eb528c", size = 227378, upload-time = "2026-09-27T12:27:55.778Z" }, { url = "https://files.pythonhosted.org/packages/74/89/01179c62d1b7e6e33bd5001566b02d7f778cf33d3ec1e81e94ca170c517f/coverage-7.16.2-cp314-cp314t-win_arm64.whl", hash = "sha256:22957cef43ce038641de78ba995de7568d2d6a37c6ddbf7fa0fd7d1ae2344d91", size = 227064, upload-time = "2026-09-27T12:27:57.496Z" }, { url = "https://files.pythonhosted.org/packages/4c/57/52935003c3f627ba6e5203d7179aad32448c10899663a30336aba8e81a2c/coverage-7.16.2-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:414c26dfdb96aac2d570a54e03008f001e32eb2d413705365503648c6bd361d8", size = 224140, upload-time = "2026-09-27T12:27:59.343Z" }, { url = "https://files.pythonhosted.org/packages/31/38/df472520f3e626524d7e2fc9d6da0afe7895a2f1489d36b48af8ca40bb41/coverage-7.16.2-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:00d3eb96e9988c45f50cccd1f1496571ac5c1f91386ac02c4d55516eeda19a24", size = 224449, upload-time = "2026-09-27T12:28:01.299Z" }, { url = "https://files.pythonhosted.org/packages/0c/aa/3be084d5b82e63ccdad4ed751e4acbae294673573e30481d29f8b7402eec/coverage-7.16.2-cp315-cp315-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4dbbd1155ca46e6e0b6b89d204428c56ef6a459af21333f365d135a2820e5a09", size = 256060, upload-time = "2026-09-27T12:28:03.185Z" }, { url = "https://files.pythonhosted.org/packages/de/29/48fca82a7ebf7ff7b2e35019cc9537e7f65e4d2aa1215cc5a8792c989251/coverage-7.16.2-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8fc15cc8d0d06e873c00ef18e1372d605f9aaf3de27d8c24e50782e75bc8b843", size = 259106, upload-time = "2026-09-27T12:28:05.15Z" }, { url = "https://files.pythonhosted.org/packages/06/3d/b2d5986f2dd53fe201aa1be2e4ab204fa1aed5101e67c0dbbb419b850aee/coverage-7.16.2-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c6afdd69218202bc1758c9a14b86b8cf1084f37ed2ca143e567a103772b16d1", size = 260674, upload-time = "2026-09-27T12:28:06.868Z" }, { url = "https://files.pythonhosted.org/packages/ce/7e/b50160be3506ead12e6480d14279af7f0f17627694300a2d1fd2c42d2ff5/coverage-7.16.2-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:aba5c63b7afdc749cc9eae943d5b868cba2b261a176378fa1c5a30bc8bc89982", size = 263139, upload-time = "2026-09-27T12:28:08.771Z" }, { url = "https://files.pythonhosted.org/packages/14/5e/7c805ac9a32606de1399bd7e9bd375aa2f973dc61b12680d9e6403c2e891/coverage-7.16.2-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9174f0af24e5eff248b9dbfe76ec5275a3d19d37edbc2810543f12cf97347a34", size = 256591, upload-time = "2026-09-27T12:28:10.842Z" }, { url = "https://files.pythonhosted.org/packages/ab/9e/76f1ed129a2daf658a3ea17122824cf2e3b91fea0460d8d3664fc5a61018/coverage-7.16.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:80e9fdb4c3d926b6ba721d4bf7435bdb869c3527ae7803290361d0ab73db13b6", size = 258708, upload-time = "2026-09-27T12:28:12.962Z" }, { url = "https://files.pythonhosted.org/packages/5a/b7/8d62e75f48b527619239a65294f842d4b7fd02a0839d43ae1de80184e2df/coverage-7.16.2-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:7b3bce4a0d05401d70b7d0d5ca783e686bc9d30e81dbd7d980d532609bf809e4", size = 256562, upload-time = "2026-09-27T12:28:14.934Z" }, { url = "https://files.pythonhosted.org/packages/b8/8d/0a15f95c3afb78e947c52644786ba4bc9de259905687dd720d5e6fae2e76/coverage-7.16.2-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:44f21e407b278efdfc1ee5e481e00518bd1d500310a30a5fbf2bcbedfef4aaf0", size = 261077, upload-time = "2026-09-27T12:28:17.215Z" }, { url = "https://files.pythonhosted.org/packages/25/00/88389987305a47d732866c07c8a500000ab574df9505e3114ac69c8d027f/coverage-7.16.2-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:59c3926585e1cd1f2190f4b2ac9014de1bbeaf0d5d0587b0dc6b0aa90d17896a", size = 256034, upload-time = "2026-09-27T12:28:19.08Z" }, { url = "https://files.pythonhosted.org/packages/92/02/34d079d4952ad461bde037d353f9a6e037a7edc45fe0f9ee8781ff73f028/coverage-7.16.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:066429634299e14dd2d511e1e85f8f9cecc500781f6b41907c0dd6f1baea7e63", size = 258034, upload-time = "2026-09-27T12:28:21.242Z" }, { url = "https://files.pythonhosted.org/packages/f6/d8/3e59a62879285b464ec1b10fd824fbc1af9ce66e842cd39974f80a0becc4/coverage-7.16.2-cp315-cp315-win32.whl", hash = "sha256:893ea9cf86cb8d2546812ac93d973aaf2ee1fb45110a873b014214fd23e3725e", size = 226252, upload-time = "2026-09-27T12:28:23.102Z" }, { url = "https://files.pythonhosted.org/packages/f4/e1/128026e1b2836e9ad6b219207ba9edf1c5e0088a7869e23088aee7fbbe7a/coverage-7.16.2-cp315-cp315-win_amd64.whl", hash = "sha256:01c6908bc613b420c26c818fe948e1b97dfd041a53c98b01c63bd8321f5c9aae", size = 226728, upload-time = "2026-09-27T12:28:25.21Z" }, { url = "https://files.pythonhosted.org/packages/a8/f4/c9fa8e7cf525ca7748ac52b0ee89331d13fe09808e45c679830708782e90/coverage-7.16.2-cp315-cp315-win_arm64.whl", hash = "sha256:967d72c835d7a8cf0af99ec813a2d06e3db6df706402f1fe85b31b437645f495", size = 226648, upload-time = "2026-09-27T12:28:27.136Z" }, { url = "https://files.pythonhosted.org/packages/a2/13/e96b045447a856666f36f9c653e2a80bdaa732aaaf72412b19aa2c26a473/coverage-7.16.2-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:98d9c97f51b334b0adce7b964442a9af33c1a00c6ac856984cc5dc8d18f81c75", size = 224858, upload-time = "2026-09-27T12:28:29.169Z" }, { url = "https://files.pythonhosted.org/packages/23/90/087f6ad1bd3df059632ca3407a4e6552ed1053ee35354de0a771acf35423/coverage-7.16.2-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:3e861f1071dcc2fec1e88bef0920f6b1eaa66a143555b4f8ab79ba2b0f30ef55", size = 225139, upload-time = "2026-09-27T12:28:31.131Z" }, { url = "https://files.pythonhosted.org/packages/7e/8e/285dcef0184358044e7cbcd810a1bdc9566bc620f54702d605477155df4a/coverage-7.16.2-cp315-cp315t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:fb9d92ecfe2d5b494367c67f7446f8b75b68d8d0c8cf3bc3e6997478be25d9e2", size = 265090, upload-time = "2026-09-27T12:28:33.04Z" }, { url = "https://files.pythonhosted.org/packages/06/b2/cc83f3a6e5789a4e89059c69555bc641c2efcde568405a1c06fc702951ab/coverage-7.16.2-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:eb57acff4a74246ae513c142d4b36e18c389c3aed8661914a53f7cd0071031b2", size = 268254, upload-time = "2026-09-27T12:28:35.135Z" }, { url = "https://files.pythonhosted.org/packages/ac/41/f548c19530f5d66ac6e3c92bbcbc49da7261de3a458b9f3e54a3efb1a0b2/coverage-7.16.2-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:444889f7f66b74e4455c0a97e0e166dd41177f1dca8c0239a47cff25e05ba7e1", size = 270695, upload-time = "2026-09-27T12:28:36.959Z" }, { url = "https://files.pythonhosted.org/packages/94/61/4dc27cf82ef96434d2874110ad0cc10ea4621025705dc5049862bd3bd181/coverage-7.16.2-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a740ea6f083c6db7b926534d159508f80ba275ab35e722522de0d18d0f56e55f", size = 271853, upload-time = "2026-09-27T12:28:38.821Z" }, { url = "https://files.pythonhosted.org/packages/38/29/bf8072b1b8bd5f2de8b21460a404460b1a2b97e80a9464c78ec0271f6199/coverage-7.16.2-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8e209591f7c41ae4a9171335cf6156afda0b21de73b02f73f5aa95b2d5fbb08d", size = 265644, upload-time = "2026-09-27T12:28:40.815Z" }, { url = "https://files.pythonhosted.org/packages/7c/2f/0aecb8721be5cdeb8afd9d6d9f6b463f074e4d8d37f00f4c42442522709f/coverage-7.16.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:396bb16e04ce04efbb3df91456ae4e3da918e69ecdf67fb711b0a0fdf35ccce0", size = 268530, upload-time = "2026-09-27T12:28:42.725Z" }, { url = "https://files.pythonhosted.org/packages/ab/0b/92b4b7628268ee711249958e68fc0328779bd3d9a7ab4715379465aedb84/coverage-7.16.2-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:9cdf19874e0d247f32f03609200370343c3c7aa260b191d8c2bb251d36198283", size = 265154, upload-time = "2026-09-27T12:28:44.684Z" }, { url = "https://files.pythonhosted.org/packages/7b/d9/41c95c1ab29b3dcd357cd1227181d1c98185632aca41ce670ce671b23a43/coverage-7.16.2-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:fd3d72233eb8b48acc94fa57d44e2d32ce8e7abed02882ccb6d855ccc4ed33ec", size = 269799, upload-time = "2026-09-27T12:28:46.672Z" }, { url = "https://files.pythonhosted.org/packages/80/07/ebeb259aa5362b033a137b86d7274ff4b109d59be8cc9913889b783bf75a/coverage-7.16.2-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:bb4ffe96aa663cee727659db5a2afeb38c95f8677b747d447b90d6d4874ea2c5", size = 265196, upload-time = "2026-09-27T12:28:48.996Z" }, { url = "https://files.pythonhosted.org/packages/b2/18/8437620f90d023680a072eee02f968055f3658bbfb7d386d0ea34cfb7f30/coverage-7.16.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:dba2edfb054f6d4a08df9d1637c39a5aa3865bca6617c13c86be21e45658a59c", size = 267266, upload-time = "2026-09-27T12:28:51.361Z" }, { url = "https://files.pythonhosted.org/packages/28/6c/f08e8ee4293e6434035424180bef4d45e028e8ecc006c61bf9453e74405e/coverage-7.16.2-cp315-cp315t-win32.whl", hash = "sha256:251aed777c47c77aba047096d4542889db089227655711dfc2b9c54ef0e15e35", size = 226583, upload-time = "2026-09-27T12:28:53.33Z" }, { url = "https://files.pythonhosted.org/packages/f7/fd/3f939c2847f4a72c20cff8b1ac33da78ea91a2d38d9b43336e60db719103/coverage-7.16.2-cp315-cp315t-win_amd64.whl", hash = "sha256:2aca0bdfa9e91621d5b09d815357bf63def4fc0e9cb66da67bf2cf93f3b1a6f5", size = 227374, upload-time = "2026-09-27T12:28:55.158Z" }, { url = "https://files.pythonhosted.org/packages/5a/35/b98cdc354c952402132e675a87f2cc3227fb68f959c84aaa491fbe15933d/coverage-7.16.2-cp315-cp315t-win_arm64.whl", hash = "sha256:b88841e654f09732804809e435b3e005a929ffd9998b872b7b213957b8759cb8", size = 227064, upload-time = "2026-09-27T12:28:57.075Z" }, { url = "https://files.pythonhosted.org/packages/3f/0c/7a64e1ac90541a8edf50daef0914848011fb057a5bf55284a4811e21939a/coverage-7.16.2-py3-none-any.whl", hash = "sha256:11d28e9123a9156cb405d8d27b44256c9a58fb5decc2073a8f17862057e3aa0f", size = 215754, upload-time = "2026-09-27T12:28:59.075Z" }, ] [[package]] name = "cryptography" version = "50.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/de/41/6cbdcf9142d00fe82836fbb51e503e58088575cf7a0fe1dbff6695bf0840/cryptography-50.0.0.tar.gz", hash = "sha256:eeac2acb5a20ed25e0ad6d1df9891a520b78b404266b6d11778f25d5d691a6c9", size = 880201, upload-time = "2026-07-31T14:25:10.11Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c5/5c/59086b4aac5e879d38ddbcf74e4be7ade89cebc3eb199a55da998c3bb46a/cryptography-50.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:031e2d5dd4bb9caa3ca9c82e5a197fd8ae680232cee62603d1a813f3f07e3d03", size = 4001252, upload-time = "2026-07-31T14:23:33.331Z" }, { url = "https://files.pythonhosted.org/packages/57/ef/8f2df13c7216bcad3e1c74e07f6e193d93e998e114f524a53877c9af27ad/cryptography-50.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fd9192b7b70c573d7f214eb1ae35e00d359f6f5e4b27c7e21e30de1fc6204645", size = 4719554, upload-time = "2026-07-31T14:23:35.611Z" }, { url = "https://files.pythonhosted.org/packages/d9/41/029086c34d91052fc3b88bcc8056f709a7c915c7a23b235a54eb800b1c97/cryptography-50.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:06a32a980526a6ab9a4b9bf8f7385800791e2bb960903cb6b530e4817509a3b7", size = 4702130, upload-time = "2026-07-31T14:23:37.635Z" }, { url = "https://files.pythonhosted.org/packages/7d/ff/b6ce0954962e7f7b969f850a883744197bb3910bdfd7b6da162eab7d9f68/cryptography-50.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a1b30560f2acc95aa8b2e06e716a13dbfc97314747b80d9707e307f77b40d6b3", size = 4725244, upload-time = "2026-07-31T14:23:39.471Z" }, { url = "https://files.pythonhosted.org/packages/06/1e/63a1027cb7fec360a182208e1b7767d5aa1fe57be3d6aa856e69a321edc0/cryptography-50.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:8d89f3976b10b4ce31118de72329025f70d2c6ead14a8217c5514dd2c6d5a78f", size = 5342265, upload-time = "2026-07-31T14:23:41.286Z" }, { url = "https://files.pythonhosted.org/packages/6b/72/a1116d683a6d7ece94590013882515de087edf9ef0e6292aae615a44df73/cryptography-50.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b42a28c1844fd9de8f3f7d540e36b66f3a9c83fceac7170ebc7a6a19edd9dcae", size = 4734609, upload-time = "2026-07-31T14:23:43.139Z" }, { url = "https://files.pythonhosted.org/packages/15/37/36a9c479bbe49acea2636c7fd3360d20f7b7e079c300352011c44850b181/cryptography-50.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:900131fafd8aead39ac7dd3a7e833be754c17a95cfd91221636949fe4eb0aa8a", size = 4356517, upload-time = "2026-07-31T14:23:44.939Z" }, { url = "https://files.pythonhosted.org/packages/32/98/8a151d64367204cbc63ec65d37502f1d9c53cf4bfc6ec3c532614dbec60d/cryptography-50.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:07949c449a1abcf60d1ee6e88956d89404c7df3c8258f46589e912988e551987", size = 4724529, upload-time = "2026-07-31T14:23:46.93Z" }, { url = "https://files.pythonhosted.org/packages/22/f6/ec13b470172126464a86bf54d2294a46d29837fc51ba3e45d4047946fb5e/cryptography-50.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f89831ef99dd7dd169ab06d63a831adb9e20a87aac6d380266bbda5823349169", size = 5299852, upload-time = "2026-07-31T14:23:48.851Z" }, { url = "https://files.pythonhosted.org/packages/da/3a/f05e32c99d440c9bb891ea0e36c9091891e36be5a9a87ab2ee6ea20729f6/cryptography-50.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:82148ec5bddac30b51a5b3c1945075f896fa022cb93f8e4a01e9f6ee95292c5f", size = 4734462, upload-time = "2026-07-31T14:23:50.861Z" }, { url = "https://files.pythonhosted.org/packages/ca/dc/bd72b26be8953f80625f63151efd38eee71c76ca6cf591c08ff34615a79e/cryptography-50.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1489e263a8048bb8b6a8bac662eb2d402ea5d2b7b4699b72f385f1e2772db105", size = 4852708, upload-time = "2026-07-31T14:23:52.715Z" }, { url = "https://files.pythonhosted.org/packages/27/20/c930314a2ab476d15dec966ec87e2e9637bb02b06106b12c0396c57bb603/cryptography-50.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7cec5b856506da6defb290f30c9ee687d5f5e8cb0bd3f6459dde43b0b4fa40ef", size = 5004179, upload-time = "2026-07-31T14:23:54.887Z" }, { url = "https://files.pythonhosted.org/packages/32/2e/c9db68a0c4bfa28e310707527c0ee3a2bd254104d2e02e68f368e197aa4c/cryptography-50.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:bd1c592e4d5974f0d08d4888e432157adba757c66da0246918e43677fafa2d30", size = 3840395, upload-time = "2026-07-31T14:23:56.677Z" }, { url = "https://files.pythonhosted.org/packages/c3/fb/951032a3bf22a5697c83183fb6294a4843772947a70e616c57b3ff5f522e/cryptography-50.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:49e7d93abdbd2990caced757e5fade25302f719c3c8fb6e6fff2dde98999fc41", size = 3989258, upload-time = "2026-07-31T14:23:58.881Z" }, { url = "https://files.pythonhosted.org/packages/d4/67/91eb047e69c5e845f2f14b8a2e4a1aab0f283cb885531e9e22c8adb176bc/cryptography-50.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:19736989797678c6af1e55cd49055cdbcb55d8f6b5583ac5335f933aba9101dc", size = 4700648, upload-time = "2026-07-31T14:24:00.702Z" }, { url = "https://files.pythonhosted.org/packages/30/82/85f0f7425c856b9f96459411eb12e74ef72df9caf6f8f15bf23a33ff131f/cryptography-50.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:80b63928fa35083b33966ce1efb70e5b9607181e49dcd1c22c8c005e319f667f", size = 4682442, upload-time = "2026-07-31T14:24:02.538Z" }, { url = "https://files.pythonhosted.org/packages/1a/28/b555a365adff1cca2fbe7b9e487d68a40de6bc67ff2cb587473eb43de0e7/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:d58c3db7cd6eed54e6c06744db55456b65ebd7492ddeae9c1e93cfca7aa857d3", size = 4707596, upload-time = "2026-07-31T14:24:04.394Z" }, { url = "https://files.pythonhosted.org/packages/72/d8/f52538140cc719df62a01cf87d1c7142318d235817109d6f4054d7c352d6/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:df2a58a472f332225671c35b0a830208b86d004f82baa8530fa3782c85646533", size = 5314552, upload-time = "2026-07-31T14:24:06.31Z" }, { url = "https://files.pythonhosted.org/packages/38/14/6120e5bd7c5aa022ad15424ba4d5c5269d0d9448ed4d55e492ea91e3c1c4/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:11b74db56cdbe3cdee6e3f6982ecb70334fa10dce99ed58bf7894aaaa3b2a037", size = 4717113, upload-time = "2026-07-31T14:24:08.349Z" }, { url = "https://files.pythonhosted.org/packages/fa/71/190bf38c3ee2e0f8efc9860ae100c9df4169742eef274b91e7aa1cb133b9/cryptography-50.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f59e38625469987d7ef6d495323c55e7db6c212eaf6112267e0d3b565a2e9c9f", size = 4338580, upload-time = "2026-07-31T14:24:10.227Z" }, { url = "https://files.pythonhosted.org/packages/3a/63/504ccfbbe61fd8aa983f7f146399cdf034c72c2fc55f5b2dfdcdcdb20c99/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ecfed7367f965a0328cfbdd70da860f15441f002f613185668c6e6ebf5a0ac11", size = 4707038, upload-time = "2026-07-31T14:24:12.169Z" }, { url = "https://files.pythonhosted.org/packages/01/77/2cf79bbfc4d12ca106437a6e170d6aaa01a373e93093118aaaef0e801bd4/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:9aa87839c383bdbab6ef865787a1fb877af8dd03464c4400322726feaaadfc6d", size = 5273110, upload-time = "2026-07-31T14:24:14.38Z" }, { url = "https://files.pythonhosted.org/packages/e5/45/8aae2972c520145377ea3559a605a899bebe227bf070b33cdb445929a9b9/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:6ba6a53445bd3cfa809ef3ef5f1589aa6ba08784a1d962bf47d0940e871dab1c", size = 4716439, upload-time = "2026-07-31T14:24:16.415Z" }, { url = "https://files.pythonhosted.org/packages/7b/20/4fe50b619a48c2525cc46e2dbc1ac490708d704be5d467bdaac6dc955682/cryptography-50.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3f5735ffe4996d28b809371756219f5354864902a3b9e7c0b9ee87041209fc9c", size = 4837383, upload-time = "2026-07-31T14:24:18.553Z" }, { url = "https://files.pythonhosted.org/packages/92/91/3a31366e183343d3703f8995c095f5734676bd6938118047e50fcf279eb4/cryptography-50.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1b4a266766514614f8aa60416e71f2fc6e575d36e7bdc90f644fadb2f4b75b95", size = 4985772, upload-time = "2026-07-31T14:24:20.385Z" }, { url = "https://files.pythonhosted.org/packages/74/9a/02ffe35b2853d121689871eb5dce862092562b3a1ed5cc98f1aaed441506/cryptography-50.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:12b9c6996425c76ea6c457ace4f3073e715b8c545add07cd1a8f3a4f90691269", size = 3816291, upload-time = "2026-07-31T14:24:22.125Z" }, { url = "https://files.pythonhosted.org/packages/03/37/73d005be173aff344af30e9fd2a576575cb2391a7101d9cd3842e1fa8cce/cryptography-50.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ccdc4a71a4dabae05de219404f9f4abc38e3b58422177ff93d0da05967dafa07", size = 4036009, upload-time = "2026-07-31T14:24:24.122Z" }, { url = "https://files.pythonhosted.org/packages/ff/c6/7a6202a534e32103a285b7834a120869557fe198d51d7cfe59754c8bda9c/cryptography-50.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:910e1d2668e7de9648f2bcee30e180db2a6b15c30f887d7c4c93ddf96e3992e3", size = 4745252, upload-time = "2026-07-31T14:24:26.118Z" }, { url = "https://files.pythonhosted.org/packages/85/4f/0fa8c2f4428198f15d9ff8d63400e27afbf94ce833f6108da1eb3753f945/cryptography-50.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a91296cb61e8df6f86d0c19cc4068228da256bf59bf86049fbd821084565327f", size = 4728939, upload-time = "2026-07-31T14:24:27.994Z" }, { url = "https://files.pythonhosted.org/packages/d1/63/54dd723490ba2dc09b299682c10b38db38f159728bcaae8c591b8af2f22d/cryptography-50.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e722f16708d854fe924790e051061f6704a472c3bac347b6fd88033ea8dd0dc5", size = 4748483, upload-time = "2026-07-31T14:24:30.254Z" }, { url = "https://files.pythonhosted.org/packages/1d/dd/7c77d26285cc7f6991efce64a0f5b4f9383bfa5dd8c5033003eaf7db4cdb/cryptography-50.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:d764dcf130c428ef66786f866dd750f53182bc608813489915e9fc106bb0c82f", size = 5367599, upload-time = "2026-07-31T14:24:32.457Z" }, { url = "https://files.pythonhosted.org/packages/46/c9/f60aed34c013f317f92817b6c171c2d22a78270fa41109bd4b08af26b194/cryptography-50.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:105110f43a471dbd0060b9c9516cb8a6a79233631a04cc2ba16f28323ac6e025", size = 4762647, upload-time = "2026-07-31T14:24:34.599Z" }, { url = "https://files.pythonhosted.org/packages/be/f3/f9a0173b139372c3a48ed98154b45cc6b9de17c789d5ab552e621c293609/cryptography-50.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:828743d939e9629bc267b8e2d08d8bb67cd4319c771a33d4b18b22dd8fb7440a", size = 4385197, upload-time = "2026-07-31T14:24:36.647Z" }, { url = "https://files.pythonhosted.org/packages/d8/36/83bb81f6e569bc38e1e4a7bc80f29b46bb9601920bc455fc8e888f5d5742/cryptography-50.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:2a8183b489dc1f7f80f135780fadc1108f14b31b8a40411c7a5b17425f65f28b", size = 4748095, upload-time = "2026-07-31T14:24:39.493Z" }, { url = "https://files.pythonhosted.org/packages/6b/16/d3008eff98c764979865834c3d386d4fd041b5f52e7f34fc29ac1a5eb515/cryptography-50.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6e7d61120573a7f2cd94cc095f9e81f6967c61ccdf194285aa143ecec8e0b708", size = 5325948, upload-time = "2026-07-31T14:24:41.556Z" }, { url = "https://files.pythonhosted.org/packages/9c/f8/d97f9603efda3888187bfdb893f26c41be4735c10631d05d284ee6b047c4/cryptography-50.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:37fdb0d0111f1e2ff07139dfb79f1b49531f8e213c46f1163dd7642979b58c47", size = 4762400, upload-time = "2026-07-31T14:24:43.636Z" }, { url = "https://files.pythonhosted.org/packages/64/a2/4615c8f7d81a00b1d6e6afe19f694e1543582349fb5f4076f6cb5dc36485/cryptography-50.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87f62a3d3b9888ed0fdde100ec06aa61ca9cd44bad9057d1dff9a516b5f5bb9", size = 4878208, upload-time = "2026-07-31T14:24:45.522Z" }, { url = "https://files.pythonhosted.org/packages/d2/1a/efcfb02f91407149a0dacffffab791f7e19bf6385f63b3666dc8b5e5c9c8/cryptography-50.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:65c2c3add92b45fd0709db8594536aea39c2a67af0e27ffcf049c498501140b7", size = 5037050, upload-time = "2026-07-31T14:24:47.697Z" }, { url = "https://files.pythonhosted.org/packages/57/30/4a22984d4f1bdfb8c054f07a92bc176b97a3134cc1d6c4b3bffb1f3688b4/cryptography-50.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:d24fead1d4d076e1bfb006dcec392074a3cd8d7b4fc8a595aa64073b2b7a96ba", size = 3874135, upload-time = "2026-07-31T14:24:50.085Z" }, ] [[package]] name = "cssselect2" version = "0.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "tinycss2" }, { name = "webencodings" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e0/20/92eaa6b0aec7189fa4b75c890640e076e9e793095721db69c5c81142c2e1/cssselect2-0.9.0.tar.gz", hash = "sha256:759aa22c216326356f65e62e791d66160a0f9c91d1424e8d8adc5e74dddfc6fb", size = 35595, upload-time = "2026-02-12T17:16:39.614Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/21/0e/8459ca4413e1a21a06c97d134bfaf18adfd27cea068813dc0faae06cbf00/cssselect2-0.9.0-py3-none-any.whl", hash = "sha256:6a99e5f91f9a016a304dd929b0966ca464bcfda15177b6fb4a118fc0fb5d9563", size = 15453, upload-time = "2026-02-12T17:16:38.317Z" }, ] [[package]] name = "cuda-pathfinder" version = "1.5.4" source = { registry = "https://pypi.org/simple" } wheels = [ { url = "https://files.pythonhosted.org/packages/11/d0/c177e29701cf1d3008d7d2b16b5fc626592ce13bd535f8795c5f57187e0e/cuda_pathfinder-1.5.4-py3-none-any.whl", hash = "sha256:9563d3175ce1828531acf4b94e1c1c7d67208c347ca002493e2654878b26f4b7", size = 51657, upload-time = "2026-04-27T22:42:07.712Z" }, ] [[package]] name = "cupy-cuda12x" version = "14.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cuda-pathfinder" }, { name = "numpy" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/d3/8c/5fe3f6719c2d4560c79c62ef6d9b7d6c34d145879ddc0c1a41f8153ad0a6/cupy_cuda12x-14.2.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:b74340aa7271f0f081f77e2e5107bac75af19b86df29213db7ada90e14428efe", size = 147490427, upload-time = "2026-08-20T02:40:00.196Z" }, { url = "https://files.pythonhosted.org/packages/7c/5b/65124de2dbaf2e85109f611a41947e39acd6dd938751c04b4c4d7bf6fc82/cupy_cuda12x-14.2.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:f82141761f2c81905d49387464ae29438887956d99063381c93a1d5d1b7d32e8", size = 136111459, upload-time = "2026-08-20T02:40:04.909Z" }, { url = "https://files.pythonhosted.org/packages/e9/18/ddea819204701024bef7fa748730702245d803847c841b737723b94fd091/cupy_cuda12x-14.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:c9571d3b5f2e65758137e210f7fb3c3b34767f0af6b6ca04035a244b6141ee12", size = 98780018, upload-time = "2026-08-20T02:40:09.465Z" }, { url = "https://files.pythonhosted.org/packages/7a/4f/dce7be227a845943d14baef3b58be49c74a465e5d9251f38840b5b1fd89a/cupy_cuda12x-14.2.0-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:cfe673f73599ee0b9c2c9de5c0bb2395d98c9238c24deafa2ddcc69cacbd6af6", size = 147070561, upload-time = "2026-08-20T02:40:14.556Z" }, { url = "https://files.pythonhosted.org/packages/c9/02/520f7b9f92114b4df7d88aa77c36db0d556caf76a362537687e3a2e42833/cupy_cuda12x-14.2.0-cp313-cp313-manylinux2014_x86_64.whl", hash = "sha256:efc1da23505e88d9834a3ddd3c00352c34e58e301f512d9dd593cc4bfbbdf7dc", size = 135714387, upload-time = "2026-08-20T02:40:19.077Z" }, { url = "https://files.pythonhosted.org/packages/29/94/2dfb330afc6756ab9a8d16e955c0458e82e769930eab01e6c491e411363d/cupy_cuda12x-14.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:dcea9f2b1887ac631a9275a61577e09d1eea26bf5f95491501c3b7528cebc592", size = 98760068, upload-time = "2026-08-20T02:40:23.43Z" }, { url = "https://files.pythonhosted.org/packages/7e/d3/f6639af54f5872d1ef0c523601c7fe76d28783e71a3e8533e096c9ca1d43/cupy_cuda12x-14.2.0-cp314-cp314-manylinux2014_aarch64.whl", hash = "sha256:ed317136439af4780f217eda0b82f25180084eb16c44854e1bc9e055f96fd429", size = 146942115, upload-time = "2026-08-20T02:40:28.484Z" }, { url = "https://files.pythonhosted.org/packages/04/5e/e6134253265fefc0a35356adcebc4e3ffa81f6c9a2a74f8f9e2de32b3018/cupy_cuda12x-14.2.0-cp314-cp314-manylinux2014_x86_64.whl", hash = "sha256:db802e4b9a85ed84fd3e84790586c06e808ee45e0214cd4e80734c09fcf93073", size = 135091580, upload-time = "2026-08-20T02:40:33.351Z" }, { url = "https://files.pythonhosted.org/packages/0a/98/4d3215440b7a0d8661295050653760b57f32c933f1ef1c81841b7329209e/cupy_cuda12x-14.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:5f08fc1d651d2446c1d18ad94f1a710224fab36d46634d4aa356423926964591", size = 99344061, upload-time = "2026-08-20T02:40:37.459Z" }, { url = "https://files.pythonhosted.org/packages/3a/e9/8ed4adeb8c64f188b9ea6fba3be62fb7999584308bdf7ec6c5e17f77b99c/cupy_cuda12x-14.2.0-cp314-cp314t-manylinux2014_aarch64.whl", hash = "sha256:9dd33f9cfc7aefbd935879bf50e95db539721a0702bdb05be3c74bd46a85ba29", size = 147438535, upload-time = "2026-08-20T02:40:42.11Z" }, { url = "https://files.pythonhosted.org/packages/5a/c9/73227968a5b01ac31eaf1d5c58b4318e4b83654ed6dac3c310c7b2075c36/cupy_cuda12x-14.2.0-cp314-cp314t-manylinux2014_x86_64.whl", hash = "sha256:8cbbd48c9cfd6b78d0a833ebbafda3e1b057c38d6acc3c6e54de0735a7364e27", size = 135455541, upload-time = "2026-08-20T02:40:46.804Z" }, { url = "https://files.pythonhosted.org/packages/2e/3d/26127dd01e08ed645a70b4084ef6dde93e6a75b0a84fddc3ac6b11b05bf7/cupy_cuda12x-14.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d14b651ed835079f8a5e273936e02eda690be7d30f2658e5f48f328322fd9d7b", size = 100242434, upload-time = "2026-08-20T02:40:51.04Z" }, ] [[package]] name = "defusedxml" version = "0.7.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, ] [[package]] name = "docker" version = "7.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pywin32", marker = "sys_platform == 'win32'" }, { name = "requests" }, { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/91/9b/4a2ea29aeba62471211598dac5d96825bb49348fa07e906ea930394a83ce/docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c", size = 117834, upload-time = "2024-05-23T11:13:57.216Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0", size = 147774, upload-time = "2024-05-23T11:13:55.01Z" }, ] [[package]] name = "docutils" version = "0.22.4" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ae/b6/03bb70946330e88ffec97aefd3ea75ba575cb2e762061e0e62a213befee8/docutils-0.22.4.tar.gz", hash = "sha256:4db53b1fde9abecbb74d91230d32ab626d94f6badfc575d6db9194a49df29968", size = 2291750, upload-time = "2025-12-18T19:00:26.443Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl", hash = "sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de", size = 633196, upload-time = "2025-12-18T19:00:18.077Z" }, ] [[package]] name = "donfig" version = "0.8.1.post1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz", hash = "sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", size = 19506, upload-time = "2024-05-23T14:14:31.513Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl", hash = "sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", size = 21592, upload-time = "2024-05-23T14:13:55.283Z" }, ] [[package]] name = "execnet" version = "2.1.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, ] [[package]] name = "flask" version = "3.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "blinker" }, { name = "click" }, { name = "itsdangerous" }, { name = "jinja2" }, { name = "markupsafe" }, { name = "werkzeug" }, ] sdist = { url = "https://files.pythonhosted.org/packages/26/00/35d85dcce6c57fdc871f3867d465d780f302a175ea360f62533f12b27e2b/flask-3.1.3.tar.gz", hash = "sha256:0ef0e52b8a9cd932855379197dd8f94047b359ca0a78695144304cb45f87c9eb", size = 759004, upload-time = "2026-02-19T05:00:57.678Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/7f/9c/34f6962f9b9e9c71f6e5ed806e0d0ff03c9d1b0b2340088a0cf4bce09b18/flask-3.1.3-py3-none-any.whl", hash = "sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c", size = 103424, upload-time = "2026-02-19T05:00:56.027Z" }, ] [[package]] name = "flask-cors" version = "6.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "flask" }, { name = "werkzeug" }, ] sdist = { url = "https://files.pythonhosted.org/packages/70/74/0fc0fa68d62f21daef41017dafab19ef4b36551521260987eb3a5394c7ba/flask_cors-6.0.2.tar.gz", hash = "sha256:6e118f3698249ae33e429760db98ce032a8bf9913638d085ca0f4c5534ad2423", size = 13472, upload-time = "2025-12-12T20:31:42.861Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/4f/af/72ad54402e599152de6d067324c46fe6a4f531c7c65baf7e96c63db55eaf/flask_cors-6.0.2-py3-none-any.whl", hash = "sha256:e57544d415dfd7da89a9564e1e3a9e515042df76e12130641ca6f3f2f03b699a", size = 13257, upload-time = "2025-12-12T20:31:41.3Z" }, ] [[package]] name = "frozenlist" version = "1.8.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782, upload-time = "2025-10-06T05:36:06.649Z" }, { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594, upload-time = "2025-10-06T05:36:07.69Z" }, { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448, upload-time = "2025-10-06T05:36:08.78Z" }, { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014, upload-time = "2025-10-06T05:36:11.394Z" }, { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909, upload-time = "2025-10-06T05:36:12.598Z" }, { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049, upload-time = "2025-10-06T05:36:14.065Z" }, { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485, upload-time = "2025-10-06T05:36:15.39Z" }, { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619, upload-time = "2025-10-06T05:36:16.558Z" }, { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320, upload-time = "2025-10-06T05:36:17.821Z" }, { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820, upload-time = "2025-10-06T05:36:19.046Z" }, { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518, upload-time = "2025-10-06T05:36:20.763Z" }, { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985, upload-time = "2025-10-06T05:36:23.661Z" }, { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591, upload-time = "2025-10-06T05:36:24.958Z" }, { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102, upload-time = "2025-10-06T05:36:26.333Z" }, { url = "https://files.pythonhosted.org/packages/2d/40/0832c31a37d60f60ed79e9dfb5a92e1e2af4f40a16a29abcc7992af9edff/frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a", size = 85717, upload-time = "2025-10-06T05:36:27.341Z" }, { url = "https://files.pythonhosted.org/packages/30/ba/b0b3de23f40bc55a7057bd38434e25c34fa48e17f20ee273bbde5e0650f3/frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7", size = 49651, upload-time = "2025-10-06T05:36:28.855Z" }, { url = "https://files.pythonhosted.org/packages/0c/ab/6e5080ee374f875296c4243c381bbdef97a9ac39c6e3ce1d5f7d42cb78d6/frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40", size = 49417, upload-time = "2025-10-06T05:36:29.877Z" }, { url = "https://files.pythonhosted.org/packages/d5/4e/e4691508f9477ce67da2015d8c00acd751e6287739123113a9fca6f1604e/frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027", size = 234391, upload-time = "2025-10-06T05:36:31.301Z" }, { url = "https://files.pythonhosted.org/packages/40/76/c202df58e3acdf12969a7895fd6f3bc016c642e6726aa63bd3025e0fc71c/frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822", size = 233048, upload-time = "2025-10-06T05:36:32.531Z" }, { url = "https://files.pythonhosted.org/packages/f9/c0/8746afb90f17b73ca5979c7a3958116e105ff796e718575175319b5bb4ce/frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121", size = 226549, upload-time = "2025-10-06T05:36:33.706Z" }, { url = "https://files.pythonhosted.org/packages/7e/eb/4c7eefc718ff72f9b6c4893291abaae5fbc0c82226a32dcd8ef4f7a5dbef/frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5", size = 239833, upload-time = "2025-10-06T05:36:34.947Z" }, { url = "https://files.pythonhosted.org/packages/c2/4e/e5c02187cf704224f8b21bee886f3d713ca379535f16893233b9d672ea71/frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e", size = 245363, upload-time = "2025-10-06T05:36:36.534Z" }, { url = "https://files.pythonhosted.org/packages/1f/96/cb85ec608464472e82ad37a17f844889c36100eed57bea094518bf270692/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11", size = 229314, upload-time = "2025-10-06T05:36:38.582Z" }, { url = "https://files.pythonhosted.org/packages/5d/6f/4ae69c550e4cee66b57887daeebe006fe985917c01d0fff9caab9883f6d0/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1", size = 243365, upload-time = "2025-10-06T05:36:40.152Z" }, { url = "https://files.pythonhosted.org/packages/7a/58/afd56de246cf11780a40a2c28dc7cbabbf06337cc8ddb1c780a2d97e88d8/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1", size = 237763, upload-time = "2025-10-06T05:36:41.355Z" }, { url = "https://files.pythonhosted.org/packages/cb/36/cdfaf6ed42e2644740d4a10452d8e97fa1c062e2a8006e4b09f1b5fd7d63/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8", size = 240110, upload-time = "2025-10-06T05:36:42.716Z" }, { url = "https://files.pythonhosted.org/packages/03/a8/9ea226fbefad669f11b52e864c55f0bd57d3c8d7eb07e9f2e9a0b39502e1/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed", size = 233717, upload-time = "2025-10-06T05:36:44.251Z" }, { url = "https://files.pythonhosted.org/packages/1e/0b/1b5531611e83ba7d13ccc9988967ea1b51186af64c42b7a7af465dcc9568/frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496", size = 39628, upload-time = "2025-10-06T05:36:45.423Z" }, { url = "https://files.pythonhosted.org/packages/d8/cf/174c91dbc9cc49bc7b7aab74d8b734e974d1faa8f191c74af9b7e80848e6/frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231", size = 43882, upload-time = "2025-10-06T05:36:46.796Z" }, { url = "https://files.pythonhosted.org/packages/c1/17/502cd212cbfa96eb1388614fe39a3fc9ab87dbbe042b66f97acb57474834/frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62", size = 39676, upload-time = "2025-10-06T05:36:47.8Z" }, { url = "https://files.pythonhosted.org/packages/d2/5c/3bbfaa920dfab09e76946a5d2833a7cbdf7b9b4a91c714666ac4855b88b4/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94", size = 89235, upload-time = "2025-10-06T05:36:48.78Z" }, { url = "https://files.pythonhosted.org/packages/d2/d6/f03961ef72166cec1687e84e8925838442b615bd0b8854b54923ce5b7b8a/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c", size = 50742, upload-time = "2025-10-06T05:36:49.837Z" }, { url = "https://files.pythonhosted.org/packages/1e/bb/a6d12b7ba4c3337667d0e421f7181c82dda448ce4e7ad7ecd249a16fa806/frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52", size = 51725, upload-time = "2025-10-06T05:36:50.851Z" }, { url = "https://files.pythonhosted.org/packages/bc/71/d1fed0ffe2c2ccd70b43714c6cab0f4188f09f8a67a7914a6b46ee30f274/frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51", size = 284533, upload-time = "2025-10-06T05:36:51.898Z" }, { url = "https://files.pythonhosted.org/packages/c9/1f/fb1685a7b009d89f9bf78a42d94461bc06581f6e718c39344754a5d9bada/frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65", size = 292506, upload-time = "2025-10-06T05:36:53.101Z" }, { url = "https://files.pythonhosted.org/packages/e6/3b/b991fe1612703f7e0d05c0cf734c1b77aaf7c7d321df4572e8d36e7048c8/frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82", size = 274161, upload-time = "2025-10-06T05:36:54.309Z" }, { url = "https://files.pythonhosted.org/packages/ca/ec/c5c618767bcdf66e88945ec0157d7f6c4a1322f1473392319b7a2501ded7/frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714", size = 294676, upload-time = "2025-10-06T05:36:55.566Z" }, { url = "https://files.pythonhosted.org/packages/7c/ce/3934758637d8f8a88d11f0585d6495ef54b2044ed6ec84492a91fa3b27aa/frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d", size = 300638, upload-time = "2025-10-06T05:36:56.758Z" }, { url = "https://files.pythonhosted.org/packages/fc/4f/a7e4d0d467298f42de4b41cbc7ddaf19d3cfeabaf9ff97c20c6c7ee409f9/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506", size = 283067, upload-time = "2025-10-06T05:36:57.965Z" }, { url = "https://files.pythonhosted.org/packages/dc/48/c7b163063d55a83772b268e6d1affb960771b0e203b632cfe09522d67ea5/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51", size = 292101, upload-time = "2025-10-06T05:36:59.237Z" }, { url = "https://files.pythonhosted.org/packages/9f/d0/2366d3c4ecdc2fd391e0afa6e11500bfba0ea772764d631bbf82f0136c9d/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e", size = 289901, upload-time = "2025-10-06T05:37:00.811Z" }, { url = "https://files.pythonhosted.org/packages/b8/94/daff920e82c1b70e3618a2ac39fbc01ae3e2ff6124e80739ce5d71c9b920/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0", size = 289395, upload-time = "2025-10-06T05:37:02.115Z" }, { url = "https://files.pythonhosted.org/packages/e3/20/bba307ab4235a09fdcd3cc5508dbabd17c4634a1af4b96e0f69bfe551ebd/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41", size = 283659, upload-time = "2025-10-06T05:37:03.711Z" }, { url = "https://files.pythonhosted.org/packages/fd/00/04ca1c3a7a124b6de4f8a9a17cc2fcad138b4608e7a3fc5877804b8715d7/frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b", size = 43492, upload-time = "2025-10-06T05:37:04.915Z" }, { url = "https://files.pythonhosted.org/packages/59/5e/c69f733a86a94ab10f68e496dc6b7e8bc078ebb415281d5698313e3af3a1/frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888", size = 48034, upload-time = "2025-10-06T05:37:06.343Z" }, { url = "https://files.pythonhosted.org/packages/16/6c/be9d79775d8abe79b05fa6d23da99ad6e7763a1d080fbae7290b286093fd/frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042", size = 41749, upload-time = "2025-10-06T05:37:07.431Z" }, { url = "https://files.pythonhosted.org/packages/f1/c8/85da824b7e7b9b6e7f7705b2ecaf9591ba6f79c1177f324c2735e41d36a2/frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0", size = 86127, upload-time = "2025-10-06T05:37:08.438Z" }, { url = "https://files.pythonhosted.org/packages/8e/e8/a1185e236ec66c20afd72399522f142c3724c785789255202d27ae992818/frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f", size = 49698, upload-time = "2025-10-06T05:37:09.48Z" }, { url = "https://files.pythonhosted.org/packages/a1/93/72b1736d68f03fda5fdf0f2180fb6caaae3894f1b854d006ac61ecc727ee/frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c", size = 49749, upload-time = "2025-10-06T05:37:10.569Z" }, { url = "https://files.pythonhosted.org/packages/a7/b2/fabede9fafd976b991e9f1b9c8c873ed86f202889b864756f240ce6dd855/frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2", size = 231298, upload-time = "2025-10-06T05:37:11.993Z" }, { url = "https://files.pythonhosted.org/packages/3a/3b/d9b1e0b0eed36e70477ffb8360c49c85c8ca8ef9700a4e6711f39a6e8b45/frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8", size = 232015, upload-time = "2025-10-06T05:37:13.194Z" }, { url = "https://files.pythonhosted.org/packages/dc/94/be719d2766c1138148564a3960fc2c06eb688da592bdc25adcf856101be7/frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686", size = 225038, upload-time = "2025-10-06T05:37:14.577Z" }, { url = "https://files.pythonhosted.org/packages/e4/09/6712b6c5465f083f52f50cf74167b92d4ea2f50e46a9eea0523d658454ae/frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e", size = 240130, upload-time = "2025-10-06T05:37:15.781Z" }, { url = "https://files.pythonhosted.org/packages/f8/d4/cd065cdcf21550b54f3ce6a22e143ac9e4836ca42a0de1022da8498eac89/frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a", size = 242845, upload-time = "2025-10-06T05:37:17.037Z" }, { url = "https://files.pythonhosted.org/packages/62/c3/f57a5c8c70cd1ead3d5d5f776f89d33110b1addae0ab010ad774d9a44fb9/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128", size = 229131, upload-time = "2025-10-06T05:37:18.221Z" }, { url = "https://files.pythonhosted.org/packages/6c/52/232476fe9cb64f0742f3fde2b7d26c1dac18b6d62071c74d4ded55e0ef94/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f", size = 240542, upload-time = "2025-10-06T05:37:19.771Z" }, { url = "https://files.pythonhosted.org/packages/5f/85/07bf3f5d0fb5414aee5f47d33c6f5c77bfe49aac680bfece33d4fdf6a246/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7", size = 237308, upload-time = "2025-10-06T05:37:20.969Z" }, { url = "https://files.pythonhosted.org/packages/11/99/ae3a33d5befd41ac0ca2cc7fd3aa707c9c324de2e89db0e0f45db9a64c26/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30", size = 238210, upload-time = "2025-10-06T05:37:22.252Z" }, { url = "https://files.pythonhosted.org/packages/b2/60/b1d2da22f4970e7a155f0adde9b1435712ece01b3cd45ba63702aea33938/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7", size = 231972, upload-time = "2025-10-06T05:37:23.5Z" }, { url = "https://files.pythonhosted.org/packages/3f/ab/945b2f32de889993b9c9133216c068b7fcf257d8595a0ac420ac8677cab0/frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806", size = 40536, upload-time = "2025-10-06T05:37:25.581Z" }, { url = "https://files.pythonhosted.org/packages/59/ad/9caa9b9c836d9ad6f067157a531ac48b7d36499f5036d4141ce78c230b1b/frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0", size = 44330, upload-time = "2025-10-06T05:37:26.928Z" }, { url = "https://files.pythonhosted.org/packages/82/13/e6950121764f2676f43534c555249f57030150260aee9dcf7d64efda11dd/frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b", size = 40627, upload-time = "2025-10-06T05:37:28.075Z" }, { url = "https://files.pythonhosted.org/packages/c0/c7/43200656ecc4e02d3f8bc248df68256cd9572b3f0017f0a0c4e93440ae23/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d", size = 89238, upload-time = "2025-10-06T05:37:29.373Z" }, { url = "https://files.pythonhosted.org/packages/d1/29/55c5f0689b9c0fb765055629f472c0de484dcaf0acee2f7707266ae3583c/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed", size = 50738, upload-time = "2025-10-06T05:37:30.792Z" }, { url = "https://files.pythonhosted.org/packages/ba/7d/b7282a445956506fa11da8c2db7d276adcbf2b17d8bb8407a47685263f90/frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930", size = 51739, upload-time = "2025-10-06T05:37:32.127Z" }, { url = "https://files.pythonhosted.org/packages/62/1c/3d8622e60d0b767a5510d1d3cf21065b9db874696a51ea6d7a43180a259c/frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c", size = 284186, upload-time = "2025-10-06T05:37:33.21Z" }, { url = "https://files.pythonhosted.org/packages/2d/14/aa36d5f85a89679a85a1d44cd7a6657e0b1c75f61e7cad987b203d2daca8/frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24", size = 292196, upload-time = "2025-10-06T05:37:36.107Z" }, { url = "https://files.pythonhosted.org/packages/05/23/6bde59eb55abd407d34f77d39a5126fb7b4f109a3f611d3929f14b700c66/frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37", size = 273830, upload-time = "2025-10-06T05:37:37.663Z" }, { url = "https://files.pythonhosted.org/packages/d2/3f/22cff331bfad7a8afa616289000ba793347fcd7bc275f3b28ecea2a27909/frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a", size = 294289, upload-time = "2025-10-06T05:37:39.261Z" }, { url = "https://files.pythonhosted.org/packages/a4/89/5b057c799de4838b6c69aa82b79705f2027615e01be996d2486a69ca99c4/frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2", size = 300318, upload-time = "2025-10-06T05:37:43.213Z" }, { url = "https://files.pythonhosted.org/packages/30/de/2c22ab3eb2a8af6d69dc799e48455813bab3690c760de58e1bf43b36da3e/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef", size = 282814, upload-time = "2025-10-06T05:37:45.337Z" }, { url = "https://files.pythonhosted.org/packages/59/f7/970141a6a8dbd7f556d94977858cfb36fa9b66e0892c6dd780d2219d8cd8/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe", size = 291762, upload-time = "2025-10-06T05:37:46.657Z" }, { url = "https://files.pythonhosted.org/packages/c1/15/ca1adae83a719f82df9116d66f5bb28bb95557b3951903d39135620ef157/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8", size = 289470, upload-time = "2025-10-06T05:37:47.946Z" }, { url = "https://files.pythonhosted.org/packages/ac/83/dca6dc53bf657d371fbc88ddeb21b79891e747189c5de990b9dfff2ccba1/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a", size = 289042, upload-time = "2025-10-06T05:37:49.499Z" }, { url = "https://files.pythonhosted.org/packages/96/52/abddd34ca99be142f354398700536c5bd315880ed0a213812bc491cff5e4/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e", size = 283148, upload-time = "2025-10-06T05:37:50.745Z" }, { url = "https://files.pythonhosted.org/packages/af/d3/76bd4ed4317e7119c2b7f57c3f6934aba26d277acc6309f873341640e21f/frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df", size = 44676, upload-time = "2025-10-06T05:37:52.222Z" }, { url = "https://files.pythonhosted.org/packages/89/76/c615883b7b521ead2944bb3480398cbb07e12b7b4e4d073d3752eb721558/frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd", size = 49451, upload-time = "2025-10-06T05:37:53.425Z" }, { url = "https://files.pythonhosted.org/packages/e0/a3/5982da14e113d07b325230f95060e2169f5311b1017ea8af2a29b374c289/frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79", size = 42507, upload-time = "2025-10-06T05:37:54.513Z" }, { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, ] [[package]] name = "fsspec" version = "2026.9.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/77/cd/9be253869fc42e764de7f3dedd6969af7d44ff9c3375214a3442a6f3fc08/fsspec-2026.9.0.tar.gz", hash = "sha256:0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe", size = 333545, upload-time = "2026-09-18T17:50:42.825Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/6c/c0/a98505f18594f1bce828bb159cec0fcf9860562f1a2c85913409fc8f3d9e/fsspec-2026.9.0-py3-none-any.whl", hash = "sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f", size = 221738, upload-time = "2026-09-18T17:50:41.341Z" }, ] [[package]] name = "ghp-import" version = "2.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "python-dateutil" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d9/29/d40217cbe2f6b1359e00c6c307bb3fc876ba74068cbab3dde77f03ca0dc4/ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343", size = 10943, upload-time = "2022-05-02T15:47:16.11Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", size = 11034, upload-time = "2022-05-02T15:47:14.552Z" }, ] [[package]] name = "google-crc32c" version = "1.9.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/fa/25/9cb0c1c31c45b893eb8f11ae70b3f4309432d59b5acaebca5dbe791729a4/google_crc32c-1.9.0.tar.gz", hash = "sha256:7b8c84c3d159ab6817fe3f74e6e6cef099c3f95dcec3abc0d8afb1404642efbe", size = 14857, upload-time = "2026-09-24T21:39:32.067Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e4/5d/0730e1b3a14d054d1466f2fec88dadf978509c749a3d96d8b069cc56d38a/google_crc32c-1.9.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:53fdafef58e230d0c946ab5f8446d123d9f548230a73b29c8b41c9546f268bc1", size = 31961, upload-time = "2026-09-24T21:19:01.724Z" }, { url = "https://files.pythonhosted.org/packages/dd/32/d085abaf2fd907121975b92245bb3480fb8be40c37d03f9d6c41857f84c3/google_crc32c-1.9.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:8b91f41645b15a720357183fa5716682ada441873e3c462c15f9714be36f146b", size = 31804, upload-time = "2026-09-24T21:22:25.81Z" }, { url = "https://files.pythonhosted.org/packages/94/78/dd1935432337e5da7af391a6fc9f161c1c8e9b9002a402b9190135fe1b59/google_crc32c-1.9.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:16865b477d7941712cb0e0aad8ad4815e984fb5fc16d3fdaef7d986e26e53c95", size = 37775, upload-time = "2026-09-24T21:38:09.249Z" }, { url = "https://files.pythonhosted.org/packages/9e/43/9db03635bb10188d93dcbab9baa2a8670a0da4e868b4370cdbd98d65fed8/google_crc32c-1.9.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3abb18297d9ef0ab120531838be0e6d68c9fa876570e11c229c48f2edac23ce7", size = 36347, upload-time = "2026-09-24T21:38:10.141Z" }, { url = "https://files.pythonhosted.org/packages/cf/eb/94dee516c846bd9382c3f566d8f8e5fb9e90599e45afeb697f9fc2533528/google_crc32c-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:fb63a8d7fa2e95dcff1ca16af2f4d88b526fa5ff72d1696285884ac2d49b6963", size = 35097, upload-time = "2026-09-24T21:39:28.934Z" }, { url = "https://files.pythonhosted.org/packages/3f/34/cb484e8b6174f130f8c6dc79c733a9dd8869b410ad6511fb6104c46b973a/google_crc32c-1.9.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:f1dc17d987ddcc5eba12a7ce48f0eb93141dea236b170c1101151396edf2f0cf", size = 31960, upload-time = "2026-09-24T21:19:02.454Z" }, { url = "https://files.pythonhosted.org/packages/af/25/3e8e567bd48448e225ea27318ccf2b94e05124e7b8b97b13eaec9e127199/google_crc32c-1.9.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:f894a2877650b56201d26a012a257b76d54a68834dc3913a93830ca8a047b075", size = 31803, upload-time = "2026-09-24T21:22:27.008Z" }, { url = "https://files.pythonhosted.org/packages/f0/18/bee0dd59ae622482dc6463636c79e4bde7c954d061c859c9256362c9931a/google_crc32c-1.9.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4488f1553a9ab7e86cdedc833374a7e904031803b995dc0bd0be48c271fa6556", size = 37776, upload-time = "2026-09-24T21:38:11.056Z" }, { url = "https://files.pythonhosted.org/packages/fd/b6/e76e80fed5f2558273c7839e622f98095c9b36c719c7147e38e3c055cb70/google_crc32c-1.9.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0568b17ed90ac596f29400d99e243fd0cc6276766183def888d1bf8d1dc13827", size = 36350, upload-time = "2026-09-24T21:38:12.138Z" }, { url = "https://files.pythonhosted.org/packages/87/34/165542bfa99dfef91a76471cc48cce74b8ff4e295722896087ab2b8e8611/google_crc32c-1.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:8583ec21d56b565d68ab2963cc7e21b3b271247c29b04286068255ef65f221bd", size = 35090, upload-time = "2026-09-24T21:39:29.764Z" }, { url = "https://files.pythonhosted.org/packages/8f/eb/43ea41f4061a1cad87b2b6559c98e960e45bf551fe66f83d833b98aaf0c9/google_crc32c-1.9.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:6a3b2c8a343c570ed8100a7627c20badfd92c6caa2067093a86be45af27f5b1b", size = 31963, upload-time = "2026-09-24T21:19:03.208Z" }, { url = "https://files.pythonhosted.org/packages/45/d2/a968c0c29ccd2b0c980ff4f9e3f7035cee28c23a1c57541825cc8221858c/google_crc32c-1.9.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:13179f7e3282617923e957b8e54b8f9c3968030f48640a9f47fd7c5c38c4a215", size = 31805, upload-time = "2026-09-24T21:22:27.917Z" }, { url = "https://files.pythonhosted.org/packages/03/73/388e493d6c3e252e37165d22efe5a1361f872a24425391b999822861b23a/google_crc32c-1.9.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:265233aff33d835f5b909584fe36ab29647b598c271b661a300001099109e53e", size = 37925, upload-time = "2026-09-24T21:38:13.32Z" }, { url = "https://files.pythonhosted.org/packages/98/36/190d32caa363ef25d685f422ed1bbf93ff1140fb22fd4d90f24cec209977/google_crc32c-1.9.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:dee799544cae42a42b17a88e38b59cf2c271051dc001da2117a8ff240ffa0548", size = 36495, upload-time = "2026-09-24T21:38:14.211Z" }, { url = "https://files.pythonhosted.org/packages/d3/fd/81cefea6adae7bd92abb23d4567d199f6485a20ec0a305ca5fa04c52b9c5/google_crc32c-1.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:af73200fa9791ccd380f3598235dba8d82b8af0905df045b3dc60b59836e8ddd", size = 35535, upload-time = "2026-09-24T21:39:30.52Z" }, { url = "https://files.pythonhosted.org/packages/c5/18/19d4f17f3f33f8fdffcb3e1e69219d6f7ec2c359c160867b04dac1d0a64d/google_crc32c-1.9.0-cp315-cp315-macosx_12_0_arm64.whl", hash = "sha256:e6e8be8a94436079cb5340f6d495d9d7ba30124d8b952703994c739c7c06e236", size = 31977, upload-time = "2026-09-24T21:19:03.976Z" }, { url = "https://files.pythonhosted.org/packages/81/b4/8010372c4b46f2ee2352dfdb630c397570cd85522a315df024ad2f9459aa/google_crc32c-1.9.0-cp315-cp315-macosx_12_0_x86_64.whl", hash = "sha256:f2b64641bca27497b986b9d87883014035aa904cb4fa333407c6752b3afee9ba", size = 31809, upload-time = "2026-09-24T21:22:29.1Z" }, { url = "https://files.pythonhosted.org/packages/c5/f8/7e33845d6b90ce1cf37cfabf25cb859277c7d3533ef1b6b1e1ca58581549/google_crc32c-1.9.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f97c3806dcea41c29c04965347b0e12481561b75e0045dc7a4f69d75dec5d9b1", size = 38359, upload-time = "2026-09-24T21:38:14.983Z" }, { url = "https://files.pythonhosted.org/packages/36/ff/556b2423f449a7515af6b8222a4d7833cbe09ff3e8d2f0b80471f5f6d02e/google_crc32c-1.9.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0abe7e202c25909869c35672ab0f2fe748a7acf276eb78577332a7c38999740f", size = 36858, upload-time = "2026-09-24T21:38:15.799Z" }, { url = "https://files.pythonhosted.org/packages/40/71/4733f1b7c921d04a2bb9b9916cf66498bf7ad0860a06289413830da83192/google_crc32c-1.9.0-cp315-cp315-win_amd64.whl", hash = "sha256:5695c8b9327e040b2aba12c6659b0acb5995314ef0af0192da66e662e011103b", size = 35534, upload-time = "2026-09-24T21:39:31.337Z" }, ] [[package]] name = "graphql-core" version = "3.2.12" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/11/7f/671c1046fe72ba5b62be2de3979ea9e61cb3dba8f1edfb880b811f8bdf8b/graphql_core-3.2.12.tar.gz", hash = "sha256:4579094d5fc8a1a59555a9b18e51b320779d9bbc63e2302c519af0c4919d9543", size = 531478, upload-time = "2026-08-27T20:25:48.607Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/4e/b6/304c572e79b9e82982dbd5091e26a89f5018e565868091d804c2142f6ba5/graphql_core-3.2.12-py3-none-any.whl", hash = "sha256:3d8f104532070485e13caa4092c1e71cda2ba6cffd96e98f285111ee10ed1e51", size = 216042, upload-time = "2026-08-27T20:25:47.234Z" }, ] [[package]] name = "griffe-inherited-docstrings" version = "1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffelib" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cb/da/fd002dc5f215cd896bfccaebe8b4aa1cdeed8ea1d9d60633685bd61ff933/griffe_inherited_docstrings-1.1.3.tar.gz", hash = "sha256:cd1f937ec9336a790e5425e7f9b92f5a5ab17f292ba86917f1c681c0704cb64e", size = 26738, upload-time = "2026-02-21T09:38:44.312Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/16/20/4bc15f242181daad1c104e0a7d33be49e712461ea89e548152be0365b9ea/griffe_inherited_docstrings-1.1.3-py3-none-any.whl", hash = "sha256:aa7f6e624515c50d9325a5cfdf4b2acac547f1889aca89092d5da7278f739695", size = 6710, upload-time = "2026-02-20T11:06:38.75Z" }, ] [[package]] name = "griffelib" version = "2.0.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/9d/82/74f4a3310cdabfbb10da554c3a672847f1ed33c6f61dd472681ce7f1fe67/griffelib-2.0.2.tar.gz", hash = "sha256:3cf20b3bc470e83763ffbf236e0076b1211bac1bc67de13daf494640f2de707e", size = 166461, upload-time = "2026-03-27T11:34:51.091Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/11/8c/c9138d881c79aa0ea9ed83cbd58d5ca75624378b38cee225dcf5c42cc91f/griffelib-2.0.2-py3-none-any.whl", hash = "sha256:925c857658fb1ba40c0772c37acbc2ab650bd794d9c1b9726922e36ea4117ea1", size = 142357, upload-time = "2026-03-27T11:34:46.275Z" }, ] [[package]] name = "hypothesis" version = "6.168.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "sortedcontainers" }, ] sdist = { url = "https://files.pythonhosted.org/packages/09/b7/13118bbc45d6d8b9d04e2de779e2a4ff23145ea39efa692b994fb874ca72/hypothesis-6.168.3.tar.gz", hash = "sha256:a43388f9067678fef6e13bdff325b6cfa6961a590498bb37f7ff31589c83bc75", size = 511022, upload-time = "2026-09-28T05:20:58.499Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/2b/cd/746a2fc1e5e5ba34f07ea6ee1ad07525a3dde5ff4836db3a7980af49a891/hypothesis-6.168.3-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d20972ca134e652e9928ecd200966a8a2857adc2812c1d74b12a872e5cd503eb", size = 791536, upload-time = "2026-09-28T05:18:57.162Z" }, { url = "https://files.pythonhosted.org/packages/c3/0f/7a158e377b69556c8e12c25c8fd811d0103e54c24a12dab1f3b9819f202e/hypothesis-6.168.3-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:eafbec09d3e87d13d8242411d1f5868b5e879f1bd6d95e233e9ff80c527bea1c", size = 787313, upload-time = "2026-09-28T05:20:18.122Z" }, { url = "https://files.pythonhosted.org/packages/45/c6/7df9104ee359e8fcd77791781c47e70794964ca69673665e0de2a6590f4c/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73c5627497968cc62d140e9fde1ea21e12cac7b64dc419fea8286cd34ff1ad4e", size = 1124036, upload-time = "2026-09-28T05:20:04.372Z" }, { url = "https://files.pythonhosted.org/packages/92/0f/8a61715404a73b9a82cb1f976803428d603cdea976a9c30266c72ad6a7ce/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dc56e6dc6eeb0aeb08ac463f847279bde2336c4786faf7ee0af4b93cd031a7", size = 1147875, upload-time = "2026-09-28T05:19:15.495Z" }, { url = "https://files.pythonhosted.org/packages/e4/3f/2b16e95cc3b9a069afa23830a012aefe377ce4a457a227e5a5026eb827ff/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:753bb501f8560d2e321ed3b62596b4496c56f15668b0a0669231ccc3e6c4e80d", size = 1149489, upload-time = "2026-09-28T05:19:18.886Z" }, { url = "https://files.pythonhosted.org/packages/f6/e7/d7cf6dc068bb02732b2a6e38a35a0dcb7f2137608198fc79d740f69d197c/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71ab606c472449cb872ef2a7acaec679ee4f651b7f0a477a04ebc85d8933ef8c", size = 1191992, upload-time = "2026-09-28T05:19:21.83Z" }, { url = "https://files.pythonhosted.org/packages/5f/26/f1e5b25dec15998e8375d722825ec3dae1075cbe7cb1f74221b2312a2e33/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26928956c54748e4dfa587333741ab750246123b93484955646ff316cca27eef", size = 1169911, upload-time = "2026-09-28T05:19:27.996Z" }, { url = "https://files.pythonhosted.org/packages/03/36/901234a49147e6bf8a5b9e54b775706a223c2b52befea1106ea2aaf15d48/hypothesis-6.168.3-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:0bdfc53c041b61c854fa3761735736b991bc2bddafee45a361cfe4d43027c1ef", size = 1129406, upload-time = "2026-09-28T05:18:42.632Z" }, { url = "https://files.pythonhosted.org/packages/55/c1/5fc9ff91ec9fba6bb527833386c66812524b5cc75fe7d4a4f9122ee1c420/hypothesis-6.168.3-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:650528e2b1e2a45e95c2df624d4b9364b8ac5a027ba84e1eea2bc5398dd01dcd", size = 1160399, upload-time = "2026-09-28T05:20:23.911Z" }, { url = "https://files.pythonhosted.org/packages/9b/d2/49ad1ef5c55547c8abfd0fc571b3493806dda530fd5cb3d8263dbad86a72/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:209dc54cdb1b4d6d7020ad8d09e44c49756361b7183adacea4d6f5705085b595", size = 1299906, upload-time = "2026-09-28T05:18:58.417Z" }, { url = "https://files.pythonhosted.org/packages/0d/5f/f98a26094c4f462c4215807bed0f9a6b5508b27a3c329bcba1bc8adc7ac7/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:af8ca98cd11dc7f9427bb90483abcd9688d4df2e8e63893a30a2423b027ebb11", size = 1425538, upload-time = "2026-09-28T05:20:49.074Z" }, { url = "https://files.pythonhosted.org/packages/d0/b8/c5b4406e3a6591e51a42f85469351d1d824e0a54b84167aec9b2353759fe/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:f8122cfdd0bc0ba843063effa22ac310bdebdb3a1319bf1e63f14baa119c72ab", size = 1377084, upload-time = "2026-09-28T05:20:06.618Z" }, { url = "https://files.pythonhosted.org/packages/e0/01/0d84a8ea469d024c15f602799c0b8410fbe03d99fdb0370449b33c3c916f/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4bedbb379eab34f792af7ee9a05aae04e9c08bbb52e0f90f5a2110d4fe4b2fbd", size = 1281162, upload-time = "2026-09-28T05:19:42.768Z" }, { url = "https://files.pythonhosted.org/packages/22/b5/ea6435038de1a795f005cb603531e8ccc053febc50180f3ab55583c9c60e/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:5a953115b9f5c95133ab2d04efffeec96e5658c3207923dca7285f7db3e6bbef", size = 1300433, upload-time = "2026-09-28T05:18:55.634Z" }, { url = "https://files.pythonhosted.org/packages/6c/78/fa6c77d9f64b6d592e69e582d83cbce7d7e56897faa4762885a2123b0166/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6173558e676ad25ed1e20507fa4024a0d816dd90f715f77006e5a28f19109026", size = 1336273, upload-time = "2026-09-28T05:19:08.273Z" }, { url = "https://files.pythonhosted.org/packages/f1/56/4acd0d778bab818c9ea336fcba768bf43bc70db7f84cc373579298df4641/hypothesis-6.168.3-cp310-abi3-win32.whl", hash = "sha256:dc66390fb12d80585aa9222bf538ce8b7aa22cf5d118250647355a1c9e8f62f4", size = 678211, upload-time = "2026-09-28T05:19:23.355Z" }, { url = "https://files.pythonhosted.org/packages/b0/db/0a02b146ad1c30f9716362f2e1a379dfd6b0f7be03ac1e22c9da24c9e2a9/hypothesis-6.168.3-cp310-abi3-win_amd64.whl", hash = "sha256:92325b276360fe86c5bf71a568c0d53a6d140b0de36dcf029f9164a17803bb24", size = 684906, upload-time = "2026-09-28T05:18:35.099Z" }, { url = "https://files.pythonhosted.org/packages/a3/c0/958deaf726848f96f52250740bf39f13f476b068e22f73e2b358eaa07532/hypothesis-6.168.3-cp310-abi3-win_arm64.whl", hash = "sha256:3cf6f1eeaf41cd8d60cf1f88fde905ca1dd77c906929a507d6ac7f66f2ccba2a", size = 683292, upload-time = "2026-09-28T05:19:57.404Z" }, { url = "https://files.pythonhosted.org/packages/7a/56/44a17266bcf1add5fba7c3df5c3920396a5e924d917fb0534a6f9e2b9b9a/hypothesis-6.168.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:bff12e036b67abc5ded682f724f4d7afaff5223190270a0763695f218c079068", size = 793313, upload-time = "2026-09-28T05:20:22.045Z" }, { url = "https://files.pythonhosted.org/packages/d8/18/1a25b11e9133cb54c02ee6cf99a61296ad461aabc7aae3399e1d0ff1f095/hypothesis-6.168.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c73c6188056e6dc110260e39d141c69c3d02f23e3ba1d8610c7a8b6f2510ec96", size = 784855, upload-time = "2026-09-28T05:19:34.496Z" }, { url = "https://files.pythonhosted.org/packages/8b/da/728d3210f0b0089337f37206a100d31ef546d1f99839e709f0a9ca443f8e/hypothesis-6.168.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb1063cb794765097b8c0add598bd34a904f44dc15db81275dd1f0ed1ac567ed", size = 1123043, upload-time = "2026-09-28T05:19:05.468Z" }, { url = "https://files.pythonhosted.org/packages/53/86/bfb9ec344a8b3ef2fa6f59bb3d30c5b3a7d86d594107670539c34ee53d1b/hypothesis-6.168.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b2d47f5d9060b049039bef0b267e88480f6468f95ee394e514e84a81ccdc5f5", size = 1169121, upload-time = "2026-09-28T05:20:40.512Z" }, { url = "https://files.pythonhosted.org/packages/3b/e0/5a76dbafbcaf8fff075f09377cddefa17289ef05dbf51878dde88385b966/hypothesis-6.168.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8367f623a98cbf90f33fd2a43b100671b152b5975bc343a578290945f52f7018", size = 1298749, upload-time = "2026-09-28T05:19:32.883Z" }, { url = "https://files.pythonhosted.org/packages/28/15/0e65af96b353f2c4bdc283c53ed0980e5ac5009afe97ef3b23c77b364664/hypothesis-6.168.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fd81241f4cd76fb18d481e87b9688b30a0cb5c30841730513323ba1c7aba1837", size = 1335271, upload-time = "2026-09-28T05:20:54.005Z" }, { url = "https://files.pythonhosted.org/packages/f5/c5/37eab2155664aa544df321f6d3940b89c6702d79a72fdc98d7b5a2286f8c/hypothesis-6.168.3-cp312-cp312-win_amd64.whl", hash = "sha256:377438de53afb94347d9845b7d90db5905c6d2a8d612108e9938e0f80503bb90", size = 682211, upload-time = "2026-09-28T05:18:51.206Z" }, { url = "https://files.pythonhosted.org/packages/0c/a2/6787da846d929e52fc3344d803299c45782dbeac287528af08380e984bf8/hypothesis-6.168.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1b230a850de63334c16654a34a2d547e0179d36b9071d4439b3e7237f6d077e7", size = 793254, upload-time = "2026-09-28T05:19:36.294Z" }, { url = "https://files.pythonhosted.org/packages/89/ba/2893f5ca42501f4d562cba3229c8994c3a3e5fac66ef60c1e0e0b5220a5d/hypothesis-6.168.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d63b0226cd3e0d8bdd97c3384b22a21934ed4d53246c1c93575dada616672499", size = 784699, upload-time = "2026-09-28T05:18:54.194Z" }, { url = "https://files.pythonhosted.org/packages/b7/aa/7d7349daf75b71f6f35876f8de115e974c5c04d600e86df1b2779b0883ab/hypothesis-6.168.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0369f5df055f96e117ab12e5f249668ff144731ab5280bc7a205fdf81f990b89", size = 1123016, upload-time = "2026-09-28T05:20:36.372Z" }, { url = "https://files.pythonhosted.org/packages/00/f0/7774e1ea072708ea46cb25c4aeee5f9978b6c271764809069e8a6789858e/hypothesis-6.168.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f076bcd0f77fdcdb7797826c879099573d03a02e228ebe649ea917081b962ac0", size = 1168989, upload-time = "2026-09-28T05:20:08.44Z" }, { url = "https://files.pythonhosted.org/packages/a7/7d/113992abed9efbd7944e3a496a382da58152dce126fddf6419ff31a0a57d/hypothesis-6.168.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b823ba1fcec8da730f29316d010b06d3f7e0c3828dcf91020e24c55e7d24652a", size = 1298626, upload-time = "2026-09-28T05:19:40.986Z" }, { url = "https://files.pythonhosted.org/packages/d5/65/3659fa5e733027e5b37a27486e57f4053535e40853d23501c422bb275043/hypothesis-6.168.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dd2849c269d674e4618590f3b48d443bd4c06b5aef3d8d869086c2e6d213d248", size = 1335184, upload-time = "2026-09-28T05:20:20.087Z" }, { url = "https://files.pythonhosted.org/packages/b4/6c/35aab2221b5ea65125340f236e1a765a9e9f3b28d89a389ea0033fed29f7/hypothesis-6.168.3-cp313-cp313-win_amd64.whl", hash = "sha256:3ef7d26f5789e691401d5f87eafed9bd2763f0dbe47af6d6b66012a509404766", size = 682173, upload-time = "2026-09-28T05:19:12.762Z" }, { url = "https://files.pythonhosted.org/packages/6e/79/27b0cb56ff5d2bc92458bf6fcebdb1f0dc01f57f043a178e9f9d74498169/hypothesis-6.168.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:e2d4c68729a13df9af4998d2652cfb5d541c5609b88c306880a5dfb284938ec2", size = 793287, upload-time = "2026-09-28T05:20:38.522Z" }, { url = "https://files.pythonhosted.org/packages/e8/bd/5342f95c3bc36586777ef8cb8eba87ac7acf0184210327fb0e6d8654e295/hypothesis-6.168.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b345f818083ec99966a43ca4f7b38feb62c6920ce28572bc2bde4948a8b7eaba", size = 784857, upload-time = "2026-09-28T05:20:42.609Z" }, { url = "https://files.pythonhosted.org/packages/4a/5f/fc774241518a0588d362680a3084275058aab86d6bc02a2c61e1073142d0/hypothesis-6.168.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0608c610fc002978fc5de0f471770d8817e9455cb8649a47983c9f8bccfc1897", size = 1123330, upload-time = "2026-09-28T05:19:29.735Z" }, { url = "https://files.pythonhosted.org/packages/b0/e6/131f16775a3dca5f4fe27f0d6ad9a9851600098a54a872d163f6ebf0b664/hypothesis-6.168.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9dcf6448b1ecc37f2b23f2d1b3ddfc9ff6b6910614c15a642dc82f419b96037b", size = 1169178, upload-time = "2026-09-28T05:19:53.436Z" }, { url = "https://files.pythonhosted.org/packages/06/61/c9f5bff8b73321c9fd12f2d69666c6e4861b97b60f5f24fdd6ac293007fd/hypothesis-6.168.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f9f094041dcce5119ebd7bab71062743ab02b6e654d056b370beda78c19e2", size = 1299127, upload-time = "2026-09-28T05:20:14.486Z" }, { url = "https://files.pythonhosted.org/packages/e4/6d/d4618f8ab12dd76c4d58ea052252759aa2bf0c4e71f4b502ba5d577e0b9f/hypothesis-6.168.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d479985fe73af97badfb72dc6d20c6a353e486a36f6d065f600026e0cf954a86", size = 1335390, upload-time = "2026-09-28T05:20:10.326Z" }, { url = "https://files.pythonhosted.org/packages/95/b6/727161e17cc297df1aeea945de05c532033b66751625bec192bc5b5ff707/hypothesis-6.168.3-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:785e2c45f8c08e274b4bf1ccae97f1a4e09407e9a68e80790cfab1d17a4a45fa", size = 624291, upload-time = "2026-09-28T05:18:36.376Z" }, { url = "https://files.pythonhosted.org/packages/77/23/2f9b506392a0b8712440bcb781abc5713be9ffac1303b6d10f3669050882/hypothesis-6.168.3-cp314-cp314-win_amd64.whl", hash = "sha256:320920b1e3dae8611eee8a03d063cf2187446f2a17c38cfb8a7fc1466f71eee2", size = 682064, upload-time = "2026-09-28T05:20:16.336Z" }, { url = "https://files.pythonhosted.org/packages/50/93/efabfd95eb2b69c0c9fa1e1c83c8290aa2715d46baeb5a7764faf09c133a/hypothesis-6.168.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:35380baa981108a7f60c4eab71e46acd8d8f58440520346a1a6aba06dca7e074", size = 791875, upload-time = "2026-09-28T05:19:37.789Z" }, { url = "https://files.pythonhosted.org/packages/1b/fc/2a0ada1623a9a33048bbf9f00a7c92513398c6b8865cba5efb854262ce30/hypothesis-6.168.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f0aaaed00438fa6673d856aaee12a4a8afbde82a9f3c5ff487cacfb064239afe", size = 783412, upload-time = "2026-09-28T05:18:48.266Z" }, { url = "https://files.pythonhosted.org/packages/41/31/73c615d37eeb12da0209555612c32ced628d860267c23cfd2691c6207aea/hypothesis-6.168.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27e6df1576bf838e7f484d4ae5cba92114497ca30afb917056bf1ea33e95675", size = 1121662, upload-time = "2026-09-28T05:19:26.225Z" }, { url = "https://files.pythonhosted.org/packages/92/54/1893344a3b8bbf83fa7c9bb7e96f6836580213854f46671cb749f7b4706c/hypothesis-6.168.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bc85014577982ec6d2e266edc5cd6e7a7d3674c648791ca983c67a52f89a0c4", size = 1167761, upload-time = "2026-09-28T05:18:49.85Z" }, { url = "https://files.pythonhosted.org/packages/26/97/a61d82968febd25a0f08ffbbfabfaeead66f68fc5fa73f3c9ece1c36fc8f/hypothesis-6.168.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ccfc29505aa1cdcc254cb9cd701fd0811fef5480addd97d4127a00df023410f7", size = 1297309, upload-time = "2026-09-28T05:19:51.711Z" }, { url = "https://files.pythonhosted.org/packages/bc/01/fe8e4cf6d39d02f4efadaa351427fa13722086cfc54a2da3579741c5177a/hypothesis-6.168.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32d0699566aaa93f9e97a44705d7164386f1e91de78d277f53bada35e78bcc96", size = 1334260, upload-time = "2026-09-28T05:19:00.121Z" }, { url = "https://files.pythonhosted.org/packages/39/6a/09177ebc62f4778f94379ecade6a51299d3d8c4f4b75ba636bf3a0202364/hypothesis-6.168.3-cp314-cp314t-win_amd64.whl", hash = "sha256:28d88fa174ecbd4ecbd7bb290f06d0db084a3971c3f511ae2830b65b5e25500f", size = 681992, upload-time = "2026-09-28T05:19:48.043Z" }, { url = "https://files.pythonhosted.org/packages/f9/f6/890bf33d608cd63348d3146a3ca83362fbf09e589c5e5230a00c403070f4/hypothesis-6.168.3-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:61f5782d807b1e6aa5c9beef1054cb2037e7d1add48a64972cd6ee447778781f", size = 791231, upload-time = "2026-09-28T05:18:43.998Z" }, { url = "https://files.pythonhosted.org/packages/03/dd/fc36b204f8aa7437c676576d9437f0422de46aa8a729e6cd5bbb0224e759/hypothesis-6.168.3-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:7aacf3cf40c7ce8f9e4924d5b57bc0b068beafdfed2347bcf160a28b4cfbba7b", size = 783171, upload-time = "2026-09-28T05:19:20.353Z" }, { url = "https://files.pythonhosted.org/packages/5c/d2/3cd087d577db67c404991d7723e64d7cd75570cdc82a0f7329aed6f2086c/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01768a03a30dc54df7fe457c0b34016c84d598fa00d5965ededab93ba4eb3408", size = 1121152, upload-time = "2026-09-28T05:18:39.095Z" }, { url = "https://files.pythonhosted.org/packages/42/6f/a05a68cc66e7a22701c50bf94fb9ddeb36f4f5d0de3cc130066709eda621/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1a8a4ffc6c6e6e577f2bfbcfebf7cffbb310283ba2532c729570a0a752cebaaa", size = 1144069, upload-time = "2026-09-28T05:20:44.705Z" }, { url = "https://files.pythonhosted.org/packages/4a/f4/fdd7a093fb2fb3a0ce24256c92ad5bf67936f4669bd06d5ceae4298150ec/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b987d73eba95183a7e59cca6d1925c588aa4307922d9852cc2b4d282a2ae4128", size = 1146692, upload-time = "2026-09-28T05:20:25.83Z" }, { url = "https://files.pythonhosted.org/packages/f1/c8/6dbd4377e935505ae4fc8ee4c7b18c69994c4bbaca015447c470218bdbee/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4569c39bd97d9573e946429ed676f3b55a7c8ed80920addd67d384a47d59b38", size = 1189480, upload-time = "2026-09-28T05:20:46.734Z" }, { url = "https://files.pythonhosted.org/packages/ba/bf/ff288b496b690000d2686dcfa7f67855e4c5a6dddb464ed8e4880be83bb2/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6042b8707a4b25bbbfe20b110258fb7549a68951e5525608a8ce06091039e9fc", size = 1167109, upload-time = "2026-09-28T05:19:24.758Z" }, { url = "https://files.pythonhosted.org/packages/d9/d1/3fee2bc29fc747fabbb27e174515592e390809ca465d3bbbabbcfa3235a8/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:f413629de94d38a7a2ad259697a6752526143cb49c2e2d7bdba19381c80693f6", size = 1126823, upload-time = "2026-09-28T05:18:52.636Z" }, { url = "https://files.pythonhosted.org/packages/58/6a/585294392fa6a6d9719446a042a777ba98ecca265d9e48cbedc10c88df0f/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f071737e4e775bebba07e319eb645a880d1e0186b4d24bad23255e849d86d483", size = 1155773, upload-time = "2026-09-28T05:20:51.268Z" }, { url = "https://files.pythonhosted.org/packages/cd/4d/fa283ff79996debf1ff08593e01f8b773eb8211b19b3d6d5d491317a65bf/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:f59a3912858d0609c26054aa1c474847c797937f5e0e1c77f0d3f08032e50f09", size = 1296671, upload-time = "2026-09-28T05:18:46.937Z" }, { url = "https://files.pythonhosted.org/packages/42/05/98f9c2f628da5afabb6c3e8df5d2d299d2d50bf306b14a6e26d3954c4951/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:f09c05a23a8025dd5cad07c2ed299a46778e72d49ff0dd5bf353bcc0f7dc1580", size = 1422001, upload-time = "2026-09-28T05:19:04.188Z" }, { url = "https://files.pythonhosted.org/packages/b8/c9/f2109ade29a7ec284d55bca328d784743e8b13e321576fd34ee7e65f99af/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:44ace770bda3a0301739fc5a413c764de790c739df1f1a7218dd049f8594d9f5", size = 1374182, upload-time = "2026-09-28T05:20:27.764Z" }, { url = "https://files.pythonhosted.org/packages/ce/6d/e05d5f72441564a3bebc71fa155deafd0fd3b015d6014ec8a00edf6e42bb/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:03b131043608f94a2578a2a896a1a72092acb7079815eef5b8513b08447e0e62", size = 1278402, upload-time = "2026-09-28T05:20:56.332Z" }, { url = "https://files.pythonhosted.org/packages/96/d7/6988a7f1f69c5c530687dce03a32c157e2d878e3a2e32af9269ce016b78a/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:e9784aca26eddfe99b03a0292320db742cc8a74200ef864fdce949f526f973cd", size = 1297770, upload-time = "2026-09-28T05:19:11.387Z" }, { url = "https://files.pythonhosted.org/packages/a2/28/5bb82b60b836bd2329e1fe01ad94efc2b14dfa806f8e50ed14b795d78770/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:2b52ac363096232bebc2add117e9178d91f4248f4cdb919fd1026b5f86a4bb16", size = 1333977, upload-time = "2026-09-28T05:20:02.483Z" }, { url = "https://files.pythonhosted.org/packages/8e/7d/d419841b8f65481ea1a50c4ba36f2670d48b8c7a51e4e8586089564cf7f6/hypothesis-6.168.3-cp315-abi3.abi3t-win32.whl", hash = "sha256:d28e3a6b511a74ce37df5274b51f36c2b274fea7365e00b16f0c81e22acd5957", size = 675394, upload-time = "2026-09-28T05:19:55.237Z" }, { url = "https://files.pythonhosted.org/packages/d8/d2/1de6a2ad100e44817f2e9a8e8ba3eeaf4769621f4f87f2d4966c4971fcc4/hypothesis-6.168.3-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:7b9638789548361a57d984f56619ac694a914c328181d911271618409261ff4a", size = 681699, upload-time = "2026-09-28T05:19:09.957Z" }, { url = "https://files.pythonhosted.org/packages/c0/79/be3370fd02734d6b1d950183ae9224580d19fa8d356b95348d60f1e3eeb7/hypothesis-6.168.3-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:65d78e4357ec48ed2c67825f06740ee3599be4cfe770a6092bed07108d679ac5", size = 679849, upload-time = "2026-09-28T05:19:06.884Z" }, ] [[package]] name = "idna" version = "3.15" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, ] [[package]] name = "imagesize" version = "2.0.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/6c/e6/7bf14eeb8f8b7251141944835abd42eb20a658d89084b7e1f3e5fe394090/imagesize-2.0.0.tar.gz", hash = "sha256:8e8358c4a05c304f1fccf7ff96f036e7243a189e9e42e90851993c558cfe9ee3", size = 1773045, upload-time = "2026-03-03T14:18:29.941Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/5f/53/fb7122b71361a0d121b669dcf3d31244ef75badbbb724af388948de543e2/imagesize-2.0.0-py2.py3-none-any.whl", hash = "sha256:5667c5bbb57ab3f1fa4bc366f4fbc971db3d5ed011fd2715fd8001f782718d96", size = 9441, upload-time = "2026-03-03T14:18:27.892Z" }, ] [[package]] name = "iniconfig" version = "2.3.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] [[package]] name = "itsdangerous" version = "2.2.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410, upload-time = "2024-04-16T21:28:15.614Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" }, ] [[package]] name = "jinja2" version = "3.1.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markupsafe" }, ] sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] [[package]] name = "jmespath" version = "1.1.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, ] [[package]] name = "joserfc" version = "1.6.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5d/ac/d4fd5b30f82900eac60d765f179f0ba005825ac462cc8ced6e13ec685ab3/joserfc-1.6.8.tar.gz", hash = "sha256:878620c553a6ebdd76ccdc356782fee3f735f21a356d079a546b42a4670ace5f", size = 232930, upload-time = "2026-05-27T03:22:37.819Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/98/8c/5cdce2cf3ce8155849baf9a5e2ce77e89dc87ec3bdb38259e5d85fbc45bd/joserfc-1.6.8-py3-none-any.whl", hash = "sha256:22fb31a69094a5e6f44632002a9df2c30c941fc6c8ce1b037e92c03de954cf9f", size = 70927, upload-time = "2026-05-27T03:22:35.796Z" }, ] [[package]] name = "jsonpatch" version = "1.33" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonpointer" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/78/18813351fe5d63acad16aec57f94ec2b70a09e53ca98145589e185423873/jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c", size = 21699, upload-time = "2023-06-26T12:07:29.144Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade", size = 12898, upload-time = "2023-06-16T21:01:28.466Z" }, ] [[package]] name = "jsonpath-ng" version = "1.8.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/32/58/250751940d75c8019659e15482d548a4aa3b6ce122c515102a4bfdac50e3/jsonpath_ng-1.8.0.tar.gz", hash = "sha256:54252968134b5e549ea5b872f1df1168bd7defe1a52fed5a358c194e1943ddc3", size = 74513, upload-time = "2026-02-24T14:42:06.182Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/03/99/33c7d78a3fb70d545fd5411ac67a651c81602cc09c9cf0df383733f068c5/jsonpath_ng-1.8.0-py3-none-any.whl", hash = "sha256:b8dde192f8af58d646fc031fac9c99fe4d00326afc4148f1f043c601a8cfe138", size = 67844, upload-time = "2026-02-28T00:53:19.637Z" }, ] [[package]] name = "jsonpointer" version = "3.1.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/18/c7/af399a2e7a67fd18d63c40c5e62d3af4e67b836a2107468b6a5ea24c4304/jsonpointer-3.1.1.tar.gz", hash = "sha256:0b801c7db33a904024f6004d526dcc53bbb8a4a0f4e32bfd10beadf60adf1900", size = 9068, upload-time = "2026-03-23T22:32:32.458Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/9e/6a/a83720e953b1682d2d109d3c2dbb0bc9bf28cc1cbc205be4ef4be5da709d/jsonpointer-3.1.1-py3-none-any.whl", hash = "sha256:8ff8b95779d071ba472cf5bc913028df06031797532f08a7d5b602d8b2a488ca", size = 7659, upload-time = "2026-03-23T22:32:31.568Z" }, ] [[package]] name = "jsonschema" version = "4.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, { name = "jsonschema-specifications" }, { name = "referencing" }, { name = "rpds-py" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, ] [[package]] name = "jsonschema-path" version = "0.4.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pathable" }, { name = "pyyaml" }, { name = "referencing" }, ] sdist = { url = "https://files.pythonhosted.org/packages/01/86/cfee6dd25843bec0760f456599a4f7e7e40221a934b9229fda0662c859bc/jsonschema_path-0.4.6.tar.gz", hash = "sha256:c89eb635f4d497c9ac328eeff359c489755838806a7d033510a692e9576f5c4b", size = 15302, upload-time = "2026-04-27T18:57:08.412Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/6c/43/3d3065c05a04bb550c143bfbb8e4fd7022cd327e1082bf257bac74923783/jsonschema_path-0.4.6-py3-none-any.whl", hash = "sha256:451354b5311fa955c3144e6e4e255388c751c0121c5570ec5bb9291dd42d08c9", size = 19565, upload-time = "2026-04-27T18:57:06.792Z" }, ] [[package]] name = "jsonschema-specifications" version = "2025.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "referencing" }, ] sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] [[package]] name = "lazy-object-proxy" version = "1.12.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/08/a2/69df9c6ba6d316cfd81fe2381e464db3e6de5db45f8c43c6a23504abf8cb/lazy_object_proxy-1.12.0.tar.gz", hash = "sha256:1f5a462d92fd0cfb82f1fab28b51bfb209fabbe6aabf7f0d51472c0c124c0c61", size = 43681, upload-time = "2025-08-22T13:50:06.783Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0d/1b/b5f5bd6bda26f1e15cd3232b223892e4498e34ec70a7f4f11c401ac969f1/lazy_object_proxy-1.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ee0d6027b760a11cc18281e702c0309dd92da458a74b4c15025d7fc490deede", size = 26746, upload-time = "2025-08-22T13:42:37.572Z" }, { url = "https://files.pythonhosted.org/packages/55/64/314889b618075c2bfc19293ffa9153ce880ac6153aacfd0a52fcabf21a66/lazy_object_proxy-1.12.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4ab2c584e3cc8be0dfca422e05ad30a9abe3555ce63e9ab7a559f62f8dbc6ff9", size = 71457, upload-time = "2025-08-22T13:42:38.743Z" }, { url = "https://files.pythonhosted.org/packages/11/53/857fc2827fc1e13fbdfc0ba2629a7d2579645a06192d5461809540b78913/lazy_object_proxy-1.12.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:14e348185adbd03ec17d051e169ec45686dcd840a3779c9d4c10aabe2ca6e1c0", size = 71036, upload-time = "2025-08-22T13:42:40.184Z" }, { url = "https://files.pythonhosted.org/packages/2b/24/e581ffed864cd33c1b445b5763d617448ebb880f48675fc9de0471a95cbc/lazy_object_proxy-1.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c4fcbe74fb85df8ba7825fa05eddca764138da752904b378f0ae5ab33a36c308", size = 69329, upload-time = "2025-08-22T13:42:41.311Z" }, { url = "https://files.pythonhosted.org/packages/78/be/15f8f5a0b0b2e668e756a152257d26370132c97f2f1943329b08f057eff0/lazy_object_proxy-1.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:563d2ec8e4d4b68ee7848c5ab4d6057a6d703cb7963b342968bb8758dda33a23", size = 70690, upload-time = "2025-08-22T13:42:42.51Z" }, { url = "https://files.pythonhosted.org/packages/5d/aa/f02be9bbfb270e13ee608c2b28b8771f20a5f64356c6d9317b20043c6129/lazy_object_proxy-1.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:53c7fd99eb156bbb82cbc5d5188891d8fdd805ba6c1e3b92b90092da2a837073", size = 26563, upload-time = "2025-08-22T13:42:43.685Z" }, { url = "https://files.pythonhosted.org/packages/f4/26/b74c791008841f8ad896c7f293415136c66cc27e7c7577de4ee68040c110/lazy_object_proxy-1.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:86fd61cb2ba249b9f436d789d1356deae69ad3231dc3c0f17293ac535162672e", size = 26745, upload-time = "2025-08-22T13:42:44.982Z" }, { url = "https://files.pythonhosted.org/packages/9b/52/641870d309e5d1fb1ea7d462a818ca727e43bfa431d8c34b173eb090348c/lazy_object_proxy-1.12.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:81d1852fb30fab81696f93db1b1e55a5d1ff7940838191062f5f56987d5fcc3e", size = 71537, upload-time = "2025-08-22T13:42:46.141Z" }, { url = "https://files.pythonhosted.org/packages/47/b6/919118e99d51c5e76e8bf5a27df406884921c0acf2c7b8a3b38d847ab3e9/lazy_object_proxy-1.12.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be9045646d83f6c2664c1330904b245ae2371b5c57a3195e4028aedc9f999655", size = 71141, upload-time = "2025-08-22T13:42:47.375Z" }, { url = "https://files.pythonhosted.org/packages/e5/47/1d20e626567b41de085cf4d4fb3661a56c159feaa73c825917b3b4d4f806/lazy_object_proxy-1.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:67f07ab742f1adfb3966c40f630baaa7902be4222a17941f3d85fd1dae5565ff", size = 69449, upload-time = "2025-08-22T13:42:48.49Z" }, { url = "https://files.pythonhosted.org/packages/58/8d/25c20ff1a1a8426d9af2d0b6f29f6388005fc8cd10d6ee71f48bff86fdd0/lazy_object_proxy-1.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:75ba769017b944fcacbf6a80c18b2761a1795b03f8899acdad1f1c39db4409be", size = 70744, upload-time = "2025-08-22T13:42:49.608Z" }, { url = "https://files.pythonhosted.org/packages/c0/67/8ec9abe15c4f8a4bcc6e65160a2c667240d025cbb6591b879bea55625263/lazy_object_proxy-1.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:7b22c2bbfb155706b928ac4d74c1a63ac8552a55ba7fff4445155523ea4067e1", size = 26568, upload-time = "2025-08-22T13:42:57.719Z" }, { url = "https://files.pythonhosted.org/packages/23/12/cd2235463f3469fd6c62d41d92b7f120e8134f76e52421413a0ad16d493e/lazy_object_proxy-1.12.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4a79b909aa16bde8ae606f06e6bbc9d3219d2e57fb3e0076e17879072b742c65", size = 27391, upload-time = "2025-08-22T13:42:50.62Z" }, { url = "https://files.pythonhosted.org/packages/60/9e/f1c53e39bbebad2e8609c67d0830cc275f694d0ea23d78e8f6db526c12d3/lazy_object_proxy-1.12.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:338ab2f132276203e404951205fe80c3fd59429b3a724e7b662b2eb539bb1be9", size = 80552, upload-time = "2025-08-22T13:42:51.731Z" }, { url = "https://files.pythonhosted.org/packages/4c/b6/6c513693448dcb317d9d8c91d91f47addc09553613379e504435b4cc8b3e/lazy_object_proxy-1.12.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8c40b3c9faee2e32bfce0df4ae63f4e73529766893258eca78548bac801c8f66", size = 82857, upload-time = "2025-08-22T13:42:53.225Z" }, { url = "https://files.pythonhosted.org/packages/12/1c/d9c4aaa4c75da11eb7c22c43d7c90a53b4fca0e27784a5ab207768debea7/lazy_object_proxy-1.12.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:717484c309df78cedf48396e420fa57fc8a2b1f06ea889df7248fdd156e58847", size = 80833, upload-time = "2025-08-22T13:42:54.391Z" }, { url = "https://files.pythonhosted.org/packages/0b/ae/29117275aac7d7d78ae4f5a4787f36ff33262499d486ac0bf3e0b97889f6/lazy_object_proxy-1.12.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a6b7ea5ea1ffe15059eb44bcbcb258f97bcb40e139b88152c40d07b1a1dfc9ac", size = 79516, upload-time = "2025-08-22T13:42:55.812Z" }, { url = "https://files.pythonhosted.org/packages/19/40/b4e48b2c38c69392ae702ae7afa7b6551e0ca5d38263198b7c79de8b3bdf/lazy_object_proxy-1.12.0-cp313-cp313t-win_amd64.whl", hash = "sha256:08c465fb5cd23527512f9bd7b4c7ba6cec33e28aad36fbbe46bf7b858f9f3f7f", size = 27656, upload-time = "2025-08-22T13:42:56.793Z" }, { url = "https://files.pythonhosted.org/packages/ef/3a/277857b51ae419a1574557c0b12e0d06bf327b758ba94cafc664cb1e2f66/lazy_object_proxy-1.12.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c9defba70ab943f1df98a656247966d7729da2fe9c2d5d85346464bf320820a3", size = 26582, upload-time = "2025-08-22T13:49:49.366Z" }, { url = "https://files.pythonhosted.org/packages/1a/b6/c5e0fa43535bb9c87880e0ba037cdb1c50e01850b0831e80eb4f4762f270/lazy_object_proxy-1.12.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6763941dbf97eea6b90f5b06eb4da9418cc088fce0e3883f5816090f9afcde4a", size = 71059, upload-time = "2025-08-22T13:49:50.488Z" }, { url = "https://files.pythonhosted.org/packages/06/8a/7dcad19c685963c652624702f1a968ff10220b16bfcc442257038216bf55/lazy_object_proxy-1.12.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fdc70d81235fc586b9e3d1aeef7d1553259b62ecaae9db2167a5d2550dcc391a", size = 71034, upload-time = "2025-08-22T13:49:54.224Z" }, { url = "https://files.pythonhosted.org/packages/12/ac/34cbfb433a10e28c7fd830f91c5a348462ba748413cbb950c7f259e67aa7/lazy_object_proxy-1.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0a83c6f7a6b2bfc11ef3ed67f8cbe99f8ff500b05655d8e7df9aab993a6abc95", size = 69529, upload-time = "2025-08-22T13:49:55.29Z" }, { url = "https://files.pythonhosted.org/packages/6f/6a/11ad7e349307c3ca4c0175db7a77d60ce42a41c60bcb11800aabd6a8acb8/lazy_object_proxy-1.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:256262384ebd2a77b023ad02fbcc9326282bcfd16484d5531154b02bc304f4c5", size = 70391, upload-time = "2025-08-22T13:49:56.35Z" }, { url = "https://files.pythonhosted.org/packages/59/97/9b410ed8fbc6e79c1ee8b13f8777a80137d4bc189caf2c6202358e66192c/lazy_object_proxy-1.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:7601ec171c7e8584f8ff3f4e440aa2eebf93e854f04639263875b8c2971f819f", size = 26988, upload-time = "2025-08-22T13:49:57.302Z" }, ] [[package]] name = "librt" version = "0.13.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/dc/2f/3908645ddddab7120b46295e541ead308109fa48dbec7d67d7a778870d60/librt-0.13.0.tar.gz", hash = "sha256:1d2a610c14ac0d0750ee0a3ab8548e83155258387891caaca04def4bf7289781", size = 211402, upload-time = "2026-07-08T12:26:29.834Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f0/f4/b2933ddae222dac338476abb872641169a5cfed2c2bb5444a5b07b32b0c3/librt-0.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:30536798f4504c0fad0885b1d371b0539abb081e4570c9d7c641cb51141b49f0", size = 150990, upload-time = "2026-07-08T12:25:02.42Z" }, { url = "https://files.pythonhosted.org/packages/90/ef/db98f744ca50e6efc9c95c70ee49b77aefac31f6a3fc7c83754a42d6a74f/librt-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:93d24ebb82aa4420b1409c389e7857bc35bd0b668007ac8172427d5c73cc8cc5", size = 155238, upload-time = "2026-07-08T12:25:03.681Z" }, { url = "https://files.pythonhosted.org/packages/03/e7/a197e7bc72baf2c61ce7fdc6906a5054dc05bd8da0819aa894e4857bf87e/librt-0.13.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb8a1adce42d8b75485a5d56a9623a50bcab995b6079f1dac59fc44034dd93d9", size = 503073, upload-time = "2026-07-08T12:25:05.049Z" }, { url = "https://files.pythonhosted.org/packages/f8/e7/7887712e27da7c1ab80fcabb1de6eb24243964f6557cae530d4b70706dbd/librt-0.13.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:0763ca2ab66058174f9dee426dc64f5e0a89c24a7df8d3fe3f1836c04e25de4b", size = 496528, upload-time = "2026-07-08T12:25:06.26Z" }, { url = "https://files.pythonhosted.org/packages/94/f0/f2283385bb6b950b26a1410f4ce51ec27231e0b3a4b925c46366d218b198/librt-0.13.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b222493da6e7b6199db9bd79502436cf5a27da3c1f7fa83c7e285444fc93fd03", size = 531786, upload-time = "2026-07-08T12:25:07.658Z" }, { url = "https://files.pythonhosted.org/packages/36/11/69ac3b54766ffba5fd7e5acebfb048d66dbe1f9f2d14516c2b3edc59cf87/librt-0.13.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fadc63331f4388c3dc90090448f682a7e9feafc11481391c1e94f2f907a3976e", size = 524393, upload-time = "2026-07-08T12:25:09.121Z" }, { url = "https://files.pythonhosted.org/packages/61/5f/d72f95fd444a926a3c14b4e24979474116988dd57a45be242077c45d3c22/librt-0.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:70d9c62a4cffd9f23396cd5ef93fc5d11b31596b9b7d6306074abe3d5fcf09bd", size = 543026, upload-time = "2026-07-08T12:25:10.459Z" }, { url = "https://files.pythonhosted.org/packages/c4/08/dcd9993ad192737a004ba263d549f8ea605b326b952e7d6205c7d4170b76/librt-0.13.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:66c0e7e6b02a155576df2c77ec933a70b72da726e248c494abf690923e624348", size = 546829, upload-time = "2026-07-08T12:25:11.716Z" }, { url = "https://files.pythonhosted.org/packages/96/d5/6d9bb2f54e4109a956b7128836529653eb9d740f784bc47ed10a02c1000e/librt-0.13.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:ac04bcd3328eb91d99dfedf6a60d9c1f15d3434e6f6daf922f0420f7d90b85c7", size = 535700, upload-time = "2026-07-08T12:25:13.144Z" }, { url = "https://files.pythonhosted.org/packages/8c/f2/10946922503858a359492fa27f13e86228bde702116a740ac7b3cd185f24/librt-0.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:db327e7271e653c32040b85ae6188059c924b57d7e1e29f935523fa017cd4e82", size = 573566, upload-time = "2026-07-08T12:25:14.336Z" }, { url = "https://files.pythonhosted.org/packages/48/a8/94f00e3c99479a18088af3685ea016c42f3c7d5d1964d8dbb40c08d7f1aa/librt-0.13.0-cp312-cp312-win32.whl", hash = "sha256:860bd1d8ba48456ce08feaf8d343a8aaeb2fa086f2bcaa2a923fa3f7a3ff9aa3", size = 106099, upload-time = "2026-07-08T12:25:16.159Z" }, { url = "https://files.pythonhosted.org/packages/c9/7b/2da9c74c1ed25a89cc4e1c8e007ea2eb4a0f1fafa3e70d757fe3242c5c5c/librt-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:e54a315caf843c8d77e388cadc56ea9ded569935ee2d2347d7ea94992e5aa6fa", size = 126934, upload-time = "2026-07-08T12:25:17.275Z" }, { url = "https://files.pythonhosted.org/packages/d0/65/aead61bbf3b5358593f9d4779d2a0e88eaf6ec191a6342dde36dd1df6371/librt-0.13.0-cp312-cp312-win_arm64.whl", hash = "sha256:c718e99a0992127af84385378460db624103b559ab260435abcfe77a4e4ed1c1", size = 112236, upload-time = "2026-07-08T12:25:18.425Z" }, { url = "https://files.pythonhosted.org/packages/67/3b/18e7b63255297a2bdc9c25c8d6d4ca8eca9f63aceb1252c0f7427ac7099e/librt-0.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a468951af16155824e88bdd8326ebe5bdb371f3ec0ac04642994b98201d914f3", size = 151027, upload-time = "2026-07-08T12:25:19.638Z" }, { url = "https://files.pythonhosted.org/packages/4d/68/e2248452c00d1a03b45fee1752cdc8f790a476efd2402b75181da88a9e61/librt-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ae01d8512cc17079e53425635327dbf3f7ff57a42c00dec348bf79791c56444c", size = 155152, upload-time = "2026-07-08T12:25:20.851Z" }, { url = "https://files.pythonhosted.org/packages/0e/16/52b1c99bf19057a062aac39c900cbb81499f6f75d6c537c14463d247ba78/librt-0.13.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32c26893cd085c1efe83219e78d866da23fb20a066101b8f68210004361d224c", size = 502499, upload-time = "2026-07-08T12:25:22.055Z" }, { url = "https://files.pythonhosted.org/packages/9f/54/b811151805c795f55e0dedee6ec687b75f9982a8105d240ea3910737a77b/librt-0.13.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5929da1981a46bcf4b28b1b9499905f0ff58e2419da402a048234e9783acbc4b", size = 496108, upload-time = "2026-07-08T12:25:23.296Z" }, { url = "https://files.pythonhosted.org/packages/8f/f8/094d6b2bd93f3fdaa54db54cc788c4a365333bddad65ab02e04da0b1d004/librt-0.13.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94b85d664d777bab6c0d709416cb42938251fda9e221b79e3a2215d85df5f4f9", size = 531576, upload-time = "2026-07-08T12:25:24.648Z" }, { url = "https://files.pythonhosted.org/packages/2e/40/541733d5755824f968f7ec39d78ffbd75d145964157ae5e69a09ec6d7326/librt-0.13.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:531b2df3e9fe96b1fcf73a6d165921e4656be5f58d631d384ebce344298368db", size = 524390, upload-time = "2026-07-08T12:25:25.898Z" }, { url = "https://files.pythonhosted.org/packages/c6/b5/255673cfdbf5ba663339d36cd863c897289ab4337577e19f9405ce059f36/librt-0.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:109b84a9edf69ad89dc1f66358659e14a031baca95e3e5b0060bd903ede8efd6", size = 543053, upload-time = "2026-07-08T12:25:27.436Z" }, { url = "https://files.pythonhosted.org/packages/9e/11/ab5005e9c9850710f21e354201bf090646349d3fabf5f951eaf70235729e/librt-0.13.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1304368a3e7ffc3e9db986796cc5326fdb5943a3567ecc137cff318e4240c0e7", size = 546387, upload-time = "2026-07-08T12:25:28.65Z" }, { url = "https://files.pythonhosted.org/packages/a2/04/a5d7ce1d1df1afd15ca283dcdf7530ac073e12d69ae8c40879dda96f7868/librt-0.13.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e4f9b472e7d308d94b62c801982065661158c6ed02790d6c7ddb4337cea0f9c1", size = 535970, upload-time = "2026-07-08T12:25:30.171Z" }, { url = "https://files.pythonhosted.org/packages/5a/76/927e267a6daa290174ac281b23c9804c8829b042ade9c6f24a065f540958/librt-0.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f836c37478f167a81200d8c8b2c920a22224564bed2c23d7aeec760965c367a", size = 573582, upload-time = "2026-07-08T12:25:31.507Z" }, { url = "https://files.pythonhosted.org/packages/10/24/b6c5213efe39c19f9e13605644d0cf063b4ddaa33ac2e45b088e23a70e2e/librt-0.13.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:4000d961ff9598ac6ea603c6c836a5ed49bc205ade5fc378b998dfe1e2c36628", size = 82189, upload-time = "2026-07-08T12:25:32.675Z" }, { url = "https://files.pythonhosted.org/packages/4c/00/d29736be177a906ac0b84a5b04b4fbfa22c776dc2f366de4172b0f968c08/librt-0.13.0-cp313-cp313-win32.whl", hash = "sha256:79e44cff71750d299d61a678e49995b0d5935a9cda238c2574daeca3ba536927", size = 106193, upload-time = "2026-07-08T12:25:33.692Z" }, { url = "https://files.pythonhosted.org/packages/c8/ac/aff6fb45393cb8912f39dfb156ef6b2d1cadb207ff465fc8f66141054be8/librt-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:54dab44a847d5ad1acd05c8a83fe518ae685516ecf4d3f7cc6e3df2a66767650", size = 126962, upload-time = "2026-07-08T12:25:34.769Z" }, { url = "https://files.pythonhosted.org/packages/d9/3a/d68cb2b334d53fd30fac81d3a489ce4ba0d9506f4df43fcf676b68352b19/librt-0.13.0-cp313-cp313-win_arm64.whl", hash = "sha256:d4cb6fbfdf874340ab5e51450753c0f817b6958a3621125ee695bbc3de866566", size = 112127, upload-time = "2026-07-08T12:25:35.981Z" }, { url = "https://files.pythonhosted.org/packages/7b/66/f49ae0d592bd45b6941e9a8bafcb6a87cddcd501ee7874707e767f01b585/librt-0.13.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:25218d94b1d2cbc0ba1d8a3f9dc9af578d9646e5ed16443a70cde1dfdcce6d71", size = 149818, upload-time = "2026-07-08T12:25:37.203Z" }, { url = "https://files.pythonhosted.org/packages/3d/50/51c76d74014d04fb95b6506d286808984b78a2f7a41039094e6b2194ac48/librt-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f26629539d4893c2957a16c41bb058e1e135c1f150f6a2e25ed047f64cf3f5c6", size = 154071, upload-time = "2026-07-08T12:25:39.399Z" }, { url = "https://files.pythonhosted.org/packages/b8/fe/f19b0f5f82d5a1f2da736586bc840abd00ce07d6388136ae80b7333883fc/librt-0.13.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4517d47b2b8af26975a406fba7d314de9696d864252e0257c6ea90238cfe27f", size = 494168, upload-time = "2026-07-08T12:25:40.641Z" }, { url = "https://files.pythonhosted.org/packages/94/bc/b8550c75775127fd31a5f20e8775997f7b527ad661fc8ddccd7497c064f7/librt-0.13.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:f19e181de5b3a1148bb3420b8c4b0b0ea0fce6950099724ad151d6cea5acc180", size = 491054, upload-time = "2026-07-08T12:25:41.905Z" }, { url = "https://files.pythonhosted.org/packages/30/14/4d0204867623df3f33f86efd3d3692ba5e01321443f4d6eab35a22697618/librt-0.13.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22034924f5b42d5a56371cf271771bfeaabf235a7a8b6264bef2d20013f786c6", size = 523006, upload-time = "2026-07-08T12:25:43.327Z" }, { url = "https://files.pythonhosted.org/packages/19/0a/c45fc9a260934696bace1ac5df1e148ac92bd71767aee3bf7cd7a4534f4c/librt-0.13.0-cp314-cp314-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c7897db4e95e22468bdda33d8e012ceacd0182abf001e6389d763f0def6286b9", size = 515058, upload-time = "2026-07-08T12:25:44.541Z" }, { url = "https://files.pythonhosted.org/packages/13/0a/50c5ce45b326854ef8fa6ae4c36cf5142e5c55315eaf9e51d0ae73ac4da3/librt-0.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1ce61b3746545029d4f5c17d6bd74b676254ad98433086c846ffb5e8fa73f007", size = 534025, upload-time = "2026-07-08T12:25:45.825Z" }, { url = "https://files.pythonhosted.org/packages/89/2d/08c413c8f93fc13b8103624fce38e5caa86cd08cbbc8465870ab287af54b/librt-0.13.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:46c330e82565962c761dbce7941be2cff7db674ee807455a8d0cadc5f9b759b0", size = 540557, upload-time = "2026-07-08T12:25:47.059Z" }, { url = "https://files.pythonhosted.org/packages/b3/c1/93af71fb4a364952210051811dd4e40174e79656b050c89cacac18af3330/librt-0.13.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:375f5af8f99cbaa99dd293af986e3d57caabc9ba81a5d3f021603764854197a1", size = 523201, upload-time = "2026-07-08T12:25:48.392Z" }, { url = "https://files.pythonhosted.org/packages/c1/6e/9766f07b676a4889d9f8bc2864e9ba5fff165653143ef4dda7df6aa34d16/librt-0.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9320d34c3376ae204b2cd176e8d4883a013934e0aef822f1aed9c536490c275d", size = 565740, upload-time = "2026-07-08T12:25:49.678Z" }, { url = "https://files.pythonhosted.org/packages/a2/1e/664e3472ce2b6e10e9b83f29d4a36eb982ff6b5a169ae7567bba3a4c4ff5/librt-0.13.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:9af313c66157a69dc69ea0059a66961692250e0dc95af9c385a48ffb770a0d16", size = 81611, upload-time = "2026-07-08T12:25:50.857Z" }, { url = "https://files.pythonhosted.org/packages/2f/d4/8582a4d65e2234673685e07309d02c230b28a85724eb0acbf13f019b7f6e/librt-0.13.0-cp314-cp314-win32.whl", hash = "sha256:f2a7253458e34f33543551394ae4fe104b497ec2a65ac266074de64c1df82e37", size = 100106, upload-time = "2026-07-08T12:25:52.03Z" }, { url = "https://files.pythonhosted.org/packages/63/ce/0cb99efe6086b46cd985dc26672166fae312a239690e75871f7fafbd3fc5/librt-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:a3dfe4edf10e8ed7e55b026a8bfc2c2a8704218b659cd4bffdf604fab966dc39", size = 121209, upload-time = "2026-07-08T12:25:53.166Z" }, { url = "https://files.pythonhosted.org/packages/26/85/4f3ccb083a3c9b0d42e223acdb3c3f507953324a59cdcab4826e8e2e3b89/librt-0.13.0-cp314-cp314-win_arm64.whl", hash = "sha256:68a5faee4bba381cb93b5961f684a514cf0053cb92308ff9c792c2fea0b174c6", size = 106404, upload-time = "2026-07-08T12:25:54.253Z" }, { url = "https://files.pythonhosted.org/packages/b2/77/333191499538c8e8189de7a4cba8e6f49ee949fd6d6e6324b21fd1522466/librt-0.13.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a38fb81d8376dfa2f8963b265fec07637802b0d01e2a127c19c66cb070fb24f5", size = 159231, upload-time = "2026-07-08T12:25:55.432Z" }, { url = "https://files.pythonhosted.org/packages/7a/9e/2aa83758f22c278b837a1d8025898434ce2b8bff36678d5330ecaef56dff/librt-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d4c8d9bd5abce34b2e75edb3bf37ab0f34e49b1f915a40ae8468eb7c85bc5b46", size = 161300, upload-time = "2026-07-08T12:25:56.585Z" }, { url = "https://files.pythonhosted.org/packages/bb/c0/86791e936553ca763d6b3c2fb4d31d596cd00e14fa631c283a40ba01559a/librt-0.13.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:387e2f1d27e89bffe0d3f520f0da0662c973fd607ca16c1808f8a5085419485e", size = 582056, upload-time = "2026-07-08T12:25:58.144Z" }, { url = "https://files.pythonhosted.org/packages/a8/d3/a9ec15984a185e000c4d2a16ba28bd623124ad4c38a10974c7ff78e3a893/librt-0.13.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:4f6db193d2e5e0ed60359b9a5a682cd67205d0d3b1e459a867dd4b5c4e7eaa7a", size = 562758, upload-time = "2026-07-08T12:25:59.544Z" }, { url = "https://files.pythonhosted.org/packages/3c/af/dbe36b78b19c06a55097f99305e4ea9458e2273e6ae16a3cbecaad7ee978/librt-0.13.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0d38604854e8d22faadf683ec6c02bb0f886e2ba56ef981a1c36ee275f21ea22", size = 602095, upload-time = "2026-07-08T12:26:00.991Z" }, { url = "https://files.pythonhosted.org/packages/2a/a8/2966891b4dd2830f5203fbee92ac2c4947653a2390ba73dfa44244fad025/librt-0.13.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:371f7ce73026815dafd51c50ce38416e91428b28c4b2ec97cd39271164b0045c", size = 593452, upload-time = "2026-07-08T12:26:02.352Z" }, { url = "https://files.pythonhosted.org/packages/61/f5/4df8bfc8405ecf8c0d525b4d69636f694bdd8620b313ec8b76e54a5926cc/librt-0.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3aaedf52171bee90860704c560bc798fe83b76247df47568e0197e9b13c735a0", size = 623729, upload-time = "2026-07-08T12:26:04.294Z" }, { url = "https://files.pythonhosted.org/packages/d6/13/9ac202dffc8db06f75d06c08c2f9f6ff054be67d21272dcc078fa1cc0c57/librt-0.13.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:96bad8725a4f196a798366c25ce075d1f7543a4ec045ffc13e6a7ec095cdab04", size = 617077, upload-time = "2026-07-08T12:26:05.845Z" }, { url = "https://files.pythonhosted.org/packages/6e/f0/ebe38610716aee5cb28efd95089bb90192096179802779381e1c5dcf239c/librt-0.13.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:6bf6a559ffe4a93bbea6cf31ddf01a7fd9ba342ef51f27beb178e318b74acd61", size = 599561, upload-time = "2026-07-08T12:26:07.21Z" }, { url = "https://files.pythonhosted.org/packages/4f/5c/c2e72e236fff7abc716d5b1753b8b8cd3ea85ac46fe17d2e7c51d4e1c723/librt-0.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:301067672387902c55f94b51d5022304b36c966ea9fe1f21caab99a9bef487c9", size = 645511, upload-time = "2026-07-08T12:26:08.562Z" }, { url = "https://files.pythonhosted.org/packages/0c/99/6203ce619dee940d6bfbe099ec3fe4be00a68e9d60f70abf906cf124fe66/librt-0.13.0-cp314-cp314t-win32.whl", hash = "sha256:5fdcf34f86de8fb66d7dc7589f96ba91c4aa46671200d400e6fd6f109a483f18", size = 104357, upload-time = "2026-07-08T12:26:09.828Z" }, { url = "https://files.pythonhosted.org/packages/52/dd/843b6314087c41657c7036d7914d8f294bdf9b580aa8513ea0588c8e9a3d/librt-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:260c33e92263fa629b4f6d3c51967a1c2158fe6c33237aaa3ebeac586b085259", size = 126998, upload-time = "2026-07-08T12:26:10.975Z" }, { url = "https://files.pythonhosted.org/packages/5f/5d/3dcec2884ba1b0806d1408612555c38dd5d68e90156b59f75f6e36435c3a/librt-0.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2f281549a4c52ac7bb97997f14353f8bd0e53a34ca0dad1c905cfd0b4a58ae99", size = 110771, upload-time = "2026-07-08T12:26:12.303Z" }, ] [[package]] name = "markdown" version = "3.10.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/2b/f4/69fa6ed85ae003c2378ffa8f6d2e3234662abd02c10d216c0ba96081a238/markdown-3.10.2.tar.gz", hash = "sha256:994d51325d25ad8aa7ce4ebaec003febcce822c3f8c911e3b17c52f7f589f950", size = 368805, upload-time = "2026-02-09T14:57:26.942Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl", hash = "sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36", size = 108180, upload-time = "2026-02-09T14:57:25.787Z" }, ] [[package]] name = "markdown-exec" version = "1.12.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pymdown-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6d/76/c47da8edb6a12b066728432fb3724109d9d91de5331df5073d12d272493f/markdown_exec-1.12.3.tar.gz", hash = "sha256:006b9cac46470a9499797bc9c579305ae4719e0a8e495e5401dfbf1e66ce7fb4", size = 77841, upload-time = "2026-07-07T09:53:13.838Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ad/a7/0279016386d611183ccc508c5688eb1e2133e8182164d7a2c6213b176f69/markdown_exec-1.12.3-py3-none-any.whl", hash = "sha256:48ac12a565f3f4331b1acd9efc48a0773e717eb7ca7c38e23c1d72ee61660de6", size = 37995, upload-time = "2026-07-07T09:53:12.619Z" }, ] [package.optional-dependencies] ansi = [ { name = "pygments-ansi-color" }, ] [[package]] name = "markdown-it-py" version = "4.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mdurl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, ] [[package]] name = "markupsafe" version = "3.0.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] [[package]] name = "mdurl" version = "0.1.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] [[package]] name = "mergedeep" version = "1.3.4" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/3a/41/580bb4006e3ed0361b8151a01d324fb03f420815446c7def45d02f74c270/mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8", size = 4661, upload-time = "2021-02-05T18:55:30.623Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/2c/19/04f9b178c2d8a15b076c8b5140708fa6ffc5601fb6f1e975537072df5b2a/mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307", size = 6354, upload-time = "2021-02-05T18:55:29.583Z" }, ] [[package]] name = "mike" version = "2.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jinja2" }, { name = "mkdocs" }, { name = "pyparsing" }, { name = "pyyaml" }, { name = "pyyaml-env-tag" }, { name = "verspec" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b4/47/fa87e9d56bef16cdfe34b059a437e8c6f7ec6f1b9c378871c3cf95ebea9c/mike-2.2.0.tar.gz", hash = "sha256:1e3858e32c0f125aac14432fc7848434358f9ae0962c5c5cde387ad47f6ad25e", size = 38450, upload-time = "2026-04-14T04:59:03.944Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/76/8e/56ccb09c7232a55403a7637caa21922f3b65901a37f5e8bdb405d0de0946/mike-2.2.0-py3-none-any.whl", hash = "sha256:e1f4981c1152eec7c2490a3401142292cc47d686194188416db2648fdfe1d040", size = 34026, upload-time = "2026-04-14T04:59:02.602Z" }, ] [[package]] name = "mkdocs" version = "1.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "ghp-import" }, { name = "jinja2" }, { name = "markdown" }, { name = "markupsafe" }, { name = "mergedeep" }, { name = "mkdocs-get-deps" }, { name = "packaging" }, { name = "pathspec" }, { name = "pyyaml" }, { name = "pyyaml-env-tag" }, { name = "watchdog" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bc/c6/bbd4f061bd16b378247f12953ffcb04786a618ce5e904b8c5a01a0309061/mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2", size = 3889159, upload-time = "2024-08-30T12:24:06.899Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/22/5b/dbc6a8cddc9cfa9c4971d59fb12bb8d42e161b7e7f8cc89e49137c5b279c/mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e", size = 3864451, upload-time = "2024-08-30T12:24:05.054Z" }, ] [[package]] name = "mkdocs-autorefs" version = "1.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown" }, { name = "markupsafe" }, { name = "mkdocs" }, ] sdist = { url = "https://files.pythonhosted.org/packages/52/c0/f641843de3f612a6b48253f39244165acff36657a91cc903633d456ae1ac/mkdocs_autorefs-1.4.4.tar.gz", hash = "sha256:d54a284f27a7346b9c38f1f852177940c222da508e66edc816a0fa55fc6da197", size = 56588, upload-time = "2026-02-10T15:23:55.105Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/28/de/a3e710469772c6a89595fc52816da05c1e164b4c866a89e3cb82fb1b67c5/mkdocs_autorefs-1.4.4-py3-none-any.whl", hash = "sha256:834ef5408d827071ad1bc69e0f39704fa34c7fc05bc8e1c72b227dfdc5c76089", size = 25530, upload-time = "2026-02-10T15:23:53.817Z" }, ] [[package]] name = "mkdocs-get-deps" version = "0.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mergedeep" }, { name = "platformdirs" }, { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ce/25/b3cccb187655b9393572bde9b09261d267c3bf2f2cdabe347673be5976a6/mkdocs_get_deps-0.2.2.tar.gz", hash = "sha256:8ee8d5f316cdbbb2834bc1df6e69c08fe769a83e040060de26d3c19fad3599a1", size = 11047, upload-time = "2026-03-10T02:46:33.632Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/88/29/744136411e785c4b0b744d5413e56555265939ab3a104c6a4b719dad33fd/mkdocs_get_deps-0.2.2-py3-none-any.whl", hash = "sha256:e7878cbeac04860b8b5e0ca31d3abad3df9411a75a32cde82f8e44b6c16ff650", size = 9555, upload-time = "2026-03-10T02:46:32.256Z" }, ] [[package]] name = "mkdocs-material" version = "9.7.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "babel" }, { name = "backrefs" }, { name = "colorama" }, { name = "jinja2" }, { name = "markdown" }, { name = "mkdocs" }, { name = "mkdocs-material-extensions" }, { name = "paginate" }, { name = "pygments" }, { name = "pymdown-extensions" }, { name = "requests" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f1/cd/c05d3a530ba7934f144fb45f7203cd236adc25c7bdcc34673d202f4b0278/mkdocs_material-9.7.7.tar.gz", hash = "sha256:c0649c065b1b0512d60aad8c10f947f8e455284475239b364b610f2deb4d0855", size = 4097923, upload-time = "2026-07-17T16:21:33.156Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ad/21/17c1bc9e6f47c972ad66fb2ac2568f99f90f1207eeb6fc3b34d094dba7b5/mkdocs_material-9.7.7-py3-none-any.whl", hash = "sha256:8ea9bb1737a5b524a5f9dcf2e1b4ebda8274ae3008aa7845720a97083bef708f", size = 9305438, upload-time = "2026-07-17T16:21:30.017Z" }, ] [package.optional-dependencies] imaging = [ { name = "cairosvg" }, { name = "pillow" }, ] [[package]] name = "mkdocs-material-extensions" version = "1.3.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/79/9b/9b4c96d6593b2a541e1cb8b34899a6d021d208bb357042823d4d2cabdbe7/mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443", size = 11847, upload-time = "2023-11-22T19:09:45.208Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/5b/54/662a4743aa81d9582ee9339d4ffa3c8fd40a4965e033d77b9da9774d3960/mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31", size = 8728, upload-time = "2023-11-22T19:09:43.465Z" }, ] [[package]] name = "mkdocs-redirects" version = "1.2.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mkdocs" }, { name = "properdocs" }, ] sdist = { url = "https://files.pythonhosted.org/packages/73/25/49725f78ca5d3026b09973f7a2b3a8b179cc2e8c15e43d5a13bc79f6b274/mkdocs_redirects-1.2.3.tar.gz", hash = "sha256:5e980330999299729a2d6a125347d1af78023d68a23681a4de3053ce7dfe2e51", size = 7712, upload-time = "2026-03-28T13:57:41.766Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c6/90/871b1cddc01d2ba1637b858eeeabc2e3013dc8df591306b5567b98ef0870/mkdocs_redirects-1.2.3-py3-none-any.whl", hash = "sha256:ec7312fff462d03ec16395d0c001006a418f8d0c21cdf2b47ff11cf839dc3ce0", size = 6245, upload-time = "2026-03-28T13:57:40.466Z" }, ] [[package]] name = "mkdocstrings" version = "1.0.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jinja2" }, { name = "markdown" }, { name = "markupsafe" }, { name = "mkdocs" }, { name = "mkdocs-autorefs" }, { name = "pymdown-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/71/f85bdf13355073ae15a7375f09879375a830553552e58c1c4b7e0bbc5c8b/mkdocstrings-1.0.6.tar.gz", hash = "sha256:a0b8c2bdd29a6416c80d717aa369bbf7831946bd9f23c2a66db1b1dbe7693dbd", size = 100649, upload-time = "2026-07-11T19:38:05.732Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/5d/5b/4c1902e8bdd5c4db63284e9d101dece4038d4025d6d88850ffe0a1578980/mkdocstrings-1.0.6-py3-none-any.whl", hash = "sha256:2703708697487d1b6d6d7b412e176fa436edf120c1bf81dc9e126b12d00893c7", size = 35787, upload-time = "2026-07-11T19:38:04.417Z" }, ] [[package]] name = "mkdocstrings-python" version = "2.0.9" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "griffelib" }, { name = "mkdocs-autorefs" }, { name = "mkdocstrings" }, ] sdist = { url = "https://files.pythonhosted.org/packages/96/11/640067d1ad713ea99c233e53e57a4c300d9885f904a61371d7ef90dea549/mkdocstrings_python-2.0.9.tar.gz", hash = "sha256:ae945637dc0618c6beedbee169f10c0234b77f322c53db3fc9fb4c29b3013038", size = 203006, upload-time = "2026-09-22T13:43:27.712Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/62/0f/f7f926cfa06317bbdbd3cf0ff82f9e3a91767352eb01519c81499fddf575/mkdocstrings_python-2.0.9-py3-none-any.whl", hash = "sha256:c0233eff3f84d78110df50541918e7ec2bcff8e1614faddea34290a90321a416", size = 105698, upload-time = "2026-09-22T13:43:26.345Z" }, ] [[package]] name = "moto" version = "5.2.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "boto3" }, { name = "botocore" }, { name = "cryptography" }, { name = "requests" }, { name = "responses" }, { name = "werkzeug" }, { name = "xmltodict" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6c/0f/1682c01ca0608c25526afb150246a3c9c1f609caccbd39758de4850e31bc/moto-5.2.3.tar.gz", hash = "sha256:a9e95c3218b6eda18e74571f1ced11cb1bc3151467d562c1da6037b9d19832cb", size = 8785514, upload-time = "2026-08-22T18:46:00.199Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/58/80/ee7ace7a49bd525455497fa2ff198111877d6f93b908b657adeb2cf057a4/moto-5.2.3-py3-none-any.whl", hash = "sha256:5ec31b3cdbb383a19d46030157bd82aaaa7a9cc5f564200f995663d347bd4728", size = 6780972, upload-time = "2026-08-22T18:45:55.719Z" }, ] [package.optional-dependencies] s3 = [ { name = "py-partiql-parser" }, { name = "pyyaml" }, ] server = [ { name = "antlr4-python3-runtime" }, { name = "aws-xray-sdk" }, { name = "cfn-lint" }, { name = "docker" }, { name = "flask" }, { name = "flask-cors" }, { name = "graphql-core" }, { name = "joserfc" }, { name = "jsonpath-ng" }, { name = "openapi-spec-validator" }, { name = "py-partiql-parser" }, { name = "pyparsing" }, { name = "pyyaml" }, { name = "setuptools" }, ] [[package]] name = "mpmath" version = "1.3.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, ] [[package]] name = "msgpack" version = "1.2.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/31/f9/c0a1c127f9049db9155afc316952ea571720dd01833ff5e4d7e8e6352dbb/msgpack-1.2.1.tar.gz", hash = "sha256:04c721c2c7448767e9e3f2520a475663d8ee0f09c31890f6d2bd70fd636a9647", size = 183960, upload-time = "2026-06-18T16:13:52.594Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/bc/dd/9e8cbd8f5582ca4b590336f2b91ee5662f6a6ca562b565abaf696a0f81ff/msgpack-1.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2ef59c659f289eddf8aa6623823f19fa2f40a4029266889eac7a2505dd210c35", size = 83531, upload-time = "2026-06-18T16:12:58.249Z" }, { url = "https://files.pythonhosted.org/packages/50/2e/ebdb85a8da151397a2790363676b7ed7c125924fe618e4c6d8befb0cc62c/msgpack-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3567748a5107cb40cdf66a275430c2f87c07777698f4bfd25c35f44d533258c", size = 82657, upload-time = "2026-06-18T16:12:59.396Z" }, { url = "https://files.pythonhosted.org/packages/26/aa/753ad8b007b464e1d8aa0c8e650b9c5f4f725e658fc5ac8a7635c55b7f6e/msgpack-1.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60926b75d00c8e816ef98f3034f484a8bc64242d66839cef4cf7e503142316a0", size = 410634, upload-time = "2026-06-18T16:13:00.383Z" }, { url = "https://files.pythonhosted.org/packages/6a/fd/6adabd4f6d5e686f97dd02ce7fce3fe4cf672cbac36b8f67ff4040e8ad8b/msgpack-1.2.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:020e881a764b20d8d7ca1a54fc01b8175519d108e3c3f194fddc200bda95951a", size = 419989, upload-time = "2026-06-18T16:13:01.776Z" }, { url = "https://files.pythonhosted.org/packages/5a/cc/85039b7b0eb168aaad7383a23c97e291a11f08351cb45a606ce865e4e3f1/msgpack-1.2.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4202c74688ca06591f78cb18988228bd4cca2cc75d57b60008372892d2f1e6e6", size = 377544, upload-time = "2026-06-18T16:13:03.637Z" }, { url = "https://files.pythonhosted.org/packages/ed/bf/35963899493b32030c85fc513b723ae66144ac70c11ebc52e889e16e3d99/msgpack-1.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8b267ce94efb76fbd1b3373511420074ee3187f0f7811bf394531de13294735a", size = 400842, upload-time = "2026-06-18T16:13:05.012Z" }, { url = "https://files.pythonhosted.org/packages/a6/df/8e2ac970c8f99264cd9997d1c73df5466bc19da3301d7dc5500862a9b089/msgpack-1.2.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e4f1d0f8f98ade9634e01fb704a408f9336c0a8f1117b369f5db83dc7551d8b1", size = 374108, upload-time = "2026-06-18T16:13:06.232Z" }, { url = "https://files.pythonhosted.org/packages/17/dd/fa8bd265110dfa51c20cb529f9e6d240a16fafe7e645004c6af2d01353ba/msgpack-1.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f02cf17a6ca1abe29b5f980644f7551f94d71f2011509b26d8625ce038f0df64", size = 414939, upload-time = "2026-06-18T16:13:07.478Z" }, { url = "https://files.pythonhosted.org/packages/2e/b9/8377a5ad8953fc0437c70cc98d9ae29f27fe5ac5109fbec0812085865735/msgpack-1.2.1-cp312-cp312-win32.whl", hash = "sha256:0c0d9802354507bcba62af19c17918e3eb437cc25e6f50657d511b5856a77aac", size = 64504, upload-time = "2026-06-18T16:13:08.822Z" }, { url = "https://files.pythonhosted.org/packages/57/7f/ce1e377df7e62461fefd9eb23bfb93a4a523f40a517b377b8f844d836828/msgpack-1.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:5c24aa15d5963051e1a5c62b12c50cd705992502b5ec1f3bece6046f33c9fc24", size = 71421, upload-time = "2026-06-18T16:13:09.828Z" }, { url = "https://files.pythonhosted.org/packages/8f/32/ebfe84c9929f08f188d56c7a2fd913406a9ddad76a634697c1c43b8112e6/msgpack-1.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:4227224aaec8f7fbcbfbd4272319347b2bb4030366502600f8c45588c5187b07", size = 64775, upload-time = "2026-06-18T16:13:11.056Z" }, { url = "https://files.pythonhosted.org/packages/b0/ac/dcddcab6f6c20ecb387ca5e980371cdb3f87ff69aeca388be97eebc4c074/msgpack-1.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0a70e3cf2804a300d921bb0940426e35f4e489a23adfb77a808892241db0a064", size = 83151, upload-time = "2026-06-18T16:13:12.173Z" }, { url = "https://files.pythonhosted.org/packages/64/71/fbcfa83a1d6a9c6091942d1cfd070962244664b87427a9a49a6897b1b219/msgpack-1.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:491cc39455ca765fad51fb451bf2915eb2cf41192ab5801ce8d67c1d614fe056", size = 82351, upload-time = "2026-06-18T16:13:13.194Z" }, { url = "https://files.pythonhosted.org/packages/e3/10/ddf7b06db879e8792d13934ddda09ff20bd2a583fd84c9b59aae9b0e650b/msgpack-1.2.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f310233ef7fb9c14e201c93639fe5f5260b005f56f0b29048e999c30935596cc", size = 407518, upload-time = "2026-06-18T16:13:14.233Z" }, { url = "https://files.pythonhosted.org/packages/79/d3/36a46a8ed992b781acbc05928bd5bee3c810cb0c3563bf81a7b0c04a1a76/msgpack-1.2.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:787c9bebb5833e8f6fc8abca3c0597683d8d87f56a8842b6b89c75a5f3176e2d", size = 416405, upload-time = "2026-06-18T16:13:15.435Z" }, { url = "https://files.pythonhosted.org/packages/f9/84/e8e9598b557c0ba6ddae901a73780a4c75ac667dddf59414b1e56a42fb34/msgpack-1.2.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dc871b997a9370d855b7394465f2f350e847a5b806dd38dcc9c989e7d87da155", size = 376257, upload-time = "2026-06-18T16:13:17.022Z" }, { url = "https://files.pythonhosted.org/packages/40/16/738fe6d875ad7e2a9429c165322a4ec088f4f273cdfae63d96a89c467961/msgpack-1.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:85f57e960d877f2977f6430896191b04a21f8901b3b4baf2e4604329f4db5402", size = 397469, upload-time = "2026-06-18T16:13:18.287Z" }, { url = "https://files.pythonhosted.org/packages/ca/be/6d5952df75a7f24f35833af764c3a6860780364cb3a0030beb8099e1b2b4/msgpack-1.2.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:1233ee2dd0cefba127583de50ea654677277047d238303521db35def3d7b2e7c", size = 372802, upload-time = "2026-06-18T16:13:19.685Z" }, { url = "https://files.pythonhosted.org/packages/e1/39/e2ef7dbf0473bcb8dc7c50bf782a892d67414877b63e47fc88eb189ef5e6/msgpack-1.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e3dc2feb0876209d9c38aa56cb1de169bd6c4348f1aa48271f241226590993e6", size = 411273, upload-time = "2026-06-18T16:13:21.028Z" }, { url = "https://files.pythonhosted.org/packages/ef/c5/133f4512a56e983a93445c836c9d94d88f3bc2e0980ff4b9e577bd8416ce/msgpack-1.2.1-cp313-cp313-win32.whl", hash = "sha256:6d09badf350af2be9d189184e04e64cf54ad93569ab3d96fca58bd3e84aad707", size = 64471, upload-time = "2026-06-18T16:13:22.293Z" }, { url = "https://files.pythonhosted.org/packages/e2/98/577e10b055096a7dd40732358cabaf7180a20c79ed1dcdbb618e4b9deac7/msgpack-1.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:33f14fba63278b714efe6ad07e50ea5f03d91537aa6a1c5f1ceca4cf44013ca9", size = 71274, upload-time = "2026-06-18T16:13:23.455Z" }, { url = "https://files.pythonhosted.org/packages/ba/ee/0c0048e7cfbef23c6a94791b8959ab28155232e7956de8a305b5ff588f05/msgpack-1.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:afc5febcd4c99effbc02b528e49d6fd0760b2b7d48c05239e345a5fa6e743d9a", size = 64795, upload-time = "2026-06-18T16:13:24.687Z" }, { url = "https://files.pythonhosted.org/packages/77/58/cce442852c6b9e1639c7c8ac8fd9143121cb32dab0f308df4d1426a8eb9c/msgpack-1.2.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:05f340e47e7e47d2da8db9b53e1bb1d294369e9ef45a747441309f6650b8351d", size = 83610, upload-time = "2026-06-18T16:13:25.724Z" }, { url = "https://files.pythonhosted.org/packages/60/5c/15b4c7a0182f75ffa90751958ba36a9c01cafee367d49a3edc10ed140b01/msgpack-1.2.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:810b916696c86ef0deb3b74588480224df4c1b071136c34183e4a2a4284d7ac7", size = 83138, upload-time = "2026-06-18T16:13:26.781Z" }, { url = "https://files.pythonhosted.org/packages/b8/a6/99e58722feaffc5f2fbcc0c8c0d1451ab9f84097f7af87291b46af2390f4/msgpack-1.2.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ca0dacff965c47afdc3749a8469d7302a8f801d6a28758d55120d75e66ce6889", size = 406090, upload-time = "2026-06-18T16:13:28.072Z" }, { url = "https://files.pythonhosted.org/packages/19/03/8c63e8cf52958534ef688625965ab04c269a6cadd8caef16758b380a821a/msgpack-1.2.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e2bf9280bceb5efca998435904b5d3e9fdbcc11d90dc9df30aec7973252b720", size = 412106, upload-time = "2026-06-18T16:13:29.427Z" }, { url = "https://files.pythonhosted.org/packages/63/d2/155d9e71b40e41fd934bc0c48b9b2770f22263e1ac20aad8e29fdca7be3f/msgpack-1.2.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6c4be5d1c02a42b066ca6ddb71adf36432868fdcdb6ee87e634e86e0674190", size = 374851, upload-time = "2026-06-18T16:13:30.631Z" }, { url = "https://files.pythonhosted.org/packages/98/48/deaf2326262a8d5ea3295ce9649912ecd3f551ba7ec8e33c665d2ba583f3/msgpack-1.2.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec0e675d59150a6269ddc9139087c722292664a37d071a849c05c473350f1f2d", size = 396168, upload-time = "2026-06-18T16:13:31.977Z" }, { url = "https://files.pythonhosted.org/packages/10/2a/b4410f906c2ec0008f1608d3ab5143afc3ad3f4e6da0fed3ea2231d0bef4/msgpack-1.2.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:dd3bfe82d53edfe4b7fc9a7ec9761e23a7a5b1dac22264505af428253c29ed24", size = 371959, upload-time = "2026-06-18T16:13:33.282Z" }, { url = "https://files.pythonhosted.org/packages/59/86/1edc67270099a528fa2093ea60fe191233cd238e4bd30cfacf7db79fc959/msgpack-1.2.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5ad5467fc3f68b5468e06c5f788d712e9f8ffc8b0cd1bcb160c105c1ee92dae7", size = 408457, upload-time = "2026-06-18T16:13:34.567Z" }, { url = "https://files.pythonhosted.org/packages/82/90/8b630fef07d8c5ab457b71ff2c217910c83d333c7a68472c186e87cc504a/msgpack-1.2.1-cp314-cp314-win32.whl", hash = "sha256:98b58bdb89c46190e4609bb36abe17c6d4105ad13f9c5f8f6f64d320f8ced3fb", size = 65942, upload-time = "2026-06-18T16:13:36.056Z" }, { url = "https://files.pythonhosted.org/packages/16/f1/467b81e98b24dd3885d7b1857728797b4ffc76a7a7483af4fb321a07de3c/msgpack-1.2.1-cp314-cp314-win_amd64.whl", hash = "sha256:74847557e28ce71bd3c438a447ca90e4b507e997ddbdef8a12a7b283b86c156b", size = 72627, upload-time = "2026-06-18T16:13:37.079Z" }, { url = "https://files.pythonhosted.org/packages/a7/1d/5d8c4c89985feb6acefb82a09e501c60392261856d2408d20bfe4f0360b1/msgpack-1.2.1-cp314-cp314-win_arm64.whl", hash = "sha256:b50b727bd652bdc37d950336c848ef20ec54a4cafc38dce19b1cd86ad625d0f7", size = 66908, upload-time = "2026-06-18T16:13:38.23Z" }, { url = "https://files.pythonhosted.org/packages/1b/02/ad2afb678b4de94496cd432b581759b756a92c1192d8c767edd6b132efdc/msgpack-1.2.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:8d00f177ca88a77c1cf848d204a38f249751650b601cb6532acc68805d8a8273", size = 86000, upload-time = "2026-06-18T16:13:39.44Z" }, { url = "https://files.pythonhosted.org/packages/54/74/0b797484013128837f3b1cbb6cea019277c4de4e377dc512b4d9a0f92940/msgpack-1.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5bb9c386f0a329c035ddbab4b72d1028bf9627add8dda41070288563d57ed1b1", size = 86544, upload-time = "2026-06-18T16:13:40.447Z" }, { url = "https://files.pythonhosted.org/packages/a9/b4/b774d7eb95561739907fec675582f83203cf41c597a418c2589b4bfb8e9d/msgpack-1.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20466cca18c49c7292a8984bc15d65857b171e7264bdcb5f96baf8be238791fc", size = 427661, upload-time = "2026-06-18T16:13:41.574Z" }, { url = "https://files.pythonhosted.org/packages/b2/f9/3243191dc9937e00756c8bc1b0272fed8f23758e43df2a3b46f533e5090f/msgpack-1.2.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:196300e7e5d6e74d50f1607ab9c06c4a1484c383cd22defd727902591f7e8dde", size = 426375, upload-time = "2026-06-18T16:13:42.936Z" }, { url = "https://files.pythonhosted.org/packages/23/c7/1693111db9944ba4ad4b67a1e788400d78a0b6af7a6523dc7e4e58f8274b/msgpack-1.2.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:575957e79cd51903a4e8495a242442949641e08f1efd5197b43bebd3ea7682b4", size = 380495, upload-time = "2026-06-18T16:13:44.306Z" }, { url = "https://files.pythonhosted.org/packages/3e/2b/92f86956a0c13e8662f7e2ad630c4eb4db07497b967589bd5245e018b2c1/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8c2ed1e48cc0f460bf3c7780e7137ff21a4e18433451916f2442c1b21036cd7d", size = 410897, upload-time = "2026-06-18T16:13:45.629Z" }, { url = "https://files.pythonhosted.org/packages/da/ea/1479f72d200313a76fc2f823a79d1e07ed052ab7b8a0280640aa7b95de42/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5f6277e5f783c36786a145e0247fc189a03f35f84b251646e53592d2bc12b355", size = 378519, upload-time = "2026-06-18T16:13:46.998Z" }, { url = "https://files.pythonhosted.org/packages/f5/4d/fa006060ffa1011d32bfae826fe766fe73e02982183601633b7121058ab3/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f9389552ecf4784886345ead0647e4edc96bee37cbab05b75540f542f766c48c", size = 419815, upload-time = "2026-06-18T16:13:48.205Z" }, { url = "https://files.pythonhosted.org/packages/2f/e1/aab6c946570496b78e67804721f3d5e2d62a93081b9b37df77764ef56347/msgpack-1.2.1-cp314-cp314t-win32.whl", hash = "sha256:c1c79a604a2969a868a78b6ebd27a887e00c624f14f66b3038e0590cb23332d1", size = 70914, upload-time = "2026-06-18T16:13:49.385Z" }, { url = "https://files.pythonhosted.org/packages/13/0a/e608956488a2af014cfe6e3d665e090b8ee42aa14b07f8f95b8880d66b09/msgpack-1.2.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f12038a35fabd52e56a3547bab42401af49a45caa6dd00b34c44de235bc93ee2", size = 77999, upload-time = "2026-06-18T16:13:50.467Z" }, { url = "https://files.pythonhosted.org/packages/d2/8a/27e2e57055176e366a46b85d02d68e7a5bcfbdd8474c9706375d965f24d3/msgpack-1.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:0adcf06ffde0777c0e1a9b771a2b1c4226ba1bbf748c8efcc02fcdeca3299107", size = 71160, upload-time = "2026-06-18T16:13:51.498Z" }, ] [[package]] name = "msgspec" version = "0.22.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/d0/e6/6dcf9306ff3c5e486578f3bf29ed11dfbdbbc2a8bf0caf7e07d392887fda/msgspec-0.22.0.tar.gz", hash = "sha256:0a13624a4969159fe35d8c2a3d377b2b61bbd8585e327440d5e52725affcce38", size = 343188, upload-time = "2026-09-29T14:14:11.422Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a4/87/3e017dca361d09ed1cd09dc981a6df21b32e830fbec3470f7486d38b6be5/msgspec-0.22.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ab1e9e7531e353653b906cdd12a0220cc288a1e8e3436aabc65f4508d91b14d9", size = 201301, upload-time = "2026-09-29T14:12:38.048Z" }, { url = "https://files.pythonhosted.org/packages/fb/02/109165edaafb895668d87177972a32ade9126a54f3736123d8e44be9096d/msgspec-0.22.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b60b43425a47eb9cfe987f6874e354ca7c760e58e295b4e2273ff03574df28a1", size = 193044, upload-time = "2026-09-29T14:12:39.46Z" }, { url = "https://files.pythonhosted.org/packages/54/a5/65de05f8804492f76ea121b21a125cdf1d97ec461c677bfa0ba354d6fbdd/msgspec-0.22.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5a169b5b03f0f2c7a296c002647db1dab75d2cd501bca34e32b71cab0261b56", size = 224035, upload-time = "2026-09-29T14:12:40.876Z" }, { url = "https://files.pythonhosted.org/packages/4a/cc/aa1a47f8c92280d37498a5ea56a2a36606d034383e3e6472d64cbb56cf85/msgspec-0.22.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99c401861c5bb3a57f7d6423ea7ed4352cd57aa3f04f4fbe9f3e3e4564a10f08", size = 230377, upload-time = "2026-09-29T14:12:42.796Z" }, { url = "https://files.pythonhosted.org/packages/61/50/f8bcdb3d613a4a4b92704297a12eba5c985cf572a64ee1a004d265759c69/msgspec-0.22.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:08826f5e5b0fa2f7a88592c396a243cfcc63d37e19f9d4fbe3b3f1be2fbdc404", size = 237390, upload-time = "2026-09-29T14:12:44.282Z" }, { url = "https://files.pythonhosted.org/packages/cf/8a/473fa423f8fdd1b810b8652594323d7301df6920b62844d860daa0feff34/msgspec-0.22.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:21460f54cee9208239b1a8421fdf25bffc77293e1daba88f585711ad839b9758", size = 227733, upload-time = "2026-09-29T14:12:45.839Z" }, { url = "https://files.pythonhosted.org/packages/03/1d/272ce23adae6c71b3f763aed3ee6e115cccc56124ed8ee0e3e3d2681e2c8/msgspec-0.22.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:cfc3d9557de9c806318725b702f3e664db33167bb42892079b693c69893fd33b", size = 236783, upload-time = "2026-09-29T14:12:47.234Z" }, { url = "https://files.pythonhosted.org/packages/f6/26/29e0b9a8605c8819a3c718158e345a616ac42c092dd7d7ab248c2f2b0a72/msgspec-0.22.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0b25dcbc108783cb72503ed705b9fbb8c3cb02ee5801923f44b5f038c91cc365", size = 232728, upload-time = "2026-09-29T14:12:48.792Z" }, { url = "https://files.pythonhosted.org/packages/e1/a6/99597c281d716da6c662b48dcc3f734669f716b41d5df2af367dac9e7c21/msgspec-0.22.0-cp312-cp312-win_amd64.whl", hash = "sha256:6ad64f5c260866b0d543f89f50cee43628989c1433c5de7ce820281fa28a2611", size = 192885, upload-time = "2026-09-29T14:12:50.274Z" }, { url = "https://files.pythonhosted.org/packages/46/80/85fff923d448b886ec3a85900c578d9367f08dad54fe48879495b4c6d055/msgspec-0.22.0-cp312-cp312-win_arm64.whl", hash = "sha256:0922714feff5300aacd8ecd65fa828317ce4bf5212b3139258c0bfc0253cd80e", size = 191223, upload-time = "2026-09-29T14:12:51.699Z" }, { url = "https://files.pythonhosted.org/packages/7f/62/5374fba2ede0408f4bd8b9b3a6c8464f8d0ea7ae9a2a064bd81ca492bd1e/msgspec-0.22.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f13c127a945479bc9db057eb253b8851075c8e1ae07ffc967bfa1c5676203a86", size = 201355, upload-time = "2026-09-29T14:12:53.145Z" }, { url = "https://files.pythonhosted.org/packages/cc/e3/357baa8d2a9164a98dfd7ef9d3a58125df0ed981be909945bdd337be7194/msgspec-0.22.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5aa24eb475d070ecbbe5b21080fc3ce4b0b76c60de25cfe0c9678d8fb44bb42f", size = 193097, upload-time = "2026-09-29T14:12:54.52Z" }, { url = "https://files.pythonhosted.org/packages/fa/1b/9cc07718d1dee8ed5e89a265801d565bc0f15ead435ccb198f9c7bf92574/msgspec-0.22.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:627bfdfe5a4b3d916b3360b30f4cddeee3a084f56593e33527c6872fa8322ff9", size = 224112, upload-time = "2026-09-29T14:12:55.983Z" }, { url = "https://files.pythonhosted.org/packages/46/64/f33fdfe95aca76601194a7064d14816c7c22c4eccc1b03a5335785895fa3/msgspec-0.22.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6c310ef83e7e291b01a63298828f848348bb99e84a1098c4b3923c05674d032", size = 230472, upload-time = "2026-09-29T14:12:57.648Z" }, { url = "https://files.pythonhosted.org/packages/8e/b3/8ceaa9981c230adf43c45a6e8da25da23a381eddc7ed05aeaca1d5e7928b/msgspec-0.22.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7c1e76c6bd523141b9c05c2f8a70979cd0efedbd68855a66f292f8892c0b8fc7", size = 237382, upload-time = "2026-09-29T14:12:59.414Z" }, { url = "https://files.pythonhosted.org/packages/88/a6/7b5c4fb39e0bf2dabc8be923c33c39b07ba769a0ce6f0afbbdfaadb1f2f2/msgspec-0.22.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc374dedd5f85a5f4de2386dc5f737894ccb8c1ac18e9566ce66fd9839e6285d", size = 227717, upload-time = "2026-09-29T14:13:00.88Z" }, { url = "https://files.pythonhosted.org/packages/b8/5b/2334ee638880e756c8bc54a1177bd65877c786433693a43594ef5ecbe2d8/msgspec-0.22.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:feafe612034d49e9144340c0b5168ee4e22c2af4aaa2c1db11ae84e1aac9543b", size = 236781, upload-time = "2026-09-29T14:13:02.468Z" }, { url = "https://files.pythonhosted.org/packages/6c/e5/b4c5323b17ecfce45350695d40fc93e16856db957a53cbcf2f53007d6e12/msgspec-0.22.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6f48317f05312bfdf78248f53933f830f07ab75cc1c813ac3ca4220cb3b5b019", size = 232777, upload-time = "2026-09-29T14:13:04.025Z" }, { url = "https://files.pythonhosted.org/packages/01/33/e591f9d3d8d6c9cfc02ae95f3e3c44920f2d18050f3f252c244e0f293a0e/msgspec-0.22.0-cp313-cp313-win_amd64.whl", hash = "sha256:0739b068f31f2004a364f97679ba91f2f5ecd6ec2a5b4b890188ab5c57d20672", size = 192829, upload-time = "2026-09-29T14:13:05.519Z" }, { url = "https://files.pythonhosted.org/packages/d1/cd/a011a5b8732cd781e2ea6da5b38d71ae4a9a329338411d1f008a58f5edbf/msgspec-0.22.0-cp313-cp313-win_arm64.whl", hash = "sha256:508278300dd4efbd21cd3a4b2b016160a5feac98bc880d3673f6c06697baaf62", size = 191258, upload-time = "2026-09-29T14:13:06.909Z" }, { url = "https://files.pythonhosted.org/packages/53/f9/ac027b35477e6b83bcee32b3d9675b37abfa130f098dd6500fa67d768852/msgspec-0.22.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:221cbcbfa4478152b91d37dcfd4830e2be92773e8139e883f43773450ebacef8", size = 201276, upload-time = "2026-09-29T14:13:08.311Z" }, { url = "https://files.pythonhosted.org/packages/13/6b/2bffffa31662b1353a62e672442865d51c291ad778352fd490de16361dc6/msgspec-0.22.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dd9568695911055440d2bb7099ed9098fc181d335daa772d0eb3fe8f31ba4efb", size = 193233, upload-time = "2026-09-29T14:13:09.943Z" }, { url = "https://files.pythonhosted.org/packages/14/bc/4066416ff6aa918d1ef9295edee0041e4629e4079ad3839bdd8a68fd87f0/msgspec-0.22.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f039ef5207b847f075a0a43020ee6140cd47505f890e47e157f2deb485c2dc96", size = 225101, upload-time = "2026-09-29T14:13:11.391Z" }, { url = "https://files.pythonhosted.org/packages/63/ba/a8d390d5bd4c7d9ccde87c95cf071ada934cc9ca2c6af4d3d50b38f2d718/msgspec-0.22.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e4f7e09cceac7dbf4c0761b8ae7df51c55b5df5e9af7aff2c895aac1ebea015", size = 230505, upload-time = "2026-09-29T14:13:12.869Z" }, { url = "https://files.pythonhosted.org/packages/9c/89/979664fdc913c624ef88a139b40e3a95ddf2a47c89e8b5c4147f69ee9c48/msgspec-0.22.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:614e2c827e0a3f934f3cf0cf4ba65210df8132b75a69a8a1f51bb3b2caf0ac5a", size = 237382, upload-time = "2026-09-29T14:13:14.317Z" }, { url = "https://files.pythonhosted.org/packages/07/3f/7d44c614376ae008ac6099be5f589b322c4ad44e32c6dbb0edd256215028/msgspec-0.22.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa3689b9dfcc663358ef23ba4299d7460f01108515b041a7d30d05908ac9c32f", size = 228962, upload-time = "2026-09-29T14:13:15.763Z" }, { url = "https://files.pythonhosted.org/packages/0b/59/bf8504e6f63f6769d01fb66f8bd856cf0ed39a07fde354f440d711640054/msgspec-0.22.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d2f950239ff1fc7322c6f9634807310265149cb168270d3ddcdda5b6ada13a28", size = 236691, upload-time = "2026-09-29T14:13:17.195Z" }, { url = "https://files.pythonhosted.org/packages/2b/40/5a9d2bde12af16a22ddbf371990a81d3e3c0dcd4bb4ef3b3f9616b033c14/msgspec-0.22.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3c789b5ccd07c0a3c09767108ee06e089b2875f2309a4569c2648f30a8d31dfa", size = 232750, upload-time = "2026-09-29T14:13:18.691Z" }, { url = "https://files.pythonhosted.org/packages/75/5d/c0e6bdb81a87f6bd56a663a330c271af7670490c80d8d635d9fa21ad1adf/msgspec-0.22.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:a66b1766311e42371e509c996c3933b161c7ae0eabdf361af5316dec197e1022", size = 136814, upload-time = "2026-09-29T14:13:20.415Z" }, { url = "https://files.pythonhosted.org/packages/b9/c0/b0cfc6d33608e5ea8871f3be31f9146c56699e737a7d8862bf018484f278/msgspec-0.22.0-cp314-cp314-win_amd64.whl", hash = "sha256:749899563d26b211379f142b8ffd7e2d7da149a51717798f0ce994dce50324f0", size = 197097, upload-time = "2026-09-29T14:13:21.869Z" }, { url = "https://files.pythonhosted.org/packages/42/1f/571f7fe7c725380605d680fc4c0084212b23d2dfcf6be0f2277f14462c56/msgspec-0.22.0-cp314-cp314-win_arm64.whl", hash = "sha256:10d0d1d464960d99a949f7ca01ef8928e51c472433a5f5ab74b2d695fb830652", size = 196779, upload-time = "2026-09-29T14:13:23.62Z" }, { url = "https://files.pythonhosted.org/packages/ab/f3/3c87372bac651b37911e0dc6926c3958949d3fcb8cec1016adbc44d948b2/msgspec-0.22.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:e79725246291516a7359caad5fb743ddc0ec66ed40d2381fb846325b5031504e", size = 205214, upload-time = "2026-09-29T14:13:25.158Z" }, { url = "https://files.pythonhosted.org/packages/43/4c/fbccd6e0fbbdf10c4d9b6bac8a26148dd5483b3ffff6d6c5a376ff1f5cb1/msgspec-0.22.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:38f7022fbe91954b31afe3888a0af1b652e0f370fafdeb1d425f4a814d789c9f", size = 196941, upload-time = "2026-09-29T14:13:26.637Z" }, { url = "https://files.pythonhosted.org/packages/55/04/8db7186d3ae8818356bc623cc132db8b77da37ce4b1345f35719c8ad5726/msgspec-0.22.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b6d3ca19a8ff28d0a67a1824e2bff7ec649ec795c80a265f20ade4caa63080de", size = 229934, upload-time = "2026-09-29T14:13:28.285Z" }, { url = "https://files.pythonhosted.org/packages/17/24/a249f3491cabbe77cc65a1a6f87c128582aa39357227149be61cac8e554f/msgspec-0.22.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8b98ae215a102cbf6635f7df45f5c4af12f77fad1f7b71b9808fcf868a5735d", size = 234378, upload-time = "2026-09-29T14:13:29.821Z" }, { url = "https://files.pythonhosted.org/packages/87/ee/6dbcb1b5de8e9d47e8f0fde9a288628dc178c1749a570b98251218fa10c4/msgspec-0.22.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e0aa0cc3f18c35bab79bd7b87fde95d6274a9deddeebd1ea541f8066a5073165", size = 243118, upload-time = "2026-09-29T14:13:31.544Z" }, { url = "https://files.pythonhosted.org/packages/79/03/7dd2d0ca988600e01fc00ad0cf20d1d44bc59369a913c988654c65f6582b/msgspec-0.22.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8c8e84789918fbc15a503b92a829115ddd7567ecd3e4778bd418c56abbb86c11", size = 234557, upload-time = "2026-09-29T14:13:33.068Z" }, { url = "https://files.pythonhosted.org/packages/74/e2/43f3c63bff1650efcaaea31466246e28b46927323fc9ff416c68cc6e4047/msgspec-0.22.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3ca7d4cd69fbb66bd2da6211d3e79d40542d196c16c6d99bf838f76767ad35be", size = 241288, upload-time = "2026-09-29T14:13:34.532Z" }, { url = "https://files.pythonhosted.org/packages/8b/70/11b93815a59674f33182dc3e873d343ca0b37e25be52ecb28f52092f1fed/msgspec-0.22.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:28f53f3604dd3e70225f7563c831628dbb03299b428f8e62aadb4b628e386874", size = 236432, upload-time = "2026-09-29T14:13:36.083Z" }, { url = "https://files.pythonhosted.org/packages/b7/82/7aad0f033f8dcb3f23868773c2ede803ae162a784828ccde75aa3f9b2f9d/msgspec-0.22.0-cp314-cp314t-win_amd64.whl", hash = "sha256:7293dee54de040cfa225c22151cc3d72f17cd674b5ebcb52f38fb9f5701592e6", size = 202062, upload-time = "2026-09-29T14:13:37.955Z" }, { url = "https://files.pythonhosted.org/packages/e3/45/cf52577926d73e2369e25927e389cb4ea1461169c489f46d3248159b5be7/msgspec-0.22.0-cp314-cp314t-win_arm64.whl", hash = "sha256:c3c510aba9015c085e514b75a9b3f1ed7c4591ae5e379655821b8bba51f30cc7", size = 201686, upload-time = "2026-09-29T14:13:39.42Z" }, { url = "https://files.pythonhosted.org/packages/c8/63/d93937e2aae34ff1ea33b62799d1963cacc1bf432d196d6130039657a122/msgspec-0.22.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:263e110955ed76fe0af2d79f819903b50a70dc0e7a752eb7aabe79d2e0a084fb", size = 202241, upload-time = "2026-09-29T14:13:40.919Z" }, { url = "https://files.pythonhosted.org/packages/3b/e2/46ece11a244cd56432eb2362ffbb8014f3f02963136d84d941f71fdc2a3f/msgspec-0.22.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:c6f06576eced70462179a4b4638e84cf69fdbba37f44d13a64a21739c131a830", size = 194232, upload-time = "2026-09-29T14:13:42.454Z" }, { url = "https://files.pythonhosted.org/packages/cf/b1/1c385f2f93006cdc2af1511cc512c347cb22e2d4f11952c205230aedf586/msgspec-0.22.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8d67582478b0eaabb899f2fb255c878ee7de57dff80eb73ab24f1865524ec441", size = 226524, upload-time = "2026-09-29T14:13:43.876Z" }, { url = "https://files.pythonhosted.org/packages/dc/fb/c80c8842d40347cacf89a60a4986b849dae1a6dfd25830441efdd6faa65b/msgspec-0.22.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:71cbbdb39631064e2f2f9e9ac2b1b69931d72276eb5f9da4ed025726296bdbb6", size = 231816, upload-time = "2026-09-29T14:13:45.329Z" }, { url = "https://files.pythonhosted.org/packages/73/ac/90bbcfd890b4bda90c93f7e1b7fc24e84b270420486d9d43ae31443d15ab/msgspec-0.22.0-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f0a5c25516e2034b2db7767081759ff8996e214def9c43b3055f61e1be1caad", size = 244241, upload-time = "2026-09-29T14:13:46.851Z" }, { url = "https://files.pythonhosted.org/packages/72/9a/eabdb5f1b5e6013b0e2f9f2a95790587f6864aa9ca37f9d7dece65b53878/msgspec-0.22.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:a1dab6a99c759d1391ab2993388c1892746a697254f4b5dc6c059ca6e3bfbc8b", size = 230198, upload-time = "2026-09-29T14:13:48.296Z" }, { url = "https://files.pythonhosted.org/packages/e9/89/9f080532d4ac52f416dd7318e55c2053cc071853d17d58e24897a5b553bf/msgspec-0.22.0-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:a52eba5c9528fd181fcec39d22b67aaa1dccc6cfe8e24d3f5d41130e6d04289d", size = 242949, upload-time = "2026-09-29T14:13:49.829Z" }, { url = "https://files.pythonhosted.org/packages/11/df/6baf9b2f3523ebe2b820820c7929fd72ec5f483a93147130338ecc353fac/msgspec-0.22.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:1e547966017265c0d23342bcf2e027305dde40ea042d16694a9b96b4f696a052", size = 233914, upload-time = "2026-09-29T14:13:51.5Z" }, { url = "https://files.pythonhosted.org/packages/bb/37/9cf650779c8c1e53291ef184c838703930a4cabb1fb37e222c85a7d49fa9/msgspec-0.22.0-cp315-cp315-win_amd64.whl", hash = "sha256:0067057df265795f742658b15dbe53f3b6f21d19dcfa53676db11088cfa41e0a", size = 197910, upload-time = "2026-09-29T14:13:53.071Z" }, { url = "https://files.pythonhosted.org/packages/f5/ce/2f78c93d4f69e0167a19c2d40d4fbf7bbd6f074e1047536735832a4368ee/msgspec-0.22.0-cp315-cp315-win_arm64.whl", hash = "sha256:05dbc8268e50c9232ec72b9af1c7b13049aade4d1197764e38c427048706e046", size = 197590, upload-time = "2026-09-29T14:13:54.47Z" }, { url = "https://files.pythonhosted.org/packages/3f/bf/282e9a443058b85b8f706c9a651e2d8cdd11cc09d16e8fa347b6c57b75bb/msgspec-0.22.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:b3113ebcceeb7693a915183c73d92c10bf5c62851dd187cab43bd025fb587419", size = 206298, upload-time = "2026-09-29T14:13:55.913Z" }, { url = "https://files.pythonhosted.org/packages/ef/2d/2e694fa46f55319007f72013b17341ea3868be1c77e7a597176b202dda92/msgspec-0.22.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:0dfadea8bdcfafc614bd031de55a8ede22b43445cfff6d8b77cc0c07d3edc8a8", size = 198145, upload-time = "2026-09-29T14:13:57.412Z" }, { url = "https://files.pythonhosted.org/packages/5b/2e/2fa279cb57cb47175ae604d572787f903d4ad3f0afa867201bbd99e6647e/msgspec-0.22.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d7a738826936c72348c613061d260446f13c82b6fd7d5d7705b6911ab8dca2f3", size = 232362, upload-time = "2026-09-29T14:13:58.817Z" }, { url = "https://files.pythonhosted.org/packages/a0/58/a7e759b11b28441c27f803b29d9b5f4b5ad85150c89354b5ede1baca9258/msgspec-0.22.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f2ddea9d78d09460f06c26a7a508adcd049761c3208776162b8eb79b8a032cff", size = 235885, upload-time = "2026-09-29T14:14:00.381Z" }, { url = "https://files.pythonhosted.org/packages/86/56/8d7ee098e94cbd9f35fa643dc497e06a4a6307b9f562cfbe48103fc3b209/msgspec-0.22.0-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:884c28c80b0a511595b29a9b04a3a230c3797369e4a033e6d5c6d9b5427f8e09", size = 248155, upload-time = "2026-09-29T14:14:01.945Z" }, { url = "https://files.pythonhosted.org/packages/b9/6d/1cabb4b8a5dbf696e2b24df9e482b2e0333bb3b1b13ebb5433813e6616ec/msgspec-0.22.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f7a923bcde480065c8e25967464cfb2a687ee67000bb43157e2d57e40eca7305", size = 236416, upload-time = "2026-09-29T14:14:03.363Z" }, { url = "https://files.pythonhosted.org/packages/ba/43/8bf0f558eb369f1f2d494b3d5ab9d0ae0907d07ecc0cdbe11b6768b02867/msgspec-0.22.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:65eea14bc65ccfeb8f3af62cb204841871e2961f002d7fa87dbe0f79dacf1c1c", size = 247292, upload-time = "2026-09-29T14:14:04.829Z" }, { url = "https://files.pythonhosted.org/packages/81/33/2fbaadf98b5510cac4bb56d2b03937e0b1fb4bfcd1ae6aba20361f299583/msgspec-0.22.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:0666a1520cab86796612e794e71107e0fbf5e8ff3ddcdfcfff8f1d94b860d2f1", size = 238220, upload-time = "2026-09-29T14:14:06.408Z" }, { url = "https://files.pythonhosted.org/packages/f1/cc/b6be6041098ab859a8472983ccc2c08339fc2ef53f28d4f5fe7f4f34276b/msgspec-0.22.0-cp315-cp315t-win_amd64.whl", hash = "sha256:885c6e0c89d6103648525fe62aa78d600054dedf7b3713d23b15d7ddb6d66a13", size = 202939, upload-time = "2026-09-29T14:14:08.079Z" }, { url = "https://files.pythonhosted.org/packages/5a/c1/664578dd98be70cd4ab1a9dcf3a181b1376b83c65ec41ee162130b58c8c0/msgspec-0.22.0-cp315-cp315t-win_arm64.whl", hash = "sha256:268594d0bae5510572599a6ab0364dd9de43c867d24a30856cd9f5edb63d8dc6", size = 202117, upload-time = "2026-09-29T14:14:09.891Z" }, ] [[package]] name = "multidict" version = "6.9.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/d6/99/1d4d69c3512d0ddbfa3a1b69cfd9a151012ab2eb4eabbb096201b1f0b7d8/multidict-6.9.1.tar.gz", hash = "sha256:0f06e60fa190aa7abd0914c2a766736fdc8e9f34878c4346338534b73d1b20e2", size = 182404, upload-time = "2026-09-21T17:59:05.362Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d9/0d/4b5afb6d3e545c9af0cdfe2db8f6f4c6664568c23d863d888674e447e6a4/multidict-6.9.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:29138fef49828542e859828107e42e50d0e587c513b7eb4b2d92bade2b0860fe", size = 98360, upload-time = "2026-09-21T17:55:11.508Z" }, { url = "https://files.pythonhosted.org/packages/89/ab/1b9ca66251899981b21138b87da9d5a9c2c81af12b1ea7d19466972f7fe2/multidict-6.9.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:19e31815d41cefc365489e591d105d2baceb2f65aa75d29471fbdbda8651e006", size = 59979, upload-time = "2026-09-21T17:55:12.866Z" }, { url = "https://files.pythonhosted.org/packages/36/eb/6ae44062466c26c8469ef43f2481a6a48d8cea0587b2d54514ec92e2adfd/multidict-6.9.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6ed30be8918e18c8bed0a2e8b70639ecf02feb61ed00ca2e41cfcb2a50fa3f42", size = 57736, upload-time = "2026-09-21T17:55:14.223Z" }, { url = "https://files.pythonhosted.org/packages/b9/5c/a67817593019257a4ac8b0d1b4c426030e637c047b0692ef405439ecea7a/multidict-6.9.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:637f4ae36264bd7b8d9a60193acddc1d735ad52e8ed53a19931ea6d921fea8e5", size = 333097, upload-time = "2026-09-21T17:55:15.591Z" }, { url = "https://files.pythonhosted.org/packages/90/bf/599ae2e6222822d88a247a8a7ae82fe6fd25d5700757b79603d5edafe6a0/multidict-6.9.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:35fc236507fb1b3138f0af5ecd5f94ed752d4d6d826248eae425f86204013eea", size = 334196, upload-time = "2026-09-21T17:55:17.015Z" }, { url = "https://files.pythonhosted.org/packages/15/10/d8aac5acacbe7f5c117866c776ec26d5f15a868759b6f37ad8e7ed3b5b02/multidict-6.9.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e58952f04772f59f11c6e007471449809a30165188669bca8fdb19dde40a8f24", size = 317465, upload-time = "2026-09-21T17:55:18.412Z" }, { url = "https://files.pythonhosted.org/packages/19/0a/714f796f7293a8b1c5c3f465a26996231d5450c54e85d64ef1258c091134/multidict-6.9.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d35a4f1c63f07fbb8c8f9946dea98b21eddf6c57421585f71d91864be3ba2a24", size = 345914, upload-time = "2026-09-21T17:55:20.306Z" }, { url = "https://files.pythonhosted.org/packages/da/b1/e37fbf769c567be277bcf32df6234035a4384677fcc1bd852752be3d6b93/multidict-6.9.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b5f8771aaaed7f80e84a4e471d2f29ab6721e4595075e54d03ae1ed951b2000a", size = 351163, upload-time = "2026-09-21T17:55:22.21Z" }, { url = "https://files.pythonhosted.org/packages/ed/5b/db08419c1e1f7c9d60cfd2787b2b517d7ae4ebbda8281b48a33eb5141467/multidict-6.9.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:976fd7689d69ec78d67d31d38d396d8adb562f7e8368279f76aed4aa451fa06d", size = 336881, upload-time = "2026-09-21T17:55:23.699Z" }, { url = "https://files.pythonhosted.org/packages/9b/07/cc9bc8a62651d2d53ab93ee4993b3a71b7cb78eb8ebfc9c757a5b6698617/multidict-6.9.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:95052e8777a86bae87c0bd0b5ab22d809e3d1d02bf69e3e66ddda5ba75a05805", size = 303405, upload-time = "2026-09-21T17:55:25.305Z" }, { url = "https://files.pythonhosted.org/packages/b6/0c/8e912afafa70e944dbb8bec4b66ca6e008511395278c0d3dd0e89567536a/multidict-6.9.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1a8adfcaf96f587ab138476eaddef95f29b8a2a8a9afbfea8d2fd62180995d02", size = 324265, upload-time = "2026-09-21T17:55:26.989Z" }, { url = "https://files.pythonhosted.org/packages/66/6a/62c2af80fb085e6234805017af857b8e913dfac7a55e9c1349c27768c58a/multidict-6.9.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1a53de2772cfb74559df2eb4456ec4eeb908435ec55a84b69370d9d745d62aa8", size = 322085, upload-time = "2026-09-21T17:55:28.732Z" }, { url = "https://files.pythonhosted.org/packages/f8/6b/35bf801b336fd960811207203ffdcdc24acc3558b3e7ff2e1c914b141e70/multidict-6.9.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f2ce963299d42fa3f22a90adc0fdf174792ffef5ff4c7ffb68260548fb05580", size = 338107, upload-time = "2026-09-21T17:55:30.303Z" }, { url = "https://files.pythonhosted.org/packages/80/41/495ef65bf5bba29d142d81b3fe1b8154b919bed70e96491b1e93c3c26f0f/multidict-6.9.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5b30ddf7234e611ca877575b62840e6af5977f92f1f9d532eedbb05a44ff8004", size = 339631, upload-time = "2026-09-21T17:55:32.012Z" }, { url = "https://files.pythonhosted.org/packages/19/bd/057fdff5f4e04dcd40a960e38f77d19d3c4b67dd243ffa5f43718edb29fc/multidict-6.9.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:63ada7ee2e9345695f9e9bc4c65d72222253f07b1ac94fd0e37555cc6f3c7f60", size = 302080, upload-time = "2026-09-21T17:55:33.817Z" }, { url = "https://files.pythonhosted.org/packages/3f/de/9ace933ee8dad808632523726f42255b09600087219e3d4ead7369820910/multidict-6.9.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3c95601ed98fad3f6e2f8fe809c3b526b0fab31ef525e00a155e227f3d17f58a", size = 341107, upload-time = "2026-09-21T17:55:35.471Z" }, { url = "https://files.pythonhosted.org/packages/d8/ac/7c1204406097bfc5c283d4a3287d61166807d189917567df4c1318484fc4/multidict-6.9.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c148e8b596000dd3e4bfe206e70f3e666be18d72032e0012555f2373c52e35d6", size = 333482, upload-time = "2026-09-21T17:55:37.002Z" }, { url = "https://files.pythonhosted.org/packages/ff/c0/a70c32ea3299ebe00f44533740cb46905717c51faf29bbd3ce8bb5886d9d/multidict-6.9.1-cp312-cp312-win32.whl", hash = "sha256:f9dad513626a33670f17cddc6078e30e311f444c957e8dbfc5b2b4603c8b4edb", size = 52548, upload-time = "2026-09-21T17:55:38.529Z" }, { url = "https://files.pythonhosted.org/packages/d7/2e/8c9c2591df01e5692ad1bf92febb082a17b1c6c3a19aa8b7ff49988fd989/multidict-6.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:a16a1dc8529f9e734a41c3b856f3eae7ebacdc061dde3f8a844e0c7889c97203", size = 59622, upload-time = "2026-09-21T17:55:39.86Z" }, { url = "https://files.pythonhosted.org/packages/d0/95/59f4472ec512bc180fd207899594c7803ece96262ff9aaa3a4b6d7da6940/multidict-6.9.1-cp312-cp312-win_arm64.whl", hash = "sha256:361f7206cf341ba94fb015688f5c8b480f8e63bd58a4c14a48aeca7851a241cc", size = 54930, upload-time = "2026-09-21T17:55:41.155Z" }, { url = "https://files.pythonhosted.org/packages/eb/45/ddf7c76860f5f553a23ad3ca38463ccf5247081618977742c8dc4625355e/multidict-6.9.1-cp313-cp313-android_24_x86_64.whl", hash = "sha256:d7bf9e43282d69561618e8a0ea33368d532ebef42f15c096f427090521dd74f3", size = 63266, upload-time = "2026-09-21T17:55:42.676Z" }, { url = "https://files.pythonhosted.org/packages/6d/d3/f4ae5945ea2de597eaeb4eca3a74c59783ace54482c9b49435442b63efa8/multidict-6.9.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:03d47df72f084f757c1cb771188d5f4e3a805e4abc4d67e32509272343ae9382", size = 55771, upload-time = "2026-09-21T17:55:44.016Z" }, { url = "https://files.pythonhosted.org/packages/93/7d/15468239920040d01c686e5ae669e6382f7bf31fc3e5e0a6b8f1c3b32d3b/multidict-6.9.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:6bc94fe17c3c56e5418f79515b786b101845f70609b0d19d0c1ba13448e5633a", size = 57116, upload-time = "2026-09-21T17:55:45.315Z" }, { url = "https://files.pythonhosted.org/packages/17/1b/b958f06aac2d8b1e485eb1c105b1b159cbf3868249e5142142451577dad2/multidict-6.9.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8f2973bbd2bebd9d2e0cd6394c1292a1a19ccd56bdcbe1e174059f1a39be5b40", size = 97692, upload-time = "2026-09-21T17:55:46.674Z" }, { url = "https://files.pythonhosted.org/packages/93/6c/d6cfe18e61010166d7237d7527c775eb9843e7078feadea45b7628751b60/multidict-6.9.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:de7738b8c0bb74c4cc16bbd7fb49fc2bcf6430dba11b3432cd52768ae40933e8", size = 59556, upload-time = "2026-09-21T17:55:48.22Z" }, { url = "https://files.pythonhosted.org/packages/46/46/9b4c1127cece0289fb207d04aae5116382ddfcb2a4dc8e0cb49a33f3c7b3/multidict-6.9.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e5ccad4b7bac125722f48d6f862bed3b514d8526deea06316bb72f152cd30a7c", size = 57357, upload-time = "2026-09-21T17:55:49.998Z" }, { url = "https://files.pythonhosted.org/packages/bc/fc/c18b07100a6064573e49e538d13d47eb2b5221f48080512df78aef54ab04/multidict-6.9.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a49ff5cdb33654cb7d6a3c377aa2a83ddefaa1db31eb10bcf3c180aa84f9af8a", size = 330433, upload-time = "2026-09-21T17:55:51.435Z" }, { url = "https://files.pythonhosted.org/packages/62/ff/52a0082adeb69656b634609d4fb0ae65456deaae4cca3d5a12656626dc50/multidict-6.9.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b22ff30006a2f28f8bff878fb93413cbe3a4d1fd517c28081d848c90e9cfd2c8", size = 332487, upload-time = "2026-09-21T17:55:52.937Z" }, { url = "https://files.pythonhosted.org/packages/39/3e/80bd4729635a7af371ff4dcc312594cf96e73f98854491be8740792de430/multidict-6.9.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3adf06c66041aa21eeb8a71e82379b74773298c8e6d3d839b151aae441a99b94", size = 316911, upload-time = "2026-09-21T17:55:54.751Z" }, { url = "https://files.pythonhosted.org/packages/15/f4/7ea4a907e053e924e2e1694d90560f79f72056504bfd5e8483695f9527d2/multidict-6.9.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:963a8d8f97057082679523d0fd4c53a38f86bc58cabe4556faef682ae53fa2fa", size = 344950, upload-time = "2026-09-21T17:55:56.5Z" }, { url = "https://files.pythonhosted.org/packages/0d/16/9642ae41fbfbdc546aa481c2b07ae1bc087a94ab5f5020b4d9ab77da401f/multidict-6.9.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b66ccc5c2cdd26e74fa5d4c29ffae424cc6148bf93ce574821783fb3b6d452c5", size = 350264, upload-time = "2026-09-21T17:55:58.073Z" }, { url = "https://files.pythonhosted.org/packages/93/f2/e06c8e42074d0a4b8419bfe92afbd1d262190b69549ecbcd2dffa4f103c2/multidict-6.9.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fd79c521f6290c69125fa2b85fa65d9e657e6a8ffaf722dc881b926bef4aa5c", size = 336213, upload-time = "2026-09-21T17:55:59.723Z" }, { url = "https://files.pythonhosted.org/packages/98/43/d7cf9ef4700e7c25244590d1b4f20cf12cbed86beba6c17be4d9e48c1099/multidict-6.9.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1d804e4caf5d5da37d6dac1325da5629ebef1e27a294c2b568b295814aa36c7b", size = 302555, upload-time = "2026-09-21T17:56:01.666Z" }, { url = "https://files.pythonhosted.org/packages/7d/3a/54209920324bc3928f5f2ba28b01a6dd957a9d48c3aaff77e38faaf5668d/multidict-6.9.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bb0d664505f4b112f384cffeee82e91e3f6448d8e574989479db4439b68cba05", size = 322247, upload-time = "2026-09-21T17:56:03.265Z" }, { url = "https://files.pythonhosted.org/packages/33/50/df96f961b178b621ab0adbf9221d4fd21534e1064fd9f5e85c1fa27fea55/multidict-6.9.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:9e14d17773b1b3c758ff153659a1824608a0cb562c45f484b5ed8a433428444a", size = 321787, upload-time = "2026-09-21T17:56:04.894Z" }, { url = "https://files.pythonhosted.org/packages/a7/9b/37f354562a8f82f9c1f63d3a94300fc82126d50320175f0702bbd544f87c/multidict-6.9.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:083735b7f395894e43adb278d5dae901448883a835ff8f1977e285fefdb10418", size = 335498, upload-time = "2026-09-21T17:56:06.783Z" }, { url = "https://files.pythonhosted.org/packages/1e/44/78e366efc6c004185295cc9eb9711c10f13eadf3b547603b9c5526329976/multidict-6.9.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b65121091567847a8cb520d364ab22ba90e00d3cc55fa9eb34bb439f0684bcd1", size = 338244, upload-time = "2026-09-21T17:56:09.162Z" }, { url = "https://files.pythonhosted.org/packages/ad/2b/ab5bd3964691abe4d14b43bdbdb8a621b77cea2ca352dc93159ec0a4575b/multidict-6.9.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:ea999ae6e80e66ad5eea287860951b033d0104ca34d6d87c7b5125ebe0e12721", size = 301641, upload-time = "2026-09-21T17:56:11.227Z" }, { url = "https://files.pythonhosted.org/packages/ea/a8/f6bc899f5aafaba755edc8e4bb934bc00b1e405f8d3fbc1dd10283738074/multidict-6.9.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:095d900c242e00fbe5f321ee072e7278b4153e78c5ce9c1efde167d62c1e4771", size = 340162, upload-time = "2026-09-21T17:56:13.076Z" }, { url = "https://files.pythonhosted.org/packages/eb/03/a8fc809ef364b8c231c065ea2ea2f97ae86d4f9a0e36b5721759fd606778/multidict-6.9.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6441cc837aea58be7d9baef1b2383eb8311ab9303f500f99ac90b584cd78bb14", size = 332826, upload-time = "2026-09-21T17:56:15.175Z" }, { url = "https://files.pythonhosted.org/packages/f2/27/32dde80245e2024e9bb7a6ca3c2fb7c695dc74014cba557cc35be6f91e24/multidict-6.9.1-cp313-cp313-win32.whl", hash = "sha256:9c4880d017555d70dea367dd49271830842d48e3891c2da97da7ce8c4abcee40", size = 52378, upload-time = "2026-09-21T17:56:16.959Z" }, { url = "https://files.pythonhosted.org/packages/be/78/1bac2987edac273a6b27fa50e9204bc3466c94a7b4f6a44828d730f8810f/multidict-6.9.1-cp313-cp313-win_amd64.whl", hash = "sha256:ac51cd64bae51c462ea58ad2492c9b8209667a4ef60c45c4a304518b67598d5d", size = 59449, upload-time = "2026-09-21T17:56:18.336Z" }, { url = "https://files.pythonhosted.org/packages/e1/32/2a77ce19eea48cb9b3521a202b513ba3349aafa702c8317be0b645ffb74b/multidict-6.9.1-cp313-cp313-win_arm64.whl", hash = "sha256:37a9ebe00c698279213d56e6c64e1962ab1e092918270649b397cac3dc196ca4", size = 54713, upload-time = "2026-09-21T17:56:19.774Z" }, { url = "https://files.pythonhosted.org/packages/ec/0f/c6041015aa2cdc11e1dff0cae58898ae5525d19e35efd65a4ecf3c6ea235/multidict-6.9.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:fc0dcb22fa9aeabfe3fa4e0430099acff985ec5d77a851382f76cc6146e780e5", size = 62336, upload-time = "2026-09-21T17:56:21.181Z" }, { url = "https://files.pythonhosted.org/packages/23/db/95bd0afb130a0149c955f1f066143528d60864e64f41ff5a1e1313609361/multidict-6.9.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:024123f0ab402ab33828e24eb80fa8f25167d0d3783ba5f287e39ed741e6abf9", size = 54892, upload-time = "2026-09-21T17:56:22.63Z" }, { url = "https://files.pythonhosted.org/packages/94/cc/8aa34ae8d09498e8da003f485c03371b55cf1bc6d56cff861b3a7de61251/multidict-6.9.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:af1b5a92315048c3e36bbebfa7d4760a9c3e910bc4166f11b20d77d20d6bcfca", size = 56251, upload-time = "2026-09-21T17:56:24.36Z" }, { url = "https://files.pythonhosted.org/packages/52/48/22baa3b95375096369b04c4348af14139570f302f951773d2c68bf0ccf2e/multidict-6.9.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b2483da477932ad1983d1d33c18bc3771c6fb00cfbaaed70a875fd547ef8e840", size = 96345, upload-time = "2026-09-21T17:56:25.777Z" }, { url = "https://files.pythonhosted.org/packages/2f/3e/b96779dcac28ec6d491fd821112a0156b519b6701bba186cdf6f6df737f4/multidict-6.9.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:db4d697b18b6ef5528b1f36bfa25072cd2a421869f5963bc0e92c8a34b9e2800", size = 58901, upload-time = "2026-09-21T17:56:27.364Z" }, { url = "https://files.pythonhosted.org/packages/f9/b4/8e6d950f02ca6ecc00ecf671f2ddb5dab7017671a8d197326f75621f5786/multidict-6.9.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:854fd2f1bc6e8a56b89910b5cd7261a8b40f13ebb31572da985ce59c7da0886d", size = 56491, upload-time = "2026-09-21T17:56:28.901Z" }, { url = "https://files.pythonhosted.org/packages/ed/8f/212ed7e03282c7bc271e3796bf985b53d5727c2827361e3dd9c5ee6b093f/multidict-6.9.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:03731f6fc036180700c9dc2308205a48e5ca6f3ff03087739ab746f294022201", size = 332297, upload-time = "2026-09-21T17:56:30.495Z" }, { url = "https://files.pythonhosted.org/packages/a0/97/555aab000e03ecba85c0a1837fd5674d8bf10949490df7800da3a40660ce/multidict-6.9.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:989261c5f1735a165f2e4e87cf6d5f17ab734fa18f9ad0383d5adfcdaefce701", size = 329197, upload-time = "2026-09-21T17:56:32.652Z" }, { url = "https://files.pythonhosted.org/packages/3d/25/f3539b5bdc147beba66015795dfe589eedcc580348fb1a8c69b1df4196b5/multidict-6.9.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b4c9e5d05b126b267ac048a89a0e2d9b48b1b648add62d4872906304a8590610", size = 308456, upload-time = "2026-09-21T17:56:34.657Z" }, { url = "https://files.pythonhosted.org/packages/1e/7d/4665cad8fc326787d218879be879d86dfe32f006c1f88ae47b04d3b52fd5/multidict-6.9.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9d0a21cf76153de8f2d96a877991d6bc59b9ab5180b949e50db73cc193b4a694", size = 343382, upload-time = "2026-09-21T17:56:36.606Z" }, { url = "https://files.pythonhosted.org/packages/41/01/e1e9abe27f492b22dede492bc423015597412b14ce074f7475f4ecd80185/multidict-6.9.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7386aa18d98d6b8af44b92173654ec469237fda35f8e8523e43b581a86f476a", size = 346848, upload-time = "2026-09-21T17:56:38.344Z" }, { url = "https://files.pythonhosted.org/packages/f8/e5/8d54118bc228e64e1087f1729647b93bbea225eda4a3f914663a6f410bca/multidict-6.9.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b69651732c64afb691e50cdc3387cae305e0eeff8804fe3e3ce203876494932a", size = 333612, upload-time = "2026-09-21T17:56:40.041Z" }, { url = "https://files.pythonhosted.org/packages/76/e1/9509dafc1fc68948e75842a74533ddfa64af57f2b17810e9cacfa68a829b/multidict-6.9.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1fee9a16d88a1c4865610de31ef5c666671020d7a81d1f510eaf3c97d00ebeba", size = 304246, upload-time = "2026-09-21T17:56:42.313Z" }, { url = "https://files.pythonhosted.org/packages/1d/b6/96b93808e1bec0b51d49d89859e49b82a94f7b2e58eca0fc44ffcafd3ae5/multidict-6.9.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7814bbee202acd3bd240204c17d8b87a4c48c81064fd8674dbe527d94d5a4290", size = 321983, upload-time = "2026-09-21T17:56:44.399Z" }, { url = "https://files.pythonhosted.org/packages/d9/1c/08e5184df026b9d8e05a6dce5da22628e46bc4a7c7f03b8bf3c6939a8c46/multidict-6.9.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:95f273bea318a194f656527ee2ed19494327bc500b8e87d9358e3222579aa28d", size = 315927, upload-time = "2026-09-21T17:56:46.208Z" }, { url = "https://files.pythonhosted.org/packages/d7/5d/9ff39e7387f6fcc9a81af5dbf055fde08149849a04f8df10ca5b086b0d39/multidict-6.9.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:73bcafa21a78d0776b3ee7cd2a63c66f968eb7db8e8d594e32d7329950f6e828", size = 336827, upload-time = "2026-09-21T17:56:48.021Z" }, { url = "https://files.pythonhosted.org/packages/5a/be/30fbb2cab22203128928066de94be247dc9a3230bf2bfc54437ac3b5f9f7/multidict-6.9.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:1a938761c77e0e6edb0c93d02f4e988d44a69e5195e5a3b893e5553311347132", size = 336317, upload-time = "2026-09-21T17:56:49.778Z" }, { url = "https://files.pythonhosted.org/packages/bf/f0/2480aef6b7d8e7ab89b064ae14f12ec069570dab3a5c13e0155912293247/multidict-6.9.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:37245ca4105386194dd1d292a6f2aae09bfe1bd7ac6f9ec25093e3cf8e9b143b", size = 303258, upload-time = "2026-09-21T17:56:51.829Z" }, { url = "https://files.pythonhosted.org/packages/4c/04/dbe59cf4ea3778600e8e70e30b31aa9118ad6da2844bc9f717804cbaa000/multidict-6.9.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f0700527dd5bfa8b7204b08330542f4f388899d3c14d885d8a368992e0eb562d", size = 337232, upload-time = "2026-09-21T17:56:53.537Z" }, { url = "https://files.pythonhosted.org/packages/f4/8c/abe55548f06f12878588f08eb803c5a391283f01243f784cb00adac79231/multidict-6.9.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6b7e54fd883671d1a8810704851c044b7173287c552b9f5c3d9e0eb9f00ae194", size = 330261, upload-time = "2026-09-21T17:56:55.335Z" }, { url = "https://files.pythonhosted.org/packages/d0/f1/ba67f668478f152bdfa18abe72e7588bd078bf8b42c172fcf542fcf45826/multidict-6.9.1-cp314-cp314-win32.whl", hash = "sha256:e81ae656b9935ac4528a71f96bb7a14d949778ed1897c573d3e7ebb9187f8841", size = 51715, upload-time = "2026-09-21T17:56:57.048Z" }, { url = "https://files.pythonhosted.org/packages/33/81/014bb2128aedc8157d2848f0ec8a0209c6b822fb88deea012834e1c4a4e8/multidict-6.9.1-cp314-cp314-win_amd64.whl", hash = "sha256:acddcac38adc8342ba48aba98896faa7928854bebb62542362138655b5367ee3", size = 58621, upload-time = "2026-09-21T17:56:58.531Z" }, { url = "https://files.pythonhosted.org/packages/66/34/8f43c03c2d8821a2320b8c560d8a49d0be2920627edfac3f546922f91969/multidict-6.9.1-cp314-cp314-win_arm64.whl", hash = "sha256:32217133dddc58c927805cf6c0731d8144584176b768042ee886d51e71860bc9", size = 53959, upload-time = "2026-09-21T17:57:00.15Z" }, { url = "https://files.pythonhosted.org/packages/08/64/4f2a5eeea10b6d42634ba168ec6efc732ef218772e57b8239cd485c202f6/multidict-6.9.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0b2fb8c349d1103863750b5d8cb5ace766917f4b35f3d883c8f778853eaa9f76", size = 107694, upload-time = "2026-09-21T17:57:01.785Z" }, { url = "https://files.pythonhosted.org/packages/32/ce/c68d08ac2096c1528ff20d77fd10f7e401db3e534ac5a1c7726e36a77c2f/multidict-6.9.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:56d834b74c993a7d7cb2b8ab33a0d55c3e0d4a3d2f2da2808a4ad3d79189711b", size = 64766, upload-time = "2026-09-21T17:57:03.52Z" }, { url = "https://files.pythonhosted.org/packages/3a/9a/9f5c270c88fe289b4876fe28f9932a70e015a7916aff1e5b0325f7597b11/multidict-6.9.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:783ba7d845d79ce976afd9c1e91a4e5714671defa198ee789e8b23316083a485", size = 62723, upload-time = "2026-09-21T17:57:05.104Z" }, { url = "https://files.pythonhosted.org/packages/7c/63/c9a0b131354dd949426166cef712d708f436a8cf261114f8d5d2aec440ca/multidict-6.9.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d718fa1b5f0d0dd75e86fbbc5b0c93ea3a5d65216c85c61cd5d7cdddfe08455", size = 290445, upload-time = "2026-09-21T17:57:06.687Z" }, { url = "https://files.pythonhosted.org/packages/9d/41/ed27d1d20ba91f4e13e8f7dcfaa592614ae0e1d5f45e03171d23c3b098ac/multidict-6.9.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a313bad717dde740959d50850c315b75fd4eb0c5e6b4dc8535db0f1c369be125", size = 304375, upload-time = "2026-09-21T17:57:08.717Z" }, { url = "https://files.pythonhosted.org/packages/66/6f/a7b26167173c3b73d6e5ebb0856c5154ad005156c3a2fa4062329921819c/multidict-6.9.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ef01d29fca550ab871fd99154f82c6472aacf8e7def272dfb07b460123850390", size = 281746, upload-time = "2026-09-21T17:57:10.713Z" }, { url = "https://files.pythonhosted.org/packages/34/1d/da27bdd49b0cb84263288f5f48dbd915f2d17fd228fee2078ca607e80277/multidict-6.9.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f3bd290711c6e9486173a6ee7cd4e7f00c3971c7908c1ec1b6e5437c5e4c6f9", size = 311499, upload-time = "2026-09-21T17:57:12.467Z" }, { url = "https://files.pythonhosted.org/packages/d0/36/ab0299a7abf53e38ff75d96c64480f1e2ff8edde985f93f7a1ad5e08bd94/multidict-6.9.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:81cdc537e0a42e3c0170752fcadbe450246d5c3b4b7231d6eb9672456605ac94", size = 313553, upload-time = "2026-09-21T17:57:14.548Z" }, { url = "https://files.pythonhosted.org/packages/5a/44/e21be702efc9c9ba097c8be874c68860b1cb6cf9c85611263039bf95b37e/multidict-6.9.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8b193bf7a443c97d81c47052f60c486071a4bdef4a573fa2514f920089414d45", size = 299898, upload-time = "2026-09-21T17:57:16.192Z" }, { url = "https://files.pythonhosted.org/packages/00/8d/e93bc0dc947c7f872e502cfaa7ad9d98f001e1c93cfa6f4940457a6e187a/multidict-6.9.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7a0c98f6a636adf0d7edd60c61589eaf52d149239af754d0bfeb0effedba53a8", size = 273054, upload-time = "2026-09-21T17:57:17.793Z" }, { url = "https://files.pythonhosted.org/packages/5e/c9/945b44493fbe7af26ffe1e43c506b376daeab070fc1a0923bef9bbf572eb/multidict-6.9.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6e6b7e3a1520c39a2772f414cd9ddf5995f77e4e823dfa1522af383addd465a3", size = 294751, upload-time = "2026-09-21T17:57:19.631Z" }, { url = "https://files.pythonhosted.org/packages/24/79/0969a9b38b814e5fa3af1b0a57708c49b3d9b3d55c208bac6e98b87bb028/multidict-6.9.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ca65cced0d67a9039e93bcd98a369920e499bf98296844ff56ead01c9085321f", size = 284523, upload-time = "2026-09-21T17:57:21.313Z" }, { url = "https://files.pythonhosted.org/packages/1d/1e/c32ddf53234c228b1a374ddabca19f542ac7f32752f5155982abe2304833/multidict-6.9.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:372c063f37480f62c1ae32dc3a1a0a5942b180c883f99789075a8a8ec3c4e709", size = 290275, upload-time = "2026-09-21T17:57:23.206Z" }, { url = "https://files.pythonhosted.org/packages/5d/09/7a99a3daafd987330abb865a2d36e432fd8bf18e790ddda55c5de15eb98f/multidict-6.9.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:827c92145b3b976b39430129c89d213b250dacbe5fce678e9a03940e6e848983", size = 303350, upload-time = "2026-09-21T17:57:25.086Z" }, { url = "https://files.pythonhosted.org/packages/2e/09/7e6ebaecc4e226d7fe27973c271d58b75c14c24e84c23736d21dc39997dd/multidict-6.9.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:9b0124b9c17e9890f0819b2e7a5f65ec9a2f5aabd6c8e7ad090c1675cf70dee6", size = 272232, upload-time = "2026-09-21T17:57:27.193Z" }, { url = "https://files.pythonhosted.org/packages/1f/6e/420e9e879b21cbdb5036214ee238efabcfab7bf262d3fc124132050f79ff/multidict-6.9.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:23136f5a564654eb61061ec6d5620a4c1ea32c8f552b65e9982a12b72bff601b", size = 300525, upload-time = "2026-09-21T17:57:29.004Z" }, { url = "https://files.pythonhosted.org/packages/b7/7f/c6b0896850b3ceb190ca302760d63148843371f027c5dde3d1d732343a08/multidict-6.9.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f2524ec55b3e65cbe235a8b3c36e2af3b635be02ed05e20c94e70c5c943c009e", size = 295933, upload-time = "2026-09-21T17:57:30.837Z" }, { url = "https://files.pythonhosted.org/packages/29/f2/f0ff3384a756227741c72f4ee94cadf1f9900df0ae961e48639f96115b1c/multidict-6.9.1-cp314-cp314t-win32.whl", hash = "sha256:35ba0263bae5dd3ad5aad767cc9afc01a8598c7dae30f1b3b2de98b1b32c28bd", size = 57776, upload-time = "2026-09-21T17:57:32.835Z" }, { url = "https://files.pythonhosted.org/packages/8d/87/013e1eed30ad46bec1eb68947bc9caf55ba2fd0d6aed04fe964e3b44c23b/multidict-6.9.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7c8d5882ba25ac0282258be435d8a05aa0cbcacfce15799154a338f847f159b9", size = 64881, upload-time = "2026-09-21T17:57:34.489Z" }, { url = "https://files.pythonhosted.org/packages/7f/95/64dd049bb4c53426e2fee7afb874daaa8b4426c41878785883a94511ecf8/multidict-6.9.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ac348379cf4de5538a0a213be1532d289aa801ec5d267c5909446b9ea2f8e2c3", size = 58582, upload-time = "2026-09-21T17:57:36.19Z" }, { url = "https://files.pythonhosted.org/packages/a2/2c/26bc1723592cfe965b575652a7749026474b0625b04b04564e1fba265dc5/multidict-6.9.1-cp315-cp315-android_24_x86_64.whl", hash = "sha256:f04551dce5a7db8c9659f2e4245494c182d0663b83661803e08d46bfcae5eda1", size = 62339, upload-time = "2026-09-21T17:57:37.967Z" }, { url = "https://files.pythonhosted.org/packages/bd/c5/614d2eb9995232d066aae7f17d1d454060b6b5d882b22fbaf6150b05ef1d/multidict-6.9.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c0c88085affd35c33e124e36930c5ad96aff9294ce195eaa0fd9cec962b64a82", size = 55009, upload-time = "2026-09-21T17:57:39.746Z" }, { url = "https://files.pythonhosted.org/packages/7f/24/f7179ecdf9cdd3581ace3292c3b883a1a500a374aa4e619bf6eb521c62e5/multidict-6.9.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a8b75dd3d3638d9a19f23e84af4ffab3b8940422c0df2da2a77005ef5aa3d7ea", size = 56273, upload-time = "2026-09-21T17:57:41.544Z" }, { url = "https://files.pythonhosted.org/packages/ac/72/c9dffcecccd1f8b9a58f960ec4eaecab62c4edb08d5bb984d4de67fe3fdb/multidict-6.9.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:006c4478de0a1876f4834e14255776286f09b9846b505fe63f67f9d173a9487c", size = 96390, upload-time = "2026-09-21T17:57:43.385Z" }, { url = "https://files.pythonhosted.org/packages/e9/3f/76542360e655cecdb7c158c7af88c69c4db0d2aac3421659d41214927229/multidict-6.9.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:2784090c30a586d5b45197bd9c32f87fb927216cde302f6cfd76d76577e90f08", size = 58900, upload-time = "2026-09-21T17:57:45.105Z" }, { url = "https://files.pythonhosted.org/packages/04/15/f28c71af461dd4ff15653a165cfd8ddaf01f3387ff80d2dd0de057d527d0/multidict-6.9.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:7c708566da8014b120a64b1eb6d200c6c0c8cb36296383723cdb6fc82038270b", size = 56540, upload-time = "2026-09-21T17:57:46.913Z" }, { url = "https://files.pythonhosted.org/packages/19/de/5ee4050c3ddad88c605a89fc1cd18e89abd40a64f183e6fb5276b6eb7f52/multidict-6.9.1-cp315-cp315-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:042fb0196047e786936a730bd302de83143950da45f2c16078da8f35e1cf7919", size = 331818, upload-time = "2026-09-21T17:57:48.825Z" }, { url = "https://files.pythonhosted.org/packages/70/e2/2d4e3c3ee70f583334ff236fe6a3babf7a05ce56169a15df65453af647c0/multidict-6.9.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2212a0c842c723d919ea4a22a9296cb6b244b386e4e6ea92adfc7fbf3095519", size = 329800, upload-time = "2026-09-21T17:57:50.91Z" }, { url = "https://files.pythonhosted.org/packages/d6/14/f64bd05667343a7250cdd99f81bda2270e3959f00832441f10c7f904e1c1/multidict-6.9.1-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fd882aa29bf402b62bf1fd7c19fd5df4b6528cf468a908864b39368572b662a9", size = 312048, upload-time = "2026-09-21T17:57:53.053Z" }, { url = "https://files.pythonhosted.org/packages/64/bf/a91256ce00199e7544e3eda79c4e9544eb7388c38fa22f11638c2e357ccb/multidict-6.9.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c57541034d12b215ab0a2bfa371d1a8a198da18176d0426c27105b9161a6862d", size = 343676, upload-time = "2026-09-21T17:57:55.069Z" }, { url = "https://files.pythonhosted.org/packages/14/48/628b978413518159228c2b2ec38645362ceb6427e935d83d716e26490a49/multidict-6.9.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d73fed4e37158ff00cd271871170b79e40138db51e625fd352fa17c6acb34f67", size = 347028, upload-time = "2026-09-21T17:57:57.157Z" }, { url = "https://files.pythonhosted.org/packages/8c/73/5f9dc2b5ac7dd7a82458ae9cd283b99c185834f23442ace295362dcc5e45/multidict-6.9.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:79da2491348b30810728050b4a8ec0416f85884125c2fd44655b3d01150d9c3e", size = 335670, upload-time = "2026-09-21T17:57:59.126Z" }, { url = "https://files.pythonhosted.org/packages/f8/84/6f319ee000f5ab3fa0055b194ad706fddd552c60c76a21d581c87db1d46c/multidict-6.9.1-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b4da208a63434d21a3df64d29758e650fc4aa8cb05848554b76949c296539cca", size = 307804, upload-time = "2026-09-21T17:58:01.2Z" }, { url = "https://files.pythonhosted.org/packages/3f/24/61be5fe1616d033532e8d63dc789c0f35b9793917834713eefe050589ba9/multidict-6.9.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:877ca17fdcfdf5c397493a71e5ff97a87bb181417fe717fdadc77c08c09301ac", size = 323409, upload-time = "2026-09-21T17:58:03.383Z" }, { url = "https://files.pythonhosted.org/packages/22/a6/eab8aaf59892169a0f4e8e78a5516a6caa6abd8e96e6410111258f4ef0ef/multidict-6.9.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:40aec5299e1ed71fbb988389059da381c3c1a60e0c649acceb2a35d9b128848e", size = 315746, upload-time = "2026-09-21T17:58:05.372Z" }, { url = "https://files.pythonhosted.org/packages/42/b1/d1e14bab80a74babf6e2ac971a878b84fdcf3a972f3ec3400ca12260fac5/multidict-6.9.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:4a409ccc42aefec904038695d5d7fd6d8f3af2721b7b6401d55135ec7d6d298e", size = 334529, upload-time = "2026-09-21T17:58:07.35Z" }, { url = "https://files.pythonhosted.org/packages/f5/50/a05b1d36f6d3363a5a334cebbf11624c9c7a69d66fd23345eb53b1d43b7e/multidict-6.9.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:0a5769559e3312dd96731fbe15b4abb6033368ac1cad5a98dadd21946a4c7d6c", size = 336455, upload-time = "2026-09-21T17:58:09.112Z" }, { url = "https://files.pythonhosted.org/packages/cd/0d/bf33cb097380252b08e1177b5ae0cab822c47ce80d236f3f724953c76721/multidict-6.9.1-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:03fac50ddfd8302175b77863a015eccfae76767cda5506eef86df559ba861e1f", size = 305865, upload-time = "2026-09-21T17:58:10.921Z" }, { url = "https://files.pythonhosted.org/packages/ba/ee/a2bd133391204b1cec5a5caf327dc9a06710439b7403220860a8fb164560/multidict-6.9.1-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:5ecace251ccfa705bf3d7d35c5032cf750f5c629809740405697bce5c118c4a4", size = 337531, upload-time = "2026-09-21T17:58:12.846Z" }, { url = "https://files.pythonhosted.org/packages/a8/32/c0972486f81a06c3bbb23063c85afaac53c51608f7e35bcb101a1006408b/multidict-6.9.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:ecc89dbd4155b2f8a47f4bbd89242a35ed15e8e0ec581cab5ac65fa38407329d", size = 332464, upload-time = "2026-09-21T17:58:14.673Z" }, { url = "https://files.pythonhosted.org/packages/69/4f/5cbbe852b45d8a7f0ab3ed042386e4894e463010eb230603596cb15d0fee/multidict-6.9.1-cp315-cp315-win32.whl", hash = "sha256:a3ffe881246d28a862f1985824f484cb7361f44d6b99c4c620436ef56462f38d", size = 51714, upload-time = "2026-09-21T17:58:16.409Z" }, { url = "https://files.pythonhosted.org/packages/d4/77/1fa118386deeae4e7a48870fa5cb073e748b0ac5265a6990651cefc48068/multidict-6.9.1-cp315-cp315-win_amd64.whl", hash = "sha256:59c123d0e948d760a5f930f316cfefa07e8d632ab84327c0693ee6a88171154f", size = 58619, upload-time = "2026-09-21T17:58:18.369Z" }, { url = "https://files.pythonhosted.org/packages/dd/ee/51c45748acfce9cb69d89dcfcd09c874b385407fd6993e885d6905659865/multidict-6.9.1-cp315-cp315-win_arm64.whl", hash = "sha256:31199204b3ced121ff5407a2c342326a5d27e3870abbf94bd80dbd2451b7bc8f", size = 53969, upload-time = "2026-09-21T17:58:20.102Z" }, { url = "https://files.pythonhosted.org/packages/aa/57/89d2d3dedfae2558853c1770c7cbcdfd5211eb4c4090b1e8cb570544a490/multidict-6.9.1-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:8879510a76940670517ea1cb589978da44b86e286ec3e50d664ef330817afce7", size = 107679, upload-time = "2026-09-21T17:58:21.98Z" }, { url = "https://files.pythonhosted.org/packages/c8/df/0c2a28870181c1762f90952b897f6db46c934ce073efe0442604cbc276db/multidict-6.9.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:aec65b53a07f580606593f877eefbb29a45939bfc0d3fe6e6d9f42b41b749f68", size = 64753, upload-time = "2026-09-21T17:58:23.622Z" }, { url = "https://files.pythonhosted.org/packages/43/7e/9c6a3e7619459407d8958f7e56ab3dda7f0738f5aed9bb4ad48076e8de83/multidict-6.9.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:14c56f73e78faa1f68bbb826197cd5871994e70e841b8590829c35912ece5c64", size = 62732, upload-time = "2026-09-21T17:58:25.46Z" }, { url = "https://files.pythonhosted.org/packages/38/eb/fd56c9d83ba0cdc3cf66f9acb278b24ef1f043b620b96d145685a3b57338/multidict-6.9.1-cp315-cp315t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:38c9986f9ce50c459b10de216a05f4bd7ed5ac63887d56e500a52bb464b861ce", size = 288648, upload-time = "2026-09-21T17:58:27.115Z" }, { url = "https://files.pythonhosted.org/packages/df/ef/51e79c2442b0f56ac4ac580108d713078b31297e028d468fc220971a305d/multidict-6.9.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:15db6a102cbaf1949cf028ecf080aac76d20bcd29ad4e092574db6c6b7af78a5", size = 304919, upload-time = "2026-09-21T17:58:29.143Z" }, { url = "https://files.pythonhosted.org/packages/6f/e4/774700cc5739353fe88a7204ed30e78a9403d98d4bb2496cc91f096d2e71/multidict-6.9.1-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:bd82c4977196681a499bb6ca9e462afbc5c91c1c15b6a991dfbd72733fea4dd2", size = 289094, upload-time = "2026-09-21T17:58:30.938Z" }, { url = "https://files.pythonhosted.org/packages/ca/41/0bec327bc3a8271825376b6d8a469d1f80e3ba1e2e730e019836e29663c3/multidict-6.9.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bb58ba73a3f96f9a3e46b1fab69929d7edbbddb3133ee74b5c3c54074a53c4f1", size = 312007, upload-time = "2026-09-21T17:58:32.964Z" }, { url = "https://files.pythonhosted.org/packages/c3/7a/9906bf5f1fe20e8e755affccc1a69ecd8aebb079ac65386e5ffc8eefbe75/multidict-6.9.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7c6eecfab7ce4cd9487ff8ba936fe38cdfd68c04faf3d5c710363c6a8e695659", size = 314657, upload-time = "2026-09-21T17:58:35.285Z" }, { url = "https://files.pythonhosted.org/packages/33/92/cf6d665a45663bbe216420fe632da10e9f4f842b07ce02bc3ddf8eb882b8/multidict-6.9.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c62e71e6289d78c0108d8aeb495f8bd3cad4bc1632fedddc9297ddf287ecc20", size = 301563, upload-time = "2026-09-21T17:58:37.472Z" }, { url = "https://files.pythonhosted.org/packages/97/4c/9f1b7945526905d58b4b2d422e2ea81326dda71f9bdc9c32c8002639234a/multidict-6.9.1-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:539c2cd5fed0947c135cd7eabaaac55f48300dfa1de0f3ca4edb5efa6606f471", size = 278540, upload-time = "2026-09-21T17:58:39.551Z" }, { url = "https://files.pythonhosted.org/packages/5c/d9/133aa472fc1be181de29a0c2e1c4cd778280882f532b404395f646742e46/multidict-6.9.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:3100d169ceb7bc8f05f89a6db11d1b21f119975fc26f29dc45a472e3569f0879", size = 294858, upload-time = "2026-09-21T17:58:41.791Z" }, { url = "https://files.pythonhosted.org/packages/bd/38/21f2305dd20ac5038fa581b5d7e0d8dbbd70dc6b50231c22ec173dacddda/multidict-6.9.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:696477ad71385c4795e3b8e4cf10b0d2c28c2a1ca6a955e031cb1e62993e9ee3", size = 285461, upload-time = "2026-09-21T17:58:43.844Z" }, { url = "https://files.pythonhosted.org/packages/ad/21/465a43c214d1d7e2d0c974e727d4b5935f8a0868838b5df9d0984c83c1dc/multidict-6.9.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:f5844e7befc707367807586f550fa97e23dcfef0728f02b98c7bc498a961a5df", size = 288964, upload-time = "2026-09-21T17:58:46.16Z" }, { url = "https://files.pythonhosted.org/packages/2d/58/72a3e8d56c1e05146d2d1841dbaacf8986375eded1e5a58d30eef191a580/multidict-6.9.1-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:abc7c2e4b47bfe6a9aea434d3fdebb9597ee636e914e92ba352f2068b9142f4c", size = 303924, upload-time = "2026-09-21T17:58:48.395Z" }, { url = "https://files.pythonhosted.org/packages/6f/b9/911084c04530a682a54536b4f738d3992a593b5ff6fea1d6a1f02f3acc6b/multidict-6.9.1-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:7ab379f95caee071a37cbd8be86d4c65accc651d391f9f97748fef006f38769c", size = 278479, upload-time = "2026-09-21T17:58:50.494Z" }, { url = "https://files.pythonhosted.org/packages/45/eb/9a333346c02c928a9e82534638bce85697c10c46b4901fcd70af19357356/multidict-6.9.1-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:557a4e1708df428ebe6c3081c83a273d275dad2446ce0d81fc648ac71afdb18d", size = 301650, upload-time = "2026-09-21T17:58:52.554Z" }, { url = "https://files.pythonhosted.org/packages/67/2f/1d936c4e45c8199db6082b6836e9182e31aa393de0de652637790831ca4b/multidict-6.9.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:475d04d5192eba487a3e2f935976340baa24529046e9c1c9c7a3b7bf80445ae1", size = 298024, upload-time = "2026-09-21T17:58:55.03Z" }, { url = "https://files.pythonhosted.org/packages/a4/0c/ca62ae2882b89a4cfeb0bbca873a4afdb5fc32994c4f8a37a7cb52ede7a0/multidict-6.9.1-cp315-cp315t-win32.whl", hash = "sha256:10083a8a0f4e1b26b599889e90b9802504ce5d3f7722f925bbb7ca47dd22a7c1", size = 57810, upload-time = "2026-09-21T17:58:57.141Z" }, { url = "https://files.pythonhosted.org/packages/02/8b/b2c2805c3eb6fa4cdf5512fb95ac6174dde71e34dd601c5be52f4df5dc82/multidict-6.9.1-cp315-cp315t-win_amd64.whl", hash = "sha256:e96ca64383efa107262ee3949f047af5ee4f1845ba09463466c04d35a83bd3ec", size = 64984, upload-time = "2026-09-21T17:58:59.446Z" }, { url = "https://files.pythonhosted.org/packages/fd/34/03f1d204d698c408f4754c969882609b6b7a902c0e3d16a7033f219cdeda/multidict-6.9.1-cp315-cp315t-win_arm64.whl", hash = "sha256:501ed8b02a5990c67a91c732843609d43a6be1f7576fcdfc867331239f37fbd3", size = 58742, upload-time = "2026-09-21T17:59:01.726Z" }, { url = "https://files.pythonhosted.org/packages/be/59/e26cb779be4c591d1a910f59d29aca9fba4de70349840a833beba2652371/multidict-6.9.1-py3-none-any.whl", hash = "sha256:7bf6478188f4e47bf5686e8a33da4ae28bf43b1b2528d9ee144d28492bfac60b", size = 19176, upload-time = "2026-09-21T17:59:03.501Z" }, ] [[package]] name = "mypy" version = "2.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "ast-serialize" }, { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, { name = "mypy-extensions" }, { name = "pathspec" }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/82/6a/878cc1097d4035f82bd516658d0c528d2a9955bc7b363afcbd0b07fea11b/mypy-2.3.1.tar.gz", hash = "sha256:47c1b1207258513a9d93495f69c8be9de73916186f0e52703e8c461b7a623419", size = 3992554, upload-time = "2026-08-15T03:03:38.549Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/85/da/d6effc4f808a842d91edc22535dc9e799d2ff6e91449168b7f47a0771f54/mypy-2.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a32bbbb940af990d3be0b8af321c7b6815bb1b3b48142fe7459b9cc5f58959ff", size = 14047547, upload-time = "2026-08-15T03:02:57.707Z" }, { url = "https://files.pythonhosted.org/packages/e4/e6/478229701dab76f26485fc8ff5d6f241f393da22447400bbc56f6946aebe/mypy-2.3.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff715e45b2231a8e85de1d163d1b42791e4d7aab8f5145f85fee1b710b735aff", size = 14216515, upload-time = "2026-08-15T03:01:26.496Z" }, { url = "https://files.pythonhosted.org/packages/8d/fe/7c42327a3b21e84681f691982cbfe43f334a3685f3b683b72c376476c4fa/mypy-2.3.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:858fc57d3d91fa728e33e7ad71def60fc6272694607b306cd3292db53ae39080", size = 15307789, upload-time = "2026-08-15T03:03:31.62Z" }, { url = "https://files.pythonhosted.org/packages/59/f4/7e597edbe01b5a56fa958ce541302dcaabfed979966f1dffedbea0ea0fc2/mypy-2.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:851833db876e7b650f93719c74b7879a08e338979c96054fdfc3bfd90a486355", size = 15548831, upload-time = "2026-08-15T03:03:15.55Z" }, { url = "https://files.pythonhosted.org/packages/a3/52/cb31e084bc0314a1e384bdd677a4b80e55af04ccac077545e2238b9d320a/mypy-2.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:4c5095a327483591c94e0c8d3ef9e50d4ab1369b541eae007c1f23bc2a41f6bb", size = 11226359, upload-time = "2026-08-15T03:03:29.002Z" }, { url = "https://files.pythonhosted.org/packages/7a/47/88fcf6217b43fa2da81a8c2611370af18141536a4f0294bbf98b457d456d/mypy-2.3.1-cp312-cp312-win_arm64.whl", hash = "sha256:bbfe022634a2a195406bd469e888d2eaf193b02ba7e607391cd7640374aaae3b", size = 10214707, upload-time = "2026-08-15T03:02:48.807Z" }, { url = "https://files.pythonhosted.org/packages/de/cf/862010ee800ca9c2bd0c4c0dacf0f092e5411824a09b8f97ad4be8fe250e/mypy-2.3.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:114dff494000f18bd10d5d95d84b8567b26da60279ecbe838131841df20e635d", size = 13964542, upload-time = "2026-08-15T03:02:21.43Z" }, { url = "https://files.pythonhosted.org/packages/75/5a/3f3a2107b41e3e92e617e25daaee121413b91e9784bea733131ed4fecc5d/mypy-2.3.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c8637731bb5eee3671eb2c3200827aa3564ed8a9309ecee4d1afe77e6d031bdb", size = 14168922, upload-time = "2026-08-15T03:03:00.351Z" }, { url = "https://files.pythonhosted.org/packages/8b/41/04dc4fe7e63d7820fa4eff272e95157d30cbea921388f3ab3fe77794cd0b/mypy-2.3.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c80fbc405ed8020f5ff3802dc18cf060197bcdd3fbdd6a26ef2fd34dfdd5226", size = 15244791, upload-time = "2026-08-15T03:02:31.089Z" }, { url = "https://files.pythonhosted.org/packages/96/fc/c3053b26b9054949285aa868cb6af8c10e7591541cacd79c5dcc06a1fcf9/mypy-2.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:84081f538ce27375045c02e3d7f81bd11d853400621ae245d87ce7b6c420ec74", size = 15501627, upload-time = "2026-08-15T03:03:34.128Z" }, { url = "https://files.pythonhosted.org/packages/70/4e/d77daab008bbc4e5001374d7928f4a260d28f0e6747af444fc4763f7a310/mypy-2.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:e9144ac16fde007096f9563eb2041b4433c2d705c4218edeb79e7e9d01035ee6", size = 11243961, upload-time = "2026-08-15T03:02:11.952Z" }, { url = "https://files.pythonhosted.org/packages/f0/f8/7eb68c136e4abd30569fe31ef2bfcb7eceae9952cab80017c04cd09f5d0c/mypy-2.3.1-cp313-cp313-win_arm64.whl", hash = "sha256:77ad9529e67dca28e511f5cd5671436584ce91f6d3bac159a353158187b986ac", size = 10213219, upload-time = "2026-08-15T03:02:26.361Z" }, { url = "https://files.pythonhosted.org/packages/be/c4/42a49d44aeff804edf1b19acce0b49e8bd1a9c57dee9605dd8d980aa43d7/mypy-2.3.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:192abaedf75da1bc0b1cef104927e70ec49c1ef0031cc4825c7ee10a438ed24d", size = 13986778, upload-time = "2026-08-15T03:01:33.69Z" }, { url = "https://files.pythonhosted.org/packages/45/13/9331fd2dfed7194d66c5304072894a8be3e51e9deda6863c1eceaa35a43d/mypy-2.3.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf678dffd16efcda2c15cbd30e9ecc0081388e29ea23687a88e686ed92638dc3", size = 14188467, upload-time = "2026-08-15T03:02:40.554Z" }, { url = "https://files.pythonhosted.org/packages/78/f7/f4a34edab45667c5465855dc585a20e87978ffa8aee711445b7239d120c6/mypy-2.3.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e036f06b41630f4c8a1d48f9ac6aa26acc65f8be089973f5519da643318f03f", size = 15225538, upload-time = "2026-08-15T03:03:09.761Z" }, { url = "https://files.pythonhosted.org/packages/40/05/534b3590757bd05794f73e07f6666c2a77b8597ffed795c94ce570096aa0/mypy-2.3.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:71af9c8a894e862b58e92abb08e53b05a384a1e5e5d6dc7cda59126211a53d82", size = 15480805, upload-time = "2026-08-15T03:01:41.134Z" }, { url = "https://files.pythonhosted.org/packages/55/da/bdfba852e2562f599624af5bb7d29e36b0b4f526f2b8bac85efe0dd1803d/mypy-2.3.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:3c80cd23d85368bdd9f37d5231dfd97d35bcbf5bf41af96ef3a9b078ad1957f9", size = 7761712, upload-time = "2026-08-15T03:02:36.008Z" }, { url = "https://files.pythonhosted.org/packages/98/31/60fc64a74cdba4f2a5d642d32317993e479163e1ac7d91b695e5d15e2264/mypy-2.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:4956f34d145e145562a0a0bf367f642bbc85c04ec2baf47ae015947c3169a85d", size = 11423968, upload-time = "2026-08-15T03:02:06.931Z" }, { url = "https://files.pythonhosted.org/packages/a9/23/eb5950b24cd26ba3b78f87707a275568d633c77dae8e61c9661be6055ca6/mypy-2.3.1-cp314-cp314-win_arm64.whl", hash = "sha256:cfb12e360242d23d91f5e978d94f58ea66acf5804c4fb6f2f794a20d4cb1b595", size = 10399323, upload-time = "2026-08-15T03:02:33.671Z" }, { url = "https://files.pythonhosted.org/packages/82/c7/f80f4e46c0b9a00eb5f78a79d49dda8bdf56a5230f7257fb33e76be04da7/mypy-2.3.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e5f1c50bb05b64e2026b52867e8d21106f01313c744a2c4ecc34c90d12e8d6e2", size = 15121308, upload-time = "2026-08-15T03:01:46.053Z" }, { url = "https://files.pythonhosted.org/packages/5d/74/9b04f17c7074cc5188f02fb63a2ca1d43fedf479e84fe3091c39061a1d7f/mypy-2.3.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:667196b352f4cf304ded4c10f90cfc179263a1acfb3cdcfa984bdfd340d498bc", size = 15536590, upload-time = "2026-08-15T03:01:35.941Z" }, { url = "https://files.pythonhosted.org/packages/26/04/c837ef6208e567774e2ed1f863f8ba6ec4817b1b6dd426315e5d559b6ec9/mypy-2.3.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9c53e395c12cad2c6d4b67d5da7c6057638a132d85c08b73646b18f802a0045", size = 16791074, upload-time = "2026-08-15T03:01:31.073Z" }, { url = "https://files.pythonhosted.org/packages/37/68/48730230afa45192d5bd429a6a2ff24a6f8dedda90fdf2b221792b54518f/mypy-2.3.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:18162b128c3f9c703cd35f5537446900b0d21a2549aa7a95d21380d2ef643fb0", size = 17069183, upload-time = "2026-08-15T03:02:28.566Z" }, { url = "https://files.pythonhosted.org/packages/1c/ea/ca23fc9c20eeda09a15c9cbcf50015d0e73f409f6ead059e42aa69a608ff/mypy-2.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:30c0477d4aab7b7f39c8397dc877f2c96b9fe5588ec379f372c56eb63d599f63", size = 12154679, upload-time = "2026-08-15T03:02:04.809Z" }, { url = "https://files.pythonhosted.org/packages/3b/67/8d982126034990869466f73b8db80dcb2234a7ac39b4dad093e047a79835/mypy-2.3.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6941ab3619377bc3f32ca02876b07d27f216f5201604b664d3937ea0fdd23bb4", size = 10969159, upload-time = "2026-08-15T03:02:38.152Z" }, { url = "https://files.pythonhosted.org/packages/ee/f7/41e7f2d8117fbc7a7587286162ffe2f688984b69c46ed63cf5f2e4fc3bae/mypy-2.3.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:6f041a6de52c9217ca125e78ba0a335cb7fd98a1c0580978e49ab2b126f70b57", size = 13990694, upload-time = "2026-08-15T03:03:21.919Z" }, { url = "https://files.pythonhosted.org/packages/06/85/8f665811a0c8f3bf6fa1d9acd665ec2d97a2bcc453ae68dcd92340941cd6/mypy-2.3.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5159ae60f5dbc3a498af5ba8365505808ac8031bc63f9e00304ad545d40bdd9b", size = 14203518, upload-time = "2026-08-15T03:01:48.455Z" }, { url = "https://files.pythonhosted.org/packages/2d/82/91b866c8546b120bff83b73a439d90d2d63ef3aff113599e6b8e4d566848/mypy-2.3.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47a8a7a0a7f6f6e63995c0ac36fa0c07b127413fdc81f0439b7f3dccafd33561", size = 15220224, upload-time = "2026-08-15T03:01:23.577Z" }, { url = "https://files.pythonhosted.org/packages/c8/78/c226c99208ee40de7c768369fa533f933afa003dfdc606ff021450724e91/mypy-2.3.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2329c0501293d4e1f33bc15d04d6304d65a1cdda967ee93a05c1e681a3923133", size = 15501512, upload-time = "2026-08-15T03:02:09.453Z" }, { url = "https://files.pythonhosted.org/packages/a9/e7/7cfb3f106c393979f4cc37ad6c0586044d50401e3c35b0c003e4f3ba6bc9/mypy-2.3.1-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:bb26deed807bdb0457cf3e3f1cd7c4a1cf9d66864eaf1b4a61e06805d4c6b1f9", size = 7761913, upload-time = "2026-08-15T03:01:55.65Z" }, { url = "https://files.pythonhosted.org/packages/99/3c/52affefa273b97939a1f474ae4a349c8718635c15b941112dfab4291b0c1/mypy-2.3.1-cp315-cp315-win_amd64.whl", hash = "sha256:375d7013876a8233b2d05be185bfa09f689696cd999ce8b1cfe6acac5c80e8a3", size = 11422533, upload-time = "2026-08-15T03:03:24.101Z" }, { url = "https://files.pythonhosted.org/packages/2a/b7/75643e70c72a5b346d8a9b1543c967ea8824df2ee3fb7ccba652c272b7bb/mypy-2.3.1-cp315-cp315-win_arm64.whl", hash = "sha256:586b3612214cceabb3c0f588c97e7d1e535393f06a60e912e994f6b3ace97523", size = 10397931, upload-time = "2026-08-15T03:02:55.265Z" }, { url = "https://files.pythonhosted.org/packages/10/ce/53be21f2d4adfcd26f63f1184a13ed797015ab463853f117e2e11e4d726f/mypy-2.3.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:ef0c6335cda9d807f8193d8ff6204a72bc909fa9882aacbca14f43cdb7188306", size = 15118669, upload-time = "2026-08-15T03:02:51.479Z" }, { url = "https://files.pythonhosted.org/packages/62/43/20de757cd42989d291a17fad607742c4c74e875ce5cea00e5a5225020ac1/mypy-2.3.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e598c8c66401d26b150872154a286e6d484cf2789c3bb28a7556806298423021", size = 15545627, upload-time = "2026-08-15T03:03:05.132Z" }, { url = "https://files.pythonhosted.org/packages/7e/fc/092bdf77ad280eaf501422f0f3b966012b528076cc13e41a774861c907d1/mypy-2.3.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eda22fd4efa9dcd39331d1dede9b5b8b8a7fd69af07592e778433da98610d29e", size = 16764157, upload-time = "2026-08-15T03:02:23.958Z" }, { url = "https://files.pythonhosted.org/packages/94/5c/c94c4d62d909b07f552d0d9356d7acc943825558e602a64822ffa2231536/mypy-2.3.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:2a0ba2e57847849fb0d1fcdabb32786d223095ed8bc121dfe322bcdb3d9c46bc", size = 17073258, upload-time = "2026-08-15T03:02:14.573Z" }, { url = "https://files.pythonhosted.org/packages/c0/f7/511a88b89e478053c02d22039bb8f3ce4183efe8fd7a4f0a5910a8bb0a32/mypy-2.3.1-cp315-cp315t-win_amd64.whl", hash = "sha256:3f7e865dd51f235f60a2dbcd8728a1c095f5ca28f095d48a725b84cd935735c4", size = 12135505, upload-time = "2026-08-15T03:02:16.714Z" }, { url = "https://files.pythonhosted.org/packages/71/bf/02573b56964ecb0f7c644f915f53c325ae15c3faec521c5adf11599a32df/mypy-2.3.1-cp315-cp315t-win_arm64.whl", hash = "sha256:8ad80807dc3ab8ea978b1b2b6e4a657194ace1d4ef03e0e731aff1abd517da29", size = 10962647, upload-time = "2026-08-15T03:01:43.712Z" }, { url = "https://files.pythonhosted.org/packages/8e/41/9675c7a1e78edecfba0b79e587a52594c56e189368261dc7b3a7fffb9527/mypy-2.3.1-py3-none-any.whl", hash = "sha256:6ed5c7e3419083268e5c9258bd1c1ef91af44a9e89374dbcaf37b775716e72eb", size = 2754338, upload-time = "2026-08-15T03:02:53.4Z" }, ] [[package]] name = "mypy-extensions" version = "1.1.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] [[package]] name = "networkx" version = "3.6.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, ] [[package]] name = "numcodecs" version = "0.17.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a6/f7/7fd3a5066fd07a1541b7937f3f9db73552f48b28a819470c5c00a0c3e472/numcodecs-0.17.1.tar.gz", hash = "sha256:5069dc01b0087c6135cbb3b043707aaf927c992fcd325248d15d907c51dd3747", size = 6290108, upload-time = "2026-10-20T19:37:58.903Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/83/42/3e48ae17b5810f4972140344d86d951c6620ff618901e4e84cb007c5e5e6/numcodecs-0.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7cc861d98827e92f93d3ff5b796ec6ee6785efadd44d721496b49e2609365457", size = 1172442, upload-time = "2026-10-20T19:37:28.792Z" }, { url = "https://files.pythonhosted.org/packages/95/e0/338a68d002397fa896581ff3c8af6ca8c6cba259cda4da2496084b1b1785/numcodecs-0.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d639064acc8f38184e763a5bfe6b5376361c3a79583c67a86a57424629a247c8", size = 979779, upload-time = "2026-10-20T19:37:30.651Z" }, { url = "https://files.pythonhosted.org/packages/73/72/ca2c75596fe84eba628510595d0b32395205ebf450870f07311adb9cd3e9/numcodecs-0.17.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bfa4d91bacec65232d02402efb82ad08e8c49298e5358d03fe205691ffcca397", size = 1385001, upload-time = "2026-10-20T19:37:32.092Z" }, { url = "https://files.pythonhosted.org/packages/76/c7/e6b131d0240345bc523873b9f16b22aaca8319abdf0acef4247ccffb3cea/numcodecs-0.17.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e3c08f6f4074eb92b3312efbf293afbbae5697d6b9d009435671a3a18e22ba5", size = 1437783, upload-time = "2026-10-20T19:37:33.861Z" }, { url = "https://files.pythonhosted.org/packages/8c/80/1fa323d38f42002bc3beb18ec3c5d1eecc8c3c703f497a9f0ea6ddf8e512/numcodecs-0.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:f65d08e20e16430dfb2a21e6a1298b09f98a0ca08a2a9e7aec25a81e4b98b7e5", size = 1497112, upload-time = "2026-10-20T19:37:35.912Z" }, { url = "https://files.pythonhosted.org/packages/80/01/325b1b78c4602f808e4733e9f88ae4b3f15813c98c9a44d46d799e543bb9/numcodecs-0.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6ca56ab612dbdfffa69c24ad510eaf3cdd794b40dd47144f96f7e0c890fd712c", size = 1171680, upload-time = "2026-10-20T19:37:37.23Z" }, { url = "https://files.pythonhosted.org/packages/fe/3c/b5350ec12911abfcc69f87113e00ee01beecda4d8550fa081e6a840a73be/numcodecs-0.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0eca3c1c135890911971f6f31e66c9dbfdb28af780c8a53aeadb160478553cd8", size = 976932, upload-time = "2026-10-20T19:37:38.588Z" }, { url = "https://files.pythonhosted.org/packages/61/3e/59c7ba952edf8e822d76b6b7933c93e7eb5edb11e273162a338c4bcf6985/numcodecs-0.17.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fed686b607d9c1afde8cd0b17d808e29d9a7c8d4ff2c12478456198f57b21e91", size = 1380146, upload-time = "2026-10-20T19:37:39.895Z" }, { url = "https://files.pythonhosted.org/packages/fb/66/d7ed1f3cf2acdd7066c79e5fbb4a71df275860df4e1951d4209c8a4bf446/numcodecs-0.17.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a67976c03dd62e1b6a5e938c61f643673a6a9a37e17800eca99d1a7ea3c89e2", size = 1431354, upload-time = "2026-10-20T19:37:41.298Z" }, { url = "https://files.pythonhosted.org/packages/26/80/ab38a26d5633da6c9217bfb04cc38025371d1e785fd6484aed56dd9a6f37/numcodecs-0.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:a5cc666eba758ae20798c6bb1ce406d68db60e3fe0d65aa67e00d130aa1d00f0", size = 1493578, upload-time = "2026-10-20T19:37:42.997Z" }, { url = "https://files.pythonhosted.org/packages/0a/cf/2a7406e1c0a8b28f885b7ff6b4f1059b1b157e1b221399ab021f3a97e720/numcodecs-0.17.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9edc7ab4a890cf24aaedb5e557c1d3fbdb697afd0135974d81b5b304ea75552d", size = 1173008, upload-time = "2026-10-20T19:37:44.286Z" }, { url = "https://files.pythonhosted.org/packages/f3/36/47934b70b187a23560fa404ff4835356e3b438cd46f28bd0a4e17abce2af/numcodecs-0.17.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0dac6eaded2c6baeb292ec76df327ea7eb222915ba50b83babf48234b09746cf", size = 978823, upload-time = "2026-10-20T19:37:45.595Z" }, { url = "https://files.pythonhosted.org/packages/fe/4b/f87f8a1b8c967a06792c052b9df0dd6db6714a42e0d8db3efa6404a72d15/numcodecs-0.17.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:71a1d46317956b8402ce06697911c41fe6b1dd5f24ee7a793830784d202b2548", size = 1388057, upload-time = "2026-10-20T19:37:46.961Z" }, { url = "https://files.pythonhosted.org/packages/c8/de/14562bb9c6d2b078570383859264b07ecf0d205139e13d37c4d56cb926d5/numcodecs-0.17.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c0bdf72039d55538c774ac2f695146ab6e36392660664904bfcce1731f0a862", size = 1433317, upload-time = "2026-10-20T19:37:48.253Z" }, { url = "https://files.pythonhosted.org/packages/d6/30/3b706e1784e3631b1d614447b3b06811a644879c9acca5dec00ab27bf2d4/numcodecs-0.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:fed097ba6c199be3c7dd5a79147d2a7ec23680967c99339a722b4d19069f527b", size = 1535610, upload-time = "2026-10-20T19:37:49.996Z" }, { url = "https://files.pythonhosted.org/packages/1d/35/69e5a27dd41f8bd916dbff5cbccaf2e4c756884f183015c10b83a63bbee5/numcodecs-0.17.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2e78cec0a8d88a93cced1e8ad1385415ddbe7f01aa5ad7f7da35b68f4f79414b", size = 1197076, upload-time = "2026-10-20T19:37:51.39Z" }, { url = "https://files.pythonhosted.org/packages/04/10/8c39a542d7572ab2398874df2b4465001e700e9d3c3ff959a42342a704e6/numcodecs-0.17.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e2b6d32c0e29a050a8f4b891c0f84a808a292b0dff0fb5602eeb5bcd6f7bd2ff", size = 1022856, upload-time = "2026-10-20T19:37:52.905Z" }, { url = "https://files.pythonhosted.org/packages/00/6b/4549e705a32d8f74a55397f29b69904ec9c04590ec4c59881786c640c0fa/numcodecs-0.17.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a853f4fde6721b35bd33d3cf6f621c4b335d07072346d24cc6af956f6f1b4db", size = 1381637, upload-time = "2026-10-20T19:37:54.493Z" }, { url = "https://files.pythonhosted.org/packages/1f/ab/5c5e298aade3ba59cff7f1e0a2afe81d180762f9467150976f0dfbb6561b/numcodecs-0.17.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b5930b538468779335640a294728da6737ed6e218316bf36a49d939ac6958e28", size = 1425742, upload-time = "2026-10-20T19:37:55.815Z" }, { url = "https://files.pythonhosted.org/packages/2c/ce/420b5e439e96d7a619c3122ef58e8847701c9d79a2b8f94702704106ff34/numcodecs-0.17.1-cp314-cp314t-win_amd64.whl", hash = "sha256:39d477687e9b3ceccde69358a62bbdbddb0972638ed90e61eff65537821f03a7", size = 1524207, upload-time = "2026-10-20T19:37:57.3Z" }, ] [package.optional-dependencies] msgpack = [ { name = "msgpack" }, ] [[package]] name = "numpy" version = "2.5.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/13/01/11703282db468b85f6f7b8c7f22d058de5970d5c7e60a3a8aaa313c3de36/numpy-2.5.3.tar.gz", hash = "sha256:df2d5874ff183595a4ba404edd04f6bd9b5505c1d7708573f6a6c17489a67563", size = 20791231, upload-time = "2026-09-06T16:27:47.073Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d6/50/8fdbb16af64895706a45f06a4068e29db732ec180f3c1375f14123359138/numpy-2.5.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cb189f09db39283b26bfd061ec16189e14f71c6755207f72a0f7540867afe5b9", size = 16994982, upload-time = "2026-09-06T16:24:29.244Z" }, { url = "https://files.pythonhosted.org/packages/60/39/789131c1188c078dcb3a1692e72e1e050c68b88ffe72c9ccaac9bcd7a9cd/numpy-2.5.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f59a878c33d6b88122d80d239bb3b845d58708750b0cb06a09aebb9b18ec696c", size = 12009327, upload-time = "2026-09-06T16:24:32.491Z" }, { url = "https://files.pythonhosted.org/packages/9c/59/a312e95696e5f601914dd8b6dd844692ba61670807417e24b68e337b5c70/numpy-2.5.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:a72f874bc9e10e4b8f80426fb49716d5141f64442a0c8418065093ec8017fbb0", size = 5445405, upload-time = "2026-09-06T16:24:35.071Z" }, { url = "https://files.pythonhosted.org/packages/30/d0/5623a1707ed4fe16e3909fe3cf5ee3da004ae677ad23d83bbf3adf1a6faf/numpy-2.5.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:fc36dc566135b5eceec4cf89758fcb719266a019ef07dae1754ae7c9f617ef3e", size = 6783213, upload-time = "2026-09-06T16:24:37.253Z" }, { url = "https://files.pythonhosted.org/packages/f1/32/84146fc020ad3c25f805f70ab60da46fe3c540a21369754a7e4369754b6f/numpy-2.5.3-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:76c2c1e6bfa5c84adc6434dfbf013aa92096a7985221762c8f11fedfd20fff58", size = 15687872, upload-time = "2026-09-06T16:24:39.751Z" }, { url = "https://files.pythonhosted.org/packages/65/af/aa78d1a88805456e212b65461354cd943197fb9acecc4c90fd12295123a3/numpy-2.5.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7e18c623bb5c95acb3b3328861272816ba199fb531921c5d6d0b675f1fde9e3", size = 16717410, upload-time = "2026-09-06T16:24:42.745Z" }, { url = "https://files.pythonhosted.org/packages/3b/24/faa79d865e69a97ba17473b23a1b74094b2259c03e820c70297293b9ea49/numpy-2.5.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4f8929ee6c96bfbd7b4ed2032e0c03af86fe1826740ab61ddabf9072d06e57ff", size = 17040975, upload-time = "2026-09-06T16:24:45.961Z" }, { url = "https://files.pythonhosted.org/packages/62/4a/8877e629445a7176297dffcaf9c485faa96a95d81728a62521ad55bd4c0f/numpy-2.5.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b5d93cf48f687479941d12b69c873ad2cc76bbd487f0091c2200636497f34034", size = 18476479, upload-time = "2026-09-06T16:24:49.35Z" }, { url = "https://files.pythonhosted.org/packages/c8/db/35e1c2d38b04cbd5b731f9d71495e055e813197669d22b612f11748d2ff9/numpy-2.5.3-cp312-cp312-win32.whl", hash = "sha256:bf63afbe037eb5d2fe87fbcc7778e61da53ebaf21d938a4515aa73b62532a5d4", size = 6133378, upload-time = "2026-09-06T16:24:51.915Z" }, { url = "https://files.pythonhosted.org/packages/3c/a1/accf6d4f0c80c5d9ba9735d6b1550e444180599f34dec69ca01360f717ad/numpy-2.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:0a59a421a32580a009e8a1751345bf829631b990dc1794b80514ab722b435def", size = 12567828, upload-time = "2026-09-06T16:24:54.255Z" }, { url = "https://files.pythonhosted.org/packages/22/43/1764aff32e4652526ae2f71fa8b3efd8d25c8a3d6926914454e47138ed1e/numpy-2.5.3-cp312-cp312-win_arm64.whl", hash = "sha256:ccb32e0525d29e8b0572eb84c9a57af0e7a4e615726927506f55063c62414034", size = 10485432, upload-time = "2026-09-06T16:24:57.278Z" }, { url = "https://files.pythonhosted.org/packages/79/e5/8fb89cd46d14e35699d13bf943a5f5f441ecee8667120a1f6105ab89e349/numpy-2.5.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:66a78fe4556c60aceda5916f9eacd638b18e9e681016ec302dcb4682d6d4d034", size = 16991061, upload-time = "2026-09-06T16:25:00.411Z" }, { url = "https://files.pythonhosted.org/packages/2f/06/9dc9e48b5e5e941c8b10350c5ff2d721da42a20517d911d15544246775ff/numpy-2.5.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92f30e89b8ee0ecf363033576c422b2f58fed6a80bed0aa48dff6d14c654663e", size = 12003676, upload-time = "2026-09-06T16:25:03.475Z" }, { url = "https://files.pythonhosted.org/packages/ab/2a/98282aa5b8f58b1157d440bb6282eed47e3632a5de53a714fbab17e659fe/numpy-2.5.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f9a2353b37a1a9e78fd82b27ad7e2a32a2d036604d18f02b05e3136c62ca3b09", size = 5439695, upload-time = "2026-09-06T16:25:05.978Z" }, { url = "https://files.pythonhosted.org/packages/a1/f9/b6533d777be9d6ffd29dc1be0867e563e6e8cc9a220ff1b716adc317f060/numpy-2.5.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:ccbc4665079665c3cf3bab4db9f6b095370cd6437d66be549b6c2a1fd19e1958", size = 6779395, upload-time = "2026-09-06T16:25:08.599Z" }, { url = "https://files.pythonhosted.org/packages/73/85/735720d04ec197c5dcfacdfc9922667c7f1f5f496a279b7ba4d7c74c4cc7/numpy-2.5.3-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c76d5dde9f445058f83d0c02af00557a4db91de9a9a57c0df87d1535001d654b", size = 15681750, upload-time = "2026-09-06T16:25:11.173Z" }, { url = "https://files.pythonhosted.org/packages/3a/1b/3b16a9bc514a440a7a0883684111dcb1ef1aee960af2ca95da8fc775f124/numpy-2.5.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5fa86b80fd24bcd1aff83ad23be44ea323de3f787be8f8b15d4a65621e25321", size = 16708577, upload-time = "2026-09-06T16:25:14.171Z" }, { url = "https://files.pythonhosted.org/packages/69/c4/386f397831b07328b639c96c5b62719346cf4baf07c68d927239752b1534/numpy-2.5.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bd4cb9ad3c7889b9b3fe0a9a9fb5d2ed26f9879bff2608d9f01aed147a20d231", size = 17042047, upload-time = "2026-09-06T16:25:17.582Z" }, { url = "https://files.pythonhosted.org/packages/5f/3e/a700ecbf36e85ae8328fd3b0e12eeddc22ed6358a64cb2bd913e0d195d65/numpy-2.5.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1302b90c0e52281681b2975adfe8a860cb7b12216a27b4b0b4207c44bf7bccf0", size = 18465724, upload-time = "2026-09-06T16:25:20.949Z" }, { url = "https://files.pythonhosted.org/packages/41/ee/38e785e88a4045f6ad1d1f2808dcdfafdca48c760260c0587bf171e29fc9/numpy-2.5.3-cp313-cp313-win32.whl", hash = "sha256:1c80eabb4035ecf4ca9cd49cde8a9fdd69a729e63e6474887d1523ade7aa277f", size = 6129003, upload-time = "2026-09-06T16:25:23.664Z" }, { url = "https://files.pythonhosted.org/packages/f3/ec/100f2b1794ede74a9b3d7ec6b9736927f56713414c1dfe19ab6c383494bf/numpy-2.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:71cad2b2a7451ab79d8f5e71b453485b6775963d5cf794179144a7463fe6e8ec", size = 12560965, upload-time = "2026-09-06T16:25:26.602Z" }, { url = "https://files.pythonhosted.org/packages/80/b1/7dc825ca94c12acebbce4c37caa5e198695eb31424bc579679f32b1bb49d/numpy-2.5.3-cp313-cp313-win_arm64.whl", hash = "sha256:8e4dd766076855b5ff7ea52fa5f07ce26286726e0f8bff446b7739d02e6ea204", size = 10482343, upload-time = "2026-09-06T16:25:29.772Z" }, { url = "https://files.pythonhosted.org/packages/70/78/cf416f15dc29375a229d9dfebf8db6e313f291580b39fa1a568b6052bb07/numpy-2.5.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:350ba9783ce969cf9f7ce6e6a9a58e1a6e2a19ca025b7ee448c4db727706212a", size = 16998686, upload-time = "2026-09-06T16:25:33.171Z" }, { url = "https://files.pythonhosted.org/packages/9e/59/abcc2d8def4fd60eec7d87f92d27c13448ffd9ab14339bcc63a0d7a2fdea/numpy-2.5.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:012e66aca395d795496446e52aeeb5866312a5d4d3f27da270e5a0b43f70dc5c", size = 12013862, upload-time = "2026-09-06T16:25:36.748Z" }, { url = "https://files.pythonhosted.org/packages/94/75/4640d2d6e4b64a049e48425a82728a41ef4adb61332d2cba68055774878b/numpy-2.5.3-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:adc1ada2662f8a5f960b8a10d9986897e7499ef07e06d4cfe7197f8cce923c07", size = 5449793, upload-time = "2026-09-06T16:25:39.476Z" }, { url = "https://files.pythonhosted.org/packages/96/cd/625b57ae33d4ca560f32cc0b47b4a5922146d9beb998ddf773900d440a73/numpy-2.5.3-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:54a115e5a73b8fc44f0cebef486365a1894b5c9760685d4558b72b7c3eb846e0", size = 6785176, upload-time = "2026-09-06T16:25:42.069Z" }, { url = "https://files.pythonhosted.org/packages/9c/72/12918652e7912ef9751e8694c88820fcd1908e0618cb23f5f3caa6004b7b/numpy-2.5.3-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be5a8381859b6da607c84f4f7d6847725f1cf1853ef8a2c9e115b7d58bef47dc", size = 15703377, upload-time = "2026-09-06T16:25:45.135Z" }, { url = "https://files.pythonhosted.org/packages/45/8f/9beacf79ca7c650688ad0baa80931adb988fe6e6e5d5903c23cc3dbd70eb/numpy-2.5.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b0521d0f4aebb6e06189451025fa17a913287b13c03d5fe05c017333b654ea5b", size = 16711928, upload-time = "2026-09-06T16:25:48.461Z" }, { url = "https://files.pythonhosted.org/packages/09/8d/41d0a56e1ac4c87495c897a211b1368691b7237aadabec8b3b8f3a74d48f/numpy-2.5.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9deb49575e5b0b94ed72c8a64ec4d033381adc27e9060ae842971f697ba96104", size = 17059507, upload-time = "2026-09-06T16:25:51.873Z" }, { url = "https://files.pythonhosted.org/packages/08/1e/0dfbc5cc251d54e2af790f254d24ec38637fa97ec7d5d11de7ffed787098/numpy-2.5.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b00eefbcf0f292945c4b4dec2ae845389ef5bcdcd596e6e4328051db5b5ba694", size = 18471002, upload-time = "2026-09-06T16:25:55.233Z" }, { url = "https://files.pythonhosted.org/packages/b5/2c/dfa40f6991f8185c8c30ffd023dfcbb11888e823cfab9557b920f3bb7bed/numpy-2.5.3-cp314-cp314-win32.whl", hash = "sha256:c2381f82999704f818e2c987a865050e285ec3621262c66d40f5a96c8f899f8e", size = 6180485, upload-time = "2026-09-06T16:25:58.157Z" }, { url = "https://files.pythonhosted.org/packages/a4/73/d2c08231e4fde7e415501fd02c715d96e98599b2d8384445933944152984/numpy-2.5.3-cp314-cp314-win_amd64.whl", hash = "sha256:2c25dfa72943e4336ddb6b0ee4277b47a0c85bede0807530ec68103bf58e2c10", size = 12698179, upload-time = "2026-09-06T16:26:00.789Z" }, { url = "https://files.pythonhosted.org/packages/5c/e9/dcdcc9b95cf5f49815055573aee1b11cfbf5299f38a180e437ded050810f/numpy-2.5.3-cp314-cp314-win_arm64.whl", hash = "sha256:15aa985ac73a8db02db7663381aa109510449d3819d37206caed27b33a65a8a6", size = 10769383, upload-time = "2026-09-06T16:26:04.011Z" }, { url = "https://files.pythonhosted.org/packages/49/c4/af8bc08a7ef4e1529a7c0cf24969accce316b783999802089a581ec99272/numpy-2.5.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ac7bb1c52d445bd4f8f7f97fefe6abc3a084dc4d63df50d79b17fa2b78e89297", size = 12132668, upload-time = "2026-09-06T16:26:07.138Z" }, { url = "https://files.pythonhosted.org/packages/c5/ae/0f15eb56d4ec5e13c1f7ff04ff407f997d1acbadb45d3e1f2e2645a8f43c/numpy-2.5.3-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e6ab667ba76450084eb64013762c438ea76d9d29cc676dcd6c2e9892ba37f841", size = 5568580, upload-time = "2026-09-06T16:26:09.828Z" }, { url = "https://files.pythonhosted.org/packages/23/fb/c72a8f25d4b6e96c354e7ab45ace3b27dc11e5d6a13b6c7d0cd6b08bf112/numpy-2.5.3-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:f7fabeb6cea87d65f3b926de33d03fb016cfdc29314c90974383b5582ae72891", size = 6882634, upload-time = "2026-09-06T16:26:12.524Z" }, { url = "https://files.pythonhosted.org/packages/07/a9/968c90ed2ab15060c338e8137f1215b5a60756ae07328e0a60d1c6734df4/numpy-2.5.3-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1fb6f8fb9ff0b3a69f52c66ce397b0246583e9f28616231b0e32ca49259a5fa6", size = 15748923, upload-time = "2026-09-06T16:26:15.092Z" }, { url = "https://files.pythonhosted.org/packages/59/08/9df04103947b95e3b6b1f2ed1a70521f325647a31b82da6a2aae3a485508/numpy-2.5.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:93e1f5447e2b1e479d7bd74701e84746b86450cff1fc368b132d195e2b8f8211", size = 16746748, upload-time = "2026-09-06T16:26:18.43Z" }, { url = "https://files.pythonhosted.org/packages/41/a0/14c8d5fe5b53a334aabb653deb391c0fef49558f491880ea300ed6785224/numpy-2.5.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c00abe94c1a69d75d827dcf1c025b25c8a45d230b3bcd77a9020883a1b047653", size = 17111561, upload-time = "2026-09-06T16:26:22.113Z" }, { url = "https://files.pythonhosted.org/packages/c4/a6/d7e96e42f01522e154c32489640f16dfc4f6181d165d05fc3bec8c2c4999/numpy-2.5.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:536f963710a4e63934d80ac0dc4f478804a83e9a84b6828018f25d09953ada33", size = 18513945, upload-time = "2026-09-06T16:26:25.401Z" }, { url = "https://files.pythonhosted.org/packages/25/39/3453afb7119d0449ef11c886874120ff180e2c337760e0e2d88f70f1a945/numpy-2.5.3-cp314-cp314t-win32.whl", hash = "sha256:4c8a6d2ebce6305fd82fbefca827775437147052a976ee7c94b36a0c1b52ac6c", size = 6335421, upload-time = "2026-09-06T16:26:28.175Z" }, { url = "https://files.pythonhosted.org/packages/99/01/22815d2b19a1a746b1d45205cffebb3fe511a18acb75fba6c88491fc9894/numpy-2.5.3-cp314-cp314t-win_amd64.whl", hash = "sha256:9a37475425b431b4d060f23b4f52cd2f3aef6bc7c654bd760adf0040eec9d435", size = 12896420, upload-time = "2026-09-06T16:26:31.265Z" }, { url = "https://files.pythonhosted.org/packages/fa/ee/a7cbba67eeaff038dc29ca8b98a88396c8b0cc9c89d4924f4a27a5c9150b/numpy-2.5.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2d8240cb4c16fd831074aa2b2cf9fc54664d826341d61c372245b96a74a49a9a", size = 10857177, upload-time = "2026-09-06T16:26:34.167Z" }, { url = "https://files.pythonhosted.org/packages/45/56/78194492883ff5eec90423fe56a3a44b154da047d88a6307f629713c584f/numpy-2.5.3-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:a6391fafaba97500887132cd582abc6e19452b1ac775a47caa7b24490e152058", size = 16996531, upload-time = "2026-09-06T16:26:37.287Z" }, { url = "https://files.pythonhosted.org/packages/11/39/dd55c0af90bbab564b09ae3b0aa60ec5c02b900fa4f1ba23440525c8b32d/numpy-2.5.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:09d5a423c71ad5feb5625844ad58050e35df43871004b52ac9c0ad44a56775be", size = 12012569, upload-time = "2026-09-06T16:26:40.707Z" }, { url = "https://files.pythonhosted.org/packages/b6/51/04f67d32e4862b281b1cb84ceeaed3421189a84fb6fb51a391cd6d5009f7/numpy-2.5.3-cp315-cp315-macosx_14_0_arm64.whl", hash = "sha256:f9579f383d1bf9df80081e72760e84960a7fd4f88cf0c9e535a8597c9bb646f5", size = 5448498, upload-time = "2026-09-06T16:26:43.435Z" }, { url = "https://files.pythonhosted.org/packages/a3/c9/25b4dc0dd1344ec26c7319e84fd4e9809d2b5628f4e12decd618036e5178/numpy-2.5.3-cp315-cp315-macosx_14_0_x86_64.whl", hash = "sha256:86bff898a431c0fb71f7610b75726e75a54d47b37edc9d537f48de63bb3c0b90", size = 6783026, upload-time = "2026-09-06T16:26:46.374Z" }, { url = "https://files.pythonhosted.org/packages/fc/c7/29285be1e5232a6e7ee3268a33c85843f5a8ee93350c6465cddd66ebbf76/numpy-2.5.3-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f3ed25271581281f2fccb1adcedfcde4c07362eec69189b50baf6f90e3ae159", size = 15697322, upload-time = "2026-09-06T16:26:49.415Z" }, { url = "https://files.pythonhosted.org/packages/55/49/bbad5335fb4996a16881f853ff3e0ba582f01720e55c89b1c06b8fc42a90/numpy-2.5.3-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffdc76bfcae6b255dff75202c5e7feaf95b40246bc0a17944facc1fecf9f79ab", size = 16708995, upload-time = "2026-09-06T16:26:53.127Z" }, { url = "https://files.pythonhosted.org/packages/ef/e9/1df35483760b04a65ea44669f89dc64f30e5aca098b48ceb8b1310b0e0fe/numpy-2.5.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:116f96cadd935c6122e9228d676fe7ede19e741f5c8bb1c3cddbe0c51ccebea2", size = 17052508, upload-time = "2026-09-06T16:26:56.464Z" }, { url = "https://files.pythonhosted.org/packages/b8/99/66e54da8265cc8be8a7382bf96edce17aaa2837d6f484432025932a3caa5/numpy-2.5.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:09ffa5d903faeaa5c4dd05009cf81c8bab9f2cb37c548b8d39b65b4cfa7c97f7", size = 18468224, upload-time = "2026-09-06T16:26:59.966Z" }, { url = "https://files.pythonhosted.org/packages/01/bc/b5e90a91c115168d793dfd2ad9c69c438c2fe7a13a437e770bc5b078e732/numpy-2.5.3-cp315-cp315-win32.whl", hash = "sha256:e01c918ac3d48e18a927cf7b14a26a3e29ff2bdf2eacb976da0aecd6a43ed034", size = 6179919, upload-time = "2026-09-06T16:27:03.166Z" }, { url = "https://files.pythonhosted.org/packages/37/ea/780748fd3985109075514ef8fc64cd25f943e40dde13a6d59141eb268fc8/numpy-2.5.3-cp315-cp315-win_amd64.whl", hash = "sha256:e931e4f499e0dc7ef29d269a8e5b35dd722e5d14be07df6240166ea7c6532fae", size = 12697656, upload-time = "2026-09-06T16:27:06.153Z" }, { url = "https://files.pythonhosted.org/packages/b3/16/407be69a2a87c8cab64d95975a8977a426a29e138f07e276ec258f0fe4e5/numpy-2.5.3-cp315-cp315-win_arm64.whl", hash = "sha256:26e15e4aecd8617dfbaecb37d223e365d7b39411fba20454be2670a96aa74cb5", size = 10767601, upload-time = "2026-09-06T16:27:09.297Z" }, { url = "https://files.pythonhosted.org/packages/44/bf/a97ffb01e41d50a32a9177aef942a4d0e389a3daf451d04e5f38ef6afb87/numpy-2.5.3-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:6cef4bb1706dfec49243c05d921eefb4e190d41e2528b30d8035ea1f36b4c24a", size = 17090092, upload-time = "2026-09-06T16:27:12.907Z" }, { url = "https://files.pythonhosted.org/packages/d1/24/136c02f2c2af9a067a84d0c3aa10c99012c0476fa5066732fa4a4202557d/numpy-2.5.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:d1c89973648c85069c5046ad460f7b8a00218b29a2e42359ac8cc63e9ab94832", size = 12129429, upload-time = "2026-09-06T16:27:16.089Z" }, { url = "https://files.pythonhosted.org/packages/fe/6c/b47582d6597789bf946d5efbeb6b9e56fd8bcbd5efc6fbf51dbe1ea31eb3/numpy-2.5.3-cp315-cp315t-macosx_14_0_arm64.whl", hash = "sha256:214045a5bf00113a146ab9ee9730c44501af6723cdf1f6830932f7b5ef2e7af0", size = 5565452, upload-time = "2026-09-06T16:27:19.868Z" }, { url = "https://files.pythonhosted.org/packages/be/b4/ef3cc6da73774202d4deae16bb321fd8298a4e0561e3539f8c4be237d916/numpy-2.5.3-cp315-cp315t-macosx_14_0_x86_64.whl", hash = "sha256:8617bbfae4486cf99c9f899966699428d19da931d06ca94ad3da986c76e15997", size = 6876736, upload-time = "2026-09-06T16:27:22.232Z" }, { url = "https://files.pythonhosted.org/packages/9e/24/e3813329498596cb842703dcacac1741612ed9fb9c4e6a3e0c7e2ebbc597/numpy-2.5.3-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:595d020938c84e320bcf40ad71089e108eac0d377cd018e14a8c094f39e98d85", size = 15745777, upload-time = "2026-09-06T16:27:25.181Z" }, { url = "https://files.pythonhosted.org/packages/4a/9e/4e7a07fd0776dc2210cdacf2010be8665194d094defc10c419d7dea794cc/numpy-2.5.3-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f24021b9f22bc6301c37b196974a92c1c18dccedb6fef3dd252e95f2d6adbe4", size = 16746949, upload-time = "2026-09-06T16:27:28.576Z" }, { url = "https://files.pythonhosted.org/packages/91/db/01674c0e20335057813a00c2ebd546ed25bff9ed7914f9bced00f8c55d94/numpy-2.5.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:71b39d9f935b6ec0f8753e3e2afb51e3efba6f2e05b68b32a40754d24bcd4a3c", size = 17108994, upload-time = "2026-09-06T16:27:31.946Z" }, { url = "https://files.pythonhosted.org/packages/45/7a/584c5e71f8d378e57cac0b033891ed65c683ef90573ba4854e8c28203db0/numpy-2.5.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:6b05c171afb3aa07adbd20abc00aea86fe375beb0fdb9ef780ec5b7f63bab1c0", size = 18512266, upload-time = "2026-09-06T16:27:35.196Z" }, { url = "https://files.pythonhosted.org/packages/a1/d2/4e1014173aa3c55e6a756e0e567290743a6ab33a288460374d7ef6bcd239/numpy-2.5.3-cp315-cp315t-win32.whl", hash = "sha256:f54660b0eb6b0b9f36e7fe1cdfdff472028dd0d14acd9b9b65098efbad059469", size = 6330292, upload-time = "2026-09-06T16:27:38.149Z" }, { url = "https://files.pythonhosted.org/packages/6c/b0/ff5658a58199b7bcaad87bf260eef6713d9d42cca4e028f935b4fc5fbac6/numpy-2.5.3-cp315-cp315t-win_amd64.whl", hash = "sha256:1aad64d99730d013cfc6debafed22783b4fc5a7f4b8bc744d2d8cf7dcc880551", size = 12884918, upload-time = "2026-09-06T16:27:40.965Z" }, { url = "https://files.pythonhosted.org/packages/fb/0b/b12a2df5d1b774bd9007a6fdff9381145b6223d37f11afc9c37ab0efd9a1/numpy-2.5.3-cp315-cp315t-win_arm64.whl", hash = "sha256:befa1ae5bd6030b3f512b43ff3fa5290bbed6b84411a44244b14adf835f5b89d", size = 10850807, upload-time = "2026-09-06T16:27:43.868Z" }, ] [[package]] name = "numpydoc" version = "1.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "sphinx" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c4/2f/5e9bf8ed0a93587b8089242b4926dfd04d4b3b0b65f04345d219a1225f03/numpydoc-1.11.0.tar.gz", hash = "sha256:26db0adaac8087e98da05c86b11198defd962aef7c21f4dffd2703c8c6eb4187", size = 101575, upload-time = "2026-09-15T16:31:08.905Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/fb/e5/f5c38d4899cd8edd5b522071c5658202077d113c3225484d779cf257970e/numpydoc-1.11.0-py3-none-any.whl", hash = "sha256:493ad2ce0d8275ca5da484ec37dbdbbc1334c3a6968dffd10ec729b51753e15e", size = 71464, upload-time = "2026-09-15T16:31:07.521Z" }, ] [[package]] name = "obstore" version = "0.11.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a2/27/aa7549157a4a681157e315534ba5ab8f167f77662166e792a6e836938f46/obstore-0.11.1.tar.gz", hash = "sha256:a5afe8b99e3b20cdc9133be7a1b381259acf0d470029f6b2fc79c3f9947ad436", size = 130828, upload-time = "2026-08-21T23:57:27.837Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0d/38/438f85772bfbcd2985845a5d00d1c910e98b15f587dae6cb1e435865eba9/obstore-0.11.1-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d5c50b755d781efebe4f9c70ee6d00858e44d95f0229b8b5174358f861d9bd7c", size = 5400146, upload-time = "2026-08-21T23:56:29.168Z" }, { url = "https://files.pythonhosted.org/packages/e7/62/edd2613649cb6b4400f5d125223ee094d2da1c4c93636453a5360c61b865/obstore-0.11.1-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:093152daa5c32b70a032f231bbc6a7eff76dda4285eed5a81d272b4485032dab", size = 4596203, upload-time = "2026-08-21T23:56:30.7Z" }, { url = "https://files.pythonhosted.org/packages/c1/52/09edd48251a26a65f786bf2def93994ffce501f3aea6724474d8de0a7f4f/obstore-0.11.1-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b97ee10456e65f166c030b5fbde8380ca508621e23b84efd77aad83afe99315a", size = 5003021, upload-time = "2026-08-21T23:56:32.085Z" }, { url = "https://files.pythonhosted.org/packages/59/8f/89a28c75d46bbb0552d34a5e84c5bc512526d45f935321f00af2e6275a99/obstore-0.11.1-cp311-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48983a143de69b11de49212caa79b5c39acbff7f592f9e85d156a0133a0e5796", size = 5240376, upload-time = "2026-08-21T23:56:33.658Z" }, { url = "https://files.pythonhosted.org/packages/50/c3/b4620899003e2472bec7baf5d2bce12cf6f6f11d02d8f17e03e093717a53/obstore-0.11.1-cp311-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b9cb988e03ff963914cf2176aee293eaaff7dcf4efcb905ce6834264b4b0884", size = 5444946, upload-time = "2026-08-21T23:56:35.093Z" }, { url = "https://files.pythonhosted.org/packages/bd/de/687685ab39ae4a70cc13ac252febfe62a387836a55e046f467c9fb231880/obstore-0.11.1-cp311-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97850f68c8417f7167549bdb705c362c825ed470298c6b04faf52258c5e9234e", size = 5304065, upload-time = "2026-08-21T23:56:36.535Z" }, { url = "https://files.pythonhosted.org/packages/b4/36/b962ad52031b5bfdd249923db6151ff2b665b08c017611db96838455f337/obstore-0.11.1-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a9f3d66dbf3c073dd6b6033c0787ef14edf78d97bff95595d6f6979692da264", size = 5556770, upload-time = "2026-08-21T23:56:38.377Z" }, { url = "https://files.pythonhosted.org/packages/e0/04/41a0f2917fcaa1e66b754fc98c92441d5a252ff9c619941ac086f9bdb4f6/obstore-0.11.1-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:2ba3bceec4263b3a70eea873abedb82e03da9599f2326e80d6505cab0c10d401", size = 5337765, upload-time = "2026-08-21T23:56:40.279Z" }, { url = "https://files.pythonhosted.org/packages/f9/39/0b618efe59ee4b58f16d2b9246674a70c390bebf665cd284b439bade9bd3/obstore-0.11.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e956fa8a953b7eb8580658d68cdf37e410356032c17697a06e44d0e8c4084a0f", size = 5544012, upload-time = "2026-08-21T23:56:41.754Z" }, { url = "https://files.pythonhosted.org/packages/57/3b/1018f6da240f3529d1cb9a836c79a91ce45349ba3cffcd6baa73296c7e6c/obstore-0.11.1-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:159a50d0f4cc53afe6f5c695313bcaf6e92ac81d7847692c8279153483272bfa", size = 5229851, upload-time = "2026-08-21T23:56:43.367Z" }, { url = "https://files.pythonhosted.org/packages/35/2c/c2fe082816b255159373b15be3f28b2515d0eb954e248e2f65449c456a14/obstore-0.11.1-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:128da07f3a1b9c70159e2b2be9e27f458a9d531d57b1856dd7c4ddb73b4c9937", size = 5361812, upload-time = "2026-08-21T23:56:44.871Z" }, { url = "https://files.pythonhosted.org/packages/43/9c/656eb5cce818e7e3985d27f4bd403feb0b6e620494775a55a0943c8d2fff/obstore-0.11.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:67b8acfbbab960c2cb14c34294a96556caa67fcff15ee77c716d5c1bd1558606", size = 5790855, upload-time = "2026-08-21T23:56:46.391Z" }, { url = "https://files.pythonhosted.org/packages/c1/5c/94ece1c902ff5cdbe1140997d1f720221a0d5defa902fdd93715297604f5/obstore-0.11.1-cp311-abi3-win_amd64.whl", hash = "sha256:e23ea15cebe5f5be5d11005043d7b5ff56848e39499681ef52f8227bd385bd51", size = 5308846, upload-time = "2026-08-21T23:56:47.774Z" }, { url = "https://files.pythonhosted.org/packages/c0/af/235730a429e55777f15f9d7fef118a6ecf02f430ed4588deb4b7e13964aa/obstore-0.11.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:73284fc8a9804596baf4d80b55c0e71a6007487bfa28d6924acd85264d5be81f", size = 5424585, upload-time = "2026-08-21T23:56:49.532Z" }, { url = "https://files.pythonhosted.org/packages/e3/3c/432ea70c061fc9401fe399391ed4f04dead4a4cf33ea7c9564ce0e299d24/obstore-0.11.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8a3f93310422b153629af929d9e88d1fc826d5e32a456de4d3a1926a770ae09c", size = 4576791, upload-time = "2026-08-21T23:56:51.014Z" }, { url = "https://files.pythonhosted.org/packages/43/c7/57dbdfa4ba5339b80fc755359bb232bdba7f82e92df6603ef383396b5f16/obstore-0.11.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d666690f0a53ae3df4be2c18af6820369a74c9c5b8960e8760f0f8c9a8f15b36", size = 4992141, upload-time = "2026-08-21T23:56:52.46Z" }, { url = "https://files.pythonhosted.org/packages/37/4b/d25a4c129250364b0ca927fbfca5146744fb3bb4de60ae9b302300cbdfef/obstore-0.11.1-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0844ab75c8413c0af2d0fd2d2f6aa42d166809b6ac4be478151cd946ce7e5d69", size = 5217707, upload-time = "2026-08-21T23:56:53.946Z" }, { url = "https://files.pythonhosted.org/packages/4b/07/0a925c886639d6edce37aafb6eba0c86820cbc9dda42e91ddd784dbe059d/obstore-0.11.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:307b5f9d64a7c00cd13371e376c05e3914216bd0818a19ff2ae05bc0135e01dc", size = 5429440, upload-time = "2026-08-21T23:56:55.507Z" }, { url = "https://files.pythonhosted.org/packages/5f/c0/38d188a00b9f2de30df26802dd5bc2ecaefdabb882fde786cff800ccf50f/obstore-0.11.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f89647953b4ea50bab3f66591f7f462cd454a5d2fbbeefd885716a2c54e13ce", size = 5308903, upload-time = "2026-08-21T23:56:56.924Z" }, { url = "https://files.pythonhosted.org/packages/60/d3/815e5a7ca59f901b6d24fb01ad3af46de37dfdfd2971c019f914bdb7f65e/obstore-0.11.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50de3116af69b6f1669cb443cc200ccdc1523a790a41f00854331b48f716cf8f", size = 5547007, upload-time = "2026-08-21T23:56:58.508Z" }, { url = "https://files.pythonhosted.org/packages/9c/a1/ec4d2291b7eb15e7ddccb2f0646ce560b8a539df4b0ab94d916e7a15a61c/obstore-0.11.1-cp314-cp314t-manylinux_2_24_aarch64.whl", hash = "sha256:eae71e1c5944ade976ce8cd1e780bed9b5c31d6d4f89cb7263e8dcff78f6cd8e", size = 5330062, upload-time = "2026-08-21T23:56:59.954Z" }, { url = "https://files.pythonhosted.org/packages/0a/09/a4a1301f2474f6569bbc0f2d337108ff2225d0b3d285894ed9a6dc953438/obstore-0.11.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ff634b5edbbf76c56ae81397aa41500a66ad0aa48b1856dcc0cf31b159172dc0", size = 5539870, upload-time = "2026-08-21T23:57:01.485Z" }, { url = "https://files.pythonhosted.org/packages/9e/8f/7dbbf935a07ff5375c33916307b77aef9d39bded35081530150822898010/obstore-0.11.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cbe509350d66249fc9e65ece4e7b1855d650f18e477721887128452b95c44b24", size = 5222673, upload-time = "2026-08-21T23:57:03.023Z" }, { url = "https://files.pythonhosted.org/packages/d5/89/4932b3dd7963a3a64fd532c30ccc836f0f4ede1a45bea9f12295299398e2/obstore-0.11.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:46122e585f48ab3f2e4fed51401a4e866279a3a3d1aee2372d598ab85ba2c539", size = 5335696, upload-time = "2026-08-21T23:57:04.439Z" }, { url = "https://files.pythonhosted.org/packages/aa/ff/f3cf31ac4ebfa648dbd2c69774579fd5da4cf4b05fa1192405dafb6aeb38/obstore-0.11.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7275e75228059b2b30772b2ba3e41562a4a92ebdd3a96619d300f98ef152119f", size = 5783641, upload-time = "2026-08-21T23:57:05.899Z" }, { url = "https://files.pythonhosted.org/packages/51/31/b352db3450e700c2c51861d139fc92c3284f7e578ca306408b054f6b4a35/obstore-0.11.1-cp314-cp314t-win_amd64.whl", hash = "sha256:13bb0b6a40931ab2da93f93ad843865d483d95acec1bd586df79313daa5a50af", size = 5290060, upload-time = "2026-08-21T23:57:07.632Z" }, ] [[package]] name = "openapi-schema-validator" version = "0.8.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonschema" }, { name = "jsonschema-specifications" }, { name = "pydantic" }, { name = "pydantic-settings" }, { name = "referencing" }, { name = "rfc3339-validator" }, ] sdist = { url = "https://files.pythonhosted.org/packages/21/4b/67b24b2b23d96ea862be2cca3632a546f67a22461200831213e80c3c6011/openapi_schema_validator-0.8.1.tar.gz", hash = "sha256:4c57266ce8cbfa37bb4eb4d62cdb7d19356c3a468e3535743c4562863e1790da", size = 23134, upload-time = "2026-03-02T08:46:29.807Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/6f/87/e9f29f463b230d4b47d65e17858c595153a8ca8c1775f16e406aa82d455d/openapi_schema_validator-0.8.1-py3-none-any.whl", hash = "sha256:0f5859794c5bfa433d478dc5ac5e5768d50adc56b14380c8a6fd3a8113e89c9b", size = 19211, upload-time = "2026-03-02T08:46:28.154Z" }, ] [[package]] name = "openapi-spec-validator" version = "0.8.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonschema" }, { name = "jsonschema-path" }, { name = "lazy-object-proxy" }, { name = "openapi-schema-validator" }, { name = "pydantic" }, { name = "pydantic-settings" }, ] sdist = { url = "https://files.pythonhosted.org/packages/79/3f/aa0c1150627b4e683ae5673486b7d5cf2623a8821601863ee389e430965a/openapi_spec_validator-0.8.5.tar.gz", hash = "sha256:93b04ef5321d5866b2502371123d86333e5c1444f051d323e02525d9e83c7622", size = 1756845, upload-time = "2026-04-24T15:25:21.334Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/64/96/d7dfe1cc0be2df22d7a97ffb0f8bb00b10d92749aa6e64ffa7cc9a041580/openapi_spec_validator-0.8.5-py3-none-any.whl", hash = "sha256:3669106361856934153991e30714616a294865a33f6411a4c25d1dc2d08cfbc2", size = 50334, upload-time = "2026-04-24T15:25:19.65Z" }, ] [[package]] name = "packaging" version = "26.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, ] [[package]] name = "paginate" version = "0.5.7" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ec/46/68dde5b6bc00c1296ec6466ab27dddede6aec9af1b99090e1107091b3b84/paginate-0.5.7.tar.gz", hash = "sha256:22bd083ab41e1a8b4f3690544afb2c60c25e5c9a63a30fa2f483f6c60c8e5945", size = 19252, upload-time = "2024-08-25T14:17:24.139Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/90/96/04b8e52da071d28f5e21a805b19cb9390aa17a47462ac87f5e2696b9566d/paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591", size = 13746, upload-time = "2024-08-25T14:17:22.55Z" }, ] [[package]] name = "pathable" version = "0.5.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/72/55/b748445cb4ea6b125626f15379be7c96d1035d4fa3e8fee362fa92298abf/pathable-0.5.0.tar.gz", hash = "sha256:d81938348a1cacb525e7c75166270644782c0fb9c8cecc16be033e71427e0ef1", size = 16655, upload-time = "2026-02-20T08:47:00.748Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/52/96/5a770e5c461462575474468e5af931cff9de036e7c2b4fea23c1c58d2cbe/pathable-0.5.0-py3-none-any.whl", hash = "sha256:646e3d09491a6351a0c82632a09c02cdf70a252e73196b36d8a15ba0a114f0a6", size = 16867, upload-time = "2026-02-20T08:46:59.536Z" }, ] [[package]] name = "pathlib-abc" version = "0.5.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/d6/cb/448649d7f25d228bf0be3a04590ab7afa77f15e056f8fa976ed05ec9a78f/pathlib_abc-0.5.2.tar.gz", hash = "sha256:fcd56f147234645e2c59c7ae22808b34c364bb231f685ddd9f96885aed78a94c", size = 33342, upload-time = "2025-10-10T18:37:20.524Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b1/29/c028a0731e202035f0e2e0bfbf1a3e46ad6c628cbb17f6f1cc9eea5d9ff1/pathlib_abc-0.5.2-py3-none-any.whl", hash = "sha256:4c9d94cf1b23af417ce7c0417b43333b06a106c01000b286c99de230d95eefbb", size = 19070, upload-time = "2025-10-10T18:37:19.437Z" }, ] [[package]] name = "pathspec" version = "1.1.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, ] [[package]] name = "pillow" version = "13.0.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ce/a4/9cc30690a394a8305e8be4c9036aa16d0dfddadd09079860b582f562496d/pillow-13.0.0.tar.gz", hash = "sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510", size = 47058631, upload-time = "2026-10-16T22:18:09.718Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/7a/b8/58d79332229482eb0d27c82973850808c19d37e5bdc14443d3417bd373f5/pillow-13.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79", size = 5568643, upload-time = "2026-10-16T22:15:44.474Z" }, { url = "https://files.pythonhosted.org/packages/39/4b/f2ed53d83aa44ac19599ac0a54897f867fafb95f406980d1e24083f8cbe3/pillow-13.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52", size = 4852329, upload-time = "2026-10-16T22:15:46.284Z" }, { url = "https://files.pythonhosted.org/packages/3b/bb/9e30e8ed18c20800ee327a12107d5249ad8dc444867b3adfcc21654e3c71/pillow-13.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d", size = 6519309, upload-time = "2026-10-16T22:15:48.14Z" }, { url = "https://files.pythonhosted.org/packages/e6/99/fdf65cc539e266ab93546c261bdf98411fe165c6e4e13f9dbf597097192b/pillow-13.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca", size = 7144542, upload-time = "2026-10-16T22:15:50.093Z" }, { url = "https://files.pythonhosted.org/packages/26/9b/f38b7c4d8f599537e2d1a6d546cec9beef7536ff8f5c49a3f9c78179d8ac/pillow-13.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620", size = 6618042, upload-time = "2026-10-16T22:15:52.054Z" }, { url = "https://files.pythonhosted.org/packages/a8/7c/45ee28531f89ee4582967d8a17f7a26d71ca64cd902b93c269590cd60a00/pillow-13.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40", size = 7228137, upload-time = "2026-10-16T22:15:54.009Z" }, { url = "https://files.pythonhosted.org/packages/01/bc/9c2a969cb1b70201d04fa17fd5aedf02a732870b8da86bceaaa30b0ca385/pillow-13.0.0-cp312-cp312-win32.whl", hash = "sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27", size = 6515423, upload-time = "2026-10-16T22:15:56.25Z" }, { url = "https://files.pythonhosted.org/packages/ff/cd/73a79ecdc822076002eb24b5e1e4a530b0bb5310fcb6d239abbec877110b/pillow-13.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b", size = 7272166, upload-time = "2026-10-16T22:15:58.597Z" }, { url = "https://files.pythonhosted.org/packages/4a/a3/be5a1236efd085b3984e4305da8f179ae9dab80c991d2ad78f2bd5c4c843/pillow-13.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51", size = 2698943, upload-time = "2026-10-16T22:16:00.637Z" }, { url = "https://files.pythonhosted.org/packages/82/e2/9132d38df320e8099e2642d9fa538f18a7dfebf68a875a31725dc2a115aa/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:e98b4f91f99b24b1da72e10621ba181f7281922384a7403094debb4199de6c00", size = 4190620, upload-time = "2026-10-16T22:16:02.527Z" }, { url = "https://files.pythonhosted.org/packages/cb/be/436dac0996876f65ef1916ffb711c5ef4af2ebc57414b6f16642e07f6b47/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:34fb3db1149c879343861831728c243c432d4a5afc8f1878ca7612ef8de2a033", size = 4290286, upload-time = "2026-10-16T22:16:04.861Z" }, { url = "https://files.pythonhosted.org/packages/60/16/7c1007162e4169ea0d5bec47e2e0b2174f56c50cf5b8c02a1dc23e41037e/pillow-13.0.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599", size = 3723169, upload-time = "2026-10-16T22:16:06.956Z" }, { url = "https://files.pythonhosted.org/packages/08/bc/578ab2eaa04705a8978bd9be0fe15c3d43a8fc95097c4f7e39540e79e4fa/pillow-13.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ef5fe6b7db3f501c56000ba8fe7c7e1a35ca1d45824fbeb95a67daa85cff4dbe", size = 5568469, upload-time = "2026-10-16T22:16:08.664Z" }, { url = "https://files.pythonhosted.org/packages/98/c8/ccede3dcc4f274b80fb49199eed1ef34ba05566c8b71976b9d1f111da2e3/pillow-13.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f11a3c1358bb3c68012a7b1f41b223c3db9f7bfde90b481df028cfe1c3363313", size = 4852187, upload-time = "2026-10-16T22:16:10.693Z" }, { url = "https://files.pythonhosted.org/packages/fb/3c/cd0770e96d20bfd990db024bcbd43897ff3e88da1f4782cf90ea231b1d4f/pillow-13.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19b66ca0a4d25cc0200b381ce824c7ea090db61427315e73cfb05dbf1b35c369", size = 6515908, upload-time = "2026-10-16T22:16:12.896Z" }, { url = "https://files.pythonhosted.org/packages/ef/fe/f5cdaa2d1c4086e757430f368c271a58ee29cfe7378e7ae7b9d964015c25/pillow-13.0.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582", size = 7142544, upload-time = "2026-10-16T22:16:14.844Z" }, { url = "https://files.pythonhosted.org/packages/fb/b5/a54f6d44adbcd23737c8c12953f42be107e4cca64e247c0ca3a5b563061c/pillow-13.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2a14cd73d5e6c927f644b0110ee81f59c4ef506259ec9183904747f2caacc81", size = 6613097, upload-time = "2026-10-16T22:16:16.893Z" }, { url = "https://files.pythonhosted.org/packages/e6/b4/9e6c1207c21b2e91db0d15c16d30f3701ce5add1bd4da0e787761e66cb6e/pillow-13.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502", size = 7224846, upload-time = "2026-10-16T22:16:19.195Z" }, { url = "https://files.pythonhosted.org/packages/78/9f/d3c3cb0518f3dd1eee0ef501535f53d15597a53c5c4118d178086143397a/pillow-13.0.0-cp313-cp313-win32.whl", hash = "sha256:12b0fa15879fe5a9d9eebe66e4e2d74128a773b1a4448010c9c714f45e04b9d8", size = 6516013, upload-time = "2026-10-16T22:16:21.389Z" }, { url = "https://files.pythonhosted.org/packages/d7/ed/1b16fb66a37e1b9e41370b93b2c614c7da761a5b2a41dc333f7af4639962/pillow-13.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:9dda0788a9932776a75d8bcc020076edc308ce2508b51f6739a45e3158b3fc42", size = 7284777, upload-time = "2026-10-16T22:16:23.393Z" }, { url = "https://files.pythonhosted.org/packages/87/7d/8d02eba9f7ecc87f8e90f731e032cc4773802f2a843763afd0c66a535ff4/pillow-13.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:40a266371ece581e5de13b6635ef78d7487824af80561b6ca27694a88c19bc0f", size = 2698826, upload-time = "2026-10-16T22:16:25.382Z" }, { url = "https://files.pythonhosted.org/packages/8e/45/fbad9c7f5e2f1745c373a434d46072bc84eddfefaa28b852425ac957fda6/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c", size = 4190721, upload-time = "2026-10-16T22:16:27.524Z" }, { url = "https://files.pythonhosted.org/packages/a2/bf/2074902a259a90692c5dbf132885d21671644cbbd3f690047516243a55db/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76", size = 4290236, upload-time = "2026-10-16T22:16:29.562Z" }, { url = "https://files.pythonhosted.org/packages/ff/a5/3d1bb93b8ea38d15d21356c1b8e9f50971ac592aaa248f7fb71739094c0e/pillow-13.0.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981", size = 3723093, upload-time = "2026-10-16T22:16:31.249Z" }, { url = "https://files.pythonhosted.org/packages/84/a1/be060135d85187b25fa8b2573786c0e45c62ae27527fff5257675eb97f4a/pillow-13.0.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4", size = 5572253, upload-time = "2026-10-16T22:16:33.103Z" }, { url = "https://files.pythonhosted.org/packages/b5/f4/0bf6f1ac9ba7ee2ffe9914aeb410a1035a0c1a3c42f122388d7fc950bc4b/pillow-13.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d", size = 4852269, upload-time = "2026-10-16T22:16:35.102Z" }, { url = "https://files.pythonhosted.org/packages/83/16/b603274617b3e6728da3a0133d4fba1ea8d7fbe0c4ea5fef9cbffa398b8b/pillow-13.0.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6", size = 6516058, upload-time = "2026-10-16T22:16:37.228Z" }, { url = "https://files.pythonhosted.org/packages/1d/f2/1371707ce438ba9a43f9d24089828494ec6f9396e28cd7903aa704dd6306/pillow-13.0.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d", size = 7142574, upload-time = "2026-10-16T22:16:39.172Z" }, { url = "https://files.pythonhosted.org/packages/b9/a9/62cf68587f910bc87c5951bcb6e9430c7643a65076adf5f18b4bf350e2bd/pillow-13.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c", size = 6614295, upload-time = "2026-10-16T22:16:41.117Z" }, { url = "https://files.pythonhosted.org/packages/6c/c9/9e9b48178649730299be0bafbf5f810a5b6977538f5a92d47e4ba5f5d608/pillow-13.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0", size = 7224841, upload-time = "2026-10-16T22:16:43.118Z" }, { url = "https://files.pythonhosted.org/packages/8f/48/c44e91384b1b6c072a714a63d00608acc16a6a968735e478e4be4384e5ac/pillow-13.0.0-cp314-cp314-win32.whl", hash = "sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055", size = 6508694, upload-time = "2026-10-16T22:16:45.883Z" }, { url = "https://files.pythonhosted.org/packages/50/e2/7ad17747517e4c813dca78257d6bb593466f8231baf1384bc98d92745edb/pillow-13.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a", size = 7281830, upload-time = "2026-10-16T22:16:47.725Z" }, { url = "https://files.pythonhosted.org/packages/41/9f/e5b51f2ce8ef4563215334035403ce61a244f93a2e65c24a940131618f87/pillow-13.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4", size = 2695468, upload-time = "2026-10-16T22:16:49.515Z" }, { url = "https://files.pythonhosted.org/packages/97/73/a53502f731f6c84805a4ce87a8881e86d07a65aee64728b900d112dc703e/pillow-13.0.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa", size = 5578196, upload-time = "2026-10-16T22:16:51.291Z" }, { url = "https://files.pythonhosted.org/packages/21/50/7e949959a56b5559c78ef172b70fc6025706acb289f3f648a15329e4fa6f/pillow-13.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f", size = 4857276, upload-time = "2026-10-16T22:16:53.399Z" }, { url = "https://files.pythonhosted.org/packages/c9/52/43ded0ac08e77e39d17bd6d894162572201b5e8fdc4a022b14c55a1960a4/pillow-13.0.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1", size = 6607008, upload-time = "2026-10-16T22:16:55.525Z" }, { url = "https://files.pythonhosted.org/packages/0d/2e/4a57ed216d5bf877387cba4d44c02aa87abe51ad2e06404aa6b597c4da39/pillow-13.0.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683", size = 7224281, upload-time = "2026-10-16T22:16:57.738Z" }, { url = "https://files.pythonhosted.org/packages/5c/3d/2929b248e135752d63aaeabf53f6de797e905b7b6263f9c1e2adade20aba/pillow-13.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058", size = 6695600, upload-time = "2026-10-16T22:16:59.842Z" }, { url = "https://files.pythonhosted.org/packages/bc/33/fb0da992cf084ed1ee6ee05e96b6191e35c351eb707ca2df874feb24ce47/pillow-13.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e", size = 7300589, upload-time = "2026-10-16T22:17:01.686Z" }, { url = "https://files.pythonhosted.org/packages/bb/af/c30dd9ef6a3dce84ecb1b2393cac629047a32e66279056c6cb46c3ffd65c/pillow-13.0.0-cp314-cp314t-win32.whl", hash = "sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15", size = 6517204, upload-time = "2026-10-16T22:17:03.722Z" }, { url = "https://files.pythonhosted.org/packages/85/d5/ef3a8ae7609c8ab26570abf18194882534b480d245a9148b895a2262c004/pillow-13.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0", size = 7286695, upload-time = "2026-10-16T22:17:05.543Z" }, { url = "https://files.pythonhosted.org/packages/84/3c/0861db0c69a03689ebe3e7599d35668a414deb5aaafd8255215e4e62e3d3/pillow-13.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692", size = 2698098, upload-time = "2026-10-16T22:17:07.674Z" }, { url = "https://files.pythonhosted.org/packages/6e/eb/e86a41962a3a31ad495a3d6a912d9002b01bba4a6fa71c28703abdca7f69/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7d5e097eabd5d4915c27ac9f6e9325bb1e01a7c94d7348b5c4451d3fc30e8521", size = 4191213, upload-time = "2026-10-16T22:17:09.449Z" }, { url = "https://files.pythonhosted.org/packages/dd/04/7e7445e706afec84ef06dc25b2c77f9cecff242c23730c62928ad1089d4a/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:51ebb529fc5e8abdd55302f6c3f8943037348ec2384329a3e55677f69ae2fdfc", size = 4290650, upload-time = "2026-10-16T22:17:11.196Z" }, { url = "https://files.pythonhosted.org/packages/46/85/d534089068ac40c0f61f650389a296c812eab745a8a733f81feec39ae955/pillow-13.0.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:c87a04e7f152ee4db5dbe15f0bfa3fe628a1aeedbe66b81c43b87f52c813f9ac", size = 3723078, upload-time = "2026-10-16T22:17:12.772Z" }, { url = "https://files.pythonhosted.org/packages/ce/2b/00b5c3b401df538b5dbe311d3899730fae5ba1e21cb8b80a128887ac6747/pillow-13.0.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:20722cd5375533ad754053363a80b8fb30c79d37930a6f7228030e237ce3bd5d", size = 5572292, upload-time = "2026-10-16T22:17:14.539Z" }, { url = "https://files.pythonhosted.org/packages/72/f5/a6b0610f946e9b54ae888ba7ad29d6e2f8d9b11f69e212d70416b7186c09/pillow-13.0.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:04a63720f79e22fd54d897c233a68314c50ca0603360f54ce2bd5db8f0f43782", size = 4852222, upload-time = "2026-10-16T22:17:16.521Z" }, { url = "https://files.pythonhosted.org/packages/1e/d0/c0ff25bb0427ec70f32aac995453321fc3c4494a46c7292ff02d9d260b04/pillow-13.0.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f8db54177035a2b5f717f94475b74907e12a96dc74f6d487f3966b0a4694071", size = 6527349, upload-time = "2026-10-16T22:17:18.534Z" }, { url = "https://files.pythonhosted.org/packages/85/11/e46fa0bcc0fc117f1eacc206b3d51f04d583fab328b4b740672d0438d83f/pillow-13.0.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f36aa77f74b14a2103622b196402c71330e4e7cf4122ee45c854e9a01e0eecf", size = 7152873, upload-time = "2026-10-16T22:17:20.415Z" }, { url = "https://files.pythonhosted.org/packages/7c/53/2bdeca705f3a58679fa9201233193c86a30238340ca29b3a152c3fcc719b/pillow-13.0.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:149a1c323ba366f90b48e6f1a6db6582eb50512df374588de56fc909abc0f0f3", size = 6626871, upload-time = "2026-10-16T22:17:22.361Z" }, { url = "https://files.pythonhosted.org/packages/14/de/3829030f209722ade3753bb456b805a0e7f885cf885b483407365c82e9ca/pillow-13.0.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e892354a424b1e82a78ba16785822bdd40de06f19446a3d4a28047df1a4f8932", size = 7236439, upload-time = "2026-10-16T22:17:24.352Z" }, { url = "https://files.pythonhosted.org/packages/44/b1/67fc37388382bc036a74f04da5bb6fcbf90ec3ac1f45893e06176c373fd3/pillow-13.0.0-cp315-cp315-win32.whl", hash = "sha256:8e459b613cf31f8980f584387b4fe47bf08e076e288d61aa8eba1f7fd4ddbb81", size = 6508688, upload-time = "2026-10-16T22:17:26.314Z" }, { url = "https://files.pythonhosted.org/packages/44/05/fdb6a79aee10861165526b41b7b440e36ee1ff58613b2fbefe3acbf06d4f/pillow-13.0.0-cp315-cp315-win_amd64.whl", hash = "sha256:d4737f50d23ddd11d6685faacbb9252887dde0bc7997127757247983d74399d6", size = 7281107, upload-time = "2026-10-16T22:17:28.198Z" }, { url = "https://files.pythonhosted.org/packages/e6/9c/6c6f7dc2a566467baf121394e83de25ac287cffe34cc030726beb4be0224/pillow-13.0.0-cp315-cp315-win_arm64.whl", hash = "sha256:2310e2adfc13c6d457d7d0f2ffdd5c56a74a317650cf765887df8d515a647ac0", size = 2695513, upload-time = "2026-10-16T22:17:30.013Z" }, { url = "https://files.pythonhosted.org/packages/23/16/e9120defdb8bff693b582d035312acc6f530cdc2bc5238a87f410efccd82/pillow-13.0.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:21c32061a5512dc13f87cc7303e120e4d77c2f70df89a35af0e3ce246e5ac57f", size = 5578129, upload-time = "2026-10-16T22:17:31.642Z" }, { url = "https://files.pythonhosted.org/packages/47/22/76faf2bb2448f48a84cf894c1e2bb1090d7d2b1aa521ece131cb2f05d631/pillow-13.0.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1c774b9c0ba0d0b356f71530677b39028878a2cae5a93e0a13779cfd17120f88", size = 4857428, upload-time = "2026-10-16T22:17:33.43Z" }, { url = "https://files.pythonhosted.org/packages/b3/64/0861ea4e03c50b14af9ed9dd8b9f388ab42f445c623a9fcbab19aef7aa55/pillow-13.0.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20822140c9a2ba9c5e42df342b0a892ab2e102fc0e3db967a133bb9ce3f898bb", size = 6612582, upload-time = "2026-10-16T22:17:35.415Z" }, { url = "https://files.pythonhosted.org/packages/5c/20/cd1e9b3f60a9f3d5ebe46da36955c26004092a501f7f2c42f9fcc26611bc/pillow-13.0.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffea6b2986a797c84b6d1d27d4f43e0b83c73fc16281a930891cd237cf0d08fe", size = 7230176, upload-time = "2026-10-16T22:17:37.362Z" }, { url = "https://files.pythonhosted.org/packages/b6/ef/ffb0a2fc85690714fb7ac662956949645d57f15bf4426f520da361e7b6e3/pillow-13.0.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:05cd2cd39055d0b9650060e4513ef4a9a0aea870083950bd046a0d6490f93a22", size = 6703640, upload-time = "2026-10-16T22:17:39.313Z" }, { url = "https://files.pythonhosted.org/packages/e0/c2/6402ee3ccb3363aeed8fdb3f9fe56703ac5377a539f0b81b53999d7e121f/pillow-13.0.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:c6fd031333dedcad7ce3e03d244093f43504f209e86925ebdb2b868fa4037630", size = 7309231, upload-time = "2026-10-16T22:17:41.021Z" }, { url = "https://files.pythonhosted.org/packages/fc/96/d6b82e8b8c06ad143b66b12e0b31051e4bf489b4fabca1f390a14fe1ebde/pillow-13.0.0-cp315-cp315t-win32.whl", hash = "sha256:c50c80e3591e5cf9a8dfa6007d4f10732f02e32263850999286e56ccd650cfb2", size = 6517200, upload-time = "2026-10-16T22:17:42.886Z" }, { url = "https://files.pythonhosted.org/packages/60/d4/0062e6005e9b270b165015aafd343364ac378ec5a685431ea42baad33357/pillow-13.0.0-cp315-cp315t-win_amd64.whl", hash = "sha256:adb3c2124c68e480f4632d06cb7caa3b48a51678475f483b4edb1c6dee29ea43", size = 7287488, upload-time = "2026-10-16T22:17:45.56Z" }, { url = "https://files.pythonhosted.org/packages/e4/81/b8eb349746be7bce47dd95337c6634e84910af91b59aaa9159c034c614fa/pillow-13.0.0-cp315-cp315t-win_arm64.whl", hash = "sha256:4a25756244bdcce3b50580baadf320fa48297f0eeb8e114ba64139a40d4d0fc8", size = 2698073, upload-time = "2026-10-16T22:17:47.556Z" }, { url = "https://files.pythonhosted.org/packages/60/da/f6d644e631c5b370c5d0b899f3ccadd2780aab070b29a110527cf4e1fd2c/pillow-13.0.0-pp312-pypy312_pp80-macosx_10_15_x86_64.whl", hash = "sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463", size = 5523188, upload-time = "2026-10-16T22:17:59.334Z" }, { url = "https://files.pythonhosted.org/packages/e4/b0/2594f202d743a35d4fe80682bfd96f277e5ca7aecb27cf64ef5bfc1d000b/pillow-13.0.0-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e", size = 4806808, upload-time = "2026-10-16T22:18:01.188Z" }, { url = "https://files.pythonhosted.org/packages/8e/ed/d504165ef79af6fafe25c76726cf30f544cc236002f7ed054f1d51adc4ff/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63", size = 5464358, upload-time = "2026-10-16T22:18:02.779Z" }, { url = "https://files.pythonhosted.org/packages/7b/fe/a10aba5503ca13e9e009b8bd0a1a4ce5584bdce246245fd6f3a83701369c/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61", size = 6043514, upload-time = "2026-10-16T22:18:04.538Z" }, { url = "https://files.pythonhosted.org/packages/c0/37/66b12dc476041fe523bca29c9d58fbf1ea92a4eaf73acfb9e993948f1191/pillow-13.0.0-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a", size = 7366030, upload-time = "2026-10-16T22:18:06.638Z" }, ] [[package]] name = "platformdirs" version = "4.9.6" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/9f/4a/0883b8e3802965322523f0b200ecf33d31f10991d0401162f4b23c698b42/platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a", size = 29400, upload-time = "2026-04-09T00:04:10.812Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917", size = 21348, upload-time = "2026-04-09T00:04:09.463Z" }, ] [[package]] name = "pluggy" version = "1.6.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] [[package]] name = "propcache" version = "0.5.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/4a/cb/e27bc2b2737a0bb49962b275efa051e8f1c35a936df7d5139b6b658b7dc9/propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba", size = 95887, upload-time = "2026-05-08T21:00:11.277Z" }, { url = "https://files.pythonhosted.org/packages/e6/13/b8ae04c59392f8d11c6cd9fb4011d1dc7c86b81225c770280300e259ffe1/propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a", size = 54654, upload-time = "2026-05-08T21:00:12.604Z" }, { url = "https://files.pythonhosted.org/packages/2c/7d/49777a3e20b55863d4794384a38acd460c04157b0a00f8602b0d508b8431/propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf", size = 55190, upload-time = "2026-05-08T21:00:13.935Z" }, { url = "https://files.pythonhosted.org/packages/44/c7/085d0cd63062e84044e3f05797749c3f8e3938ff3aeb0eb2f69d43fafc91/propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144", size = 59995, upload-time = "2026-05-08T21:00:15.526Z" }, { url = "https://files.pythonhosted.org/packages/9c/42/32cf8e3009e92b2645cf1e944f701e8ea4e924dffde1ee26db860bcbf7e4/propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9", size = 63422, upload-time = "2026-05-08T21:00:16.824Z" }, { url = "https://files.pythonhosted.org/packages/9e/1b/f112433f99fc979431b87a39ef169e3f8df070d99a72792c56d6937ac48b/propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42", size = 64342, upload-time = "2026-05-08T21:00:18.362Z" }, { url = "https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476", size = 61639, upload-time = "2026-05-08T21:00:19.692Z" }, { url = "https://files.pythonhosted.org/packages/cc/da/4d775080b1490c0ae604acda868bd71aabe3a89ed16f2aa4339eb8a283e7/propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba", size = 61588, upload-time = "2026-05-08T21:00:21.155Z" }, { url = "https://files.pythonhosted.org/packages/04/ac/f076982cbe2195ee9cf32de5a1e46951d9fb399fc207f390562dd0fd8fb2/propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a", size = 60029, upload-time = "2026-05-08T21:00:22.713Z" }, { url = "https://files.pythonhosted.org/packages/70/60/189be62e0dd898dce3b331e1b8c7a543cd3a405ac0c81fe8ee8a9d5d77e1/propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64", size = 56774, upload-time = "2026-05-08T21:00:24.001Z" }, { url = "https://files.pythonhosted.org/packages/ea/9e/93377b9c7939c1ffae98f878dee955efadfd638078bc86dbc21f9d52f651/propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913", size = 63532, upload-time = "2026-05-08T21:00:25.545Z" }, { url = "https://files.pythonhosted.org/packages/14/f9/590ef6cfb9b8028d516d287812ece32bb0bc5f11fbb9c8bf6b2e6313fec8/propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1", size = 61592, upload-time = "2026-05-08T21:00:27.186Z" }, { url = "https://files.pythonhosted.org/packages/b4/5e/70958b3034c297a630bba2f17ca7abc2d5f39a803ad7e370ab79d1ecd022/propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33", size = 64788, upload-time = "2026-05-08T21:00:28.8Z" }, { url = "https://files.pythonhosted.org/packages/12/fd/77fe5936d8c3086ca9048f7f415f122ed82e53884a9ec193646b42deef06/propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a", size = 62514, upload-time = "2026-05-08T21:00:30.098Z" }, { url = "https://files.pythonhosted.org/packages/cf/74/66bd798b5b3be70aa1b391f5cc9d6a0a5532d7fd3b19ec0b213e72e6ad9d/propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031", size = 39018, upload-time = "2026-05-08T21:00:31.622Z" }, { url = "https://files.pythonhosted.org/packages/61/7c/5c0d34aa3024694d6dcb9271cdbdd08c4e47c1c0ad95ec7e7bc74cdea145/propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42", size = 42322, upload-time = "2026-05-08T21:00:32.918Z" }, { url = "https://files.pythonhosted.org/packages/4d/91/875812f1a3feb20ceba818ef39fbe4d92f1081e04ac815c822496d0d038b/propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84", size = 38172, upload-time = "2026-05-08T21:00:35.124Z" }, { url = "https://files.pythonhosted.org/packages/c5/09/f049e45385503fe67db75a6b6186a7b9f0c3930366dc960522c312a825b1/propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a", size = 94457, upload-time = "2026-05-08T21:00:36.355Z" }, { url = "https://files.pythonhosted.org/packages/6b/65/83d1d05655baf63113731bd5a1008435e14f8d1e5a06cbe4ec5b23ad7a31/propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117", size = 53835, upload-time = "2026-05-08T21:00:38.072Z" }, { url = "https://files.pythonhosted.org/packages/a9/12/a6ba6482bb5ea3260c000c9b20881c95fa11c6b30173715668259f844ed7/propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098", size = 54545, upload-time = "2026-05-08T21:00:39.319Z" }, { url = "https://files.pythonhosted.org/packages/a9/19/7fa086f5764c59ec8a8e157cd93aa8497acc00aba9dcdec56bfffb32602d/propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4", size = 59886, upload-time = "2026-05-08T21:00:40.621Z" }, { url = "https://files.pythonhosted.org/packages/a1/e4/5d7663dc8235956c8f5281698a3af1d351d8820341ddd890f59d9a9127f2/propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e", size = 63261, upload-time = "2026-05-08T21:00:41.775Z" }, { url = "https://files.pythonhosted.org/packages/4a/4a/15a03adee24d6350da4292caeac44c34c033d2afe5e87eb370f38854560f/propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7", size = 64184, upload-time = "2026-05-08T21:00:43.018Z" }, { url = "https://files.pythonhosted.org/packages/8b/c6/979176efdaa3d239e36d503d5af63a0a773b36662ed8f52e5b6a6d9fd40e/propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d", size = 61534, upload-time = "2026-05-08T21:00:44.507Z" }, { url = "https://files.pythonhosted.org/packages/c8/22/63e8cd1bae4c2d2be6493b6b7d10566ddafad88137cfbc99964a1119853c/propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a", size = 61500, upload-time = "2026-05-08T21:00:45.796Z" }, { url = "https://files.pythonhosted.org/packages/60/5a/28e5d9acbac1cc9ccb67045e8c1b943aa8d79fdf39c93bd73cacd68008ea/propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2", size = 59994, upload-time = "2026-05-08T21:00:47.093Z" }, { url = "https://files.pythonhosted.org/packages/f3/40/db650677f554a95b9c01a7c9d93d629e93a15562f5deb4573c9ee136fed2/propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa", size = 56884, upload-time = "2026-05-08T21:00:48.376Z" }, { url = "https://files.pythonhosted.org/packages/80/45/70b39b89516ff8b96bf732fa6fded8cef20f293cb1508690101c3c07ec51/propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853", size = 63464, upload-time = "2026-05-08T21:00:49.954Z" }, { url = "https://files.pythonhosted.org/packages/f9/e2/fa59d3a89eac5534293124af4f1d0d0ada091ce4a0ab4610ce03fd2bdd8d/propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a", size = 61588, upload-time = "2026-05-08T21:00:51.281Z" }, { url = "https://files.pythonhosted.org/packages/0b/97/efb547a55c4bc7381cfb202d6a2239ac621045277bc1ea5dfd3a7f0516c0/propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704", size = 64667, upload-time = "2026-05-08T21:00:52.602Z" }, { url = "https://files.pythonhosted.org/packages/92/56/f5c7d9b4b7595d5127da38974d791b2153f3d1eae6c674af3583ace92ad3/propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4", size = 62463, upload-time = "2026-05-08T21:00:54.303Z" }, { url = "https://files.pythonhosted.org/packages/bd/3b/484a3a65fc9f9f60c41dcd17b428bace5389544e2c680994534a20755066/propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d", size = 38621, upload-time = "2026-05-08T21:00:55.808Z" }, { url = "https://files.pythonhosted.org/packages/1c/fd/3f0f10dba4dabad3bf53102be007abf55481067952bde0fdddff439e7c61/propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757", size = 41649, upload-time = "2026-05-08T21:00:57.061Z" }, { url = "https://files.pythonhosted.org/packages/90/ec/6ce619cc32bb500a482f811f9cd509368b4e58e638d13f2c68f370d6b475/propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f", size = 37636, upload-time = "2026-05-08T21:00:58.646Z" }, { url = "https://files.pythonhosted.org/packages/1b/82/c1d268bbbf2ef981c5bf0fbbe746db617c66e3bcefe431a1aa8943fbe23a/propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d", size = 98872, upload-time = "2026-05-08T21:00:59.889Z" }, { url = "https://files.pythonhosted.org/packages/f4/d4/52c871e73e864e6b34c0e2d58ac1ec5ccd149497ddc7ad2137ae98323a35/propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa", size = 56257, upload-time = "2026-05-08T21:01:01.195Z" }, { url = "https://files.pythonhosted.org/packages/67/f0/9b90ca2a210b3d09bcfcd96ecd0f55545c091535abce2a45de2775cfd357/propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94", size = 56696, upload-time = "2026-05-08T21:01:02.941Z" }, { url = "https://files.pythonhosted.org/packages/9d/0e/6e9d4ba07c8e56e21ddec1e75f12148142b21ca83a51871babce095334f4/propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164", size = 62378, upload-time = "2026-05-08T21:01:04.475Z" }, { url = "https://files.pythonhosted.org/packages/65/19/c10badaa463dde8a27ce884f8ee2ec37e6035b7c9f5ff0c8f74f06f08dac/propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f", size = 65283, upload-time = "2026-05-08T21:01:05.959Z" }, { url = "https://files.pythonhosted.org/packages/b0/b6/93bea99ca80e19cef6512a8580e5b7857bbe09422d9daa7fd4ef5723306c/propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c", size = 66616, upload-time = "2026-05-08T21:01:07.228Z" }, { url = "https://files.pythonhosted.org/packages/83/e4/5c7462e50625f051f37fb38b8224f7639f667184bbd34424ec83819bb1b7/propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc", size = 63773, upload-time = "2026-05-08T21:01:08.514Z" }, { url = "https://files.pythonhosted.org/packages/ca/b6/99238894047b13c823be25027e736626cd414a52a5e30d2c3347c2733529/propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f", size = 63664, upload-time = "2026-05-08T21:01:09.874Z" }, { url = "https://files.pythonhosted.org/packages/85/1e/a3a1a63116a2b8edb415a8bb9a6f0c34bd03830b1e18e8ce2904e1dc1cf4/propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb", size = 62643, upload-time = "2026-05-08T21:01:11.132Z" }, { url = "https://files.pythonhosted.org/packages/e4/03/893cf147de2fc6543c5eaa07ad833170e7e2a2385725bbebe8c0503723bb/propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751", size = 59595, upload-time = "2026-05-08T21:01:12.387Z" }, { url = "https://files.pythonhosted.org/packages/86/3b/04c1a2e12c57766568ba75ba72b3bf2042818d4c1425fab6fc07155c7cff/propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836", size = 65711, upload-time = "2026-05-08T21:01:13.676Z" }, { url = "https://files.pythonhosted.org/packages/1c/34/80f8d0099f8d6bacc4de1624c85672681c8cd1149ca2da0e38fd120b817f/propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f", size = 64247, upload-time = "2026-05-08T21:01:14.936Z" }, { url = "https://files.pythonhosted.org/packages/f3/1a/8b08f3a5f1037e9e370c55883ceeeee0f6dd0416fb2d2d67b8bfc91f2a79/propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55", size = 67102, upload-time = "2026-05-08T21:01:16.281Z" }, { url = "https://files.pythonhosted.org/packages/34/68/8bdb7bb7756d76e005490649d10e4a8369e610c74d619f71e1aedf889e9c/propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568", size = 64964, upload-time = "2026-05-08T21:01:17.57Z" }, { url = "https://files.pythonhosted.org/packages/0a/aa/50fb0b5d3968b61a510926ff8b8465f1d6e976b3ab74496d7a4b9fc42515/propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191", size = 42546, upload-time = "2026-05-08T21:01:18.946Z" }, { url = "https://files.pythonhosted.org/packages/ae/4c/0ddbae64321bd4a95bcbfc19307238016b5b1fee645c84626c8d539e5b74/propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7", size = 46330, upload-time = "2026-05-08T21:01:20.162Z" }, { url = "https://files.pythonhosted.org/packages/00/d9/9cddc8efb78d8af264c5ec9f6d10b62f57c515feda8d321595f56010fb23/propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96", size = 40521, upload-time = "2026-05-08T21:01:21.399Z" }, { url = "https://files.pythonhosted.org/packages/e2/ea/23ee535d90ce8bcc465a3028eb3cc0ce3bd1005f4bb27710b30587de798d/propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999", size = 94662, upload-time = "2026-05-08T21:01:22.683Z" }, { url = "https://files.pythonhosted.org/packages/b5/06/c5a52f419b5d8972f8d46a7577476090d8e3263ff589ce40b5ca4968d5be/propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e", size = 53928, upload-time = "2026-05-08T21:01:23.986Z" }, { url = "https://files.pythonhosted.org/packages/63/b1/4260d67d6bd85e58a66b72d54ce15d5de789b6f3870cc6bedf8ff9667401/propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539", size = 54650, upload-time = "2026-05-08T21:01:25.305Z" }, { url = "https://files.pythonhosted.org/packages/70/06/2f46c318e3307cd7a6a7481def374ce838c0fe20084b39dd54b0879d0e99/propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e", size = 59912, upload-time = "2026-05-08T21:01:26.545Z" }, { url = "https://files.pythonhosted.org/packages/4c/29/fe1aebec2ce57ab985a9c382bded1124431f85078113aa222c5d278430d4/propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979", size = 63300, upload-time = "2026-05-08T21:01:27.937Z" }, { url = "https://files.pythonhosted.org/packages/b4/18/2334b26768b6c82be8c69e83671b767d5ef426aa09b0cba6c2ea47816774/propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80", size = 64208, upload-time = "2026-05-08T21:01:29.484Z" }, { url = "https://files.pythonhosted.org/packages/2b/76/7f1bfd6afff4c5e38e36a3c6d68eb5f4b7311ea80baf693db78d95b603c4/propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825", size = 61633, upload-time = "2026-05-08T21:01:31.068Z" }, { url = "https://files.pythonhosted.org/packages/c4/46/b3ff8aba2b4953a3e50de2cf72f1b5748b8eca93b15f3dc2c84339084c09/propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39", size = 61724, upload-time = "2026-05-08T21:01:32.374Z" }, { url = "https://files.pythonhosted.org/packages/c5/01/814cfcafbcff954f94c01cf30e097ddc88a076b5440fbcf4570753437d40/propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4", size = 60069, upload-time = "2026-05-08T21:01:33.67Z" }, { url = "https://files.pythonhosted.org/packages/da/68/5c6f7622d510cc666a300687e06fd060c1a43361c0c9b20d284f06d8096a/propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5", size = 57099, upload-time = "2026-05-08T21:01:34.915Z" }, { url = "https://files.pythonhosted.org/packages/55/27/9cb0b4c679124085327957d42521c99dba04c88c90c3e55a6f0b633ebccc/propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702", size = 63391, upload-time = "2026-05-08T21:01:36.231Z" }, { url = "https://files.pythonhosted.org/packages/f0/9d/7258aaa5bdf60fc6f27591eef6fe52768cb0beda7140be477c8b12c9794a/propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3", size = 61626, upload-time = "2026-05-08T21:01:37.545Z" }, { url = "https://files.pythonhosted.org/packages/8e/0d/41c602003e8a9b16fe1e7eadf62c7bfba9d5474370b24200bf48b315f45f/propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5", size = 64781, upload-time = "2026-05-08T21:01:38.83Z" }, { url = "https://files.pythonhosted.org/packages/8b/f3/38e66b1856e9bd079deea015bc4a55f7767c0e4db2f7dcf69e7e680ba4ce/propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4", size = 62570, upload-time = "2026-05-08T21:01:40.415Z" }, { url = "https://files.pythonhosted.org/packages/95/ca/bbfe9b910ce57dde8bb4876b4520fc02a4e89497c10de26be936758a3aaa/propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0", size = 39436, upload-time = "2026-05-08T21:01:41.654Z" }, { url = "https://files.pythonhosted.org/packages/61/d2/45c9defbaa1ea297035d9d4cce9e8f80daafbf19319c6007f157c6256ea9/propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c", size = 42373, upload-time = "2026-05-08T21:01:43.041Z" }, { url = "https://files.pythonhosted.org/packages/44/68/9ea5103f41d5217d7d6ec24db90018e23aebec070c3f9a6e54d12b841fd8/propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0", size = 38554, upload-time = "2026-05-08T21:01:44.336Z" }, { url = "https://files.pythonhosted.org/packages/8a/81/fadf555f42d3b762eea8a53950b0489fdc0aa9da5f8ed9e10ce0a4e01b48/propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb", size = 99395, upload-time = "2026-05-08T21:01:45.883Z" }, { url = "https://files.pythonhosted.org/packages/f5/c9/c61e134a686949cf7971af3a390148b1156f7be81c73bc0cd12c873e2d48/propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078", size = 56653, upload-time = "2026-05-08T21:01:47.307Z" }, { url = "https://files.pythonhosted.org/packages/cb/73/daf935ea7048ddd7ec8eec5345b4a40b619d2d178b3c0a0900796bc3c794/propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa", size = 56914, upload-time = "2026-05-08T21:01:48.573Z" }, { url = "https://files.pythonhosted.org/packages/79/9f/aba959b435ea18617edd7cf0a7ad0b9c574b8fc7e3d2cd55fb59cb255d33/propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917", size = 62567, upload-time = "2026-05-08T21:01:49.903Z" }, { url = "https://files.pythonhosted.org/packages/6c/a1/859942de9a791ff42f6141736f5b37749b8f53e65edfa49638c67dd67e6a/propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe", size = 65542, upload-time = "2026-05-08T21:01:51.204Z" }, { url = "https://files.pythonhosted.org/packages/b5/61/315bc0fd6c0fc7f80a528b8afd209e5fc4a875ea79571b91b8f50f442907/propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03", size = 66845, upload-time = "2026-05-08T21:01:52.539Z" }, { url = "https://files.pythonhosted.org/packages/47/f7/9f8122e3132e8e354ac41975ef8f1099be7d5a16bc7ae562734e993665c0/propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335", size = 63985, upload-time = "2026-05-08T21:01:53.847Z" }, { url = "https://files.pythonhosted.org/packages/c8/54/c317819ec157cbf6f35df9df9657a6f82daf34d5faf15948b2f639c2192e/propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285", size = 63999, upload-time = "2026-05-08T21:01:55.179Z" }, { url = "https://files.pythonhosted.org/packages/5a/56/387e3f7dfce0a9233df41fb888aa1c30222cb4bbbf09537c02dd9bd85fe2/propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837", size = 62779, upload-time = "2026-05-08T21:01:57.489Z" }, { url = "https://files.pythonhosted.org/packages/a1/9c/596784cb5824ed61ee960d3f8655a3f0993e107c6e98ab6c818b7fb92ccb/propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8", size = 59796, upload-time = "2026-05-08T21:01:58.736Z" }, { url = "https://files.pythonhosted.org/packages/c2/3d/1a6cfa1726a48542c1e8784a0761421476a5b68e09b7f36bf95eb954aaba/propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366", size = 66023, upload-time = "2026-05-08T21:02:00.228Z" }, { url = "https://files.pythonhosted.org/packages/e4/0e/05fd6990369477076e4e280bcb970de760fddf0161a46e988bc95f7940ec/propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56", size = 64448, upload-time = "2026-05-08T21:02:01.888Z" }, { url = "https://files.pythonhosted.org/packages/cd/86/5f8da315a4309c62c10c0b2516b17492d5d3bbe1bb862b96604db67e2a37/propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d", size = 67329, upload-time = "2026-05-08T21:02:03.484Z" }, { url = "https://files.pythonhosted.org/packages/da/d3/3368efe79ab21f0cdf86ef49895811c9cc933131d4cde1f28a624e22e712/propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2", size = 65172, upload-time = "2026-05-08T21:02:04.745Z" }, { url = "https://files.pythonhosted.org/packages/d5/07/127e8b0bacfb325396196f9d976a22453049b89b9b2b08477cc3145faa44/propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821", size = 43813, upload-time = "2026-05-08T21:02:06.025Z" }, { url = "https://files.pythonhosted.org/packages/88/fb/46dad6c0ae49ed230ab1b16c890c2b6314e2403e6c412976f4a72d64a527/propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370", size = 47764, upload-time = "2026-05-08T21:02:07.353Z" }, { url = "https://files.pythonhosted.org/packages/e7/c4/a47d0a63aa309d10d59ede6e9d4cff03a344a79d1f0f4cd0cd74997b53e0/propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6", size = 41140, upload-time = "2026-05-08T21:02:09.065Z" }, { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, ] [[package]] name = "properdocs" version = "1.6.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "ghp-import" }, { name = "jinja2" }, { name = "markdown" }, { name = "markupsafe" }, { name = "packaging" }, { name = "pathspec" }, { name = "platformdirs" }, { name = "pyyaml" }, { name = "pyyaml-env-tag" }, { name = "watchdog" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ec/29/f27a4e1eddf72ed3db6e47818fbafe6debbf09fd7051f9c1a007239b46ef/properdocs-1.6.7.tar.gz", hash = "sha256:adc7b16e562890af0e098a7e5b02e3a81c20894a87d6a28d345c9300de73c26e", size = 276141, upload-time = "2026-03-20T20:07:48.167Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/bd/4d/fc923f5c85318ee8cc903566dc4e0ebe41b2dfc1d2ecf5546db232397ed6/properdocs-1.6.7-py3-none-any.whl", hash = "sha256:6fa0cfa2e01bf338f684892c8a506cf70ea88ae7f3479c933b6fa20168101cbd", size = 225406, upload-time = "2026-03-20T20:07:46.875Z" }, ] [[package]] name = "py-cpuinfo2" version = "10.1.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/dc/97/a8b1ddada14c8280a047c0746f95cb05d94a31b1a331cea22bcdc2b2a82d/py_cpuinfo2-10.1.1.tar.gz", hash = "sha256:7861133863663f16e06eca63b12904ef100b5760415e92372dac0162799a4771", size = 100840, upload-time = "2026-03-25T21:49:40.797Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/23/0a/ba69d2dde1ae12ef1d389ea5a216384c5ff6ef7a1e7a48d1e9b6686f6790/py_cpuinfo2-10.1.1-py3-none-any.whl", hash = "sha256:adc53396bfb206e6498d078ec2ab407f85799ecd819584ac36a8f80a2d4d762d", size = 23791, upload-time = "2026-03-25T21:49:39.574Z" }, ] [[package]] name = "py-partiql-parser" version = "0.6.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/56/7a/a0f6bda783eb4df8e3dfd55973a1ac6d368a89178c300e1b5b91cd181e5e/py_partiql_parser-0.6.3.tar.gz", hash = "sha256:09cecf916ce6e3da2c050f0cb6106166de42c33d34a078ec2eb19377ea70389a", size = 17456, upload-time = "2025-10-18T13:56:13.441Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c9/33/a7cbfccc39056a5cf8126b7aab4c8bafbedd4f0ca68ae40ecb627a2d2cd3/py_partiql_parser-0.6.3-py2.py3-none-any.whl", hash = "sha256:deb0769c3346179d2f590dcbde556f708cdb929059fb654bad75f4cf6e07f582", size = 23752, upload-time = "2025-10-18T13:56:12.256Z" }, ] [[package]] name = "pycparser" version = "3.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, ] [[package]] name = "pydantic" version = "2.12.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-types" }, { name = "pydantic-core" }, { name = "typing-extensions" }, { name = "typing-inspection" }, ] sdist = { url = "https://files.pythonhosted.org/packages/69/44/36f1a6e523abc58ae5f928898e4aca2e0ea509b5aa6f6f392a5d882be928/pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49", size = 821591, upload-time = "2025-11-26T15:11:46.471Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d", size = 463580, upload-time = "2025-11-26T15:11:44.605Z" }, ] [[package]] name = "pydantic-core" version = "2.41.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, ] [[package]] name = "pydantic-settings" version = "2.14.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "python-dotenv" }, { name = "typing-inspection" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5c/b5/8f48e906c3e0205276e8bd8cb7512217a87b2685304d64be27cad5b3019f/pydantic_settings-2.14.2.tar.gz", hash = "sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f", size = 237700, upload-time = "2026-06-19T13:44:56.324Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/77/c1/6e422f34e569cf8e18df68d1939c81c099d2b61e4f7d9621c8a77560799c/pydantic_settings-2.14.2-py3-none-any.whl", hash = "sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440", size = 61715, upload-time = "2026-06-19T13:44:55.02Z" }, ] [[package]] name = "pygments" version = "2.20.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] [[package]] name = "pygments-ansi-color" version = "0.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/f9/7f417aaee98a74b4f757f2b72971245181fcf25d824d2e7a190345669eaf/pygments-ansi-color-0.3.0.tar.gz", hash = "sha256:7018954cf5b11d1e734383a1bafab5af613213f246109417fee3f76da26d5431", size = 7317, upload-time = "2023-05-18T22:44:35.792Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e6/17/8306a0bcd8c88d7761c2e73e831b0be026cd6873ce1f12beb3b4c9a03ffa/pygments_ansi_color-0.3.0-py3-none-any.whl", hash = "sha256:7eb063feaecadad9d4d1fd3474cbfeadf3486b64f760a8f2a00fc25392180aba", size = 10242, upload-time = "2023-05-18T22:44:34.287Z" }, ] [[package]] name = "pymdown-extensions" version = "11.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown" }, { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/21/a9/5f0c535ba3b08fe09270c16808e053a968868242ecbd5676d4e3a488bf28/pymdown_extensions-11.0.1.tar.gz", hash = "sha256:dd2905ae6fc5b75582fafb139a1266ffc754705efa902aa50067fa7ff4f94ec0", size = 857113, upload-time = "2026-07-02T17:59:22.955Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d6/54/da572c98c0b77626a91b5d3b89f0231d8bff5125c225420908632f8b342d/pymdown_extensions-11.0.1-py3-none-any.whl", hash = "sha256:db3943a62bab7e03af1364f0c4083e64b91fb097675a4b6cceccfbe9a77e5eb2", size = 269455, upload-time = "2026-07-02T17:59:21.271Z" }, ] [[package]] name = "pyparsing" version = "3.3.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, ] [[package]] name = "pytest" version = "9.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] [[package]] name = "pytest-accept" version = "0.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "astor" }, { name = "pytest" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6f/42/573e6dcd8d1a5fc779d525268d5036f710d018cdeefbae2bd5559687a6d5/pytest_accept-0.3.0.tar.gz", hash = "sha256:6f4e03e2492621e10b7678a331828274498cca04686b15b0f3e704e16e651068", size = 29744, upload-time = "2026-06-11T18:44:39.03Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/06/40/79f6f13e616f580b9c35110376818c76cacc54b70d2965bf1f82be336c66/pytest_accept-0.3.0-py3-none-any.whl", hash = "sha256:5e73f27020853b861a63dc630dfe63124af4bd60e0e0bb692e1376b99798f2b5", size = 39846, upload-time = "2026-06-11T18:44:37.965Z" }, ] [[package]] name = "pytest-asyncio" version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pytest" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, ] [[package]] name = "pytest-benchmark" version = "5.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "py-cpuinfo2" }, { name = "pytest" }, ] sdist = { url = "https://files.pythonhosted.org/packages/63/8f/83a15e40dbc34a580ee56eb56983cae5394c6e94d50cf28fe268e457be25/pytest_benchmark-5.3.0.tar.gz", hash = "sha256:358444d4e89be901ee2b6404fb043ac3d7684002ad7f3563cc153fca6339c965", size = 375410, upload-time = "2026-08-23T17:45:08.891Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/eb/42/7e80f7cfa191e0a766d1de99b4661847415ad5db34f8209d81fd42175b59/pytest_benchmark-5.3.0-py3-none-any.whl", hash = "sha256:920ab1dfcffa718d49aa15ba144c7e357bda59216a0dc308016cc1c7236f719d", size = 48401, upload-time = "2026-08-23T17:45:07.094Z" }, ] [[package]] name = "pytest-codspeed" version = "5.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pytest" }, { name = "rich" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e1/b4/cf932fcd1960a2fd6d9b09eb403253a8709aeee975961afa6299239a830e/pytest_codspeed-5.0.3.tar.gz", hash = "sha256:91afef90e6a96b013495e4702ef5d6358614a449e71008cdc194ef668778b92f", size = 324571, upload-time = "2026-05-22T16:20:49.231Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c2/22/456c48160b761d5028c8afa119f085a9fc42855a783a13d73918078969f0/pytest_codspeed-5.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2eeb25fb1ac3f73c4de50e739e78fea396b89782bdb740bf2a7cd2df21f8d4ee", size = 366255, upload-time = "2026-05-22T16:20:56.214Z" }, { url = "https://files.pythonhosted.org/packages/74/33/ac7441fa937c9d9f158083a8c46920a5a5c81ed3c5f96240fc8d650db5c2/pytest_codspeed-5.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73c5c9d98a3372a42611989ccfa437cce3842431ac6d6b9ab42c4f0e59c070f7", size = 932325, upload-time = "2026-05-22T16:21:08.814Z" }, { url = "https://files.pythonhosted.org/packages/77/bc/8b994adcb9e9016e7d9a808056a3dd9cca21441e432ef456eae2b697d7fe/pytest_codspeed-5.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2e0ab65df73e837666d12357280ca50ff6d6ac03ea5266703be518b68170edf", size = 934885, upload-time = "2026-05-22T16:21:01.444Z" }, { url = "https://files.pythonhosted.org/packages/dc/8e/e032451e9e0a06b0c4bff53105f62b693d9a54595dd8c024693741ce3380/pytest_codspeed-5.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6524c57fec279a22ffef6112af404036afc71b4704758ae9f0abda429b8478d4", size = 366253, upload-time = "2026-05-22T16:20:46.192Z" }, { url = "https://files.pythonhosted.org/packages/a9/7b/ae76fd8ac656b9695806a6aafd5f22ec32e6ce20e266a58f9112e01d3cd8/pytest_codspeed-5.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c383c9121deb58a69f174188e9e4488ffc0daced0ed276abf87747182511901", size = 932360, upload-time = "2026-05-22T16:20:30.589Z" }, { url = "https://files.pythonhosted.org/packages/a6/4a/dfd43d943fdb143be4fd62f34c2793ba349dc27aa188e521d19d629aa7ab/pytest_codspeed-5.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4bcdb4b6522738152885ef067e0c8524d5699828d780fb6f464cdb3db44369c", size = 934928, upload-time = "2026-05-22T16:20:38.62Z" }, { url = "https://files.pythonhosted.org/packages/04/6a/fdcec19c7f267c195f147c51d3fd2245f6b8d09b80495ed0a90c008e0842/pytest_codspeed-5.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:25464363c7f9b9bd5022e969c0addba616fa40ac9b8f0fc9e030c4538863b32d", size = 366259, upload-time = "2026-05-22T16:21:06.039Z" }, { url = "https://files.pythonhosted.org/packages/6a/96/c6b03b81dcd21ae3d6b32cca0b3c10149fa378eb21b338d4b63c9eb8050b/pytest_codspeed-5.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:efd43f82ea03ced8488a767ded9473f050791ab7783ea8654107e1e0ac66af40", size = 932395, upload-time = "2026-05-22T16:21:04.804Z" }, { url = "https://files.pythonhosted.org/packages/96/08/56ad8f1cc7d6962f8a680141b361e93467a2abc53d976cd9d5e1edd740e3/pytest_codspeed-5.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:782f9985b6f6b45b8bc20152d206d3a52b56dd088ba81cb70a71f0b39841be9e", size = 934994, upload-time = "2026-05-22T16:20:28.809Z" }, { url = "https://files.pythonhosted.org/packages/0b/54/9096c4545f09da94b1b00f3be2fe4952949e86c9bcafca9a29b26aed1a75/pytest_codspeed-5.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9aa0815b90196f3c20d736ea8691381e97f12bbe8c7d87af10a351e434b452cb", size = 366311, upload-time = "2026-05-22T16:20:41.791Z" }, { url = "https://files.pythonhosted.org/packages/a7/3c/24c53f67a38ad48cb087105ac30a8aa0923223ee274ea9bf2dc705edaa59/pytest_codspeed-5.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:85505c96a3477c346ec2d2b7dced8478f4c651e2b1666ee102d53a832b511853", size = 933169, upload-time = "2026-05-22T16:20:43.178Z" }, { url = "https://files.pythonhosted.org/packages/d1/de/2213f868fa7694f743f96cccbc07e757f45c920c523cccc2da97bc8652df/pytest_codspeed-5.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20eba63765be9d1b6cacbbfad84b87d49eb04b357a7045a0899880da181f81e3", size = 935522, upload-time = "2026-05-22T16:21:03.398Z" }, { url = "https://files.pythonhosted.org/packages/df/85/5dfea1c031d6cccc11653464828edf205c30f798caf5b2a85375aacd914a/pytest_codspeed-5.0.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:ec9fa6f0af0a9feb0e0bd517fb59ef28f806fbd50c0c6900ac26cbb4d080eba5", size = 366275, upload-time = "2026-05-22T16:20:59.463Z" }, { url = "https://files.pythonhosted.org/packages/3c/2b/af4d1b612f03b98a6cf3c7d5f62678917a60110a8bf380d49ab408b31137/pytest_codspeed-5.0.3-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8df77b3409f54f4a268f77f3ff74992fe1d995cdbaf2cecf8ad74d32db217ce7", size = 932537, upload-time = "2026-05-22T16:20:54.945Z" }, { url = "https://files.pythonhosted.org/packages/f5/a2/c7ec45e36a61b418efb2a3cccaa67a0c2fcf1f21d5880f64c33114f0c249/pytest_codspeed-5.0.3-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5d8695a227ea1c3a41d25db5b3fe720bf1b4808bd38862be811a4efd902c792", size = 934153, upload-time = "2026-05-22T16:21:07.494Z" }, { url = "https://files.pythonhosted.org/packages/cf/c7/d5bada9618a0af56a5c8065fc61280849cab8e7c1e24025807a51c3157ce/pytest_codspeed-5.0.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bf4cc4178cbace8f4d2bd240408276bc4da3850ac5fcb5fb5f8a74ab417615bb", size = 366339, upload-time = "2026-05-22T16:20:51.968Z" }, { url = "https://files.pythonhosted.org/packages/a8/37/fb27aeb40a81320e7349553b877a21333c897b27c8dfe215630452908f36/pytest_codspeed-5.0.3-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:abe793da40f87295d33988673d34f06ea569848b44490b847552cd416816258a", size = 933055, upload-time = "2026-05-22T16:20:44.861Z" }, { url = "https://files.pythonhosted.org/packages/f3/d9/6f2d69e96deaf0475a695fc9195af59e7a3b5fab50782855e65c63a7bc28/pytest_codspeed-5.0.3-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c3a9ed38dfa776443b86f4b49a982e8443d0953db4974bd2673d63cc904ae1ad", size = 934481, upload-time = "2026-05-22T16:20:58.264Z" }, { url = "https://files.pythonhosted.org/packages/c5/b2/1d2a993c532146dce9eca5b5942d51898021c3579ce18b2454f932a915f8/pytest_codspeed-5.0.3-py3-none-any.whl", hash = "sha256:fe2ea83c924c2250675b75686c3ee456b8cf0208d83d552e182a195fdf467378", size = 74033, upload-time = "2026-05-22T16:20:26.814Z" }, ] [[package]] name = "pytest-cov" version = "7.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "coverage" }, { name = "pluggy" }, { name = "pytest" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, ] [[package]] name = "pytest-reportlog" version = "1.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pytest" }, ] sdist = { url = "https://files.pythonhosted.org/packages/93/d5/2f4a73822efaad1d6aea6660468d6290963e5f61d2f9d9ca707f7f0b1c13/pytest_reportlog-1.0.0.tar.gz", hash = "sha256:75aec3a92bb53456c3e028605a636579d26f31c6f1e035ad9f706c203cfcb74e", size = 5646, upload-time = "2025-11-11T16:05:15.98Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/05/70/807cbdac584629623c1c2f76b7a39a343e3989510cd52385f1f40581e963/pytest_reportlog-1.0.0-py3-none-any.whl", hash = "sha256:3fc837ef3be6e50f33b52aaf99f88bfbb2ec525febbc3990389d24bbfba28753", size = 6015, upload-time = "2025-11-11T16:05:14.585Z" }, ] [[package]] name = "pytest-xdist" version = "3.8.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "execnet" }, { name = "pytest" }, ] sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, ] [[package]] name = "python-dateutil" version = "2.9.0.post0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "six" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] [[package]] name = "python-dotenv" version = "1.2.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, ] [[package]] name = "pywin32" version = "311" source = { registry = "https://pypi.org/simple" } wheels = [ { url = "https://files.pythonhosted.org/packages/e7/ab/01ea1943d4eba0f850c3c61e78e8dd59757ff815ff3ccd0a84de5f541f42/pywin32-311-cp312-cp312-win32.whl", hash = "sha256:750ec6e621af2b948540032557b10a2d43b0cee2ae9758c54154d711cc852d31", size = 8706543, upload-time = "2025-07-14T20:13:20.765Z" }, { url = "https://files.pythonhosted.org/packages/d1/a8/a0e8d07d4d051ec7502cd58b291ec98dcc0c3fff027caad0470b72cfcc2f/pywin32-311-cp312-cp312-win_amd64.whl", hash = "sha256:b8c095edad5c211ff31c05223658e71bf7116daa0ecf3ad85f3201ea3190d067", size = 9495040, upload-time = "2025-07-14T20:13:22.543Z" }, { url = "https://files.pythonhosted.org/packages/ba/3a/2ae996277b4b50f17d61f0603efd8253cb2d79cc7ae159468007b586396d/pywin32-311-cp312-cp312-win_arm64.whl", hash = "sha256:e286f46a9a39c4a18b319c28f59b61de793654af2f395c102b4f819e584b5852", size = 8710102, upload-time = "2025-07-14T20:13:24.682Z" }, { url = "https://files.pythonhosted.org/packages/a5/be/3fd5de0979fcb3994bfee0d65ed8ca9506a8a1260651b86174f6a86f52b3/pywin32-311-cp313-cp313-win32.whl", hash = "sha256:f95ba5a847cba10dd8c4d8fefa9f2a6cf283b8b88ed6178fa8a6c1ab16054d0d", size = 8705700, upload-time = "2025-07-14T20:13:26.471Z" }, { url = "https://files.pythonhosted.org/packages/e3/28/e0a1909523c6890208295a29e05c2adb2126364e289826c0a8bc7297bd5c/pywin32-311-cp313-cp313-win_amd64.whl", hash = "sha256:718a38f7e5b058e76aee1c56ddd06908116d35147e133427e59a3983f703a20d", size = 9494700, upload-time = "2025-07-14T20:13:28.243Z" }, { url = "https://files.pythonhosted.org/packages/04/bf/90339ac0f55726dce7d794e6d79a18a91265bdf3aa70b6b9ca52f35e022a/pywin32-311-cp313-cp313-win_arm64.whl", hash = "sha256:7b4075d959648406202d92a2310cb990fea19b535c7f4a78d3f5e10b926eeb8a", size = 8709318, upload-time = "2025-07-14T20:13:30.348Z" }, { url = "https://files.pythonhosted.org/packages/c9/31/097f2e132c4f16d99a22bfb777e0fd88bd8e1c634304e102f313af69ace5/pywin32-311-cp314-cp314-win32.whl", hash = "sha256:b7a2c10b93f8986666d0c803ee19b5990885872a7de910fc460f9b0c2fbf92ee", size = 8840714, upload-time = "2025-07-14T20:13:32.449Z" }, { url = "https://files.pythonhosted.org/packages/90/4b/07c77d8ba0e01349358082713400435347df8426208171ce297da32c313d/pywin32-311-cp314-cp314-win_amd64.whl", hash = "sha256:3aca44c046bd2ed8c90de9cb8427f581c479e594e99b5c0bb19b29c10fd6cb87", size = 9656800, upload-time = "2025-07-14T20:13:34.312Z" }, { url = "https://files.pythonhosted.org/packages/c0/d2/21af5c535501a7233e734b8af901574572da66fcc254cb35d0609c9080dd/pywin32-311-cp314-cp314-win_arm64.whl", hash = "sha256:a508e2d9025764a8270f93111a970e1d0fbfc33f4153b388bb649b7eec4f9b42", size = 8932540, upload-time = "2025-07-14T20:13:36.379Z" }, ] [[package]] name = "pyyaml" version = "6.0.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] [[package]] name = "pyyaml-env-tag" version = "1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/2e/79c822141bfd05a853236b504869ebc6b70159afc570e1d5a20641782eaa/pyyaml_env_tag-1.1.tar.gz", hash = "sha256:2eb38b75a2d21ee0475d6d97ec19c63287a7e140231e4214969d0eac923cd7ff", size = 5737, upload-time = "2025-05-13T15:24:01.64Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/04/11/432f32f8097b03e3cd5fe57e88efb685d964e2e5178a48ed61e841f7fdce/pyyaml_env_tag-1.1-py3-none-any.whl", hash = "sha256:17109e1a528561e32f026364712fee1264bc2ea6715120891174ed1b980d2e04", size = 4722, upload-time = "2025-05-13T15:23:59.629Z" }, ] [[package]] name = "referencing" version = "0.37.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, { name = "rpds-py" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, ] [[package]] name = "regex" version = "2026.5.9" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/dc/0e/49aee608ad09480e7fd276898c99ec6192985fa331abe4eb3a986094490b/regex-2026.5.9.tar.gz", hash = "sha256:a8234aa23ec39894bfe4a3f1b85616a7032481964a13ac6fc9f10de4f6fca270", size = 416074, upload-time = "2026-05-09T23:15:19.37Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/50/9b/6550044bc44e17c84d312c031c2ec42fbdb6a4ec4e29093be3a172d08772/regex-2026.5.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:57eeeb05db7979413dec5438f2db21d7ecbba787cde7a711df1a6f6df672aa06", size = 490451, upload-time = "2026-05-09T23:12:34.72Z" }, { url = "https://files.pythonhosted.org/packages/1e/95/fc7ba4303b5a0f92446a12ee6778ef2c6c799233f5060042a31bf390cfe9/regex-2026.5.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:398c521292f4c7fb807001dcd54694d3a1fcafc179a36ad9cc56f98df85930b6", size = 292112, upload-time = "2026-05-09T23:12:36.285Z" }, { url = "https://files.pythonhosted.org/packages/54/4b/ee27938d1b2c443e89a9a10e00d2d19aa5ee300cd3d61140644e93bb083e/regex-2026.5.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f7a7c26137296beba7784de6eba69c6a93a63ccebc385e4962fe67e267a91225", size = 289599, upload-time = "2026-05-09T23:12:38.089Z" }, { url = "https://files.pythonhosted.org/packages/d8/dd/ba103dc19614e25f3880800ca67ce093d6e21b325d72b8383c7bf906e9fa/regex-2026.5.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6441cc660d76107934a09c22167200839a0e89604a6297f78a974e66e931d2c0", size = 796732, upload-time = "2026-05-09T23:12:40.062Z" }, { url = "https://files.pythonhosted.org/packages/cf/e7/f035b4fd858b050b0080bf302968dc0f59ba34e391872d54936758e6844e/regex-2026.5.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:91328f1c23d47595ca3ef0a7557fa129c5a23404b775c770697d2f35b33e0107", size = 865440, upload-time = "2026-05-09T23:12:42.059Z" }, { url = "https://files.pythonhosted.org/packages/0a/51/8cd301ecc899aea28124357f729f4272f44de7806fc7ca02490bfbe253e8/regex-2026.5.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:93a7860539414dddaefba2b40f8771765ae17949d4c7182b876ce429e11a8309", size = 912329, upload-time = "2026-05-09T23:12:44.373Z" }, { url = "https://files.pythonhosted.org/packages/cc/1e/3fbe2fa1e8cebd62f3bb7d3321cff1640aca2e240b51d9bd624aad949260/regex-2026.5.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd2810d22146b6d838acc5ec15602cb6b47920aa4e33015df3868eedfd20bab8", size = 801239, upload-time = "2026-05-09T23:12:46.268Z" }, { url = "https://files.pythonhosted.org/packages/17/2f/6f6008682bf2cf98040a0d3153a8e557b6ab728d7713d045cee4ce544ab8/regex-2026.5.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daff2bdbaf1d23e52fdff7c0b7bc2048b68f978df6a4d107ac981f94caef2e66", size = 777054, upload-time = "2026-05-09T23:12:48.051Z" }, { url = "https://files.pythonhosted.org/packages/19/2b/eee0d20a6842ba04df4b8847a920b57ef56853f14ef85405473e586b605a/regex-2026.5.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4eeb011098fcb77af513dcef521a3dbecbf8849b1e38940759d293b7a93f5026", size = 785098, upload-time = "2026-05-09T23:12:49.851Z" }, { url = "https://files.pythonhosted.org/packages/4a/98/6fc1e6410feefb92159edaed5041992bfe390e8d26c721865434acbca558/regex-2026.5.9-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ea9c8ecfa1b73c73b626534d6626e5340d429630943672b8480724f44e84b962", size = 860095, upload-time = "2026-05-09T23:12:51.666Z" }, { url = "https://files.pythonhosted.org/packages/18/a3/bd855e0f2cb1a978ecf6fa6bb69632dd9c3f6ea3b81cde62fde14c9daec7/regex-2026.5.9-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:cd2846168eb9ee3c513902bc8225409cb1caab31d04728b145171fa1625d9621", size = 765762, upload-time = "2026-05-09T23:12:53.413Z" }, { url = "https://files.pythonhosted.org/packages/dc/66/0ae8c092e60b14c79d24f8e0b7f0aea5bfbffdcab00b5483d13404d3c3a5/regex-2026.5.9-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:39617fb0cde9c0e6306dc70e3bfc096f3da793219879f7ae7aa341a69fbdcf6d", size = 852100, upload-time = "2026-05-09T23:12:55.256Z" }, { url = "https://files.pythonhosted.org/packages/21/de/8dfde60fc1b21c946a893ba273403b72617edb261370cb1087099a83f088/regex-2026.5.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fd03c4f0e33280d15cae17159b899245d6b7c53d21def19b263b39655061f5ce", size = 789479, upload-time = "2026-05-09T23:12:57.573Z" }, { url = "https://files.pythonhosted.org/packages/c3/1c/bdcc98f9a4af4fdd166c74941174619ccff4726d3ce32faa8e9a2ecd38dd/regex-2026.5.9-cp312-cp312-win32.whl", hash = "sha256:164eba9b755ea6f244b0d881196fbc1fac09714e9782c9e2732b813142033c8e", size = 266699, upload-time = "2026-05-09T23:12:59.14Z" }, { url = "https://files.pythonhosted.org/packages/78/87/240d36864f9e48ace85f72e79ced97ceb7f27ce87739a947dcb834b4e6bc/regex-2026.5.9-cp312-cp312-win_amd64.whl", hash = "sha256:86f40a5d6444db30a125c9c9177e6b25dad981cbc37451fd838f145e6edac92e", size = 277783, upload-time = "2026-05-09T23:13:00.789Z" }, { url = "https://files.pythonhosted.org/packages/4f/b5/7b30f312b0669dff5beebe5b0989dc2d1a312b1a44fab852199c387a5b96/regex-2026.5.9-cp312-cp312-win_arm64.whl", hash = "sha256:96f5f58b54a063d7ea9dca08e1cf57bfe10499c4d579ee672da284f57f5f0070", size = 270513, upload-time = "2026-05-09T23:13:02.426Z" }, { url = "https://files.pythonhosted.org/packages/aa/da/797e91ecec6f84135da778ddce78c20e0af5d2a15c26f87a81bc3eadb6db/regex-2026.5.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d626b84406444b165fc0ba981604edea39f0588ff1f92baa23fe50799ea9afdb", size = 490303, upload-time = "2026-05-09T23:13:04.382Z" }, { url = "https://files.pythonhosted.org/packages/44/da/bf30abaaa737b58f4a4b8c4a03659e02fd92092c822e0197ed9e0daab917/regex-2026.5.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d7bdc0ab8f3dd7e1b4f9ab88634e13374669db86bb3c72e8292f07ae313f539f", size = 292019, upload-time = "2026-05-09T23:13:06.022Z" }, { url = "https://files.pythonhosted.org/packages/2d/e7/d0eaf5713828417b9e5648cf81fa9bacd4961f6ab98c380c2034f8716e35/regex-2026.5.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a8820737949116ffff55fe18f9fc644530063ba6ebfcb8314239416e78f1347c", size = 289468, upload-time = "2026-05-09T23:13:08.214Z" }, { url = "https://files.pythonhosted.org/packages/d3/9b/b3fdd62b003baa1a9b593cd8c8699c9651c2e80cc21a5c715707983c42d7/regex-2026.5.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa0fbdbac82cb3e4450d0ccde7d7a35607f4cb2dd9fba4b8b69bfaf8c9fa6aed", size = 796749, upload-time = "2026-05-09T23:13:10.573Z" }, { url = "https://files.pythonhosted.org/packages/d4/30/66ab84588765f5b4b271a9ca09ef7ce2b87caa95176ec3d2ad65d7bc4902/regex-2026.5.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:57e8915c7986aa33d25e4d3629cef711cd2863f2961b10409f0c04cb8b7d9020", size = 865445, upload-time = "2026-05-09T23:13:12.523Z" }, { url = "https://files.pythonhosted.org/packages/1a/89/f05169e8588aac365f35ffc7f3bc3184f095ef4cfded7cfaa3c7fd5dbd89/regex-2026.5.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:508f56a89ba9cb26e4168cbc37dbd60a28d82430a9e18ad1d25fe0883c314ca2", size = 912322, upload-time = "2026-05-09T23:13:14.281Z" }, { url = "https://files.pythonhosted.org/packages/30/e1/c93444052cf41581f3c884ab3fb5823daf0992f11cd4388d4275ca610558/regex-2026.5.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b6d189041f15691cfa2b6c4290448ec221244d225b3f5fe9e7771b34ffcdf6e2", size = 801269, upload-time = "2026-05-09T23:13:16.569Z" }, { url = "https://files.pythonhosted.org/packages/50/fe/0cf96b882f540e62e8b9956599798203d599c44cf4c77917ca27400ff69b/regex-2026.5.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e82db382b44d0111b22601c509c89f64434816c9e0eef9d1989cda8cc6ff1c04", size = 777085, upload-time = "2026-05-09T23:13:18.675Z" }, { url = "https://files.pythonhosted.org/packages/23/5c/d78d4924e7fc875557b9e9b768423925fdfaac5549d06da7810019a9bd26/regex-2026.5.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2acfb48634f64996b57f90f39afa692ff362162722581921fe92239a59960f3c", size = 785153, upload-time = "2026-05-09T23:13:20.525Z" }, { url = "https://files.pythonhosted.org/packages/bf/e0/5214774090e7b4524dcea3e3c4aa74141d43043f8beb49c1599db1c8b53a/regex-2026.5.9-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d29eebfc9525db68cad3c97eedd7f754fa265aa5cd0cf4f863b2421e1b48fc9f", size = 860164, upload-time = "2026-05-09T23:13:22.263Z" }, { url = "https://files.pythonhosted.org/packages/6e/e1/4a57a83350319b1271f0d7a249b8672513ed928b237a741631270de6caea/regex-2026.5.9-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:debb893095e944091c16e641a6e33c1b0f4cb61ab945ec5afbf53ce7068834d8", size = 765731, upload-time = "2026-05-09T23:13:24.277Z" }, { url = "https://files.pythonhosted.org/packages/12/f4/499e74a20c156fc75836ee04a72a38d1a063978f600937f9760467beb1b0/regex-2026.5.9-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d659eee77986549c9ea45b861c7567e44d6287c3dc9a4565478853f7b9fe2ff6", size = 852062, upload-time = "2026-05-09T23:13:26.125Z" }, { url = "https://files.pythonhosted.org/packages/5b/92/7eebc0d0a01e78629695f342ba17e0deaff8fb45e79cc0d7b98287da6e3e/regex-2026.5.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2efa205e6d98b24d1f3ab395c11aa15cdf10935bca283d0285e0499c284fba21", size = 789577, upload-time = "2026-05-09T23:13:27.814Z" }, { url = "https://files.pythonhosted.org/packages/05/a4/018e71f7d2ad48c1ebe6d3ae0026f9b7cb4802fd15c7cc02fdf724355102/regex-2026.5.9-cp313-cp313-win32.whl", hash = "sha256:f3844f134e834076677dd369976e9f5068679fcb8e50102fdf6b7ac96a3ec127", size = 266691, upload-time = "2026-05-09T23:13:29.549Z" }, { url = "https://files.pythonhosted.org/packages/e6/1d/861a93719fb9ee7dbfc3761b3797b7a3e112a5d42c6129459d2d741be9b5/regex-2026.5.9-cp313-cp313-win_amd64.whl", hash = "sha256:3527bb4942d2c14552155406cdedd906567456821848aed1cb4933a391bf5eca", size = 277747, upload-time = "2026-05-09T23:13:31.859Z" }, { url = "https://files.pythonhosted.org/packages/d9/c6/0a2436ae4da1ba76e51cb98943c6838a9a721faa40ebe2dce07694ae34e3/regex-2026.5.9-cp313-cp313-win_arm64.whl", hash = "sha256:56a33f191f17d8c417f99945ebdc1e691d3af9605d86ec68c7e54a57e3e17af6", size = 270500, upload-time = "2026-05-09T23:13:33.525Z" }, { url = "https://files.pythonhosted.org/packages/e8/e9/d21346f7b60ed58789371358ed66b09d00f832e1bd7c06e55d9da5679882/regex-2026.5.9-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:01f28d868834624c934b8d2e0aa1c8341337e37831f4a012f18a5afcba4cbaf3", size = 494172, upload-time = "2026-05-09T23:13:35.935Z" }, { url = "https://files.pythonhosted.org/packages/c4/43/fd1177a2032037c681baecdb3422ee4e1424aec4e4f470ef47793d325274/regex-2026.5.9-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:48036f6374aaa79eb3b754ec29c61d1c6b1606749d705a13f8854fa2539671f6", size = 293952, upload-time = "2026-05-09T23:13:38.307Z" }, { url = "https://files.pythonhosted.org/packages/f2/7d/9fbf919768368d3f8a4f6c692cf2aa61e482b2b81ec6a298ace4cbf02480/regex-2026.5.9-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b96350aa424e79d4fd6b567b344dcbe2b2d6bfc48dfe7717587e1fa6d43da6ff", size = 292314, upload-time = "2026-05-09T23:13:40.353Z" }, { url = "https://files.pythonhosted.org/packages/e2/6c/e41bfeecb589716843e7c4df09ba46ff2a42961457afece19059d85caeef/regex-2026.5.9-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f3af7a4903c5c04a11a196a5aa75cdd7dd3f8508132f9fb3259d9f5908e3b88", size = 811681, upload-time = "2026-05-09T23:13:42.543Z" }, { url = "https://files.pythonhosted.org/packages/87/83/a5c1c525fba0aa656e88ad0face0b1829788ef4c2fb6b26df58aa1151b84/regex-2026.5.9-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7e87577720152d2caae19fe2baaf1f8d5ca12091e9e229f03915c37d1e4b9178", size = 871135, upload-time = "2026-05-09T23:13:44.326Z" }, { url = "https://files.pythonhosted.org/packages/18/d4/80882e799e440dd878b0979cbebf8fa4d54624a332c83037c7a701649e3f/regex-2026.5.9-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c8b9b9d294cfea3cd19c718ade7cc93492b2c4991abd9a68d0b3477ae6d8e100", size = 917265, upload-time = "2026-05-09T23:13:47.295Z" }, { url = "https://files.pythonhosted.org/packages/ae/ff/8db60211e2286e396aad7dc7725356c502bff0901ea05bd6cdc2e1a042b9/regex-2026.5.9-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:728d8bfd28a8845c8b6bc5dc7ce010453d206396786c0765c2740cb65f37791e", size = 816311, upload-time = "2026-05-09T23:13:49.885Z" }, { url = "https://files.pythonhosted.org/packages/4c/47/742ef579c61730f8d268e5cf1f9ce0e37e2ea041ad0f5644724f2378e463/regex-2026.5.9-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7e30b874d341fac767d7df5a0870540541c2c054b80cfaac116e8d367a8a7ff2", size = 785498, upload-time = "2026-05-09T23:13:52.25Z" }, { url = "https://files.pythonhosted.org/packages/7f/ab/cb0999802dcb0fb95b1ab005e8d4163d8afdd67efc2cb6b6630ac13f8cb1/regex-2026.5.9-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:fd190e88a895a8901325fad284a3f74ea52b1da8525b76cc811fa9b1edf0ce2b", size = 801348, upload-time = "2026-05-09T23:13:54.127Z" }, { url = "https://files.pythonhosted.org/packages/7d/62/8ca59a24c55bc34d166eefaf3717bd77772f329fdbf984d86581e0a3571c/regex-2026.5.9-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:8e76e8161ad00694cfce6767d5dea860c6391ac5b83e5c3a39661e696f11fc7e", size = 866493, upload-time = "2026-05-09T23:13:56.067Z" }, { url = "https://files.pythonhosted.org/packages/8d/3d/30f2ae62cef3278bb5bb821f467277a55fb73f01032cf85997e15e8289a8/regex-2026.5.9-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ddda5340e6c01a293027dd46232fa79eaff1b48058ce7a98f572b6445b088041", size = 772811, upload-time = "2026-05-09T23:13:57.867Z" }, { url = "https://files.pythonhosted.org/packages/d8/ae/7d2089bcd78ad0c0161bc684339df50032acb438a7bd3305e7ddb1193cec/regex-2026.5.9-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:205109e96b3cf5adf8f4cd62bedde9487feb282b9497a3535451e5a24cd706a0", size = 856584, upload-time = "2026-05-09T23:13:59.679Z" }, { url = "https://files.pythonhosted.org/packages/a9/29/92ff47f75990131ea4f24ba17819e5a9d141e10819807e09addd73409af6/regex-2026.5.9-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dfbe4579b9f08036aa7d101d1835437a20783574ac66327e6b29b4018a138081", size = 803453, upload-time = "2026-05-09T23:14:01.978Z" }, { url = "https://files.pythonhosted.org/packages/04/99/eff29f1037dcab36702c9ee5d6858cf1ce2336ea8ea2987f64245b99ea5e/regex-2026.5.9-cp313-cp313t-win32.whl", hash = "sha256:ed2c9e8068b614c574d8d30e543d617cf5379b0535d46f97ef00e904745a08b5", size = 269951, upload-time = "2026-05-09T23:14:03.661Z" }, { url = "https://files.pythonhosted.org/packages/0e/9d/8870b8981d27b22cda77bb26a5ac7ebfa9c7d9e0dea195a834a82380e748/regex-2026.5.9-cp313-cp313t-win_amd64.whl", hash = "sha256:b46b0f094dc1d3b90356c85a0bd2c9bafc4a6a190b9d6f8ddd5a033b6e088ed4", size = 281240, upload-time = "2026-05-09T23:14:05.56Z" }, { url = "https://files.pythonhosted.org/packages/72/b1/3379415e8f135c13ac551353397cc4fe97b4978f3cac73c5fcbcded548b8/regex-2026.5.9-cp313-cp313t-win_arm64.whl", hash = "sha256:872acc074bd29ffc9913ecdfedf6ea77502312ca44a4aa0d3779089c6069d8de", size = 272383, upload-time = "2026-05-09T23:14:07.843Z" }, { url = "https://files.pythonhosted.org/packages/13/3e/9c3cd292d8808b3645a2ce517e200179b6d0e903f176300bd8b542e14de5/regex-2026.5.9-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:1bd7587a2948b4085195d5a3374eaf4a425dc3e55784c038175355ecf3bbbf8a", size = 490376, upload-time = "2026-05-09T23:14:09.64Z" }, { url = "https://files.pythonhosted.org/packages/60/70/d43ee8a2ca0a8b68d167f21658b85520ac0574617c7f320367c5047f7556/regex-2026.5.9-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:dea2e88e1cce4522496cce630e11e67b98b7076620bc4336c3f674bc21a375f4", size = 291964, upload-time = "2026-05-09T23:14:11.424Z" }, { url = "https://files.pythonhosted.org/packages/21/91/9d50b433828d8e74196904e168a43abf1e6e88b2a15d47ed742456720c37/regex-2026.5.9-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2099f7e7ff7b6aa3192312650a56e91cc091e49d50b04e4f6f8b6e28b3b27f1c", size = 289682, upload-time = "2026-05-09T23:14:13.123Z" }, { url = "https://files.pythonhosted.org/packages/3e/d2/b835e3cafbb9d977736912436259ff551d60919f7d7b3d37d46659c63564/regex-2026.5.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecd353045824e4477562a2ac718c25799cdaaa41f7aa925a806a8a3e6848a5b9", size = 796996, upload-time = "2026-05-09T23:14:14.923Z" }, { url = "https://files.pythonhosted.org/packages/2c/a6/9f992d00019166b9de01c546dd4549bc679f2a68df11b877740b0760b7c2/regex-2026.5.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:65c8c8c37377794bd5b2f3ebe51919042bf17aec802e23c833d89782ed0c78af", size = 866089, upload-time = "2026-05-09T23:14:17.757Z" }, { url = "https://files.pythonhosted.org/packages/e0/08/4d32af657e049b19cb62b02e46e38fe1518797bfb2203ee93a510b21b0dc/regex-2026.5.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5b73ab8afcf66c622db143d1c6fda4e58e4d537ee4f125229ad47b1ab80f34c0", size = 911530, upload-time = "2026-05-09T23:14:20.353Z" }, { url = "https://files.pythonhosted.org/packages/d9/27/2af43dd1dc201d1fecefda64a45f4ad0995855b92724f795a777b402ee69/regex-2026.5.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0de5cf193997384ed2ca6f1cd4f78055b255d93d82d5a8cd6ba0d11c10b167e4", size = 800643, upload-time = "2026-05-09T23:14:22.265Z" }, { url = "https://files.pythonhosted.org/packages/a4/dd/23a249047013b5321d4a60c4d2437462086f601b061776a525e5fba2a59f/regex-2026.5.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d641a8c9a61618047796d572a39a79b26167b0411d2c3031937b2fe2d081e2cf", size = 777223, upload-time = "2026-05-09T23:14:24.179Z" }, { url = "https://files.pythonhosted.org/packages/94/6a/e85ed9538cd19586d0465076a4578a12e093ce776d15f3f8ce92733a8dd6/regex-2026.5.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:24b2355ef5cc9aa5b8f07d17704face1c166fdcc2290fa7bd6e6c925655a8346", size = 785760, upload-time = "2026-05-09T23:14:26.065Z" }, { url = "https://files.pythonhosted.org/packages/2a/c4/f25473209438638e947c55f9156fd8f236f74169229028cc99116380868e/regex-2026.5.9-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:a24852d3c29ad9e47593593d8a247c44ccc3d0548ef12c822d6ed0810affe676", size = 860891, upload-time = "2026-05-09T23:14:28.17Z" }, { url = "https://files.pythonhosted.org/packages/f9/f7/f4f86e3c74419c37370e91f150ae0c2ef7d34b2e0e4cdd5da046a02e4022/regex-2026.5.9-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:916714069da19329ef7de197dcbc77bb3104145c7c2c864dbfbe318f46b88b14", size = 765891, upload-time = "2026-05-09T23:14:30.06Z" }, { url = "https://files.pythonhosted.org/packages/26/70/704d8e13765939146b1cd0ef4e2feb71d7929727d2290f026eed10095955/regex-2026.5.9-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:fa411799ca8da32a8d38d020a88faa5b6f91657d284761352940ecf9f7c3bbdd", size = 851380, upload-time = "2026-05-09T23:14:32.123Z" }, { url = "https://files.pythonhosted.org/packages/26/29/1a13582a8460038edc38e49f64ceb0dd7c60f5caba77571f4bf6601965d9/regex-2026.5.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1e6da47d679b7010ef27556b6e0f99771b744936db1792a10ceac6547ae1503e", size = 789350, upload-time = "2026-05-09T23:14:34.799Z" }, { url = "https://files.pythonhosted.org/packages/73/56/3dcafe34fc72e271d62ad9a291801e88a1457bb251c132f15fcc2e5aad1a/regex-2026.5.9-cp314-cp314-win32.whl", hash = "sha256:98bd73080e8756255137e1bd3f3f00295bbc5aa383c0e0f973920e9134d7c4ad", size = 272130, upload-time = "2026-05-09T23:14:36.729Z" }, { url = "https://files.pythonhosted.org/packages/d0/9c/02eebf0be95efe416c664db7fb8b6b05b7a0b06a7544f2884f2558b0526f/regex-2026.5.9-cp314-cp314-win_amd64.whl", hash = "sha256:ff8d372ac2acdc048d1c19916f27ee61bc5722728458ba6ca5052f2c72d51763", size = 280999, upload-time = "2026-05-09T23:14:39.126Z" }, { url = "https://files.pythonhosted.org/packages/70/5a/1dd1abee76cb7a846a0bcf42fdc87e5720c3c33c24f3e37814310a513d9f/regex-2026.5.9-cp314-cp314-win_arm64.whl", hash = "sha256:e1d93bf647916292e8edcec150c07ddf3dc50179ccaf770c04a7f9e452155372", size = 273500, upload-time = "2026-05-09T23:14:41.059Z" }, { url = "https://files.pythonhosted.org/packages/86/c1/c5f619b0057a7965cb78ec559c1d7a45ce8c99a35bea95483d64959a93d9/regex-2026.5.9-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:83d0ee4a57d1c87cb549e195ec300b8f0ec3a82eba66d835e4e2ed8634fe4499", size = 494269, upload-time = "2026-05-09T23:14:42.869Z" }, { url = "https://files.pythonhosted.org/packages/05/2c/5d01f1aee33de4bbe60c8452945bfc8477ca7c5ae4450f6bfe711036cb36/regex-2026.5.9-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:d3d7eb5c9a7f6df82ed3cfac9beb93882a5cbcb5b8b157b56cb2b3b276574ac1", size = 293954, upload-time = "2026-05-09T23:14:44.822Z" }, { url = "https://files.pythonhosted.org/packages/7a/fe/e8988b2ae2108c6ef71bd4aa8d87fbe257976dd0810e826cd75f701c68b6/regex-2026.5.9-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:075160bf16658e16d35233300b8453aac25de4cbea808d22348b6979668e924d", size = 292405, upload-time = "2026-05-09T23:14:47.211Z" }, { url = "https://files.pythonhosted.org/packages/79/34/d2b0937faa7859263f7f0a3c6b103a1296306be6952dc173d0154e9a2f49/regex-2026.5.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45375819235558a4ff1c4971dc32881f022613abdb180128f5cb4768c1765a1c", size = 811855, upload-time = "2026-05-09T23:14:49.21Z" }, { url = "https://files.pythonhosted.org/packages/80/fe/daf53a47457a8486db66c66c01ceb9c2303eecee3f87197f1e77eb1a736d/regex-2026.5.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ead4b163ac30a29574510cd4b3e2e985ac5290c05fc7095557d6a5f403fc31b5", size = 871189, upload-time = "2026-05-09T23:14:51.555Z" }, { url = "https://files.pythonhosted.org/packages/1c/75/058fc4470cbfbf57d800aff1a0022b929a3f9fa553ee10a0cdf2070eb31f/regex-2026.5.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8c6e4218fbdfbcd4f6c19efca40930d24a621bf4b48cb76bc6640543bd28ef20", size = 917485, upload-time = "2026-05-09T23:14:53.633Z" }, { url = "https://files.pythonhosted.org/packages/88/e7/179cfda3a28bc843b5c6cfe7f79f23489c791ed95f151083803660878432/regex-2026.5.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6351571c8a42b505eb555c0dc47d740d0fb66977dc142919eea6f4325b7c56a0", size = 816369, upload-time = "2026-05-09T23:14:56.198Z" }, { url = "https://files.pythonhosted.org/packages/41/90/6f0cc422071688266d344fca8462d787cba0a2c144acb25721f9a61ec265/regex-2026.5.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:002205cafd2a9e78c6290c7d1df277bf3277b3b7a30e0b4bb0dac2e2e3f7cb2d", size = 785869, upload-time = "2026-05-09T23:14:58.602Z" }, { url = "https://files.pythonhosted.org/packages/02/67/a31f1760f09c27b251ef39e9beb541f462cf977381d067faa764c2c0e393/regex-2026.5.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8abd33fef90b2a9efac5557d6033ca82d1195ed3a15fea5af15ba7b463c6a63b", size = 801427, upload-time = "2026-05-09T23:15:00.642Z" }, { url = "https://files.pythonhosted.org/packages/e3/c4/1a80654597b6bc1e1ea0494824c31200e8a956abe290afae9b19a166a148/regex-2026.5.9-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:31037c82eccb44b7ea2e9e221d7c01429430e989a1f4b91ea5a855f6017b509a", size = 866482, upload-time = "2026-05-09T23:15:03.384Z" }, { url = "https://files.pythonhosted.org/packages/d1/11/960724e06482c08466ff5611e242e86f80062949cdf6b4b9cc317b9dd93d/regex-2026.5.9-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5604dfd046dc37eca90250fc3be938b076c8059fa772ac0ed6f499b0f0fb0415", size = 773022, upload-time = "2026-05-09T23:15:05.625Z" }, { url = "https://files.pythonhosted.org/packages/50/a8/a9979c3e7918280e93159ebcab5ef1a65116dd4f3bd6091be0eae4a126e8/regex-2026.5.9-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:0e1b1b4e496afbb24f4a62aba855ee4f88f25578927697b340702e48c9ee6bc2", size = 856642, upload-time = "2026-05-09T23:15:07.966Z" }, { url = "https://files.pythonhosted.org/packages/fe/d4/a9b732f2f0072c0ab12227483abb24fffcb9f73f8a2b203df0a6d0434735/regex-2026.5.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:be3372b9df6ddecff6486d37e19095a7b4973137caf5512407a89f4455361f41", size = 803552, upload-time = "2026-05-09T23:15:10.215Z" }, { url = "https://files.pythonhosted.org/packages/d5/fe/1b3113817447a1d4155e4ac76d2e072f42c0bcba2f43fa8a0e756ea2cd91/regex-2026.5.9-cp314-cp314t-win32.whl", hash = "sha256:3ddd90103f9e5c471c49c7852ecc1fe27c7e45eb99e977aefe7caa4e779f4f58", size = 275746, upload-time = "2026-05-09T23:15:12.609Z" }, { url = "https://files.pythonhosted.org/packages/92/73/93d42045302636c91f2e5ef588b65b84b01428f28ec77de256b1dfdfbe5c/regex-2026.5.9-cp314-cp314t-win_amd64.whl", hash = "sha256:ca518ed29c46eecba6010b15f1b9a479314d2de409536e71b6a13aa04e3b8a77", size = 285685, upload-time = "2026-05-09T23:15:15.086Z" }, { url = "https://files.pythonhosted.org/packages/da/80/35b4c33c804a165a7f55289afda3ea9e3eb6d15800341a2d66455c0f1f30/regex-2026.5.9-cp314-cp314t-win_arm64.whl", hash = "sha256:5e41809d2683fcde7d5a8c87a6567ba1fb1ce0de9f31bff578de00a4b2d76daa", size = 275713, upload-time = "2026-05-09T23:15:16.98Z" }, ] [[package]] name = "requests" version = "2.34.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, { name = "charset-normalizer" }, { name = "idna" }, { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] [[package]] name = "responses" version = "0.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyyaml" }, { name = "requests" }, { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9f/b4/b7e040379838cc71bf5aabdb26998dfbe5ee73904c92c1c161faf5de8866/responses-0.26.0.tar.gz", hash = "sha256:c7f6923e6343ef3682816ba421c006626777893cb0d5e1434f674b649bac9eb4", size = 81303, upload-time = "2026-02-19T14:38:05.574Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ce/04/7f73d05b556da048923e31a0cc878f03be7c5425ed1f268082255c75d872/responses-0.26.0-py3-none-any.whl", hash = "sha256:03ec4409088cd5c66b71ecbbbd27fe2c58ddfad801c66203457b3e6a04868c37", size = 35099, upload-time = "2026-02-19T14:38:03.847Z" }, ] [[package]] name = "rfc3339-validator" version = "0.1.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "six" }, ] sdist = { url = "https://files.pythonhosted.org/packages/28/ea/a9387748e2d111c3c2b275ba970b735e04e15cdb1eb30693b6b5708c4dbd/rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b", size = 5513, upload-time = "2021-05-12T16:37:54.178Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa", size = 3490, upload-time = "2021-05-12T16:37:52.536Z" }, ] [[package]] name = "rich" version = "15.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown-it-py" }, { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, ] [[package]] name = "roman-numerals" version = "4.1.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ae/f9/41dc953bbeb056c17d5f7a519f50fdf010bd0553be2d630bc69d1e022703/roman_numerals-4.1.0.tar.gz", hash = "sha256:1af8b147eb1405d5839e78aeb93131690495fe9da5c91856cb33ad55a7f1e5b2", size = 9077, upload-time = "2025-12-17T18:25:34.381Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl", hash = "sha256:647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7", size = 7676, upload-time = "2025-12-17T18:25:33.098Z" }, ] [[package]] name = "rpds-py" version = "0.30.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469, upload-time = "2025-11-30T20:24:38.837Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/03/e7/98a2f4ac921d82f33e03f3835f5bf3a4a40aa1bfdc57975e74a97b2b4bdd/rpds_py-0.30.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a161f20d9a43006833cd7068375a94d035714d73a172b681d8881820600abfad", size = 375086, upload-time = "2025-11-30T20:22:17.93Z" }, { url = "https://files.pythonhosted.org/packages/4d/a1/bca7fd3d452b272e13335db8d6b0b3ecde0f90ad6f16f3328c6fb150c889/rpds_py-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6abc8880d9d036ecaafe709079969f56e876fcf107f7a8e9920ba6d5a3878d05", size = 359053, upload-time = "2025-11-30T20:22:19.297Z" }, { url = "https://files.pythonhosted.org/packages/65/1c/ae157e83a6357eceff62ba7e52113e3ec4834a84cfe07fa4b0757a7d105f/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca28829ae5f5d569bb62a79512c842a03a12576375d5ece7d2cadf8abe96ec28", size = 390763, upload-time = "2025-11-30T20:22:21.661Z" }, { url = "https://files.pythonhosted.org/packages/d4/36/eb2eb8515e2ad24c0bd43c3ee9cd74c33f7ca6430755ccdb240fd3144c44/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1010ed9524c73b94d15919ca4d41d8780980e1765babf85f9a2f90d247153dd", size = 408951, upload-time = "2025-11-30T20:22:23.408Z" }, { url = "https://files.pythonhosted.org/packages/d6/65/ad8dc1784a331fabbd740ef6f71ce2198c7ed0890dab595adb9ea2d775a1/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f", size = 514622, upload-time = "2025-11-30T20:22:25.16Z" }, { url = "https://files.pythonhosted.org/packages/63/8e/0cfa7ae158e15e143fe03993b5bcd743a59f541f5952e1546b1ac1b5fd45/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d948b135c4693daff7bc2dcfc4ec57237a29bd37e60c2fabf5aff2bbacf3e2f1", size = 414492, upload-time = "2025-11-30T20:22:26.505Z" }, { url = "https://files.pythonhosted.org/packages/60/1b/6f8f29f3f995c7ffdde46a626ddccd7c63aefc0efae881dc13b6e5d5bb16/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23", size = 394080, upload-time = "2025-11-30T20:22:27.934Z" }, { url = "https://files.pythonhosted.org/packages/6d/d5/a266341051a7a3ca2f4b750a3aa4abc986378431fc2da508c5034d081b70/rpds_py-0.30.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:2e6ecb5a5bcacf59c3f912155044479af1d0b6681280048b338b28e364aca1f6", size = 408680, upload-time = "2025-11-30T20:22:29.341Z" }, { url = "https://files.pythonhosted.org/packages/10/3b/71b725851df9ab7a7a4e33cf36d241933da66040d195a84781f49c50490c/rpds_py-0.30.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8fa71a2e078c527c3e9dc9fc5a98c9db40bcc8a92b4e8858e36d329f8684b51", size = 423589, upload-time = "2025-11-30T20:22:31.469Z" }, { url = "https://files.pythonhosted.org/packages/00/2b/e59e58c544dc9bd8bd8384ecdb8ea91f6727f0e37a7131baeff8d6f51661/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73c67f2db7bc334e518d097c6d1e6fed021bbc9b7d678d6cc433478365d1d5f5", size = 573289, upload-time = "2025-11-30T20:22:32.997Z" }, { url = "https://files.pythonhosted.org/packages/da/3e/a18e6f5b460893172a7d6a680e86d3b6bc87a54c1f0b03446a3c8c7b588f/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5ba103fb455be00f3b1c2076c9d4264bfcb037c976167a6047ed82f23153f02e", size = 599737, upload-time = "2025-11-30T20:22:34.419Z" }, { url = "https://files.pythonhosted.org/packages/5c/e2/714694e4b87b85a18e2c243614974413c60aa107fd815b8cbc42b873d1d7/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee9c752c0364588353e627da8a7e808a66873672bcb5f52890c33fd965b394", size = 563120, upload-time = "2025-11-30T20:22:35.903Z" }, { url = "https://files.pythonhosted.org/packages/6f/ab/d5d5e3bcedb0a77f4f613706b750e50a5a3ba1c15ccd3665ecc636c968fd/rpds_py-0.30.0-cp312-cp312-win32.whl", hash = "sha256:1ab5b83dbcf55acc8b08fc62b796ef672c457b17dbd7820a11d6c52c06839bdf", size = 223782, upload-time = "2025-11-30T20:22:37.271Z" }, { url = "https://files.pythonhosted.org/packages/39/3b/f786af9957306fdc38a74cef405b7b93180f481fb48453a114bb6465744a/rpds_py-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:a090322ca841abd453d43456ac34db46e8b05fd9b3b4ac0c78bcde8b089f959b", size = 240463, upload-time = "2025-11-30T20:22:39.021Z" }, { url = "https://files.pythonhosted.org/packages/f3/d2/b91dc748126c1559042cfe41990deb92c4ee3e2b415f6b5234969ffaf0cc/rpds_py-0.30.0-cp312-cp312-win_arm64.whl", hash = "sha256:669b1805bd639dd2989b281be2cfd951c6121b65e729d9b843e9639ef1fd555e", size = 230868, upload-time = "2025-11-30T20:22:40.493Z" }, { url = "https://files.pythonhosted.org/packages/ed/dc/d61221eb88ff410de3c49143407f6f3147acf2538c86f2ab7ce65ae7d5f9/rpds_py-0.30.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f83424d738204d9770830d35290ff3273fbb02b41f919870479fab14b9d303b2", size = 374887, upload-time = "2025-11-30T20:22:41.812Z" }, { url = "https://files.pythonhosted.org/packages/fd/32/55fb50ae104061dbc564ef15cc43c013dc4a9f4527a1f4d99baddf56fe5f/rpds_py-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7536cd91353c5273434b4e003cbda89034d67e7710eab8761fd918ec6c69cf8", size = 358904, upload-time = "2025-11-30T20:22:43.479Z" }, { url = "https://files.pythonhosted.org/packages/58/70/faed8186300e3b9bdd138d0273109784eea2396c68458ed580f885dfe7ad/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2771c6c15973347f50fece41fc447c054b7ac2ae0502388ce3b6738cd366e3d4", size = 389945, upload-time = "2025-11-30T20:22:44.819Z" }, { url = "https://files.pythonhosted.org/packages/bd/a8/073cac3ed2c6387df38f71296d002ab43496a96b92c823e76f46b8af0543/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136", size = 407783, upload-time = "2025-11-30T20:22:46.103Z" }, { url = "https://files.pythonhosted.org/packages/77/57/5999eb8c58671f1c11eba084115e77a8899d6e694d2a18f69f0ba471ec8b/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76fec018282b4ead0364022e3c54b60bf368b9d926877957a8624b58419169b7", size = 515021, upload-time = "2025-11-30T20:22:47.458Z" }, { url = "https://files.pythonhosted.org/packages/e0/af/5ab4833eadc36c0a8ed2bc5c0de0493c04f6c06de223170bd0798ff98ced/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:692bef75a5525db97318e8cd061542b5a79812d711ea03dbc1f6f8dbb0c5f0d2", size = 414589, upload-time = "2025-11-30T20:22:48.872Z" }, { url = "https://files.pythonhosted.org/packages/b7/de/f7192e12b21b9e9a68a6d0f249b4af3fdcdff8418be0767a627564afa1f1/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9027da1ce107104c50c81383cae773ef5c24d296dd11c99e2629dbd7967a20c6", size = 394025, upload-time = "2025-11-30T20:22:50.196Z" }, { url = "https://files.pythonhosted.org/packages/91/c4/fc70cd0249496493500e7cc2de87504f5aa6509de1e88623431fec76d4b6/rpds_py-0.30.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:9cf69cdda1f5968a30a359aba2f7f9aa648a9ce4b580d6826437f2b291cfc86e", size = 408895, upload-time = "2025-11-30T20:22:51.87Z" }, { url = "https://files.pythonhosted.org/packages/58/95/d9275b05ab96556fefff73a385813eb66032e4c99f411d0795372d9abcea/rpds_py-0.30.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a4796a717bf12b9da9d3ad002519a86063dcac8988b030e405704ef7d74d2d9d", size = 422799, upload-time = "2025-11-30T20:22:53.341Z" }, { url = "https://files.pythonhosted.org/packages/06/c1/3088fc04b6624eb12a57eb814f0d4997a44b0d208d6cace713033ff1a6ba/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d4c2aa7c50ad4728a094ebd5eb46c452e9cb7edbfdb18f9e1221f597a73e1e7", size = 572731, upload-time = "2025-11-30T20:22:54.778Z" }, { url = "https://files.pythonhosted.org/packages/d8/42/c612a833183b39774e8ac8fecae81263a68b9583ee343db33ab571a7ce55/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba81a9203d07805435eb06f536d95a266c21e5b2dfbf6517748ca40c98d19e31", size = 599027, upload-time = "2025-11-30T20:22:56.212Z" }, { url = "https://files.pythonhosted.org/packages/5f/60/525a50f45b01d70005403ae0e25f43c0384369ad24ffe46e8d9068b50086/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:945dccface01af02675628334f7cf49c2af4c1c904748efc5cf7bbdf0b579f95", size = 563020, upload-time = "2025-11-30T20:22:58.2Z" }, { url = "https://files.pythonhosted.org/packages/0b/5d/47c4655e9bcd5ca907148535c10e7d489044243cc9941c16ed7cd53be91d/rpds_py-0.30.0-cp313-cp313-win32.whl", hash = "sha256:b40fb160a2db369a194cb27943582b38f79fc4887291417685f3ad693c5a1d5d", size = 223139, upload-time = "2025-11-30T20:23:00.209Z" }, { url = "https://files.pythonhosted.org/packages/f2/e1/485132437d20aa4d3e1d8b3fb5a5e65aa8139f1e097080c2a8443201742c/rpds_py-0.30.0-cp313-cp313-win_amd64.whl", hash = "sha256:806f36b1b605e2d6a72716f321f20036b9489d29c51c91f4dd29a3e3afb73b15", size = 240224, upload-time = "2025-11-30T20:23:02.008Z" }, { url = "https://files.pythonhosted.org/packages/24/95/ffd128ed1146a153d928617b0ef673960130be0009c77d8fbf0abe306713/rpds_py-0.30.0-cp313-cp313-win_arm64.whl", hash = "sha256:d96c2086587c7c30d44f31f42eae4eac89b60dabbac18c7669be3700f13c3ce1", size = 230645, upload-time = "2025-11-30T20:23:03.43Z" }, { url = "https://files.pythonhosted.org/packages/ff/1b/b10de890a0def2a319a2626334a7f0ae388215eb60914dbac8a3bae54435/rpds_py-0.30.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:eb0b93f2e5c2189ee831ee43f156ed34e2a89a78a66b98cadad955972548be5a", size = 364443, upload-time = "2025-11-30T20:23:04.878Z" }, { url = "https://files.pythonhosted.org/packages/0d/bf/27e39f5971dc4f305a4fb9c672ca06f290f7c4e261c568f3dea16a410d47/rpds_py-0.30.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:922e10f31f303c7c920da8981051ff6d8c1a56207dbdf330d9047f6d30b70e5e", size = 353375, upload-time = "2025-11-30T20:23:06.342Z" }, { url = "https://files.pythonhosted.org/packages/40/58/442ada3bba6e8e6615fc00483135c14a7538d2ffac30e2d933ccf6852232/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdc62c8286ba9bf7f47befdcea13ea0e26bf294bda99758fd90535cbaf408000", size = 383850, upload-time = "2025-11-30T20:23:07.825Z" }, { url = "https://files.pythonhosted.org/packages/14/14/f59b0127409a33c6ef6f5c1ebd5ad8e32d7861c9c7adfa9a624fc3889f6c/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47f9a91efc418b54fb8190a6b4aa7813a23fb79c51f4bb84e418f5476c38b8db", size = 392812, upload-time = "2025-11-30T20:23:09.228Z" }, { url = "https://files.pythonhosted.org/packages/b3/66/e0be3e162ac299b3a22527e8913767d869e6cc75c46bd844aa43fb81ab62/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3587eb9b17f3789ad50824084fa6f81921bbf9a795826570bda82cb3ed91f2", size = 517841, upload-time = "2025-11-30T20:23:11.186Z" }, { url = "https://files.pythonhosted.org/packages/3d/55/fa3b9cf31d0c963ecf1ba777f7cf4b2a2c976795ac430d24a1f43d25a6ba/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39c02563fc592411c2c61d26b6c5fe1e51eaa44a75aa2c8735ca88b0d9599daa", size = 408149, upload-time = "2025-11-30T20:23:12.864Z" }, { url = "https://files.pythonhosted.org/packages/60/ca/780cf3b1a32b18c0f05c441958d3758f02544f1d613abf9488cd78876378/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a1234d8febafdfd33a42d97da7a43f5dcb120c1060e352a3fbc0c6d36e2083", size = 383843, upload-time = "2025-11-30T20:23:14.638Z" }, { url = "https://files.pythonhosted.org/packages/82/86/d5f2e04f2aa6247c613da0c1dd87fcd08fa17107e858193566048a1e2f0a/rpds_py-0.30.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:eb2c4071ab598733724c08221091e8d80e89064cd472819285a9ab0f24bcedb9", size = 396507, upload-time = "2025-11-30T20:23:16.105Z" }, { url = "https://files.pythonhosted.org/packages/4b/9a/453255d2f769fe44e07ea9785c8347edaf867f7026872e76c1ad9f7bed92/rpds_py-0.30.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bdfdb946967d816e6adf9a3d8201bfad269c67efe6cefd7093ef959683c8de0", size = 414949, upload-time = "2025-11-30T20:23:17.539Z" }, { url = "https://files.pythonhosted.org/packages/a3/31/622a86cdc0c45d6df0e9ccb6becdba5074735e7033c20e401a6d9d0e2ca0/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c77afbd5f5250bf27bf516c7c4a016813eb2d3e116139aed0096940c5982da94", size = 565790, upload-time = "2025-11-30T20:23:19.029Z" }, { url = "https://files.pythonhosted.org/packages/1c/5d/15bbf0fb4a3f58a3b1c67855ec1efcc4ceaef4e86644665fff03e1b66d8d/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:61046904275472a76c8c90c9ccee9013d70a6d0f73eecefd38c1ae7c39045a08", size = 590217, upload-time = "2025-11-30T20:23:20.885Z" }, { url = "https://files.pythonhosted.org/packages/6d/61/21b8c41f68e60c8cc3b2e25644f0e3681926020f11d06ab0b78e3c6bbff1/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c5f36a861bc4b7da6516dbdf302c55313afa09b81931e8280361a4f6c9a2d27", size = 555806, upload-time = "2025-11-30T20:23:22.488Z" }, { url = "https://files.pythonhosted.org/packages/f9/39/7e067bb06c31de48de3eb200f9fc7c58982a4d3db44b07e73963e10d3be9/rpds_py-0.30.0-cp313-cp313t-win32.whl", hash = "sha256:3d4a69de7a3e50ffc214ae16d79d8fbb0922972da0356dcf4d0fdca2878559c6", size = 211341, upload-time = "2025-11-30T20:23:24.449Z" }, { url = "https://files.pythonhosted.org/packages/0a/4d/222ef0b46443cf4cf46764d9c630f3fe4abaa7245be9417e56e9f52b8f65/rpds_py-0.30.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f14fc5df50a716f7ece6a80b6c78bb35ea2ca47c499e422aa4463455dd96d56d", size = 225768, upload-time = "2025-11-30T20:23:25.908Z" }, { url = "https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:68f19c879420aa08f61203801423f6cd5ac5f0ac4ac82a2368a9fcd6a9a075e0", size = 362099, upload-time = "2025-11-30T20:23:27.316Z" }, { url = "https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ec7c4490c672c1a0389d319b3a9cfcd098dcdc4783991553c332a15acf7249be", size = 353192, upload-time = "2025-11-30T20:23:29.151Z" }, { url = "https://files.pythonhosted.org/packages/bf/c4/76eb0e1e72d1a9c4703c69607cec123c29028bff28ce41588792417098ac/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f251c812357a3fed308d684a5079ddfb9d933860fc6de89f2b7ab00da481e65f", size = 384080, upload-time = "2025-11-30T20:23:30.785Z" }, { url = "https://files.pythonhosted.org/packages/72/87/87ea665e92f3298d1b26d78814721dc39ed8d2c74b86e83348d6b48a6f31/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac98b175585ecf4c0348fd7b29c3864bda53b805c773cbf7bfdaffc8070c976f", size = 394841, upload-time = "2025-11-30T20:23:32.209Z" }, { url = "https://files.pythonhosted.org/packages/77/ad/7783a89ca0587c15dcbf139b4a8364a872a25f861bdb88ed99f9b0dec985/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3e62880792319dbeb7eb866547f2e35973289e7d5696c6e295476448f5b63c87", size = 516670, upload-time = "2025-11-30T20:23:33.742Z" }, { url = "https://files.pythonhosted.org/packages/5b/3c/2882bdac942bd2172f3da574eab16f309ae10a3925644e969536553cb4ee/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e7fc54e0900ab35d041b0601431b0a0eb495f0851a0639b6ef90f7741b39a18", size = 408005, upload-time = "2025-11-30T20:23:35.253Z" }, { url = "https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47e77dc9822d3ad616c3d5759ea5631a75e5809d5a28707744ef79d7a1bcfcad", size = 382112, upload-time = "2025-11-30T20:23:36.842Z" }, { url = "https://files.pythonhosted.org/packages/cf/8e/1da49d4a107027e5fbc64daeab96a0706361a2918da10cb41769244b805d/rpds_py-0.30.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b4dc1a6ff022ff85ecafef7979a2c6eb423430e05f1165d6688234e62ba99a07", size = 399049, upload-time = "2025-11-30T20:23:38.343Z" }, { url = "https://files.pythonhosted.org/packages/df/5a/7ee239b1aa48a127570ec03becbb29c9d5a9eb092febbd1699d567cae859/rpds_py-0.30.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4559c972db3a360808309e06a74628b95eaccbf961c335c8fe0d590cf587456f", size = 415661, upload-time = "2025-11-30T20:23:40.263Z" }, { url = "https://files.pythonhosted.org/packages/70/ea/caa143cf6b772f823bc7929a45da1fa83569ee49b11d18d0ada7f5ee6fd6/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ed177ed9bded28f8deb6ab40c183cd1192aa0de40c12f38be4d59cd33cb5c65", size = 565606, upload-time = "2025-11-30T20:23:42.186Z" }, { url = "https://files.pythonhosted.org/packages/64/91/ac20ba2d69303f961ad8cf55bf7dbdb4763f627291ba3d0d7d67333cced9/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ad1fa8db769b76ea911cb4e10f049d80bf518c104f15b3edb2371cc65375c46f", size = 591126, upload-time = "2025-11-30T20:23:44.086Z" }, { url = "https://files.pythonhosted.org/packages/21/20/7ff5f3c8b00c8a95f75985128c26ba44503fb35b8e0259d812766ea966c7/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:46e83c697b1f1c72b50e5ee5adb4353eef7406fb3f2043d64c33f20ad1c2fc53", size = 553371, upload-time = "2025-11-30T20:23:46.004Z" }, { url = "https://files.pythonhosted.org/packages/72/c7/81dadd7b27c8ee391c132a6b192111ca58d866577ce2d9b0ca157552cce0/rpds_py-0.30.0-cp314-cp314-win32.whl", hash = "sha256:ee454b2a007d57363c2dfd5b6ca4a5d7e2c518938f8ed3b706e37e5d470801ed", size = 215298, upload-time = "2025-11-30T20:23:47.696Z" }, { url = "https://files.pythonhosted.org/packages/3e/d2/1aaac33287e8cfb07aab2e6b8ac1deca62f6f65411344f1433c55e6f3eb8/rpds_py-0.30.0-cp314-cp314-win_amd64.whl", hash = "sha256:95f0802447ac2d10bcc69f6dc28fe95fdf17940367b21d34e34c737870758950", size = 228604, upload-time = "2025-11-30T20:23:49.501Z" }, { url = "https://files.pythonhosted.org/packages/e8/95/ab005315818cc519ad074cb7784dae60d939163108bd2b394e60dc7b5461/rpds_py-0.30.0-cp314-cp314-win_arm64.whl", hash = "sha256:613aa4771c99f03346e54c3f038e4cc574ac09a3ddfb0e8878487335e96dead6", size = 222391, upload-time = "2025-11-30T20:23:50.96Z" }, { url = "https://files.pythonhosted.org/packages/9e/68/154fe0194d83b973cdedcdcc88947a2752411165930182ae41d983dcefa6/rpds_py-0.30.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:7e6ecfcb62edfd632e56983964e6884851786443739dbfe3582947e87274f7cb", size = 364868, upload-time = "2025-11-30T20:23:52.494Z" }, { url = "https://files.pythonhosted.org/packages/83/69/8bbc8b07ec854d92a8b75668c24d2abcb1719ebf890f5604c61c9369a16f/rpds_py-0.30.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a1d0bc22a7cdc173fedebb73ef81e07faef93692b8c1ad3733b67e31e1b6e1b8", size = 353747, upload-time = "2025-11-30T20:23:54.036Z" }, { url = "https://files.pythonhosted.org/packages/ab/00/ba2e50183dbd9abcce9497fa5149c62b4ff3e22d338a30d690f9af970561/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d08f00679177226c4cb8c5265012eea897c8ca3b93f429e546600c971bcbae7", size = 383795, upload-time = "2025-11-30T20:23:55.556Z" }, { url = "https://files.pythonhosted.org/packages/05/6f/86f0272b84926bcb0e4c972262f54223e8ecc556b3224d281e6598fc9268/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5965af57d5848192c13534f90f9dd16464f3c37aaf166cc1da1cae1fd5a34898", size = 393330, upload-time = "2025-11-30T20:23:57.033Z" }, { url = "https://files.pythonhosted.org/packages/cb/e9/0e02bb2e6dc63d212641da45df2b0bf29699d01715913e0d0f017ee29438/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a4e86e34e9ab6b667c27f3211ca48f73dba7cd3d90f8d5b11be56e5dbc3fb4e", size = 518194, upload-time = "2025-11-30T20:23:58.637Z" }, { url = "https://files.pythonhosted.org/packages/ee/ca/be7bca14cf21513bdf9c0606aba17d1f389ea2b6987035eb4f62bd923f25/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d3e6b26f2c785d65cc25ef1e5267ccbe1b069c5c21b8cc724efee290554419", size = 408340, upload-time = "2025-11-30T20:24:00.2Z" }, { url = "https://files.pythonhosted.org/packages/c2/c7/736e00ebf39ed81d75544c0da6ef7b0998f8201b369acf842f9a90dc8fce/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:626a7433c34566535b6e56a1b39a7b17ba961e97ce3b80ec62e6f1312c025551", size = 383765, upload-time = "2025-11-30T20:24:01.759Z" }, { url = "https://files.pythonhosted.org/packages/4a/3f/da50dfde9956aaf365c4adc9533b100008ed31aea635f2b8d7b627e25b49/rpds_py-0.30.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:acd7eb3f4471577b9b5a41baf02a978e8bdeb08b4b355273994f8b87032000a8", size = 396834, upload-time = "2025-11-30T20:24:03.687Z" }, { url = "https://files.pythonhosted.org/packages/4e/00/34bcc2565b6020eab2623349efbdec810676ad571995911f1abdae62a3a0/rpds_py-0.30.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe5fa731a1fa8a0a56b0977413f8cacac1768dad38d16b3a296712709476fbd5", size = 415470, upload-time = "2025-11-30T20:24:05.232Z" }, { url = "https://files.pythonhosted.org/packages/8c/28/882e72b5b3e6f718d5453bd4d0d9cf8df36fddeb4ddbbab17869d5868616/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74a3243a411126362712ee1524dfc90c650a503502f135d54d1b352bd01f2404", size = 565630, upload-time = "2025-11-30T20:24:06.878Z" }, { url = "https://files.pythonhosted.org/packages/3b/97/04a65539c17692de5b85c6e293520fd01317fd878ea1995f0367d4532fb1/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3e8eeb0544f2eb0d2581774be4c3410356eba189529a6b3e36bbbf9696175856", size = 591148, upload-time = "2025-11-30T20:24:08.445Z" }, { url = "https://files.pythonhosted.org/packages/85/70/92482ccffb96f5441aab93e26c4d66489eb599efdcf96fad90c14bbfb976/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dbd936cde57abfee19ab3213cf9c26be06d60750e60a8e4dd85d1ab12c8b1f40", size = 556030, upload-time = "2025-11-30T20:24:10.956Z" }, { url = "https://files.pythonhosted.org/packages/20/53/7c7e784abfa500a2b6b583b147ee4bb5a2b3747a9166bab52fec4b5b5e7d/rpds_py-0.30.0-cp314-cp314t-win32.whl", hash = "sha256:dc824125c72246d924f7f796b4f63c1e9dc810c7d9e2355864b3c3a73d59ade0", size = 211570, upload-time = "2025-11-30T20:24:12.735Z" }, { url = "https://files.pythonhosted.org/packages/d0/02/fa464cdfbe6b26e0600b62c528b72d8608f5cc49f96b8d6e38c95d60c676/rpds_py-0.30.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27f4b0e92de5bfbc6f86e43959e6edd1425c33b5e69aab0984a72047f2bcf1e3", size = 226532, upload-time = "2025-11-30T20:24:14.634Z" }, ] [[package]] name = "ruff" version = "0.16.9" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/96/bf/c935ca98e73fe8ce65b87ef08a280c0c1e85295d569228c15e87d8fdfaf1/ruff-0.16.9.tar.gz", hash = "sha256:12b625c6cfba78d285d9f48eda5f053374f1e53cb10ef17342a383750db99161", size = 4948764, upload-time = "2026-09-24T20:37:49.416Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0d/26/df51322b52ee1ada7eff2d071ea09d11d5c2d1dcc9f02594f5785c4d1635/ruff-0.16.9-py3-none-linux_armv6l.whl", hash = "sha256:95e6f022090368ab3b824c36276839c53b2adf1a3f4c09fefc33dfc400f6da96", size = 10082922, upload-time = "2026-09-24T20:37:13.045Z" }, { url = "https://files.pythonhosted.org/packages/a5/27/7bf51f5a7aa375e9f339280a303aab44ca75dc1525f1cdc5991761685b0f/ruff-0.16.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:a5f27be168556594a86d2f415db0cf43f5291917849318f873c7e2791f7a8c67", size = 10236360, upload-time = "2026-09-24T20:37:16.049Z" }, { url = "https://files.pythonhosted.org/packages/b6/63/09659283f92f02dff45809da194a70da2f688d87c55d8875c4fae3536072/ruff-0.16.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1632eb1d6197f33bd00b1acbc5b71009e89a8895c158e2d2b03a834fac964ab6", size = 9892940, upload-time = "2026-09-24T20:37:17.957Z" }, { url = "https://files.pythonhosted.org/packages/24/58/98de1b72ec172f5f8f1731236fe21585b3998bf7dfe9fcc44ae9ba626012/ruff-0.16.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3f951b14d865d5952c89d40a5ca07e87abe24fa5453299878411e127748fb1c", size = 10032114, upload-time = "2026-09-24T20:37:19.942Z" }, { url = "https://files.pythonhosted.org/packages/c8/7d/f1e17c54ab59d4bad1dce8ee3e22a7a1d0ef4745240decacdcf3832b5bb2/ruff-0.16.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:447fc07e1573afff7cb02803462b12b6c8ece7cf10e2cd78565fa6d7a1c0bf8d", size = 9910227, upload-time = "2026-09-24T20:37:21.872Z" }, { url = "https://files.pythonhosted.org/packages/34/19/436f647a65075bbd3bab2668b3bdaa5120559b294694018cdcefabbbf30b/ruff-0.16.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a3e039a6a40ed976c491722b60e0ae4a4aa1a86057f540ee7a37a5d19ae9120", size = 10547484, upload-time = "2026-09-24T20:37:24.229Z" }, { url = "https://files.pythonhosted.org/packages/03/59/38430a6bc2f6d8095447ac39625cf8b6e9344a47e6c26225c8ba1bff3ffb/ruff-0.16.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4684dded7db60aa57cb118fa158630f5feade4af5782903b6053484bdf9bd129", size = 11412367, upload-time = "2026-09-24T20:37:26.307Z" }, { url = "https://files.pythonhosted.org/packages/c8/bd/bbb6d7fc7f208c8b8c50dd5dc8206e4cfdb1e7a8fb852606a3adf370c880/ruff-0.16.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d29c934357e45642fda2f34c0b1f4025b4a6c01e15e4bf0016879d60078a142c", size = 10869787, upload-time = "2026-09-24T20:37:28.35Z" }, { url = "https://files.pythonhosted.org/packages/bc/b8/9c543074918061abbefc3bd139bee22de35abedb00dde0f2d27288838962/ruff-0.16.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a21713e629d3e5bdb2f5c2def1cc7f04f47fa8e1a7eb0571b4a28e1da64bc728", size = 10406494, upload-time = "2026-09-24T20:37:30.624Z" }, { url = "https://files.pythonhosted.org/packages/35/7a/5a8851bd146e7ccf8fd4b003f6c75c11f8fbdb0e60673b45097986b6bf41/ruff-0.16.9-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:7baa24ef5fc8e77aa93879e1d3f43754a01ae488e869f1ae30cf431afd4d2452", size = 10590083, upload-time = "2026-09-24T20:37:32.439Z" }, { url = "https://files.pythonhosted.org/packages/87/f0/4c3467188f23f806960b46fa76575a7cd0514c9ba90562650b71efc96980/ruff-0.16.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a41aac6230aadfaa133bdfa1614488531ffa3e0837567ae04c0da2058a9c0f9e", size = 10119151, upload-time = "2026-09-24T20:37:34.581Z" }, { url = "https://files.pythonhosted.org/packages/15/34/5a4def5adea572ce6aea0bb64f21f928ee317b80e0db747d7979b01d7261/ruff-0.16.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:c2529fb5896d49115b0e9aa8f887490b34bbe76baf879ec2264ac59406869ce7", size = 9911544, upload-time = "2026-09-24T20:37:36.796Z" }, { url = "https://files.pythonhosted.org/packages/62/5d/d15ebea7499eef9373318c0ee6ca127832927c6529731f6d48e18dce7ca9/ruff-0.16.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:41e3870277694177429b56406d65dfbdb2c2802c52b715edaf6a0b829c69d4ee", size = 10269884, upload-time = "2026-09-24T20:37:38.857Z" }, { url = "https://files.pythonhosted.org/packages/d1/56/c5d3cd119ded7a3c7aba0e961b69cb3df701c91662c98ad694467d060ce1/ruff-0.16.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:8adbe4e58af167f767d7b2ba5e83c42e878350796cf78c2f5e14ab9903a92588", size = 10749366, upload-time = "2026-09-24T20:37:41.042Z" }, { url = "https://files.pythonhosted.org/packages/ac/fe/734ec7527029ac757ecf821f143c9f3fcf69149c21f53a044a899b430f5a/ruff-0.16.9-py3-none-win32.whl", hash = "sha256:0e1dbc2073624dee6618d41d0098690a7244654af746704b64759e12b6b6b385", size = 10152355, upload-time = "2026-09-24T20:37:43.025Z" }, { url = "https://files.pythonhosted.org/packages/14/21/26e4643629b3ebb44f0a06f9c9a53058d63d989415f63a9a3c28e2ee7f22/ruff-0.16.9-py3-none-win_amd64.whl", hash = "sha256:6bd40fec8cd4c8a3d4dd589bd8ad4e6320c13c29234159bfd959a40d529d597b", size = 10592965, upload-time = "2026-09-24T20:37:44.944Z" }, { url = "https://files.pythonhosted.org/packages/51/60/5fb1a39dbb5ae314d5f59bc7348a63c1d5c20f3cd83914c4b5cb0be31d2d/ruff-0.16.9-py3-none-win_arm64.whl", hash = "sha256:ed1a252039200f57a59eebc063b54beabea67bfbaaca0eeaa7f54b5fbcda2284", size = 10458649, upload-time = "2026-09-24T20:37:46.882Z" }, ] [[package]] name = "s3fs" version = "2026.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiobotocore" }, { name = "aiohttp" }, { name = "fsspec" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f7/fe/ab89ab94cc1293849bc1a60876b589007bdd43f59a06ca309e717bd45dc7/s3fs-2026.9.0.tar.gz", hash = "sha256:ec078ff0122d28fd21d36ea4987794e4f42ecd6e8785eeaa083208771c1464e5", size = 89163, upload-time = "2026-09-18T19:02:54.117Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/bc/9c/15fb18bd708949dc953fe120be8d3c7a1b1b140c2592a9c81ce7961e9ecf/s3fs-2026.9.0-py3-none-any.whl", hash = "sha256:eb185fd98e20a1c9375087d9b2a6ef506f0cd01ae0c0c94efc79f88403fb81fe", size = 33126, upload-time = "2026-09-18T19:02:52.933Z" }, ] [[package]] name = "s3transfer" version = "0.17.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9b/ec/7c692cde9125b77e84b307354d4fb705f98b8ccad59a036d5957ca75bfc3/s3transfer-0.17.0.tar.gz", hash = "sha256:9edeb6d1c3c2f89d6050348548834ad8289610d886e5bf7b7207728bd43ce33a", size = 155337, upload-time = "2026-04-29T22:07:36.33Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/87/72/c6c32d2b657fa3dad1de340254e14390b1e334ce38268b7ad51abda3c8c2/s3transfer-0.17.0-py3-none-any.whl", hash = "sha256:ce3801712acf4ad3e89fb9990df97b4972e93f4b3b0004d214be5bce12814c20", size = 86811, upload-time = "2026-04-29T22:07:34.966Z" }, ] [[package]] name = "setuptools" version = "84.1.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/73/eb/7e5f628ee2eb168fecbf8026de7856b2543d6d609cc85f086b9a91ac93d6/setuptools-84.1.0.tar.gz", hash = "sha256:23657205b63d4d31395ab917815247f694aafad28cd21316bf5fd45a51048ea0", size = 1171783, upload-time = "2026-10-19T20:11:43.489Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/18/02/b4bbb90f8f29e334898ff45c6d9b1fffaa6f378017052b3023a06262b5f7/setuptools-84.1.0-py3-none-any.whl", hash = "sha256:5231689c6ac8156cd155cfb98c3fd6fb252c3e7e9bd3d78b64d3c55f132624c9", size = 819380, upload-time = "2026-10-19T20:11:41.684Z" }, ] [[package]] name = "shellingham" version = "1.5.4" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] [[package]] name = "six" version = "1.17.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] [[package]] name = "snowballstemmer" version = "3.0.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/75/a7/9810d872919697c9d01295633f5d574fb416d47e535f258272ca1f01f447/snowballstemmer-3.0.1.tar.gz", hash = "sha256:6d5eeeec8e9f84d4d56b847692bacf79bc2c8e90c7f80ca4444ff8b6f2e52895", size = 105575, upload-time = "2025-05-09T16:34:51.843Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl", hash = "sha256:6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064", size = 103274, upload-time = "2025-05-09T16:34:50.371Z" }, ] [[package]] name = "sortedcontainers" version = "2.4.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, ] [[package]] name = "sphinx" version = "9.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "alabaster" }, { name = "babel" }, { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "docutils" }, { name = "imagesize" }, { name = "jinja2" }, { name = "packaging" }, { name = "pygments" }, { name = "requests" }, { name = "roman-numerals" }, { name = "snowballstemmer" }, { name = "sphinxcontrib-applehelp" }, { name = "sphinxcontrib-devhelp" }, { name = "sphinxcontrib-htmlhelp" }, { name = "sphinxcontrib-jsmath" }, { name = "sphinxcontrib-qthelp" }, { name = "sphinxcontrib-serializinghtml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/bd/f08eb0f4eed5c83f1ba2a3bd18f7745a2b1525fad70660a1c00224ec468a/sphinx-9.1.0.tar.gz", hash = "sha256:7741722357dd75f8190766926071fed3bdc211c74dd2d7d4df5404da95930ddb", size = 8718324, upload-time = "2025-12-31T15:09:27.646Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/73/f7/b1884cb3188ab181fc81fa00c266699dab600f927a964df02ec3d5d1916a/sphinx-9.1.0-py3-none-any.whl", hash = "sha256:c84fdd4e782504495fe4f2c0b3413d6c2bf388589bb352d439b2a3bb99991978", size = 3921742, upload-time = "2025-12-31T15:09:25.561Z" }, ] [[package]] name = "sphinxcontrib-applehelp" version = "2.0.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ba/6e/b837e84a1a704953c62ef8776d45c3e8d759876b4a84fe14eba2859106fe/sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1", size = 20053, upload-time = "2024-07-29T01:09:00.465Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5", size = 119300, upload-time = "2024-07-29T01:08:58.99Z" }, ] [[package]] name = "sphinxcontrib-devhelp" version = "2.0.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/f6/d2/5beee64d3e4e747f316bae86b55943f51e82bb86ecd325883ef65741e7da/sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad", size = 12967, upload-time = "2024-07-29T01:09:23.417Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2", size = 82530, upload-time = "2024-07-29T01:09:21.945Z" }, ] [[package]] name = "sphinxcontrib-htmlhelp" version = "2.1.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/43/93/983afd9aa001e5201eab16b5a444ed5b9b0a7a010541e0ddfbbfd0b2470c/sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9", size = 22617, upload-time = "2024-07-29T01:09:37.889Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8", size = 98705, upload-time = "2024-07-29T01:09:36.407Z" }, ] [[package]] name = "sphinxcontrib-jsmath" version = "1.0.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/b2/e8/9ed3830aeed71f17c026a07a5097edcf44b692850ef215b161b8ad875729/sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8", size = 5787, upload-time = "2019-01-21T16:10:16.347Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", size = 5071, upload-time = "2019-01-21T16:10:14.333Z" }, ] [[package]] name = "sphinxcontrib-qthelp" version = "2.0.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/68/bc/9104308fc285eb3e0b31b67688235db556cd5b0ef31d96f30e45f2e51cae/sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab", size = 17165, upload-time = "2024-07-29T01:09:56.435Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb", size = 88743, upload-time = "2024-07-29T01:09:54.885Z" }, ] [[package]] name = "sphinxcontrib-serializinghtml" version = "2.0.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/3b/44/6716b257b0aa6bfd51a1b31665d1c205fb12cb5ad56de752dfa15657de2f/sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d", size = 16080, upload-time = "2024-07-29T01:10:09.332Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", size = 92072, upload-time = "2024-07-29T01:10:08.203Z" }, ] [[package]] name = "sympy" version = "1.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mpmath" }, ] sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, ] [[package]] name = "tinycss2" version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "webencodings" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/fd/7a5ee21fd08ff70d3d33a5781c255cbe779659bd03278feb98b19ee550f4/tinycss2-1.4.0.tar.gz", hash = "sha256:10c0972f6fc0fbee87c3edb76549357415e94548c1ae10ebccdea16fb404a9b7", size = 87085, upload-time = "2024-10-24T14:58:29.895Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e6/34/ebdc18bae6aa14fbee1a08b63c015c72b64868ff7dae68808ab500c492e2/tinycss2-1.4.0-py3-none-any.whl", hash = "sha256:3a49cf47b7675da0b15d0c6e1df8df4ebd96e9394bb905a5775adb0d884c5289", size = 26610, upload-time = "2024-10-24T14:58:28.029Z" }, ] [[package]] name = "tomlkit" version = "0.15.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/94/96/e07752635b98536177fa1f37671c8f3cdde2e724c6bcf6034b2cfb571565/tomlkit-0.15.1.tar.gz", hash = "sha256:e25bbf38843005246210a12982776f27f99cb9be67160e14434d0c0d21ee1e97", size = 180129, upload-time = "2026-07-17T01:48:04.562Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/13/bc/8c13eb66537dce1d2bd3a57132902f38d0e7f5bb46fa9f4daed9fe9d76ee/tomlkit-0.15.1-py3-none-any.whl", hash = "sha256:177a05aece5a8ca5266fd3c448abb47b8d352f09d477d3ca8332db4d89b24304", size = 49449, upload-time = "2026-07-17T01:48:05.728Z" }, ] [[package]] name = "towncrier" version = "26.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "jinja2" }, ] sdist = { url = "https://files.pythonhosted.org/packages/92/d4/e85ce614bf45b1a7fe13b82cc687819c2eaac5a1dd94b84834322095fd98/towncrier-26.9.0.tar.gz", hash = "sha256:ace9031631c718cc0709107b8755e6e31350eb8acdc877af43df3177addb9a55", size = 79338, upload-time = "2026-09-04T12:57:25.341Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/82/88/f9340b545dafa64de053798e760515a2736fbb60092cf9aa6b146ccf3ede/towncrier-26.9.0-py3-none-any.whl", hash = "sha256:ae4d223e6aadaff98d29b7f09e58d9c9ccf4cd3b455a2d2e0486d432cd8bf660", size = 67030, upload-time = "2026-09-04T12:57:23.986Z" }, ] [[package]] name = "typer" version = "0.27.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc" }, { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "rich" }, { name = "shellingham" }, ] sdist = { url = "https://files.pythonhosted.org/packages/16/f7/57713ba479fd405eb76de31404b2c744c289e336b2d999511ebf51e496f7/typer-0.27.2.tar.gz", hash = "sha256:269b7eb9d3c202ca84b4bc9618cb04ebb43d3d4d1e567e4c768607232c05f945", size = 204045, upload-time = "2026-08-28T10:26:55.046Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/dc/bf/205d0004930ede8f542fb58f601526fccf4ae7626075ca1e6c4de5d3d652/typer-0.27.2-py3-none-any.whl", hash = "sha256:b3a5fc4342d5fc8fda8fc3010b1cf117e9249aab7fae800c2eff62fd3842d97d", size = 123130, upload-time = "2026-08-28T10:26:53.752Z" }, ] [[package]] name = "typing-extensions" version = "4.16.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] [[package]] name = "typing-inspection" version = "0.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] [[package]] name = "universal-pathlib" version = "0.3.10" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fsspec" }, { name = "pathlib-abc" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3d/6e/d997a70ee8f4c61f9a7e2f4f8af721cf072a3326848fc881b05187e52558/universal_pathlib-0.3.10.tar.gz", hash = "sha256:4487cbc90730a48cfb64f811d99e14b6faed6d738420cd5f93f59f48e6930bfb", size = 261110, upload-time = "2026-02-22T14:40:58.87Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/dd/1a/5d9a402b39ec892d856bbdd9db502ff73ce28cdf4aff72eb1ce1d6843506/universal_pathlib-0.3.10-py3-none-any.whl", hash = "sha256:dfaf2fb35683d2eb1287a3ed7b215e4d6016aa6eaf339c607023d22f90821c66", size = 83528, upload-time = "2026-02-22T14:40:57.316Z" }, ] [[package]] name = "urllib3" version = "2.8.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, ] [[package]] name = "uv" version = "0.13.4" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/79/55/609455d21e87430c11fe734abfced50a3bdf177b2f49d41fbb19c199cbc3/uv-0.13.4.tar.gz", hash = "sha256:b0bff5b9a5061f7a35b1d0c1a94c19af134222c48eb7aadc42f85a038be1b248", size = 7482486, upload-time = "2026-10-20T15:12:06.801Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/2c/bd/4349da5356b17a1c0e36c8552c809925f86f6292de9b1f599f01e8e8093d/uv-0.13.4-py3-none-linux_armv6l.whl", hash = "sha256:205a4f93e568fb24f45a2f3834f1c0a8f1adb5f396bf473c71030ed707dce048", size = 18272390, upload-time = "2026-10-20T15:11:02.468Z" }, { url = "https://files.pythonhosted.org/packages/21/f2/0cb77a1ae43959772f8f7873eb1f4061907fbb1694ce55c79b231f639b1d/uv-0.13.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b4bbacea78c9db136cd2ec3147cbfd9ed910b4db5e1877109bd74d0e73bb2eaf", size = 17384113, upload-time = "2026-10-20T15:11:07.172Z" }, { url = "https://files.pythonhosted.org/packages/26/44/5a8733dd1b2e8d6f9cffd1ff6428a3e11dcc7e146a7657e32cfb8d9fb4a4/uv-0.13.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:95f4eb2b991b53b50ec2b18ab612fca8ba1e7a75a47af0331fb8bcd21da3eb03", size = 14433723, upload-time = "2026-10-20T15:11:10.185Z" }, { url = "https://files.pythonhosted.org/packages/f8/39/be14f051bf34a1822d634819890a2d0cba3b69101e633fb7b9f54d7b4020/uv-0.13.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:e90ab42825587821096cb0d071e387468da38cfa22818c91f9c0b3c0453048e8", size = 17816443, upload-time = "2026-10-20T15:11:13.674Z" }, { url = "https://files.pythonhosted.org/packages/97/a0/5df0f8bc6367dd37b98432a7c462e2918554f6db2a71211574852b8f0727/uv-0.13.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:3a9dcb6762c3ba249d8c084315ba03839082af3ece3536cc4164ec735c351d68", size = 17828025, upload-time = "2026-10-20T15:11:16.806Z" }, { url = "https://files.pythonhosted.org/packages/8b/9c/2fed5357700b7c304b94214979e4efe680c3bc6e337a97eedaa372fe0947/uv-0.13.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8ae6ff2b07892379dc767c9064c2b8bbfac88e767edac7ce28e1b0f15d86abb4", size = 17843737, upload-time = "2026-10-20T15:11:20.043Z" }, { url = "https://files.pythonhosted.org/packages/4c/2e/ee026284f096236dc3e1abed0316f9edab65c0ea176ea0072c064e62478d/uv-0.13.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:25cc14ff4fd8335d3be2cc493a621a003fdafcb0fb1c953c905daabe4def5ab8", size = 18576350, upload-time = "2026-10-20T15:11:24.035Z" }, { url = "https://files.pythonhosted.org/packages/6d/f1/67fbdb649c8c5d0cb38ea74829080479b77c8631cda5d77fc1b28f027ca7/uv-0.13.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:77062b02856c74697f48328d8a6583ab07e6da0a993e86c2db46f3d1da27c637", size = 19831234, upload-time = "2026-10-20T15:11:27.627Z" }, { url = "https://files.pythonhosted.org/packages/bd/00/7d49e3a1a3e98e5244fa38e0c964dc8de568bf3bb101257a63dc2602efed/uv-0.13.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fb0e2028d9a8929884344afb7d91d0f65d9783f640d28aede89fca3602bedde9", size = 19702886, upload-time = "2026-10-20T15:11:31.128Z" }, { url = "https://files.pythonhosted.org/packages/4b/a1/de02c0913726728eca56ee258875cbba9c7b0572c63f494fd9f55c7852e3/uv-0.13.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05e4332791b4d55b6dde8695b7fa6079b26991519f44760887c5dfbd5ca4fe53", size = 17181168, upload-time = "2026-10-20T15:11:34.583Z" }, { url = "https://files.pythonhosted.org/packages/a8/cd/f4a3c7377674366798c56aabd0f9a1f05687c3939f5ba857a3ea6093ff84/uv-0.13.4-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:7b3cdf868e5415631fe797b648c0d6945d38d388f23367bdd0b6332e80a1fcb0", size = 16502020, upload-time = "2026-10-20T15:11:37.88Z" }, { url = "https://files.pythonhosted.org/packages/f9/f4/ce79657c5a0f51b602db9a663529baafff03b9fe8cc41e09ad3f7246b88a/uv-0.13.4-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:4e6b08578c920b86910897f36bb6da7229c8e7e0187976570e2a538cb28b777e", size = 18000980, upload-time = "2026-10-20T15:11:41.59Z" }, { url = "https://files.pythonhosted.org/packages/21/e6/7d77022d5661f8e01578bc5423bf02af408220e7e7ac5ee15eb94742c7cb/uv-0.13.4-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:d86a5dec8ab30c70519c71ca5085c8aaf28664836cbc600951cedaa3fc678cbf", size = 18179083, upload-time = "2026-10-20T15:11:45.191Z" }, { url = "https://files.pythonhosted.org/packages/f4/40/1bf0324800d38009774ba087a6fbffddc76a3e5bf42c081b9298b1236fb5/uv-0.13.4-py3-none-musllinux_1_1_i686.whl", hash = "sha256:5d89d1028c2e22977d9215cbb79cfe12559cb0c8a507a12e2f3a7e47930cf1bf", size = 18010389, upload-time = "2026-10-20T15:11:48.963Z" }, { url = "https://files.pythonhosted.org/packages/1e/ec/f352ca2cc3b4b6191bb4ec8cdf4bc190eed0ba8bda4c33b7fd66c5d33cec/uv-0.13.4-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:52b5c753a7c2d5f8323a70bb4c93118988356331d0758afa6446c81f974e7196", size = 19004998, upload-time = "2026-10-20T15:11:52.628Z" }, { url = "https://files.pythonhosted.org/packages/9e/4f/fc05ff4292edfe785b47874ec86d3d83ca2bf735cff8a9d2170ffb66ad83/uv-0.13.4-py3-none-win32.whl", hash = "sha256:d333f01852b817ba05612de31474739da1af7e5463d119b30a9c4331eba1f408", size = 16190502, upload-time = "2026-10-20T15:11:56.657Z" }, { url = "https://files.pythonhosted.org/packages/2f/b9/b526eab6c72eacc705a58378f6850fde046b28c86fed01bd42da61367c6b/uv-0.13.4-py3-none-win_amd64.whl", hash = "sha256:b2ff80397273fff0e02c20eacae6882834c811827d6fc08f658db156fe754386", size = 15331833, upload-time = "2026-10-20T15:11:59.733Z" }, { url = "https://files.pythonhosted.org/packages/7c/f7/4d5fa89d3d10ff6a84e547ef6e02c45b2f2d50e66682da778acf60db63f1/uv-0.13.4-py3-none-win_arm64.whl", hash = "sha256:deffa5a4afdb48f3e44fb26e80377d256b330ddcad0373daff4b54f6162b43a4", size = 15690151, upload-time = "2026-10-20T15:12:02.74Z" }, ] [[package]] name = "verspec" version = "0.1.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e7/44/8126f9f0c44319b2efc65feaad589cadef4d77ece200ae3c9133d58464d0/verspec-0.1.0.tar.gz", hash = "sha256:c4504ca697b2056cdb4bfa7121461f5a0e81809255b41c03dda4ba823637c01e", size = 27123, upload-time = "2020-11-30T02:24:09.646Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a4/ce/3b6fee91c85626eaf769d617f1be9d2e15c1cca027bbdeb2e0d751469355/verspec-0.1.0-py3-none-any.whl", hash = "sha256:741877d5633cc9464c45a469ae2a31e801e6dbbaa85b9675d481cda100f11c31", size = 19640, upload-time = "2020-11-30T02:24:08.387Z" }, ] [[package]] name = "watchdog" version = "6.0.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220, upload-time = "2024-11-01T14:07:13.037Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/39/ea/3930d07dafc9e286ed356a679aa02d777c06e9bfd1164fa7c19c288a5483/watchdog-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdd4e6f14b8b18c334febb9c4425a878a2ac20efd1e0b231978e7b150f92a948", size = 96471, upload-time = "2024-11-01T14:06:37.745Z" }, { url = "https://files.pythonhosted.org/packages/12/87/48361531f70b1f87928b045df868a9fd4e253d9ae087fa4cf3f7113be363/watchdog-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c7c15dda13c4eb00d6fb6fc508b3c0ed88b9d5d374056b239c4ad1611125c860", size = 88449, upload-time = "2024-11-01T14:06:39.748Z" }, { url = "https://files.pythonhosted.org/packages/5b/7e/8f322f5e600812e6f9a31b75d242631068ca8f4ef0582dd3ae6e72daecc8/watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0", size = 89054, upload-time = "2024-11-01T14:06:41.009Z" }, { url = "https://files.pythonhosted.org/packages/68/98/b0345cabdce2041a01293ba483333582891a3bd5769b08eceb0d406056ef/watchdog-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:490ab2ef84f11129844c23fb14ecf30ef3d8a6abafd3754a6f75ca1e6654136c", size = 96480, upload-time = "2024-11-01T14:06:42.952Z" }, { url = "https://files.pythonhosted.org/packages/85/83/cdf13902c626b28eedef7ec4f10745c52aad8a8fe7eb04ed7b1f111ca20e/watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:76aae96b00ae814b181bb25b1b98076d5fc84e8a53cd8885a318b42b6d3a5134", size = 88451, upload-time = "2024-11-01T14:06:45.084Z" }, { url = "https://files.pythonhosted.org/packages/fe/c4/225c87bae08c8b9ec99030cd48ae9c4eca050a59bf5c2255853e18c87b50/watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a175f755fc2279e0b7312c0035d52e27211a5bc39719dd529625b1930917345b", size = 89057, upload-time = "2024-11-01T14:06:47.324Z" }, { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079, upload-time = "2024-11-01T14:06:59.472Z" }, { url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078, upload-time = "2024-11-01T14:07:01.431Z" }, { url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076, upload-time = "2024-11-01T14:07:02.568Z" }, { url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077, upload-time = "2024-11-01T14:07:03.893Z" }, { url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078, upload-time = "2024-11-01T14:07:05.189Z" }, { url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077, upload-time = "2024-11-01T14:07:06.376Z" }, { url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078, upload-time = "2024-11-01T14:07:07.547Z" }, { url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065, upload-time = "2024-11-01T14:07:09.525Z" }, { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070, upload-time = "2024-11-01T14:07:10.686Z" }, { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, ] [[package]] name = "webencodings" version = "0.5.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47/webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923", size = 9721, upload-time = "2017-04-05T20:21:34.189Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", size = 11774, upload-time = "2017-04-05T20:21:32.581Z" }, ] [[package]] name = "werkzeug" version = "3.1.9" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markupsafe" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a4/34/4dd12fc8bb7d61c91467ec3efe415ffa7d5456f799954b40c5bbaeae470e/werkzeug-3.1.9.tar.gz", hash = "sha256:55ca7c70a75689be937aa27f8ff4b018f06ff4838fc73045560bf0f5a1291060", size = 940188, upload-time = "2026-09-27T18:33:41.637Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a1/38/df03f564f43cec2684823f3cccae1a652ee7face1cbaa76fb223096e64d7/werkzeug-3.1.9-py3-none-any.whl", hash = "sha256:6392e50c78460ba618e5b21f08a71f59c99ce99cdc6cf6e3dd7e6ccca8754fab", size = 228700, upload-time = "2026-09-27T18:33:39.685Z" }, ] [[package]] name = "wrapt" version = "2.1.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/2e/64/925f213fdcbb9baeb1530449ac71a4d57fc361c053d06bf78d0c5c7cd80c/wrapt-2.1.2.tar.gz", hash = "sha256:3996a67eecc2c68fd47b4e3c564405a5777367adfd9b8abb58387b63ee83b21e", size = 81678, upload-time = "2026-03-06T02:53:25.134Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/4c/b6/1db817582c49c7fcbb7df6809d0f515af29d7c2fbf57eb44c36e98fb1492/wrapt-2.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ff2aad9c4cda28a8f0653fc2d487596458c2a3f475e56ba02909e950a9efa6a9", size = 61255, upload-time = "2026-03-06T02:52:45.663Z" }, { url = "https://files.pythonhosted.org/packages/a2/16/9b02a6b99c09227c93cd4b73acc3678114154ec38da53043c0ddc1fba0dc/wrapt-2.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6433ea84e1cfacf32021d2a4ee909554ade7fd392caa6f7c13f1f4bf7b8e8748", size = 61848, upload-time = "2026-03-06T02:53:48.728Z" }, { url = "https://files.pythonhosted.org/packages/af/aa/ead46a88f9ec3a432a4832dfedb84092fc35af2d0ba40cd04aea3889f247/wrapt-2.1.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c20b757c268d30d6215916a5fa8461048d023865d888e437fab451139cad6c8e", size = 121433, upload-time = "2026-03-06T02:54:40.328Z" }, { url = "https://files.pythonhosted.org/packages/3a/9f/742c7c7cdf58b59085a1ee4b6c37b013f66ac33673a7ef4aaed5e992bc33/wrapt-2.1.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79847b83eb38e70d93dc392c7c5b587efe65b3e7afcc167aa8abd5d60e8761c8", size = 123013, upload-time = "2026-03-06T02:53:26.58Z" }, { url = "https://files.pythonhosted.org/packages/e8/44/2c3dd45d53236b7ed7c646fcf212251dc19e48e599debd3926b52310fafb/wrapt-2.1.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f8fba1bae256186a83d1875b2b1f4e2d1242e8fac0f58ec0d7e41b26967b965c", size = 117326, upload-time = "2026-03-06T02:53:11.547Z" }, { url = "https://files.pythonhosted.org/packages/74/e2/b17d66abc26bd96f89dec0ecd0ef03da4a1286e6ff793839ec431b9fae57/wrapt-2.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e3d3b35eedcf5f7d022291ecd7533321c4775f7b9cd0050a31a68499ba45757c", size = 121444, upload-time = "2026-03-06T02:54:09.5Z" }, { url = "https://files.pythonhosted.org/packages/3c/62/e2977843fdf9f03daf1586a0ff49060b1b2fc7ff85a7ea82b6217c1ae36e/wrapt-2.1.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6f2c5390460de57fa9582bc8a1b7a6c86e1a41dfad74c5225fc07044c15cc8d1", size = 116237, upload-time = "2026-03-06T02:54:03.884Z" }, { url = "https://files.pythonhosted.org/packages/88/dd/27fc67914e68d740bce512f11734aec08696e6b17641fef8867c00c949fc/wrapt-2.1.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7dfa9f2cf65d027b951d05c662cc99ee3bd01f6e4691ed39848a7a5fffc902b2", size = 120563, upload-time = "2026-03-06T02:53:20.412Z" }, { url = "https://files.pythonhosted.org/packages/ec/9f/b750b3692ed2ef4705cb305bd68858e73010492b80e43d2a4faa5573cbe7/wrapt-2.1.2-cp312-cp312-win32.whl", hash = "sha256:eba8155747eb2cae4a0b913d9ebd12a1db4d860fc4c829d7578c7b989bd3f2f0", size = 58198, upload-time = "2026-03-06T02:53:37.732Z" }, { url = "https://files.pythonhosted.org/packages/8e/b2/feecfe29f28483d888d76a48f03c4c4d8afea944dbee2b0cd3380f9df032/wrapt-2.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:1c51c738d7d9faa0b3601708e7e2eda9bf779e1b601dce6c77411f2a1b324a63", size = 60441, upload-time = "2026-03-06T02:52:47.138Z" }, { url = "https://files.pythonhosted.org/packages/44/e1/e328f605d6e208547ea9fd120804fcdec68536ac748987a68c47c606eea8/wrapt-2.1.2-cp312-cp312-win_arm64.whl", hash = "sha256:c8e46ae8e4032792eb2f677dbd0d557170a8e5524d22acc55199f43efedd39bf", size = 58836, upload-time = "2026-03-06T02:53:22.053Z" }, { url = "https://files.pythonhosted.org/packages/4c/7a/d936840735c828b38d26a854e85d5338894cda544cb7a85a9d5b8b9c4df7/wrapt-2.1.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:787fd6f4d67befa6fe2abdffcbd3de2d82dfc6fb8a6d850407c53332709d030b", size = 61259, upload-time = "2026-03-06T02:53:41.922Z" }, { url = "https://files.pythonhosted.org/packages/5e/88/9a9b9a90ac8ca11c2fdb6a286cb3a1fc7dd774c00ed70929a6434f6bc634/wrapt-2.1.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4bdf26e03e6d0da3f0e9422fd36bcebf7bc0eeb55fdf9c727a09abc6b9fe472e", size = 61851, upload-time = "2026-03-06T02:52:48.672Z" }, { url = "https://files.pythonhosted.org/packages/03/a9/5b7d6a16fd6533fed2756900fc8fc923f678179aea62ada6d65c92718c00/wrapt-2.1.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bbac24d879aa22998e87f6b3f481a5216311e7d53c7db87f189a7a0266dafffb", size = 121446, upload-time = "2026-03-06T02:54:14.013Z" }, { url = "https://files.pythonhosted.org/packages/45/bb/34c443690c847835cfe9f892be78c533d4f32366ad2888972c094a897e39/wrapt-2.1.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16997dfb9d67addc2e3f41b62a104341e80cac52f91110dece393923c0ebd5ca", size = 123056, upload-time = "2026-03-06T02:54:10.829Z" }, { url = "https://files.pythonhosted.org/packages/93/b9/ff205f391cb708f67f41ea148545f2b53ff543a7ac293b30d178af4d2271/wrapt-2.1.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:162e4e2ba7542da9027821cb6e7c5e068d64f9a10b5f15512ea28e954893a267", size = 117359, upload-time = "2026-03-06T02:53:03.623Z" }, { url = "https://files.pythonhosted.org/packages/1f/3d/1ea04d7747825119c3c9a5e0874a40b33594ada92e5649347c457d982805/wrapt-2.1.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f29c827a8d9936ac320746747a016c4bc66ef639f5cd0d32df24f5eacbf9c69f", size = 121479, upload-time = "2026-03-06T02:53:45.844Z" }, { url = "https://files.pythonhosted.org/packages/78/cc/ee3a011920c7a023b25e8df26f306b2484a531ab84ca5c96260a73de76c0/wrapt-2.1.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:a9dd9813825f7ecb018c17fd147a01845eb330254dff86d3b5816f20f4d6aaf8", size = 116271, upload-time = "2026-03-06T02:54:46.356Z" }, { url = "https://files.pythonhosted.org/packages/98/fd/e5ff7ded41b76d802cf1191288473e850d24ba2e39a6ec540f21ae3b57cb/wrapt-2.1.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6f8dbdd3719e534860d6a78526aafc220e0241f981367018c2875178cf83a413", size = 120573, upload-time = "2026-03-06T02:52:50.163Z" }, { url = "https://files.pythonhosted.org/packages/47/c5/242cae3b5b080cd09bacef0591691ba1879739050cc7c801ff35c8886b66/wrapt-2.1.2-cp313-cp313-win32.whl", hash = "sha256:5c35b5d82b16a3bc6e0a04349b606a0582bc29f573786aebe98e0c159bc48db6", size = 58205, upload-time = "2026-03-06T02:53:47.494Z" }, { url = "https://files.pythonhosted.org/packages/12/69/c358c61e7a50f290958809b3c61ebe8b3838ea3e070d7aac9814f95a0528/wrapt-2.1.2-cp313-cp313-win_amd64.whl", hash = "sha256:f8bc1c264d8d1cf5b3560a87bbdd31131573eb25f9f9447bb6252b8d4c44a3a1", size = 60452, upload-time = "2026-03-06T02:53:30.038Z" }, { url = "https://files.pythonhosted.org/packages/8e/66/c8a6fcfe321295fd8c0ab1bd685b5a01462a9b3aa2f597254462fc2bc975/wrapt-2.1.2-cp313-cp313-win_arm64.whl", hash = "sha256:3beb22f674550d5634642c645aba4c72a2c66fb185ae1aebe1e955fae5a13baf", size = 58842, upload-time = "2026-03-06T02:52:52.114Z" }, { url = "https://files.pythonhosted.org/packages/da/55/9c7052c349106e0b3f17ae8db4b23a691a963c334de7f9dbd60f8f74a831/wrapt-2.1.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0fc04bc8664a8bc4c8e00b37b5355cffca2535209fba1abb09ae2b7c76ddf82b", size = 63075, upload-time = "2026-03-06T02:53:19.108Z" }, { url = "https://files.pythonhosted.org/packages/09/a8/ce7b4006f7218248dd71b7b2b732d0710845a0e49213b18faef64811ffef/wrapt-2.1.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a9b9d50c9af998875a1482a038eb05755dfd6fe303a313f6a940bb53a83c3f18", size = 63719, upload-time = "2026-03-06T02:54:33.452Z" }, { url = "https://files.pythonhosted.org/packages/e4/e5/2ca472e80b9e2b7a17f106bb8f9df1db11e62101652ce210f66935c6af67/wrapt-2.1.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2d3ff4f0024dd224290c0eabf0240f1bfc1f26363431505fb1b0283d3b08f11d", size = 152643, upload-time = "2026-03-06T02:52:42.721Z" }, { url = "https://files.pythonhosted.org/packages/36/42/30f0f2cefca9d9cbf6835f544d825064570203c3e70aa873d8ae12e23791/wrapt-2.1.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3278c471f4468ad544a691b31bb856374fbdefb7fee1a152153e64019379f015", size = 158805, upload-time = "2026-03-06T02:54:25.441Z" }, { url = "https://files.pythonhosted.org/packages/bb/67/d08672f801f604889dcf58f1a0b424fe3808860ede9e03affc1876b295af/wrapt-2.1.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a8914c754d3134a3032601c6984db1c576e6abaf3fc68094bb8ab1379d75ff92", size = 145990, upload-time = "2026-03-06T02:53:57.456Z" }, { url = "https://files.pythonhosted.org/packages/68/a7/fd371b02e73babec1de6ade596e8cd9691051058cfdadbfd62a5898f3295/wrapt-2.1.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ff95d4264e55839be37bafe1536db2ab2de19da6b65f9244f01f332b5286cfbf", size = 155670, upload-time = "2026-03-06T02:54:55.309Z" }, { url = "https://files.pythonhosted.org/packages/86/2d/9fe0095dfdb621009f40117dcebf41d7396c2c22dca6eac779f4c007b86c/wrapt-2.1.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:76405518ca4e1b76fbb1b9f686cff93aebae03920cc55ceeec48ff9f719c5f67", size = 144357, upload-time = "2026-03-06T02:54:24.092Z" }, { url = "https://files.pythonhosted.org/packages/0e/b6/ec7b4a254abbe4cde9fa15c5d2cca4518f6b07d0f1b77d4ee9655e30280e/wrapt-2.1.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c0be8b5a74c5824e9359b53e7e58bef71a729bacc82e16587db1c4ebc91f7c5a", size = 150269, upload-time = "2026-03-06T02:53:31.268Z" }, { url = "https://files.pythonhosted.org/packages/6e/6b/2fabe8ebf148f4ee3c782aae86a795cc68ffe7d432ef550f234025ce0cfa/wrapt-2.1.2-cp313-cp313t-win32.whl", hash = "sha256:f01277d9a5fc1862f26f7626da9cf443bebc0abd2f303f41c5e995b15887dabd", size = 59894, upload-time = "2026-03-06T02:54:15.391Z" }, { url = "https://files.pythonhosted.org/packages/ca/fb/9ba66fc2dedc936de5f8073c0217b5d4484e966d87723415cc8262c5d9c2/wrapt-2.1.2-cp313-cp313t-win_amd64.whl", hash = "sha256:84ce8f1c2104d2f6daa912b1b5b039f331febfeee74f8042ad4e04992bd95c8f", size = 63197, upload-time = "2026-03-06T02:54:41.943Z" }, { url = "https://files.pythonhosted.org/packages/c0/1c/012d7423c95d0e337117723eb8ecf73c622ce15a97847e84cf3f8f26cd7e/wrapt-2.1.2-cp313-cp313t-win_arm64.whl", hash = "sha256:a93cd767e37faeddbe07d8fc4212d5cba660af59bdb0f6372c93faaa13e6e679", size = 60363, upload-time = "2026-03-06T02:54:48.093Z" }, { url = "https://files.pythonhosted.org/packages/39/25/e7ea0b417db02bb796182a5316398a75792cd9a22528783d868755e1f669/wrapt-2.1.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1370e516598854e5b4366e09ce81e08bfe94d42b0fd569b88ec46cc56d9164a9", size = 61418, upload-time = "2026-03-06T02:53:55.706Z" }, { url = "https://files.pythonhosted.org/packages/ec/0f/fa539e2f6a770249907757eaeb9a5ff4deb41c026f8466c1c6d799088a9b/wrapt-2.1.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6de1a3851c27e0bd6a04ca993ea6f80fc53e6c742ee1601f486c08e9f9b900a9", size = 61914, upload-time = "2026-03-06T02:52:53.37Z" }, { url = "https://files.pythonhosted.org/packages/53/37/02af1867f5b1441aaeda9c82deed061b7cd1372572ddcd717f6df90b5e93/wrapt-2.1.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:de9f1a2bbc5ac7f6012ec24525bdd444765a2ff64b5985ac6e0692144838542e", size = 120417, upload-time = "2026-03-06T02:54:30.74Z" }, { url = "https://files.pythonhosted.org/packages/c3/b7/0138a6238c8ba7476c77cf786a807f871672b37f37a422970342308276e7/wrapt-2.1.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:970d57ed83fa040d8b20c52fe74a6ae7e3775ae8cff5efd6a81e06b19078484c", size = 122797, upload-time = "2026-03-06T02:54:51.539Z" }, { url = "https://files.pythonhosted.org/packages/e1/ad/819ae558036d6a15b7ed290d5b14e209ca795dd4da9c58e50c067d5927b0/wrapt-2.1.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3969c56e4563c375861c8df14fa55146e81ac11c8db49ea6fb7f2ba58bc1ff9a", size = 117350, upload-time = "2026-03-06T02:54:37.651Z" }, { url = "https://files.pythonhosted.org/packages/8b/2d/afc18dc57a4600a6e594f77a9ae09db54f55ba455440a54886694a84c71b/wrapt-2.1.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:57d7c0c980abdc5f1d98b11a2aa3bb159790add80258c717fa49a99921456d90", size = 121223, upload-time = "2026-03-06T02:54:35.221Z" }, { url = "https://files.pythonhosted.org/packages/b9/5b/5ec189b22205697bc56eb3b62aed87a1e0423e9c8285d0781c7a83170d15/wrapt-2.1.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:776867878e83130c7a04237010463372e877c1c994d449ca6aaafeab6aab2586", size = 116287, upload-time = "2026-03-06T02:54:19.654Z" }, { url = "https://files.pythonhosted.org/packages/f7/2d/f84939a7c9b5e6cdd8a8d0f6a26cabf36a0f7e468b967720e8b0cd2bdf69/wrapt-2.1.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:fab036efe5464ec3291411fabb80a7a39e2dd80bae9bcbeeca5087fdfa891e19", size = 119593, upload-time = "2026-03-06T02:54:16.697Z" }, { url = "https://files.pythonhosted.org/packages/0b/fe/ccd22a1263159c4ac811ab9374c061bcb4a702773f6e06e38de5f81a1bdc/wrapt-2.1.2-cp314-cp314-win32.whl", hash = "sha256:e6ed62c82ddf58d001096ae84ce7f833db97ae2263bff31c9b336ba8cfe3f508", size = 58631, upload-time = "2026-03-06T02:53:06.498Z" }, { url = "https://files.pythonhosted.org/packages/65/0a/6bd83be7bff2e7efaac7b4ac9748da9d75a34634bbbbc8ad077d527146df/wrapt-2.1.2-cp314-cp314-win_amd64.whl", hash = "sha256:467e7c76315390331c67073073d00662015bb730c566820c9ca9b54e4d67fd04", size = 60875, upload-time = "2026-03-06T02:53:50.252Z" }, { url = "https://files.pythonhosted.org/packages/6c/c0/0b3056397fe02ff80e5a5d72d627c11eb885d1ca78e71b1a5c1e8c7d45de/wrapt-2.1.2-cp314-cp314-win_arm64.whl", hash = "sha256:da1f00a557c66225d53b095a97eace0fc5349e3bfda28fa34ffae238978ee575", size = 59164, upload-time = "2026-03-06T02:53:59.128Z" }, { url = "https://files.pythonhosted.org/packages/71/ed/5d89c798741993b2371396eb9d4634f009ff1ad8a6c78d366fe2883ea7a6/wrapt-2.1.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:62503ffbc2d3a69891cf29beeaccdb4d5e0a126e2b6a851688d4777e01428dbb", size = 63163, upload-time = "2026-03-06T02:52:54.873Z" }, { url = "https://files.pythonhosted.org/packages/c6/8c/05d277d182bf36b0a13d6bd393ed1dec3468a25b59d01fba2dd70fe4d6ae/wrapt-2.1.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c7e6cd120ef837d5b6f860a6ea3745f8763805c418bb2f12eeb1fa6e25f22d22", size = 63723, upload-time = "2026-03-06T02:52:56.374Z" }, { url = "https://files.pythonhosted.org/packages/f4/27/6c51ec1eff4413c57e72d6106bb8dec6f0c7cdba6503d78f0fa98767bcc9/wrapt-2.1.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3769a77df8e756d65fbc050333f423c01ae012b4f6731aaf70cf2bef61b34596", size = 152652, upload-time = "2026-03-06T02:53:23.79Z" }, { url = "https://files.pythonhosted.org/packages/db/4c/d7dd662d6963fc7335bfe29d512b02b71cdfa23eeca7ab3ac74a67505deb/wrapt-2.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a76d61a2e851996150ba0f80582dd92a870643fa481f3b3846f229de88caf044", size = 158807, upload-time = "2026-03-06T02:53:35.742Z" }, { url = "https://files.pythonhosted.org/packages/b4/4d/1e5eea1a78d539d346765727422976676615814029522c76b87a95f6bcdd/wrapt-2.1.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6f97edc9842cf215312b75fe737ee7c8adda75a89979f8e11558dfff6343cc4b", size = 146061, upload-time = "2026-03-06T02:52:57.574Z" }, { url = "https://files.pythonhosted.org/packages/89/bc/62cabea7695cd12a288023251eeefdcb8465056ddaab6227cb78a2de005b/wrapt-2.1.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4006c351de6d5007aa33a551f600404ba44228a89e833d2fadc5caa5de8edfbf", size = 155667, upload-time = "2026-03-06T02:53:39.422Z" }, { url = "https://files.pythonhosted.org/packages/e9/99/6f2888cd68588f24df3a76572c69c2de28287acb9e1972bf0c83ce97dbc1/wrapt-2.1.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:a9372fc3639a878c8e7d87e1556fa209091b0a66e912c611e3f833e2c4202be2", size = 144392, upload-time = "2026-03-06T02:54:22.41Z" }, { url = "https://files.pythonhosted.org/packages/40/51/1dfc783a6c57971614c48e361a82ca3b6da9055879952587bc99fe1a7171/wrapt-2.1.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3144b027ff30cbd2fca07c0a87e67011adb717eb5f5bd8496325c17e454257a3", size = 150296, upload-time = "2026-03-06T02:54:07.848Z" }, { url = "https://files.pythonhosted.org/packages/6c/38/cbb8b933a0201076c1f64fc42883b0023002bdc14a4964219154e6ff3350/wrapt-2.1.2-cp314-cp314t-win32.whl", hash = "sha256:3b8d15e52e195813efe5db8cec156eebe339aaf84222f4f4f051a6c01f237ed7", size = 60539, upload-time = "2026-03-06T02:54:00.594Z" }, { url = "https://files.pythonhosted.org/packages/82/dd/e5176e4b241c9f528402cebb238a36785a628179d7d8b71091154b3e4c9e/wrapt-2.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:08ffa54146a7559f5b8df4b289b46d963a8e74ed16ba3687f99896101a3990c5", size = 63969, upload-time = "2026-03-06T02:54:39Z" }, { url = "https://files.pythonhosted.org/packages/5c/99/79f17046cf67e4a95b9987ea129632ba8bcec0bc81f3fb3d19bdb0bd60cd/wrapt-2.1.2-cp314-cp314t-win_arm64.whl", hash = "sha256:72aaa9d0d8e4ed0e2e98019cea47a21f823c9dd4b43c7b77bba6679ffcca6a00", size = 60554, upload-time = "2026-03-06T02:53:14.132Z" }, { url = "https://files.pythonhosted.org/packages/1a/c7/8528ac2dfa2c1e6708f647df7ae144ead13f0a31146f43c7264b4942bf12/wrapt-2.1.2-py3-none-any.whl", hash = "sha256:b8fd6fa2b2c4e7621808f8c62e8317f4aae56e59721ad933bac5239d913cf0e8", size = 43993, upload-time = "2026-03-06T02:53:12.905Z" }, ] [[package]] name = "xmltodict" version = "1.0.4" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/19/70/80f3b7c10d2630aa66414bf23d210386700aa390547278c789afa994fd7e/xmltodict-1.0.4.tar.gz", hash = "sha256:6d94c9f834dd9e44514162799d344d815a3a4faec913717a9ecbfa5be1bb8e61", size = 26124, upload-time = "2026-02-22T02:21:22.074Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/38/34/98a2f52245f4d47be93b580dae5f9861ef58977d73a79eb47c58f1ad1f3a/xmltodict-1.0.4-py3-none-any.whl", hash = "sha256:a4a00d300b0e1c59fc2bfccb53d7b2e88c32f200df138a0dd2229f842497026a", size = 13580, upload-time = "2026-02-22T02:21:21.039Z" }, ] [[package]] name = "yarl" version = "1.25.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "idna" }, { name = "multidict" }, { name = "propcache" }, ] sdist = { url = "https://files.pythonhosted.org/packages/75/16/e8be8e2fb175bbf41a0680381a319f1199fae256588241a2ac8677eafb49/yarl-1.25.1.tar.gz", hash = "sha256:03dd38de09bc213e9a8b29761eec33ee1d5318dac0e49d8af36e4d27830e23a7", size = 246245, upload-time = "2026-09-15T19:35:02.264Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/75/b3/cd32ac66ae622b854c2df0ac52106dda220d361b65a64fde7d5b3684aa3f/yarl-1.25.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:94d7aa6debf92a1dd14cb5280b083a764169a13cfb23a452111160274ed989f4", size = 144798, upload-time = "2026-09-15T19:31:01.821Z" }, { url = "https://files.pythonhosted.org/packages/61/fb/a2c52a8007c2051ba74662afb112ecf3d00346af4c25e33df9d80fd14fb8/yarl-1.25.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:83d4a37e4b95da4d8bda930d6d35b75b4cdadbacbb4980cae290ea3100b5d51d", size = 104583, upload-time = "2026-09-15T19:31:04.05Z" }, { url = "https://files.pythonhosted.org/packages/be/dd/ee38aec8e09fdf957e50d4085453fbe202f56c6c3b4cf07b81cdb4f09ee9/yarl-1.25.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e029648f9c951db30e98a7d7ec90835db88ec4b32820efe2a9bdc2287e032eb6", size = 104325, upload-time = "2026-09-15T19:31:06.338Z" }, { url = "https://files.pythonhosted.org/packages/1e/b3/058dbfb1857b484c9cf9cc135659f50b85ce66e03c99e44dc2f7b6161f55/yarl-1.25.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d781294bb815ecb5ea57ff6bbf8038e0a31a95fdf3e1788f66e0dc100d64b58", size = 115358, upload-time = "2026-09-15T19:31:08.593Z" }, { url = "https://files.pythonhosted.org/packages/db/39/29693446cf0cf6b15a0e2f75a5d40f93c56819b05b0622196f45e95b5cc0/yarl-1.25.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e12c538e00e7c1b286a07061046b90e8124e6a9793efae2c70db6a4aad07faad", size = 107658, upload-time = "2026-09-15T19:31:10.802Z" }, { url = "https://files.pythonhosted.org/packages/86/b3/3c4dd7e1af43b931fba95e0a722737f2ea94a6d199c802585282831d7abd/yarl-1.25.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7e4de3ac4adbad3d0bc7c6f4360a7dbff5de2f15e3b723be3198074e17fd9c40", size = 122660, upload-time = "2026-09-15T19:31:12.84Z" }, { url = "https://files.pythonhosted.org/packages/bd/b5/1b60dbc3cfc9c5712b15148c206748f2bc93953ffdbe25ea75b63dfc89c9/yarl-1.25.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:419f392a1da624877975709e3864dfe833af6cc7671b39318086d456e288380c", size = 126506, upload-time = "2026-09-15T19:31:15.088Z" }, { url = "https://files.pythonhosted.org/packages/bc/7b/ca212cbe170ac8b96e45317ecbcf9c3c3ecf0cdec98d5b088a9c4088929b/yarl-1.25.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6f117789d22dce188e5754e8bc65b7e6ebf8cb73963b9fa761f672a5883769d", size = 117050, upload-time = "2026-09-15T19:31:17.241Z" }, { url = "https://files.pythonhosted.org/packages/cb/c3/72b4938cdbe619ad71ac156182faef4908846b84dc3ca4dbb4c4e6f84014/yarl-1.25.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:80e47012e730da131c9f059c80936783f9659aae22dc31c03c0595590d11ed54", size = 114174, upload-time = "2026-09-15T19:31:19.294Z" }, { url = "https://files.pythonhosted.org/packages/e8/43/268717870f9ba0cc9701a95181587f6dc8c5f387aab4aeecc83158f38a79/yarl-1.25.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e80f557716fd765439577131e526b8942ffc2c07bdbc5e39fa62f660ba1e963f", size = 114944, upload-time = "2026-09-15T19:31:21.414Z" }, { url = "https://files.pythonhosted.org/packages/da/84/baa5bf504d51fe062c4bcaf62936da97fffb43285978d0b39984824231fd/yarl-1.25.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f61964f235a43738bfac50da46fc4254943a7eea3051aeb0b6fc7c992c29fadc", size = 108263, upload-time = "2026-09-15T19:31:23.388Z" }, { url = "https://files.pythonhosted.org/packages/a4/28/779a2ed9e0152a601a27039bed9aead3f0b79797a67e2c44bfa444622dd8/yarl-1.25.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e546fe1d4a93ebc2910f0d768baff19faa09843ab3f2036a67ed6e69fae4419d", size = 122184, upload-time = "2026-09-15T19:31:25.343Z" }, { url = "https://files.pythonhosted.org/packages/f8/1f/118e9e5b8f07694d63fd3222e801d7782270003f1a222aa798df3f8d5933/yarl-1.25.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:cce0727fd5ac04d372fa9bbfde9febc2bcf209aadfcf0468e45dec72719895d1", size = 114001, upload-time = "2026-09-15T19:31:27.465Z" }, { url = "https://files.pythonhosted.org/packages/0f/ae/a4cf1cf372313734b17996d4007f9f73596e7a178b9485802e5494ecf484/yarl-1.25.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af4ea5b37403ef4e30f3927eaed540db942bde01d8d3ff083527c0704d1c9c68", size = 120565, upload-time = "2026-09-15T19:31:29.47Z" }, { url = "https://files.pythonhosted.org/packages/05/79/ad94f93ca731bc9e44d321833ab96b82a4f9f5f63cf773f81a4aeea5ecc1/yarl-1.25.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:68782fdb4027b8d1eee25ec35e9a6db05e863b899eb0310b3a33b6c3fef55707", size = 117060, upload-time = "2026-09-15T19:31:31.367Z" }, { url = "https://files.pythonhosted.org/packages/bb/cc/51a7b4abf4ac593b8e7eb3794b28e5a35ae26eed8bc04787628d215af82f/yarl-1.25.1-cp312-cp312-win_amd64.whl", hash = "sha256:7d575b54cb3863ef9bc290ea4b009999d55dc237326131e4853cf33e888fee03", size = 102593, upload-time = "2026-09-15T19:31:33.329Z" }, { url = "https://files.pythonhosted.org/packages/9d/21/0941a6b93a58b59a1ec75e5333bf06929b671309c43c0cd201c172d9c39f/yarl-1.25.1-cp312-cp312-win_arm64.whl", hash = "sha256:bc3ac7bf569f6b64dad04dd7808c7872dae8a97df657856eac05e9b7e3614a85", size = 97697, upload-time = "2026-09-15T19:31:35.855Z" }, { url = "https://files.pythonhosted.org/packages/7b/ed/2f3129bbcc9a5c8ba12cc2b29d8060a3bab9c8043c456cfd4b5ca3188890/yarl-1.25.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:25868beca8b6765f8f7d0e11fe6dd7c66dd4b0793b9500286d20cc92352126a5", size = 143623, upload-time = "2026-09-15T19:31:37.966Z" }, { url = "https://files.pythonhosted.org/packages/17/e1/f1bc3390fdca352826676b531d0712736f156919090206700421d46b2c37/yarl-1.25.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:10b2fd95332f0d716d5eee3c9fb2ce8eada19082de7fee83d32e37992fd75c26", size = 104011, upload-time = "2026-09-15T19:31:40.25Z" }, { url = "https://files.pythonhosted.org/packages/a8/aa/50acc5c3e5da04172ae3c281c75405af4d2ca911e16120ab0563f4dffb66/yarl-1.25.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0f12afda4eea8c8994a76d4df1875c765194f5fbe8a9d197929ea303caee29ec", size = 103677, upload-time = "2026-09-15T19:31:42.46Z" }, { url = "https://files.pythonhosted.org/packages/30/d2/7d1e0ab9f8390e1fbcede5a6dbf70d23c96ad09b8c5567f3a514d1ddb0e2/yarl-1.25.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:14b79a30a93a3ce2e8832603fd0ab780ada281b0ba5110b519a634f2d7d7d1fc", size = 115392, upload-time = "2026-09-15T19:31:44.371Z" }, { url = "https://files.pythonhosted.org/packages/71/e1/5ba1e3a2a22139213655e760919038e8ed7e2d4a99826d0bbddb3beb96e5/yarl-1.25.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4bd6340d20ae2c7ca719b87b426e808e90743b676d05d4c26c4fb5ca71f41184", size = 107493, upload-time = "2026-09-15T19:31:46.273Z" }, { url = "https://files.pythonhosted.org/packages/f5/53/780653d5e0f73831f467cf13548912e5eec97f21dc49fc8daf21da027df4/yarl-1.25.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:126a2533570c554719ca40a1288fdee1700b6bc82e7131aa69fa85252d92e651", size = 122537, upload-time = "2026-09-15T19:31:48.654Z" }, { url = "https://files.pythonhosted.org/packages/03/92/d54fa70236c6036271c9c9c09fd978df5cbe3ef49ef6c46e9b833476d215/yarl-1.25.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a3faadac7d812ddac258feb57b9846b60c1b437c4f4b9ad42595c6f6fe4390df", size = 126170, upload-time = "2026-09-15T19:31:50.872Z" }, { url = "https://files.pythonhosted.org/packages/0e/b7/a82a49bf88340b837ef6972b508a1604ae377b9e6904b46b10cf5f1cf925/yarl-1.25.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be80550d9bfe83d9b62398a37081a90434e6df2d978ec345c3d2820de6beddab", size = 117012, upload-time = "2026-09-15T19:31:53.189Z" }, { url = "https://files.pythonhosted.org/packages/ef/78/5d684b411e3f3602464ee9b538db48205038f8605872985f61efb809ced0/yarl-1.25.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e07595c7d6f4db270ceede356a1bd1c07a34f1c26f958d1ed0cd7b48e0d2bba3", size = 114950, upload-time = "2026-09-15T19:31:55.694Z" }, { url = "https://files.pythonhosted.org/packages/2f/11/51d82b852c64f7fad0fc7a7ff3031517204887e874c722bbca839c0b23ac/yarl-1.25.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:eb96ed1ae6c7d072d60840c0434aef07a2df611812810807fbc54263a6053e9a", size = 115428, upload-time = "2026-09-15T19:31:57.966Z" }, { url = "https://files.pythonhosted.org/packages/e4/49/9d1978049bf646b9ea918313926453c6901b71c92f097467777d47d36a88/yarl-1.25.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3feb99222553a8cbedfa52c2f59dd84c3f50d5b582c728d522caf8d72769a54b", size = 108428, upload-time = "2026-09-15T19:32:00.048Z" }, { url = "https://files.pythonhosted.org/packages/43/35/7b8f1ebb45d7ec3dda7d1909bf44f458de41ef91e2937f107733582a5166/yarl-1.25.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a2ed0ba415ccdf08f14bf544cb78346d0f76086707ffee24921a2c84dbf1305a", size = 121961, upload-time = "2026-09-15T19:32:02.436Z" }, { url = "https://files.pythonhosted.org/packages/63/d6/d8b689ab7ca26edeb85f6ff28812aac7a25376eefc1780e303a7bfbaceff/yarl-1.25.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2b49375d22299b0a834c2bca72f39aaecc270d96fb24c30424899676f487b22a", size = 114961, upload-time = "2026-09-15T19:32:04.456Z" }, { url = "https://files.pythonhosted.org/packages/cf/d5/1a1798ea4dc6b7ee3260010a27907ebc697c95dae99817d817ed446d24aa/yarl-1.25.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ef74070ac553c59eb4f04258722066d6c6135b7baa03b2e9f2da65c096e96d98", size = 120036, upload-time = "2026-09-15T19:32:06.5Z" }, { url = "https://files.pythonhosted.org/packages/91/8d/b1b35ed7903da6669b1d367cb2c09436acd4ff508029b4f39a0c0c2058fc/yarl-1.25.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0a66db89ea473abeac4b70523cafd94db3772380e565f9d28af7a179b7af71fa", size = 117276, upload-time = "2026-09-15T19:32:09.401Z" }, { url = "https://files.pythonhosted.org/packages/3a/8f/4db01cef62caff0d7a4593ed694fb8a41a27a11158cab80d290221f13e57/yarl-1.25.1-cp313-cp313-win_amd64.whl", hash = "sha256:1f51020b2eb8a003c84925638ec63c21a750a4bddd3a22ec8eac6a742dadf1b9", size = 101945, upload-time = "2026-09-15T19:32:11.545Z" }, { url = "https://files.pythonhosted.org/packages/c0/5e/3ce00497c5c0babb74d4130c10c3828ccd215b4819d12020c42429f991ac/yarl-1.25.1-cp313-cp313-win_arm64.whl", hash = "sha256:b10dd0557ba422715b5206b3743192135a6022acca8baec51aa127d0a75db8fe", size = 97270, upload-time = "2026-09-15T19:32:14.127Z" }, { url = "https://files.pythonhosted.org/packages/80/cf/54023edfab7aa773b860503db0c56e962ccab0922803ee97988c176ea090/yarl-1.25.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:a9ca696eb02e5c02a8afd872ada510eba9b7fe6e68b9572c2e9a9b1941e31e2e", size = 143975, upload-time = "2026-09-15T19:32:16.416Z" }, { url = "https://files.pythonhosted.org/packages/d7/a8/e6c1be0e6761d0f2d10bbf33a3e1e02b99dc83874d92945d7b461a72481e/yarl-1.25.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a5877f2255aab518ebe528289037699201d5dc5f045f2396cb30aa02db22f57f", size = 104018, upload-time = "2026-09-15T19:32:18.364Z" }, { url = "https://files.pythonhosted.org/packages/6e/bb/dda344765ffd3430afe1a1c66c866a57fae67786537d4f14607df6505ac1/yarl-1.25.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7a5c3115595995779ee21f2567035793911c3802a43c74f3fbb0314929ec67ac", size = 104156, upload-time = "2026-09-15T19:32:20.459Z" }, { url = "https://files.pythonhosted.org/packages/e5/5f/ed1538bcd06009fe990d6d283dd7667f639e62a81e35c6d8c6ef6c08fb3c/yarl-1.25.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77e5099b99b37f3cf79c246998ca9f7313a78054cd1809ec46bc1afad47e1c4c", size = 116025, upload-time = "2026-09-15T19:32:22.766Z" }, { url = "https://files.pythonhosted.org/packages/a2/af/2185daf56b99830d3356ecfada46faaa49945de6626e842b7728088d4980/yarl-1.25.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6efaf45df6a849cef613a03a94c845647456662f85438c886bb67a9c027c8c2c", size = 106985, upload-time = "2026-09-15T19:32:24.749Z" }, { url = "https://files.pythonhosted.org/packages/c1/65/bc1ae564fb4b04a30b6a8f250e787772581c57e4c3d5cf07ac3359de3103/yarl-1.25.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d5f90e44653c4e0f78501ed9bb7d3fce835a8d62b7c6ed0cb16557534087e743", size = 123030, upload-time = "2026-09-15T19:32:27.084Z" }, { url = "https://files.pythonhosted.org/packages/6a/3e/e2afcde10d74e53b3fa889960991efb3019beda2b1682a01de720a302056/yarl-1.25.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:632da579b2d879f6bad20f2cfa35ded1efe2f4f77f8abb26a6234a5b236acd2f", size = 126765, upload-time = "2026-09-15T19:32:29.332Z" }, { url = "https://files.pythonhosted.org/packages/a2/be/415b00c0fe5a0615b062a456b26623d7ec91c2bee20faea1a14045aa0469/yarl-1.25.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:30eec96e8a91bd588ce897c9543f6d5d8d34b28fbcba28a4dedf20ebeae9fe57", size = 117199, upload-time = "2026-09-15T19:32:31.49Z" }, { url = "https://files.pythonhosted.org/packages/97/27/3d8c63ddd3e8bcfd033748ab93876678ce59bacd66e4cb1ed851c9c5b37e/yarl-1.25.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:12b6bc4906e11f5e1a1cdcb12296e7afbd366c783cc8073403cd2fb74334e453", size = 115187, upload-time = "2026-09-15T19:32:34.137Z" }, { url = "https://files.pythonhosted.org/packages/39/b7/7a81d0be1a502a26a0d4326c6f2ecb736c824f570ea1c6529f2b0b227b50/yarl-1.25.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9d6ed3d17bccce4c05343e1ca8da13bc5c02c812a4e7282ddd05e8769322d3fc", size = 116085, upload-time = "2026-09-15T19:32:36.438Z" }, { url = "https://files.pythonhosted.org/packages/f0/69/39fff459916aa0fab42215dc47b759586fd80f94aa56dfc4a7c15ba6e0dc/yarl-1.25.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:f38a70074041d3b7e138e452799f5174198bae5bd5ab2000917badf403908c5f", size = 107996, upload-time = "2026-09-15T19:32:38.959Z" }, { url = "https://files.pythonhosted.org/packages/c0/39/80b9a55a3335590451d9ecf3eb593a8c635351f4c905ef056d7e8a8fd9e7/yarl-1.25.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ca89e4e21854ed27ec753297dde84b16c9f8e53b14a4866fb44457d643c19f8", size = 122549, upload-time = "2026-09-15T19:32:41.151Z" }, { url = "https://files.pythonhosted.org/packages/42/7d/a179c6757818bb59372a4adafd09f7f26a3b4a0f04c3ae404b544c0b0c82/yarl-1.25.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:1ab7618921a93767387a4b83776f751588f5b5ae9bb5bc96620e2e2e00bca868", size = 115107, upload-time = "2026-09-15T19:32:43.072Z" }, { url = "https://files.pythonhosted.org/packages/32/2b/a773ac867e4ab53a98ed98e5cefe3bae31e6f550252ca9d1de266f1a40c5/yarl-1.25.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0ae12ff2b805fa02c4dab838005caef735e39986322698c48588d3beacb65c62", size = 120666, upload-time = "2026-09-15T19:32:45.061Z" }, { url = "https://files.pythonhosted.org/packages/bc/41/52be6505e85b0f76b4f85b01b5de7e06a0512201abc2c95e14e099549174/yarl-1.25.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:90c30ed53546da833c700115c0064c22120d1b1560f474699fd31f22dd668233", size = 117505, upload-time = "2026-09-15T19:32:47.177Z" }, { url = "https://files.pythonhosted.org/packages/f5/01/349c0386caedbbe488d519f252df54efac8a1459282d466c474bdd84a620/yarl-1.25.1-cp314-cp314-win_amd64.whl", hash = "sha256:acfa7e22aa6c6e7a5996a41d275bfa01efa7ea56ab890590280e9063e2cf5c1b", size = 103446, upload-time = "2026-09-15T19:32:49.615Z" }, { url = "https://files.pythonhosted.org/packages/5c/f0/8ec63180f77912f0dc4e5a42760cb8c08d20da1d5ace3578a01b84d1f3d8/yarl-1.25.1-cp314-cp314-win_arm64.whl", hash = "sha256:8e7d98cdbb6d71e726f7d525952867096053d1f290dd4e3c50d7d313a136f414", size = 99159, upload-time = "2026-09-15T19:32:51.686Z" }, { url = "https://files.pythonhosted.org/packages/47/7d/92d2220d6886b70ab1ed8579533ac2af2dfac716d5d929001daff7986df9/yarl-1.25.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d21f0fa80a02d05299207eeaafef345d812ace96d5306e4ef265e1d419a615fa", size = 150071, upload-time = "2026-09-15T19:32:53.911Z" }, { url = "https://files.pythonhosted.org/packages/64/fc/b245e448124bcda9340df38e3553fa222b50260fca027a84095e9bd8642d/yarl-1.25.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:17c9877a89fb6e2bca6f9087eb24cd7fb434653946ef5075e470d23d49b52287", size = 106780, upload-time = "2026-09-15T19:32:56.443Z" }, { url = "https://files.pythonhosted.org/packages/51/e2/9a6ce2e334ebf218a30335ae76fb1696459430d42f733b8cb0d7d65b84d3/yarl-1.25.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:29273edf1530e397bd07cb784db1fbe0d2590b77569f2e24679a9c0a2d763b94", size = 107361, upload-time = "2026-09-15T19:32:58.827Z" }, { url = "https://files.pythonhosted.org/packages/ed/70/66e8c76b569b450d16e190f15071c916c3df70b0e33927e415ac497cf0c2/yarl-1.25.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b7abffdf37af1cec6a2ad69b827aa84320db5894791bc8ed932dc93fb274b7e9", size = 114396, upload-time = "2026-09-15T19:33:02.24Z" }, { url = "https://files.pythonhosted.org/packages/73/23/0d82838a05c57fdc05bc8b66e8c92dcc0df15e27463a5f163142d521c682/yarl-1.25.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2239a02249d9326655419e0168a28ca9008938eaab31dc29fc875c217927a6c0", size = 104882, upload-time = "2026-09-15T19:33:04.494Z" }, { url = "https://files.pythonhosted.org/packages/86/d4/ea08615c4edaa6049a13a2f1128944d068d1893abda7d708d4d7ea01599a/yarl-1.25.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:664ec6a520b74a1df2810666eb67695fcb77fa663e6ea0a25aaf2e529cb24dfa", size = 119485, upload-time = "2026-09-15T19:33:06.583Z" }, { url = "https://files.pythonhosted.org/packages/1a/82/0898bdce9b1ae403b308b9c733d0d24af4a3464270c2c081f457b16c3e0d/yarl-1.25.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4f1c91f5a5980a937ff8e238e98e6897e1ad74a4b1e2c0d68c73b5ffbb3f5c0b", size = 122490, upload-time = "2026-09-15T19:33:08.653Z" }, { url = "https://files.pythonhosted.org/packages/d1/38/97d79b81c342b78246cfedb74809e68841f3198d21653e10d3232bd9c622/yarl-1.25.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c88edaec8c349ad4c5ad4c486a3defcc4b80ceb2f074436ffa0a87caf5e76a6", size = 115336, upload-time = "2026-09-15T19:33:11.056Z" }, { url = "https://files.pythonhosted.org/packages/8e/9d/2577896554cd310dc470adb6da0b7dd0b435cb63e2565204a7ac240e504c/yarl-1.25.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:35dcbea443fafb3eece757ad4e514560ddeb6c34cfae1582c620d7b293d7feee", size = 111825, upload-time = "2026-09-15T19:33:13.204Z" }, { url = "https://files.pythonhosted.org/packages/29/6b/7ac49d8ba84a5c4bd73415a4c949d22c749cb3762579b3d50e48019a78aa/yarl-1.25.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:882569ff613758cac762a457a5d72d6e211b28d4bcfea89d1d71ea942b02eac0", size = 114655, upload-time = "2026-09-15T19:33:15.553Z" }, { url = "https://files.pythonhosted.org/packages/e5/18/e5942a16723f5b72f9b1297fd5a85a54f6300cd15c0dcb5005b90cd89156/yarl-1.25.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d0f1489233a254bb3643d2f05de7d59019254d81daeca6b9162fe9edef57e0c7", size = 106395, upload-time = "2026-09-15T19:33:17.599Z" }, { url = "https://files.pythonhosted.org/packages/7b/2d/549fa46240781513ebc47ae7eb418df428a163a2a3d644cc9cbb3ecb7846/yarl-1.25.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f41753a76f4f63927d03a0d8ba8f5ce0f2083bec29a8cfaccc55371b1564b96b", size = 119277, upload-time = "2026-09-15T19:33:19.973Z" }, { url = "https://files.pythonhosted.org/packages/76/16/4763f78dcdc0b3b9fb3842b04afe72b9320857c6a69300c62a0eab03d119/yarl-1.25.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8fb0eb4955adf0579001581f2f71a126e8781ba61bcd120f127b0401163c6c2d", size = 112504, upload-time = "2026-09-15T19:33:22.464Z" }, { url = "https://files.pythonhosted.org/packages/ae/b4/974e3edfe0d188393ce1cb9de400111c63fe61f4eb3b772a500d84c970d1/yarl-1.25.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a1e32763e641a1566507d90a8d3b19bfc3cc04a9d4e5ae3e32189874ed4b58a3", size = 116243, upload-time = "2026-09-15T19:33:24.788Z" }, { url = "https://files.pythonhosted.org/packages/b0/aa/157b940428da80c104ca09666a740e51c94963df65d5b112e06b52e4d7a8/yarl-1.25.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65b5b2066651b7432d389e9799d979c703bcc6ef44266bb8153ef54e91e4aab3", size = 115822, upload-time = "2026-09-15T19:33:26.886Z" }, { url = "https://files.pythonhosted.org/packages/7e/af/19fbdce41412e1b96825544cc52cd7029d3724655d0988237972f078bd29/yarl-1.25.1-cp314-cp314t-win_amd64.whl", hash = "sha256:734f6e5400352ac4254456003d462866c684703570929cff7a7bde015d0cb371", size = 107386, upload-time = "2026-09-15T19:33:29.009Z" }, { url = "https://files.pythonhosted.org/packages/2a/99/f6431c8968e89be608d74b28ae2d024521b2953f27dd44e0dece5e04f67a/yarl-1.25.1-cp314-cp314t-win_arm64.whl", hash = "sha256:287e99ff5aa4dc1c7630bfc683ded6f106d756c99dec432a2d7f197a784f51c6", size = 102094, upload-time = "2026-09-15T19:33:31.151Z" }, { url = "https://files.pythonhosted.org/packages/c7/3b/4f51eab40c2eabea6c3d5b121dff4b8988dc35087732ffede12d2be8b8dd/yarl-1.25.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:9b1bdaae98bc016825dd3c9d8ee1832f829b3341f9cc6ebd1a1b0a7fef7367cc", size = 143875, upload-time = "2026-09-15T19:33:33.52Z" }, { url = "https://files.pythonhosted.org/packages/41/05/bbd58fc063f5f299a883f810760b265ca26c8167c91cb9a494d0fe2387e1/yarl-1.25.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:e7011b8fb8c4054bf0c12e5edc6cd83778b0028e99ce59b18586ed036f92cfdc", size = 104028, upload-time = "2026-09-15T19:33:36.22Z" }, { url = "https://files.pythonhosted.org/packages/12/ee/2fba0aecb52e7020e189f684148783aa0b9cfa3b3bfb0b400646eef70ad4/yarl-1.25.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:f074e8d4aa0a5798920ddb6de3d08b228c614ff3724c3e8bd7577f4bafea867b", size = 104041, upload-time = "2026-09-15T19:33:38.86Z" }, { url = "https://files.pythonhosted.org/packages/38/35/884beab53ed88c7247d1671972b5ef116f7351fe0c7e6de8c3558372cb16/yarl-1.25.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7d42e7e3ca399555578b4d617e3a6ecf13371b3743a115995fa010c7bf341459", size = 116018, upload-time = "2026-09-15T19:33:44.265Z" }, { url = "https://files.pythonhosted.org/packages/e2/cc/1a91b685afb55cc18608443ace95280e96e263a97565811732b6788d3269/yarl-1.25.1-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:aa4ed3dd308548f9e707d9caaf005d2d7f8c1e7868f858dfeb47fe76e16b391d", size = 107033, upload-time = "2026-09-15T19:33:46.45Z" }, { url = "https://files.pythonhosted.org/packages/c5/c1/58b379fcb1d68d907b7fcf75200c44321896509b2a6a74abbb4b19d864d2/yarl-1.25.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:42a66563d8cc056ee32e6191e05097a7b2b3bc302e0bc3133daf8710eb18bd26", size = 123257, upload-time = "2026-09-15T19:33:48.631Z" }, { url = "https://files.pythonhosted.org/packages/d7/2d/1fe96cf5c2aeab10095e48f38585cf5a8451fb7253234822398e52aa5336/yarl-1.25.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:98d370568f393215d605304cdb77b3d5539bd192c75b623c7304c42c8d6d8273", size = 126745, upload-time = "2026-09-15T19:33:50.999Z" }, { url = "https://files.pythonhosted.org/packages/ad/60/8674394ce43f4dadae573a1d6f451716438e9eab7d7fe8d643c673a32d85/yarl-1.25.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23bf5b403c879a54964e0feac7285688e04bb220074878d737d331522da0a5bf", size = 117255, upload-time = "2026-09-15T19:33:53.456Z" }, { url = "https://files.pythonhosted.org/packages/2f/72/0faa30e02605d56127d42bb987dcc97da3863b7bf70b9bfbf5f739c05e30/yarl-1.25.1-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d45673badd08456d0340e9364eddafe1c53a9d2896424294de4d7dd71ad3ee57", size = 115166, upload-time = "2026-09-15T19:33:55.669Z" }, { url = "https://files.pythonhosted.org/packages/8c/90/9a46eac564c437e128285c5c1d7bb385d268394e9209d84f6bf4a14471ef/yarl-1.25.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:0136d640dfa9b0523853e411430a99f8a91eca85774c6420285a33b755bc6de3", size = 116082, upload-time = "2026-09-15T19:33:57.78Z" }, { url = "https://files.pythonhosted.org/packages/ad/38/4b1a686a3758878f93d2f1ea943f5a165f3555769cd16e761cfd0efdca17/yarl-1.25.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:59ba3a6e1aa8cfe5adf4bd270fd965db21955401b7ca6f1696010c55ed4daec2", size = 108048, upload-time = "2026-09-15T19:34:00.25Z" }, { url = "https://files.pythonhosted.org/packages/da/14/f348eb967f31a58348612a2b93bd8a2ab664548e2b5e879cac7f592f7201/yarl-1.25.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:87796fedc3ba97ec14fab55acb48584276e6c1e4c1e89c422bda62c838e754a9", size = 122775, upload-time = "2026-09-15T19:34:02.455Z" }, { url = "https://files.pythonhosted.org/packages/ab/e9/4f7b79700f88cb9e8bb66f8b54f9bce1844c013a2c39fdc47112e9334c95/yarl-1.25.1-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:bd0912757081f89b107d6c00b2ff8a194401b0b87eadcf4481de2b865a8fd44f", size = 115096, upload-time = "2026-09-15T19:34:05.283Z" }, { url = "https://files.pythonhosted.org/packages/cf/37/f9cb020331997d3eb887bd28d5410ecfd3d80bf163c23d7cec490d78dade/yarl-1.25.1-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:b51c159a9794633f5e0db7ecec7b2b6e3734eca1f5d17dc989ff3552a43ff78b", size = 120655, upload-time = "2026-09-15T19:34:07.382Z" }, { url = "https://files.pythonhosted.org/packages/12/83/52fceb22891a41f168db7ec22fd1d81e06b6a0b8d9f70921bd3e785defd0/yarl-1.25.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:319e070a01db9920fb63761843f96a104c8e2b9427266731810dc1e22595b17c", size = 117488, upload-time = "2026-09-15T19:34:09.988Z" }, { url = "https://files.pythonhosted.org/packages/f5/5c/ce6c4ff1247fcbe4b33d462c23a097106d909b173fde7042bc52290466e2/yarl-1.25.1-cp315-cp315-win_amd64.whl", hash = "sha256:a2059a2d891bd156bc5184e7ab7a56e78a84dfcfdeac8c501b552533ad1c36ee", size = 103434, upload-time = "2026-09-15T19:34:12.56Z" }, { url = "https://files.pythonhosted.org/packages/3c/a1/766a906b0704fb26d52b19dc22bed48a8ba0544203b70dcf44da350e8194/yarl-1.25.1-cp315-cp315-win_arm64.whl", hash = "sha256:a78b50b4f7918a3de71105d5c0b93bbc57bb8339a4d03a9dfd449f9068e76f3d", size = 99155, upload-time = "2026-09-15T19:34:15.132Z" }, { url = "https://files.pythonhosted.org/packages/bd/d3/a1d09b32cb6ab14f66b44939f5b4255b8b9e747aef3974af1d5d80ccc2fd/yarl-1.25.1-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:b5402a340723fa7da00b5cff987ddab61276be6d11251ea71ae02bcac54890d8", size = 149284, upload-time = "2026-09-15T19:34:17.452Z" }, { url = "https://files.pythonhosted.org/packages/7f/3b/fe554d879692650bca70bfbc0df124e82e4d2bb7456f698c7756f1279a96/yarl-1.25.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:eda19ea5ee88742f47a2340816e6f2d40b53bed3ab5b69794769f36af9f35bb4", size = 106399, upload-time = "2026-09-15T19:34:21.474Z" }, { url = "https://files.pythonhosted.org/packages/e1/4b/e7af56177ac8d40094c82d7728224c0b8472157d50d362e5fb3b014b2bc8/yarl-1.25.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:75baa6cf9b6d1c52f3e111a130e202fd8cf0a5b3a066c3f73d615e885092e4ec", size = 106968, upload-time = "2026-09-15T19:34:23.619Z" }, { url = "https://files.pythonhosted.org/packages/da/4f/2df41fd738d46f23ef829ae8b4468d94bb6070038fc6dfab6165ed44fea8/yarl-1.25.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbcef5a9119ef653653132cccaf999b30a0af6f33bb0a4ba80bec30056868487", size = 114717, upload-time = "2026-09-15T19:34:25.879Z" }, { url = "https://files.pythonhosted.org/packages/69/ea/002b66df53bbd1aed1c23358ff99c9bdc744fe3f4d2740e1b2fdd7192885/yarl-1.25.1-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7efc9f082dfed77c316edffa9deb52888e1bc6789171887cc1f68e06d65465c8", size = 105198, upload-time = "2026-09-15T19:34:28.204Z" }, { url = "https://files.pythonhosted.org/packages/b1/7c/95c8bc0c8f97d71e59c94525ad60d76f5c57d3f2820f08137ca8b9f0542a/yarl-1.25.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe01645169a2112aa1d4ebc3e4c5f029c5c8f97adfc32e5d37c993b39a994d75", size = 120271, upload-time = "2026-09-15T19:34:30.5Z" }, { url = "https://files.pythonhosted.org/packages/5d/7a/6fe9da56ec77927baa669fd86c39c567ce6205bab53d581082c6744c8ae7/yarl-1.25.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1c557dfd5e3db046053a0bdc72261ade790ebe8e2c7a41b36b0ca1f14cb95f3", size = 123572, upload-time = "2026-09-15T19:34:32.73Z" }, { url = "https://files.pythonhosted.org/packages/8b/83/35f222d17fa70a14c7c74fdf112ccf5515e0c2a87082b1f9b99f7693bf57/yarl-1.25.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ce4d6ccafb33d39bd78444612d14938ead674c25702ded2ee9c54a47735d225", size = 115228, upload-time = "2026-09-15T19:34:35.344Z" }, { url = "https://files.pythonhosted.org/packages/da/6f/fbaaf619423578a7d898d0f226ae47bc1906c293865c416d83c17b828b0e/yarl-1.25.1-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:80a063f8297fc796296f00f100be520f209b23dc98f93ce8eba6ee7122598209", size = 111618, upload-time = "2026-09-15T19:34:37.656Z" }, { url = "https://files.pythonhosted.org/packages/ba/78/7383278f1b3cf8e0496bd95b3281a7b09b89217b6b428db24c6b99b3deca/yarl-1.25.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:7cb414a73e21a7ab58254926073f2930cb22f5b4314ea4260a687e2b3fd4dce3", size = 114839, upload-time = "2026-09-15T19:34:40.099Z" }, { url = "https://files.pythonhosted.org/packages/62/49/5506e5b6d29aab91bd845cc9016d88c3d3f81b81bc242b8100bdd5737825/yarl-1.25.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:85a18376073f8a39aa07be34f9fc77e2869aa72c55c441efdd2cf79a0407504d", size = 106212, upload-time = "2026-09-15T19:34:42.768Z" }, { url = "https://files.pythonhosted.org/packages/46/8a/19877c193b7c5929f4b07118c18bbe390f3fadd0f59dd98c0cd12b31fa5c/yarl-1.25.1-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:77716e245c90f058466a05e6a465bb8600f767a8f4b18b4d40f3aff958e5f73c", size = 119985, upload-time = "2026-09-15T19:34:44.976Z" }, { url = "https://files.pythonhosted.org/packages/4e/6c/0a46fbbf9ecbcbd0cc20d2193394254b9e19817f22c814aab60f99847400/yarl-1.25.1-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:1e80dcf1446e1b080b1932b0d103c464a04112f5bc31f0f983ad418172063cde", size = 112081, upload-time = "2026-09-15T19:34:47.45Z" }, { url = "https://files.pythonhosted.org/packages/ef/30/93f5d471230c74ccd06255d0842739f86551f937f9e63a5e947853c6244a/yarl-1.25.1-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:bdc8d8b8c22e9e43ac68316b5e6cf083dec537f4ec213cb4aa967b583bc3fa64", size = 116995, upload-time = "2026-09-15T19:34:49.972Z" }, { url = "https://files.pythonhosted.org/packages/2b/80/c386593035ee3f9c6c6af0847b5578f2830c674794a9d7701b744a3ebd42/yarl-1.25.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:dfbf531053a0935f2e871bcd4753f90313688772ff8c017f5ea402e315a78c1f", size = 115570, upload-time = "2026-09-15T19:34:52.628Z" }, { url = "https://files.pythonhosted.org/packages/38/02/eef443559563ef8f2e10469387b8b1e97cb5efee95b288a56da60801f7ee/yarl-1.25.1-cp315-cp315t-win_amd64.whl", hash = "sha256:b13b88747769537f3d32e89e3a735da10c0a9e35d7322928c701b5f93d3afffd", size = 106811, upload-time = "2026-09-15T19:34:54.935Z" }, { url = "https://files.pythonhosted.org/packages/88/91/41e284ca2cf5211e05dae031d126a3668aea88fa759df56e7e35c6ad25ba/yarl-1.25.1-cp315-cp315t-win_arm64.whl", hash = "sha256:783dd1467083f4d3f7722ad6a313f24c173e7571372738fcb7a6e6d1ba48df25", size = 101804, upload-time = "2026-09-15T19:34:57.231Z" }, { url = "https://files.pythonhosted.org/packages/54/22/318c7980066769c6bcd9221ed2248294f5698811da099013098c670565ed/yarl-1.25.1-py3-none-any.whl", hash = "sha256:681c758b0490f9e96b78e5fa8e8dc6e648e9185bb6eaebe73183c33ea0c445f3", size = 63617, upload-time = "2026-09-15T19:34:59.616Z" }, ] [[package]] name = "zarr" source = { editable = "." } dependencies = [ { name = "donfig" }, { name = "google-crc32c" }, { name = "msgspec" }, { name = "numcodecs" }, { name = "numpy" }, { name = "packaging" }, { name = "typing-extensions" }, ] [package.optional-dependencies] cast-value-rs = [ { name = "cast-value-rs" }, ] cli = [ { name = "typer" }, ] gpu = [ { name = "cupy-cuda12x", marker = "sys_platform != 'darwin'" }, ] optional = [ { name = "universal-pathlib" }, ] remote = [ { name = "fsspec" }, { name = "obstore" }, ] [package.dev-dependencies] dev = [ { name = "astroid" }, { name = "botocore" }, { name = "coverage" }, { name = "fsspec" }, { name = "griffe-inherited-docstrings" }, { name = "hypothesis" }, { name = "markdown-exec", extra = ["ansi"] }, { name = "mike" }, { name = "mkdocs" }, { name = "mkdocs-material", extra = ["imaging"] }, { name = "mkdocs-redirects" }, { name = "mkdocstrings" }, { name = "mkdocstrings-python" }, { name = "moto", extra = ["s3", "server"] }, { name = "mypy" }, { name = "numcodecs", extra = ["msgpack"] }, { name = "numpydoc" }, { name = "obstore" }, { name = "pytest" }, { name = "pytest-accept" }, { name = "pytest-asyncio" }, { name = "pytest-benchmark" }, { name = "pytest-codspeed" }, { name = "pytest-cov" }, { name = "pytest-reportlog" }, { name = "pytest-xdist" }, { name = "requests" }, { name = "ruff" }, { name = "s3fs" }, { name = "tomlkit" }, { name = "towncrier" }, { name = "universal-pathlib" }, { name = "uv" }, ] docs = [ { name = "astroid" }, { name = "griffe-inherited-docstrings" }, { name = "markdown-exec", extra = ["ansi"] }, { name = "mike" }, { name = "mkdocs" }, { name = "mkdocs-material", extra = ["imaging"] }, { name = "mkdocs-redirects" }, { name = "mkdocstrings" }, { name = "mkdocstrings-python" }, { name = "numcodecs", extra = ["msgpack"] }, { name = "pytest" }, { name = "ruff" }, { name = "s3fs" }, { name = "towncrier" }, ] release = [ { name = "towncrier" }, ] remote-tests = [ { name = "botocore" }, { name = "coverage" }, { name = "fsspec" }, { name = "hypothesis" }, { name = "moto", extra = ["s3", "server"] }, { name = "numpydoc" }, { name = "obstore" }, { name = "pytest" }, { name = "pytest-accept" }, { name = "pytest-asyncio" }, { name = "pytest-benchmark" }, { name = "pytest-codspeed" }, { name = "pytest-cov" }, { name = "pytest-reportlog" }, { name = "pytest-xdist" }, { name = "requests" }, { name = "s3fs" }, { name = "tomlkit" }, { name = "uv" }, ] test = [ { name = "coverage" }, { name = "hypothesis" }, { name = "numpydoc" }, { name = "pytest" }, { name = "pytest-accept" }, { name = "pytest-asyncio" }, { name = "pytest-benchmark" }, { name = "pytest-codspeed" }, { name = "pytest-cov" }, { name = "pytest-reportlog" }, { name = "pytest-xdist" }, { name = "tomlkit" }, { name = "uv" }, ] [package.metadata] requires-dist = [ { name = "cast-value-rs", marker = "extra == 'cast-value-rs'", specifier = ">=0.4.2" }, { name = "cupy-cuda12x", marker = "sys_platform != 'darwin' and extra == 'gpu'" }, { name = "donfig", specifier = ">=0.8" }, { name = "fsspec", marker = "extra == 'remote'", specifier = ">=2023.10.0" }, { name = "google-crc32c", specifier = ">=1.5" }, { name = "msgspec", specifier = ">=0.19" }, { name = "numcodecs", specifier = ">=0.16.6,!=0.17.0" }, { name = "numpy", specifier = ">=2" }, { name = "obstore", marker = "extra == 'remote'", specifier = ">=0.5.1" }, { name = "packaging", specifier = ">=22.0" }, { name = "typer", marker = "extra == 'cli'" }, { name = "typing-extensions", specifier = ">=4.14" }, { name = "universal-pathlib", marker = "extra == 'optional'" }, ] provides-extras = ["cast-value-rs", "cli", "gpu", "optional", "remote"] [package.metadata.requires-dev] dev = [ { name = "astroid", specifier = "==4.3.3" }, { name = "botocore" }, { name = "coverage", specifier = "==7.16.2" }, { name = "fsspec", specifier = ">=2023.10.0" }, { name = "griffe-inherited-docstrings", specifier = "==1.1.3" }, { name = "hypothesis", specifier = "==6.168.3" }, { name = "markdown-exec", extras = ["ansi"], specifier = "==1.12.3" }, { name = "mike", specifier = "==2.2.0" }, { name = "mkdocs", specifier = "==1.6.1" }, { name = "mkdocs-material", extras = ["imaging"], specifier = "==9.7.7" }, { name = "mkdocs-redirects", specifier = "==1.2.3" }, { name = "mkdocstrings", specifier = "==1.0.6" }, { name = "mkdocstrings-python", specifier = "==2.0.9" }, { name = "moto", extras = ["s3", "server"], specifier = "==5.2.3" }, { name = "mypy", specifier = "==2.3.1" }, { name = "numcodecs", extras = ["msgpack"] }, { name = "numpydoc", specifier = "==1.11.0" }, { name = "obstore", specifier = ">=0.5.1" }, { name = "pytest", specifier = "==9.1.1" }, { name = "pytest-accept", specifier = "==0.3.0" }, { name = "pytest-asyncio", specifier = "==1.4.0" }, { name = "pytest-benchmark", specifier = "==5.3.0" }, { name = "pytest-codspeed", specifier = "==5.0.3" }, { name = "pytest-cov", specifier = "==7.1.0" }, { name = "pytest-reportlog", specifier = "==1.0.0" }, { name = "pytest-xdist", specifier = "==3.8.0" }, { name = "requests", specifier = "==2.34.2" }, { name = "ruff", specifier = "==0.16.9" }, { name = "s3fs", specifier = ">=2023.10.0" }, { name = "tomlkit", specifier = "==0.15.1" }, { name = "towncrier", specifier = "==26.9.0" }, { name = "universal-pathlib" }, { name = "uv", specifier = "==0.13.4" }, ] docs = [ { name = "astroid", specifier = "==4.3.3" }, { name = "griffe-inherited-docstrings", specifier = "==1.1.3" }, { name = "markdown-exec", extras = ["ansi"], specifier = "==1.12.3" }, { name = "mike", specifier = "==2.2.0" }, { name = "mkdocs", specifier = "==1.6.1" }, { name = "mkdocs-material", extras = ["imaging"], specifier = "==9.7.7" }, { name = "mkdocs-redirects", specifier = "==1.2.3" }, { name = "mkdocstrings", specifier = "==1.0.6" }, { name = "mkdocstrings-python", specifier = "==2.0.9" }, { name = "numcodecs", extras = ["msgpack"] }, { name = "pytest", specifier = "==9.1.1" }, { name = "ruff", specifier = "==0.16.9" }, { name = "s3fs", specifier = ">=2023.10.0" }, { name = "towncrier", specifier = "==26.9.0" }, ] release = [{ name = "towncrier", specifier = "==26.9.0" }] remote-tests = [ { name = "botocore" }, { name = "coverage", specifier = "==7.16.2" }, { name = "fsspec", specifier = ">=2023.10.0" }, { name = "hypothesis", specifier = "==6.168.3" }, { name = "moto", extras = ["s3", "server"], specifier = "==5.2.3" }, { name = "numpydoc", specifier = "==1.11.0" }, { name = "obstore", specifier = ">=0.5.1" }, { name = "pytest", specifier = "==9.1.1" }, { name = "pytest-accept", specifier = "==0.3.0" }, { name = "pytest-asyncio", specifier = "==1.4.0" }, { name = "pytest-benchmark", specifier = "==5.3.0" }, { name = "pytest-codspeed", specifier = "==5.0.3" }, { name = "pytest-cov", specifier = "==7.1.0" }, { name = "pytest-reportlog", specifier = "==1.0.0" }, { name = "pytest-xdist", specifier = "==3.8.0" }, { name = "requests", specifier = "==2.34.2" }, { name = "s3fs", specifier = ">=2023.10.0" }, { name = "tomlkit", specifier = "==0.15.1" }, { name = "uv", specifier = "==0.13.4" }, ] test = [ { name = "coverage", specifier = "==7.16.2" }, { name = "hypothesis", specifier = "==6.168.3" }, { name = "numpydoc", specifier = "==1.11.0" }, { name = "pytest", specifier = "==9.1.1" }, { name = "pytest-accept", specifier = "==0.3.0" }, { name = "pytest-asyncio", specifier = "==1.4.0" }, { name = "pytest-benchmark", specifier = "==5.3.0" }, { name = "pytest-codspeed", specifier = "==5.0.3" }, { name = "pytest-cov", specifier = "==7.1.0" }, { name = "pytest-reportlog", specifier = "==1.0.0" }, { name = "pytest-xdist", specifier = "==3.8.0" }, { name = "tomlkit", specifier = "==0.15.1" }, { name = "uv", specifier = "==0.13.4" }, ]